gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

list self dir

Browse source 
https://github.com/chrislusf/seaweedfs/issues/3086
This commit is contained in:
Konstantin Lebedev 2022-05-30 18:09:08 +05:00
parent 26f3646961
commit 62e5e3822c
1 changed files with 76 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
weed/s3api/s3api_objects_list_handlers.go
View file

@ -135,8 +135,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket) bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)
reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir) reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir)
if strings.HasSuffix(reqDir, "/") { if strings.HasSuffix(reqDir, "/") {
// remove trailing "/" reqDir = strings.TrimSuffix(reqDir, "/")
reqDir = reqDir[:len(reqDir)-1]
} }
var contents []ListEntry var contents []ListEntry
@ -148,24 +147,52 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
// check filer // check filer
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
_, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, func(dir string, entry *filer_pb.Entry) { _, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
if entry.IsDirectory && entry.Attributes.Mime == "" { if entry.IsDirectory {
if delimiter == "/" { if delimiter == "/" {
commonPrefixes = append(commonPrefixes, PrefixEntry{ commonPrefixes = append(commonPrefixes, PrefixEntry{
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
}) })
} }
} else { return
}
storageClass := "STANDARD" storageClass := "STANDARD"
if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok { if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok {
storageClass = string(v) storageClass = string(v)
} }
entryName := entry.Name contents = append(contents, ListEntry{
if entry.IsDirectory { Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
entryName += "/" LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + filer.ETag(entry) + "\"",
Size: int64(filer.FileSize(entry)),
Owner: CanonicalUser{
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
DisplayName: entry.Attributes.UserName,
},
StorageClass: StorageClass(storageClass),
})
})
if doErr != nil {
return doErr
}
if !isTruncated {
nextMarker = ""
}
if len(contents) == 0 && maxKeys > 0 {
if strings.HasSuffix(originalPrefix, "/") && prefix == "" {
reqDir, prefix = filepath.Split(strings.TrimSuffix(reqDir, "/"))
reqDir = strings.TrimSuffix(reqDir, "/")
}
_, _, _, doErr = s3a.doListFilerEntries(client, reqDir, prefix, 1, prefix, delimiter, true, func(dir string, entry *filer_pb.Entry) {
if entry.IsDirectory && entry.Attributes.Mime != "" && entry.Name == prefix {
storageClass := "STANDARD"
if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok {
storageClass = string(v)
} }
contents = append(contents, ListEntry{ contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s/%s", dir, entryName)[len(bucketPrefix):], Key: fmt.Sprintf("%s/%s", dir, entry.Name+"/")[len(bucketPrefix):],
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
ETag: "\"" + filer.ETag(entry) + "\"", ETag: "\"" + filer.ETag(entry) + "\"",
Size: int64(filer.FileSize(entry)), Size: int64(filer.FileSize(entry)),
@ -177,12 +204,6 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
}) })
} }
}) })
if doErr != nil {
return doErr
}
if !isTruncated {
nextMarker = ""
} }
response = ListBucketResult{ response = ListBucketResult{
@ -203,7 +224,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
return return
} }
func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) { func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, inclusiveStartFrom bool, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
// invariants // invariants
// prefix and marker should be under dir, marker may contain "/" // prefix and marker should be under dir, marker may contain "/"
// maxKeys should be updated for each recursion // maxKeys should be updated for each recursion
@ -218,8 +239,8 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
if strings.Contains(marker, "/") { if strings.Contains(marker, "/") {
sepIndex := strings.Index(marker, "/") sepIndex := strings.Index(marker, "/")
subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:] subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
// println("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys) glog.V(4).Infoln("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys)
subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, eachEntryFn) subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, false, eachEntryFn)
if subErr != nil { if subErr != nil {
err = subErr err = subErr
return return
@ -241,7 +262,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
Prefix: prefix, Prefix: prefix,
Limit: uint32(maxKeys + 1), Limit: uint32(maxKeys + 1),
StartFromFileName: marker, StartFromFileName: marker,
InclusiveStartFrom: false, InclusiveStartFrom: inclusiveStartFrom,
} }
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@ -270,11 +291,13 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
nextMarker = entry.Name nextMarker = entry.Name
if entry.IsDirectory { if entry.IsDirectory {
// println("ListEntries", dir, "dir:", entry.Name) // println("ListEntries", dir, "dir:", entry.Name)
if entry.Name != ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys if entry.Name == ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys
continue
}
if delimiter != "/" { if delimiter != "/" {
eachEntryFn(dir, entry) eachEntryFn(dir, entry)
// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter) // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter)
subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, eachEntryFn) subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, false, eachEntryFn)
if subErr != nil { if subErr != nil {
err = fmt.Errorf("doListFilerEntries2: %v", subErr) err = fmt.Errorf("doListFilerEntries2: %v", subErr)
return return
@ -293,12 +316,11 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
glog.Errorf("check empty folder %s: %v", dir, err) glog.Errorf("check empty folder %s: %v", dir, err)
} }
} }
if !isEmpty || entry.Attributes.Mime != "" { if !isEmpty {
eachEntryFn(dir, entry) eachEntryFn(dir, entry)
counter++ counter++
} }
} }
}
} else { } else {
// println("ListEntries", dir, "file:", entry.Name) // println("ListEntries", dir, "file:", entry.Name)
eachEntryFn(dir, entry) eachEntryFn(dir, entry)