mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
parent
26f3646961
commit
62e5e3822c
|
@ -135,8 +135,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
|
||||||
bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)
|
bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)
|
||||||
reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir)
|
reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir)
|
||||||
if strings.HasSuffix(reqDir, "/") {
|
if strings.HasSuffix(reqDir, "/") {
|
||||||
// remove trailing "/"
|
reqDir = strings.TrimSuffix(reqDir, "/")
|
||||||
reqDir = reqDir[:len(reqDir)-1]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var contents []ListEntry
|
var contents []ListEntry
|
||||||
|
@ -148,34 +147,30 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
|
||||||
// check filer
|
// check filer
|
||||||
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||||
|
|
||||||
_, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, func(dir string, entry *filer_pb.Entry) {
|
_, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
|
||||||
if entry.IsDirectory && entry.Attributes.Mime == "" {
|
if entry.IsDirectory {
|
||||||
if delimiter == "/" {
|
if delimiter == "/" {
|
||||||
commonPrefixes = append(commonPrefixes, PrefixEntry{
|
commonPrefixes = append(commonPrefixes, PrefixEntry{
|
||||||
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
|
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
} else {
|
return
|
||||||
storageClass := "STANDARD"
|
|
||||||
if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok {
|
|
||||||
storageClass = string(v)
|
|
||||||
}
|
|
||||||
entryName := entry.Name
|
|
||||||
if entry.IsDirectory {
|
|
||||||
entryName += "/"
|
|
||||||
}
|
|
||||||
contents = append(contents, ListEntry{
|
|
||||||
Key: fmt.Sprintf("%s/%s", dir, entryName)[len(bucketPrefix):],
|
|
||||||
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
|
|
||||||
ETag: "\"" + filer.ETag(entry) + "\"",
|
|
||||||
Size: int64(filer.FileSize(entry)),
|
|
||||||
Owner: CanonicalUser{
|
|
||||||
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
|
|
||||||
DisplayName: entry.Attributes.UserName,
|
|
||||||
},
|
|
||||||
StorageClass: StorageClass(storageClass),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
storageClass := "STANDARD"
|
||||||
|
if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok {
|
||||||
|
storageClass = string(v)
|
||||||
|
}
|
||||||
|
contents = append(contents, ListEntry{
|
||||||
|
Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
|
||||||
|
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
|
||||||
|
ETag: "\"" + filer.ETag(entry) + "\"",
|
||||||
|
Size: int64(filer.FileSize(entry)),
|
||||||
|
Owner: CanonicalUser{
|
||||||
|
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
|
||||||
|
DisplayName: entry.Attributes.UserName,
|
||||||
|
},
|
||||||
|
StorageClass: StorageClass(storageClass),
|
||||||
|
})
|
||||||
})
|
})
|
||||||
if doErr != nil {
|
if doErr != nil {
|
||||||
return doErr
|
return doErr
|
||||||
|
@ -185,6 +180,32 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
|
||||||
nextMarker = ""
|
nextMarker = ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(contents) == 0 && maxKeys > 0 {
|
||||||
|
if strings.HasSuffix(originalPrefix, "/") && prefix == "" {
|
||||||
|
reqDir, prefix = filepath.Split(strings.TrimSuffix(reqDir, "/"))
|
||||||
|
reqDir = strings.TrimSuffix(reqDir, "/")
|
||||||
|
}
|
||||||
|
_, _, _, doErr = s3a.doListFilerEntries(client, reqDir, prefix, 1, prefix, delimiter, true, func(dir string, entry *filer_pb.Entry) {
|
||||||
|
if entry.IsDirectory && entry.Attributes.Mime != "" && entry.Name == prefix {
|
||||||
|
storageClass := "STANDARD"
|
||||||
|
if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok {
|
||||||
|
storageClass = string(v)
|
||||||
|
}
|
||||||
|
contents = append(contents, ListEntry{
|
||||||
|
Key: fmt.Sprintf("%s/%s", dir, entry.Name+"/")[len(bucketPrefix):],
|
||||||
|
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
|
||||||
|
ETag: "\"" + filer.ETag(entry) + "\"",
|
||||||
|
Size: int64(filer.FileSize(entry)),
|
||||||
|
Owner: CanonicalUser{
|
||||||
|
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
|
||||||
|
DisplayName: entry.Attributes.UserName,
|
||||||
|
},
|
||||||
|
StorageClass: StorageClass(storageClass),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
response = ListBucketResult{
|
response = ListBucketResult{
|
||||||
Name: bucket,
|
Name: bucket,
|
||||||
Prefix: originalPrefix,
|
Prefix: originalPrefix,
|
||||||
|
@ -203,7 +224,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
|
func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, inclusiveStartFrom bool, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
|
||||||
// invariants
|
// invariants
|
||||||
// prefix and marker should be under dir, marker may contain "/"
|
// prefix and marker should be under dir, marker may contain "/"
|
||||||
// maxKeys should be updated for each recursion
|
// maxKeys should be updated for each recursion
|
||||||
|
@ -218,8 +239,8 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
||||||
if strings.Contains(marker, "/") {
|
if strings.Contains(marker, "/") {
|
||||||
sepIndex := strings.Index(marker, "/")
|
sepIndex := strings.Index(marker, "/")
|
||||||
subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
|
subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
|
||||||
// println("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys)
|
glog.V(4).Infoln("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys)
|
||||||
subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, eachEntryFn)
|
subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, false, eachEntryFn)
|
||||||
if subErr != nil {
|
if subErr != nil {
|
||||||
err = subErr
|
err = subErr
|
||||||
return
|
return
|
||||||
|
@ -241,7 +262,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
||||||
Prefix: prefix,
|
Prefix: prefix,
|
||||||
Limit: uint32(maxKeys + 1),
|
Limit: uint32(maxKeys + 1),
|
||||||
StartFromFileName: marker,
|
StartFromFileName: marker,
|
||||||
InclusiveStartFrom: false,
|
InclusiveStartFrom: inclusiveStartFrom,
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
@ -270,33 +291,34 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
||||||
nextMarker = entry.Name
|
nextMarker = entry.Name
|
||||||
if entry.IsDirectory {
|
if entry.IsDirectory {
|
||||||
// println("ListEntries", dir, "dir:", entry.Name)
|
// println("ListEntries", dir, "dir:", entry.Name)
|
||||||
if entry.Name != ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys
|
if entry.Name == ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys
|
||||||
if delimiter != "/" {
|
continue
|
||||||
|
}
|
||||||
|
if delimiter != "/" {
|
||||||
|
eachEntryFn(dir, entry)
|
||||||
|
// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter)
|
||||||
|
subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, false, eachEntryFn)
|
||||||
|
if subErr != nil {
|
||||||
|
err = fmt.Errorf("doListFilerEntries2: %v", subErr)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated)
|
||||||
|
counter += subCounter
|
||||||
|
nextMarker = entry.Name + "/" + subNextMarker
|
||||||
|
if subIsTruncated {
|
||||||
|
isTruncated = true
|
||||||
|
return
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
var isEmpty bool
|
||||||
|
if !s3a.option.AllowEmptyFolder {
|
||||||
|
if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil {
|
||||||
|
glog.Errorf("check empty folder %s: %v", dir, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !isEmpty {
|
||||||
eachEntryFn(dir, entry)
|
eachEntryFn(dir, entry)
|
||||||
// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter)
|
counter++
|
||||||
subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, eachEntryFn)
|
|
||||||
if subErr != nil {
|
|
||||||
err = fmt.Errorf("doListFilerEntries2: %v", subErr)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated)
|
|
||||||
counter += subCounter
|
|
||||||
nextMarker = entry.Name + "/" + subNextMarker
|
|
||||||
if subIsTruncated {
|
|
||||||
isTruncated = true
|
|
||||||
return
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
var isEmpty bool
|
|
||||||
if !s3a.option.AllowEmptyFolder {
|
|
||||||
if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil {
|
|
||||||
glog.Errorf("check empty folder %s: %v", dir, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !isEmpty || entry.Attributes.Mime != "" {
|
|
||||||
eachEntryFn(dir, entry)
|
|
||||||
counter++
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
Loading…
Reference in a new issue