gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

s3 fix get list of dir object key with slash suffix (#4391)

Browse source 
* s3 fix get list of dir object key with slash suffix
https://github.com/seaweedfs/seaweedfs/issues/3086

* list only entry dir eq prefix

---------

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
This commit is contained in:
Konstantin Lebedev 2023-04-12 11:52:35 +05:00 committed by GitHub
parent ccc030b245
commit 44ad07276c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 22 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
weed/s3api/s3api_objects_list_handlers.go
View file

@ -141,7 +141,8 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
var doErr error
var nextMarker string
cursor := &ListingCursor{
maxKeys: maxKeys,
maxKeys: maxKeys,
prefixEndsOnDelimiter: strings.HasSuffix(originalPrefix, "/"),
}
// check filer
@ -199,7 +200,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
nextMarker = requestDir + "/" + nextMarker
}
break
} else if empty {
} else if empty || strings.HasSuffix(originalPrefix, "/") {
nextMarker = ""
break
} else {
@ -227,8 +228,9 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
}
type ListingCursor struct {
maxKeys int
isTruncated bool
maxKeys int
isTruncated bool
prefixEndsOnDelimiter bool
}
// the prefix and marker may be in different directories
@ -236,7 +238,7 @@ type ListingCursor struct {
func normalizePrefixMarker(prefix, marker string) (alignedDir, alignedPrefix, alignedMarker string) {
// alignedDir should not end with "/"
// alignedDir, alignedPrefix, alignedMarker should only have "/" in middle
prefix = strings.TrimLeft(prefix, "/")
prefix = strings.Trim(prefix, "/")
marker = strings.TrimLeft(marker, "/")
if prefix == "" {
return "", "", marker
@ -264,6 +266,7 @@ func normalizePrefixMarker(prefix, marker string) (alignedDir, alignedPrefix, al
}
return
}
func toDirAndName(dirAndName string) (dir, name string) {
sepIndex := strings.LastIndex(dirAndName, "/")
if sepIndex >= 0 {
@ -273,6 +276,7 @@ func toDirAndName(dirAndName string) (dir, name string) {
}
return
}
func toParentAndDescendants(dirAndName string) (dir, name string) {
sepIndex := strings.Index(dirAndName, "/")
if sepIndex >= 0 {
@ -287,7 +291,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
// invariants
// prefix and marker should be under dir, marker may contain "/"
// maxKeys should be updated for each recursion
// glog.V(4).Infof("doListFilerEntries dir: %s, prefix: %s, marker %s, maxKeys: %d, prefixEndsOnDelimiter: %+v", dir, prefix, marker, cursor.maxKeys, cursor.prefixEndsOnDelimiter)
if prefix == "/" && delimiter == "/" {
return
}
@ -319,6 +323,9 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
StartFromFileName: marker,
InclusiveStartFrom: inclusiveStartFrom,
}
if cursor.prefixEndsOnDelimiter {
request.Limit = uint32(1)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -344,8 +351,15 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
}
entry := resp.Entry
nextMarker = entry.Name
if cursor.prefixEndsOnDelimiter {
if entry.Name == prefix && entry.IsDirectory {
cursor.prefixEndsOnDelimiter = false
} else {
continue
}
}
if entry.IsDirectory {
// println("ListEntries", dir, "dir:", entry.Name)
// glog.V(4).Infof("List Dir Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
if entry.Name == s3_constants.MultipartUploadsFolder { // FIXME no need to apply to all directories. this extra also affects maxKeys
continue
}
@ -375,7 +389,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
}
} else {
eachEntryFn(dir, entry)
// println("ListEntries", dir, "file:", entry.Name, "maxKeys", cursor.maxKeys)
// glog.V(4).Infof("List File Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
}
}
return