From 44ad07276c719328d1f11aa06d1fca48ce9ddcba Mon Sep 17 00:00:00 2001
From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com>
Date: Wed, 12 Apr 2023 11:52:35 +0500
Subject: [PATCH] s3 fix get list of dir object key with slash suffix (#4391)

* s3 fix get list of dir object key with slash suffix
https://github.com/seaweedfs/seaweedfs/issues/3086

* list only entry dir eq prefix

---------

Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co>
---
 weed/s3api/s3api_objects_list_handlers.go | 30 +++++++++++++++++------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go
index b9eddd921..50e19ba04 100644
--- a/weed/s3api/s3api_objects_list_handlers.go
+++ b/weed/s3api/s3api_objects_list_handlers.go
@@ -141,7 +141,8 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 	var doErr error
 	var nextMarker string
 	cursor := &ListingCursor{
-		maxKeys: maxKeys,
+		maxKeys:               maxKeys,
+		prefixEndsOnDelimiter: strings.HasSuffix(originalPrefix, "/"),
 	}
 
 	// check filer
@@ -199,7 +200,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 					nextMarker = requestDir + "/" + nextMarker
 				}
 				break
-			} else if empty {
+			} else if empty || strings.HasSuffix(originalPrefix, "/") {
 				nextMarker = ""
 				break
 			} else {
@@ -227,8 +228,9 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 }
 
 type ListingCursor struct {
-	maxKeys     int
-	isTruncated bool
+	maxKeys               int
+	isTruncated           bool
+	prefixEndsOnDelimiter bool
 }
 
 // the prefix and marker may be in different directories
@@ -236,7 +238,7 @@ type ListingCursor struct {
 func normalizePrefixMarker(prefix, marker string) (alignedDir, alignedPrefix, alignedMarker string) {
 	// alignedDir should not end with "/"
 	// alignedDir, alignedPrefix, alignedMarker should only have "/" in middle
-	prefix = strings.TrimLeft(prefix, "/")
+	prefix = strings.Trim(prefix, "/")
 	marker = strings.TrimLeft(marker, "/")
 	if prefix == "" {
 		return "", "", marker
@@ -264,6 +266,7 @@ func normalizePrefixMarker(prefix, marker string) (alignedDir, alignedPrefix, al
 	}
 	return
 }
+
 func toDirAndName(dirAndName string) (dir, name string) {
 	sepIndex := strings.LastIndex(dirAndName, "/")
 	if sepIndex >= 0 {
@@ -273,6 +276,7 @@ func toDirAndName(dirAndName string) (dir, name string) {
 	}
 	return
 }
+
 func toParentAndDescendants(dirAndName string) (dir, name string) {
 	sepIndex := strings.Index(dirAndName, "/")
 	if sepIndex >= 0 {
@@ -287,7 +291,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 	// invariants
 	//   prefix and marker should be under dir, marker may contain "/"
 	//   maxKeys should be updated for each recursion
-
+	// glog.V(4).Infof("doListFilerEntries dir: %s, prefix: %s, marker %s, maxKeys: %d, prefixEndsOnDelimiter: %+v", dir, prefix, marker, cursor.maxKeys, cursor.prefixEndsOnDelimiter)
 	if prefix == "/" && delimiter == "/" {
 		return
 	}
@@ -319,6 +323,9 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 		StartFromFileName:  marker,
 		InclusiveStartFrom: inclusiveStartFrom,
 	}
+	if cursor.prefixEndsOnDelimiter {
+		request.Limit = uint32(1)
+	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -344,8 +351,15 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 		}
 		entry := resp.Entry
 		nextMarker = entry.Name
+		if cursor.prefixEndsOnDelimiter {
+			if entry.Name == prefix && entry.IsDirectory {
+				cursor.prefixEndsOnDelimiter = false
+			} else {
+				continue
+			}
+		}
 		if entry.IsDirectory {
-			// println("ListEntries", dir, "dir:", entry.Name)
+			// glog.V(4).Infof("List Dir Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
 			if entry.Name == s3_constants.MultipartUploadsFolder { // FIXME no need to apply to all directories. this extra also affects maxKeys
 				continue
 			}
@@ -375,7 +389,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 			}
 		} else {
 			eachEntryFn(dir, entry)
-			// println("ListEntries", dir, "file:", entry.Name, "maxKeys", cursor.maxKeys)
+			// glog.V(4).Infof("List File Entries %s, file: %s, maxKeys %d", dir, entry.Name, cursor.maxKeys)
 		}
 	}
 	return