From c07820178f492c9f344bfce2d929c646dcdc7638 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 7 Jun 2022 14:43:10 +0500 Subject: [PATCH] fix s3 tests bucket_list_delimiter_prefix bucket_list_delimiter_prefix_underscore bucket_list_delimiter_prefix_ends_with_delimiter --- weed/pb/filer_pb/filer_pb_helper.go | 4 ++ weed/s3api/s3api_object_handlers.go | 8 ++- weed/s3api/s3api_objects_list_handlers.go | 72 ++++++++++++++--------- 3 files changed, 53 insertions(+), 31 deletions(-) diff --git a/weed/pb/filer_pb/filer_pb_helper.go b/weed/pb/filer_pb/filer_pb_helper.go index 5f613a55d..2c9526b80 100644 --- a/weed/pb/filer_pb/filer_pb_helper.go +++ b/weed/pb/filer_pb/filer_pb_helper.go @@ -17,6 +17,10 @@ func (entry *Entry) IsInRemoteOnly() bool { return len(entry.Chunks) == 0 && entry.RemoteEntry != nil && entry.RemoteEntry.RemoteSize > 0 } +func (entry *Entry) IsDirectoryKeyObject() bool { + return entry.IsDirectory && entry.Attributes != nil && entry.Attributes.Mime != "" +} + func (entry *Entry) FileMode() (fileMode os.FileMode) { if entry != nil && entry.Attributes != nil { fileMode = os.FileMode(entry.Attributes.FileMode) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 5e34803cc..84ed1c92b 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -92,16 +92,20 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) } defer dataReader.Close() + objectContentType := r.Header.Get("Content-Type") if strings.HasSuffix(object, "/") { if err := s3a.mkdir(s3a.option.BucketsPath, bucket+strings.TrimSuffix(object, "/"), func(entry *filer_pb.Entry) { - entry.Attributes.Mime = r.Header.Get("Content-Type") + if objectContentType == "" { + objectContentType = "httpd/unix-directory" + } + entry.Attributes.Mime = objectContentType }); err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } } else { uploadUrl := s3a.toFilerUrl(bucket, object) - if r.Header.Get("Content-Type") == "" { + if objectContentType == "" { dataReader = mimeDetect(r, dataReader) } diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 22d15aebc..6b934bccd 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -133,6 +133,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m reqDir = reqDir[1:] } bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket) + bucketPrefixLen := len(bucketPrefix) reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir) if strings.HasSuffix(reqDir, "/") { reqDir = strings.TrimSuffix(reqDir, "/") @@ -147,14 +148,14 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m // check filer err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - _, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) { + _, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, false, false, bucketPrefixLen, func(dir string, entry *filer_pb.Entry) { if entry.IsDirectory { if delimiter == "/" { commonPrefixes = append(commonPrefixes, PrefixEntry{ - Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], + Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[bucketPrefixLen:], }) } - if entry.Attributes.Mime == "" || !strings.HasSuffix(entry.Name, "/") { + if !(entry.IsDirectoryKeyObject() && strings.HasSuffix(entry.Name, "/")) { return } } @@ -163,7 +164,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m storageClass = string(v) } contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], + Key: fmt.Sprintf("%s/%s", dir, entry.Name)[bucketPrefixLen:], LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), ETag: "\"" + filer.ETag(entry) + "\"", Size: int64(filer.FileSize(entry)), @@ -174,6 +175,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m StorageClass: StorageClass(storageClass), }) }) + glog.V(4).Infof("end doListFilerEntries isTruncated:%v nextMarker:%v reqDir: %v prefix: %v", isTruncated, nextMarker, reqDir, prefix) if doErr != nil { return doErr } @@ -187,14 +189,14 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m reqDir, prefix = filepath.Split(strings.TrimSuffix(reqDir, "/")) reqDir = strings.TrimSuffix(reqDir, "/") } - _, _, _, doErr = s3a.doListFilerEntries(client, reqDir, prefix, 1, prefix, delimiter, true, func(dir string, entry *filer_pb.Entry) { - if entry.IsDirectory && entry.Attributes.Mime != "" && entry.Name == prefix { + _, _, _, doErr = s3a.doListFilerEntries(client, reqDir, prefix, 1, prefix, delimiter, true, false, bucketPrefixLen, func(dir string, entry *filer_pb.Entry) { + if entry.IsDirectoryKeyObject() && entry.Name == prefix { storageClass := "STANDARD" if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { storageClass = string(v) } contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], + Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[bucketPrefixLen:], LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), ETag: "\"" + fmt.Sprintf("%x", entry.Attributes.Md5) + "\"", Size: int64(filer.FileSize(entry)), @@ -211,6 +213,10 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m } } + if len(nextMarker) > 0 { + nextMarker = nextMarker[bucketPrefixLen:] + } + response = ListBucketResult{ Name: bucket, Prefix: originalPrefix, @@ -229,7 +235,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m return } -func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, inclusiveStartFrom bool, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) { +func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, inclusiveStartFrom bool, subEntries bool, bucketPrefixLen int, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) { // invariants // prefix and marker should be under dir, marker may contain "/" // maxKeys should be updated for each recursion @@ -242,20 +248,27 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d } if strings.Contains(marker, "/") { - sepIndex := strings.Index(marker, "/") - subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:] - glog.V(4).Infoln("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys) - subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, false, eachEntryFn) - if subErr != nil { - err = subErr - return + if strings.HasSuffix(marker, "/") { + marker = strings.TrimSuffix(marker, "/") + } + sepIndex := strings.Index(marker, "/") + if sepIndex != -1 { + subPrefix, subMarker := marker[0:sepIndex], marker[sepIndex+1:] + subDir := fmt.Sprintf("%s/%s", dir[0:bucketPrefixLen-1], subPrefix) + if strings.HasPrefix(subDir, dir) { + subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, subDir, "", maxKeys, subMarker, delimiter, false, false, bucketPrefixLen, eachEntryFn) + if subErr != nil { + err = subErr + return + } + counter += subCounter + isTruncated = isTruncated || subIsTruncated + maxKeys -= subCounter + nextMarker = subNextMarker + // finished processing this sub directory + marker = subPrefix + } } - counter += subCounter - isTruncated = isTruncated || subIsTruncated - maxKeys -= subCounter - nextMarker = subDir + "/" + subNextMarker - // finished processing this sub directory - marker = subDir } if maxKeys <= 0 { return @@ -293,45 +306,46 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d return } entry := resp.Entry - nextMarker = entry.Name + nextMarker = dir + "/" + entry.Name if entry.IsDirectory { // println("ListEntries", dir, "dir:", entry.Name) if entry.Name == ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys continue } - if delimiter != "/" { + if delimiter == "" { eachEntryFn(dir, entry) // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter) - subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, false, eachEntryFn) + subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, false, true, bucketPrefixLen, eachEntryFn) if subErr != nil { err = fmt.Errorf("doListFilerEntries2: %v", subErr) return } // println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated) - if subCounter == 0 && entry.Attributes.Mime != "" { + if subCounter == 0 && entry.IsDirectoryKeyObject() { entry.Name += "/" eachEntryFn(dir, entry) counter++ } counter += subCounter - nextMarker = entry.Name + "/" + subNextMarker + nextMarker = subNextMarker if subIsTruncated { isTruncated = true return } - } else { + } else if delimiter == "/" { var isEmpty bool - if !s3a.option.AllowEmptyFolder && entry.Attributes.Mime == "" { + if !s3a.option.AllowEmptyFolder && !entry.IsDirectoryKeyObject() { if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil { glog.Errorf("check empty folder %s: %v", dir, err) } } if !isEmpty { + nextMarker += "/" eachEntryFn(dir, entry) counter++ } } - } else { + } else if !(delimiter == "/" && subEntries) { // println("ListEntries", dir, "file:", entry.Name) eachEntryFn(dir, entry) counter++