s3 API add ListObjectsV2Handler

weed/s3api/s3api_objects_list_handlers.go

@@ -18,6 +18,39 @@ const (
 	maxObjectListSizeLimit = 1000 // Limit number of objects in a listObjectsResponse.
 )
 
+func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
+
+	// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
+
+	// collect parameters
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	originalPrefix, marker, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())
+
+	if maxKeys < 0 {
+		writeErrorResponse(w, ErrInvalidMaxKeys, r.URL)
+		return
+	}
+	if delimiter != "" && delimiter != "/" {
+		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		return
+	}
+
+	if marker == "" {
+		marker = startAfter
+	}
+
+	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker)
+
+	if err != nil {
+		writeErrorResponse(w, ErrInternalError, r.URL)
+		return
+	}
+
+	writeSuccessResponseXML(w, encodeResponse(response))
+}
+
 func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
 
 	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
@@ -34,19 +67,31 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 	}
 	if delimiter != "" && delimiter != "/" {
 		writeErrorResponse(w, ErrNotImplemented, r.URL)
+		return
 	}
 
+	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker)
+
+	if err != nil {
+		writeErrorResponse(w, ErrInternalError, r.URL)
+		return
+	}
+
+	writeSuccessResponseXML(w, encodeResponse(response))
+}
+
+func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response ListBucketResponse, err error) {
+
 	// convert full path prefix into directory name and prefix for entry name
 	dir, prefix := filepath.Split(originalPrefix)
 
 	// check filer
-	var response ListBucketResponse
+	err = s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-	err := s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.ListEntriesRequest{
 			Directory:          fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, dir),
 			Prefix:             prefix,
-			Limit:              uint32(maxKeys),
+			Limit:              uint32(maxKeys + 1),
 			StartFromFileName:  marker,
 			InclusiveStartFrom: false,
 		}
@@ -59,7 +104,16 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 
 		var contents []ListEntry
 		var commonPrefixes []PrefixEntry
+		var counter int
+		var lastEntryName string
+		var isTruncated bool
 		for _, entry := range resp.Entries {
+			counter++
+			if counter > maxKeys {
+				isTruncated = true
+				break
+			}
+			lastEntryName = entry.Name
 			if entry.IsDirectory {
 				commonPrefixes = append(commonPrefixes, PrefixEntry{
 					Prefix: fmt.Sprintf("%s%s/", dir, entry.Name),
@@ -82,11 +136,11 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 			ListBucketResponse: ListBucketResult{
 				Name:           bucket,
 				Prefix:         originalPrefix,
-				Marker:         marker, // TODO
+				Marker:         marker,
-				NextMarker:     "",     // TODO
+				NextMarker:     lastEntryName,
 				MaxKeys:        maxKeys,
-				Delimiter:      delimiter,
+				Delimiter:      "/",
-				IsTruncated:    false, // TODO
+				IsTruncated:    isTruncated,
 				Contents:       contents,
 				CommonPrefixes: commonPrefixes,
 			},
@@ -95,12 +149,21 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 		return nil
 	})
 
-	if err != nil {
-		writeErrorResponse(w, ErrInternalError, r.URL)
 	return
-	}
+}
 
-	writeSuccessResponseXML(w, encodeResponse(response))
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int) {
+	prefix = values.Get("prefix")
+	token = values.Get("continuation-token")
+	startAfter = values.Get("start-after")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-keys") != "" {
+		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	} else {
+		maxkeys = maxObjectListSizeLimit
+	}
+	fetchOwner = values.Get("fetch-owner") == "true"
+	return
 }
 
 func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int) {

weed/s3api/s3api_server.go

@@ -45,20 +45,23 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 
 		// PutObject
 		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(s3a.PutObjectHandler)
-		// GetObject
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.GetObjectHandler)
-		// HeadObject
-		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(s3a.HeadObjectHandler)
-		// DeleteObject
-		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(s3a.DeleteObjectHandler)
-
 		// PutBucket
 		bucket.Methods("PUT").HandlerFunc(s3a.PutBucketHandler)
-		// DeleteBucket
+
-		bucket.Methods("DELETE").HandlerFunc(s3a.DeleteBucketHandler)
+		// HeadObject
+		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(s3a.HeadObjectHandler)
 		// HeadBucket
 		bucket.Methods("HEAD").HandlerFunc(s3a.HeadBucketHandler)
 
+		// DeleteObject
+		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(s3a.DeleteObjectHandler)
+		// DeleteBucket
+		bucket.Methods("DELETE").HandlerFunc(s3a.DeleteBucketHandler)
+
+		// GetObject
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.GetObjectHandler)
+		// ListObjectsV2
+		bucket.Methods("GET").HandlerFunc(s3a.ListObjectsV2Handler).Queries("list-type", "2")
 		// ListObjectsV1 (Legacy)
 		bucket.Methods("GET").HandlerFunc(s3a.ListObjectsV1Handler)
 
@@ -81,10 +84,6 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 
 			// ListMultipartUploads
 			bucket.Methods("GET").HandlerFunc(s3a.ListMultipartUploadsHandler).Queries("uploads", "")
-			// ListObjectsV2
-			bucket.Methods("GET").HandlerFunc(s3a.ListObjectsV2Handler).Queries("list-type", "2")
-			// ListObjectsV1 (Legacy)
-			bucket.Methods("GET").HandlerFunc(s3a.ListObjectsV1Handler)
 			// DeleteMultipleObjects
 			bucket.Methods("POST").HandlerFunc(s3a.DeleteMultipleObjectsHandler).Queries("delete", "")