From b54a65ba5afcec13743442357a8544ec8cf876f5 Mon Sep 17 00:00:00 2001 From: zerospiel Date: Thu, 3 Feb 2022 17:17:05 +0300 Subject: [PATCH] weed/s3api: added new bucket handlers for more compatibility with AWS S3 Protocol Otherwise any requests to the underlying handlers results in calls to ListObjects (v1) that may intensively load gateway and volume servers. Added the following handlers with default responses: - GetBucketLocation - GetBucketRequestPayment Added the following handlers with NotFound and NotImplemented responses: - PutBucketAcl - GetBucketPolicy - PutBucketPolicy - DeleteBucketPolicy - GetBucketCors - PutBucketCors - DeleteBucketCors --- weed/s3api/s3api_bucket_handlers.go | 19 +++++++-- weed/s3api/s3api_bucket_skip_handlers.go | 49 ++++++++++++++++++++++++ weed/s3api/s3api_server.go | 44 ++++++++++++--------- weed/s3api/s3api_xsd_generated.go | 4 ++ weed/s3api/s3err/s3-error.go | 1 + weed/s3api/s3err/s3api_errors.go | 12 ++++++ 6 files changed, 107 insertions(+), 22 deletions(-) create mode 100644 weed/s3api/s3api_bucket_skip_handlers.go diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 3233f4006..815e1b76b 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -4,13 +4,14 @@ import ( "context" "encoding/xml" "fmt" - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" - "github.com/chrislusf/seaweedfs/weed/storage/needle" "math" "net/http" "time" + "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" + "github.com/chrislusf/seaweedfs/weed/storage/needle" + xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" @@ -309,3 +310,15 @@ func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *h s3err.WriteEmptyResponse(w, r, http.StatusNoContent) } + +// GetBucketLocationHandler Get bucket location +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html +func (s3a *S3ApiServer) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) { + writeSuccessResponseXML(w, r, LocationConstraint{}) +} + +// GetBucketRequestPaymentHandler Get bucket location +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketRequestPayment.html +func (s3a *S3ApiServer) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) { + writeSuccessResponseXML(w, r, RequestPaymentConfiguration{Payer: "BucketOwner"}) +} diff --git a/weed/s3api/s3api_bucket_skip_handlers.go b/weed/s3api/s3api_bucket_skip_handlers.go new file mode 100644 index 000000000..f4ca1177d --- /dev/null +++ b/weed/s3api/s3api_bucket_skip_handlers.go @@ -0,0 +1,49 @@ +package s3api + +import ( + "net/http" + + "github.com/chrislusf/seaweedfs/weed/s3api/s3err" +) + +// GetBucketCorsHandler Get bucket CORS +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketCors.html +func (s3a *S3ApiServer) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchCORSConfiguration) +} + +// PutBucketCorsHandler Put bucket CORS +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketCors.html +func (s3a *S3ApiServer) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) +} + +// DeleteBucketCorsHandler Delete bucket CORS +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketCors.html +func (s3a *S3ApiServer) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, http.StatusNoContent) +} + +// GetBucketPolicyHandler Get bucket Policy +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketPolicy.html +func (s3a *S3ApiServer) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucketPolicy) +} + +// PutBucketPolicyHandler Put bucket Policy +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketPolicy.html +func (s3a *S3ApiServer) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) +} + +// DeleteBucketPolicyHandler Delete bucket Policy +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketPolicy.html +func (s3a *S3ApiServer) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, http.StatusNoContent) +} + +// PutBucketAclHandler Put bucket ACL +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html +func (s3a *S3ApiServer) PutBucketAclHandler(w http.ResponseWriter, r *http.Request) { + s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) +} diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index c560fbea2..df015e803 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -2,16 +2,16 @@ package s3api import ( "fmt" - "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/security" - "github.com/chrislusf/seaweedfs/weed/util" "net/http" "strings" "time" "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/pb" . "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util" "github.com/gorilla/mux" "google.golang.org/grpc" ) @@ -134,6 +134,28 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // GetBucketACL bucket.Methods("GET").HandlerFunc(s3a.iam.Auth(s3a.GetBucketAclHandler, ACTION_READ)).Queries("acl", "") + // PutBucketACL + bucket.Methods("PUT").HandlerFunc(s3a.iam.Auth(s3a.PutBucketAclHandler, ACTION_WRITE)).Queries("acl", "") + // GetBucketPolicy + bucket.Methods("GET").HandlerFunc(s3a.iam.Auth(s3a.GetBucketPolicyHandler, ACTION_READ)).Queries("policy", "") + + // PutBucketPolicy + bucket.Methods("PUT").HandlerFunc(s3a.iam.Auth(s3a.PutBucketPolicyHandler, ACTION_WRITE)).Queries("policy", "") + // DeleteBucketPolicy + bucket.Methods("DELETE").HandlerFunc(s3a.iam.Auth(s3a.DeleteBucketPolicyHandler, ACTION_WRITE)).Queries("policy", "") + + // GetBucketCors + bucket.Methods("GET").HandlerFunc(s3a.iam.Auth(s3a.GetBucketCorsHandler, ACTION_READ)).Queries("cors", "") + // PutBucketCors + bucket.Methods("PUT").HandlerFunc(s3a.iam.Auth(s3a.PutBucketCorsHandler, ACTION_WRITE)).Queries("cors", "") + // DeleteBucketCors + bucket.Methods("DELETE").HandlerFunc(s3a.iam.Auth(s3a.DeleteBucketCorsHandler, ACTION_WRITE)).Queries("cors", "") + + // GetBucketLocation + bucket.Methods("GET").HandlerFunc(s3a.iam.Auth(s3a.GetBucketLocationHandler, ACTION_READ)).Queries("location", "") + + // GetBucketRequestPayment + bucket.Methods("GET").HandlerFunc(s3a.iam.Auth(s3a.GetBucketRequestPaymentHandler, ACTION_READ)).Queries("requestPayment", "") // GetObjectACL bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.iam.Auth(s3a.GetObjectAclHandler, ACTION_READ)).Queries("acl", "") @@ -154,22 +176,6 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // DeleteBucket bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketHandler, ACTION_WRITE), "DELETE")) - - /* - - // not implemented - // GetBucketLocation - bucket.Methods("GET").HandlerFunc(s3a.GetBucketLocationHandler).Queries("location", "") - // GetBucketPolicy - bucket.Methods("GET").HandlerFunc(s3a.GetBucketPolicyHandler).Queries("policy", "") - // GetObjectACL - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.GetObjectACLHandler).Queries("acl", "") - // PutBucketPolicy - bucket.Methods("PUT").HandlerFunc(s3a.PutBucketPolicyHandler).Queries("policy", "") - // DeleteBucketPolicy - bucket.Methods("DELETE").HandlerFunc(s3a.DeleteBucketPolicyHandler).Queries("policy", "") - */ - } // ListBuckets diff --git a/weed/s3api/s3api_xsd_generated.go b/weed/s3api/s3api_xsd_generated.go index a8e4ef404..dd6a32ff2 100644 --- a/weed/s3api/s3api_xsd_generated.go +++ b/weed/s3api/s3api_xsd_generated.go @@ -646,6 +646,10 @@ type ListVersionsResult struct { CommonPrefixes []PrefixEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CommonPrefixes,omitempty"` } +type LocationConstraint struct { + LocationConstraint string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint"` +} + type LoggingSettings struct { TargetBucket string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ TargetBucket"` TargetPrefix string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ TargetPrefix"` diff --git a/weed/s3api/s3err/s3-error.go b/weed/s3api/s3err/s3-error.go index 224378ec5..b87764742 100644 --- a/weed/s3api/s3err/s3-error.go +++ b/weed/s3api/s3err/s3-error.go @@ -58,4 +58,5 @@ var s3ErrorResponseMap = map[string]string{ "InvalidDuration": "Duration provided in the request is invalid.", "XAmzContentSHA256Mismatch": "The provided 'x-amz-content-sha256' header does not match what was computed.", // Add new API errors here. + "NoSuchCORSConfiguration": "The CORS configuration does not exist", } diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index 3063df844..8d02f15b1 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -51,6 +51,8 @@ const ( ErrBucketAlreadyExists ErrBucketAlreadyOwnedByYou ErrNoSuchBucket + ErrNoSuchBucketPolicy + ErrNoSuchCORSConfiguration ErrNoSuchLifecycleConfiguration ErrNoSuchKey ErrNoSuchUpload @@ -164,6 +166,16 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "The specified bucket does not exist", HTTPStatusCode: http.StatusNotFound, }, + ErrNoSuchBucketPolicy: { + Code: "NoSuchBucketPolicy", + Description: "The bucket policy does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrNoSuchCORSConfiguration: { + Code: "NoSuchCORSConfiguration", + Description: "The CORS configuration does not exist", + HTTPStatusCode: http.StatusNotFound, + }, ErrNoSuchLifecycleConfiguration: { Code: "NoSuchLifecycleConfiguration", Description: "The lifecycle configuration does not exist",