From 62563a895ac4c9f438683946341e3e6ffa0a3906 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 20 Sep 2020 16:00:01 -0700 Subject: [PATCH] refactoring --- weed/s3api/auth_signature_v4.go | 3 +- weed/s3api/policy/post-policy_test.go | 2 - weed/s3api/policy/postpolicyform.go | 2 +- .../s3api/s3api_object_handlers_postpolicy.go | 3 +- weed/s3api/s3api_server.go | 40 +++++++++---------- weed/s3api/stats.go | 6 ++- weed/server/filer_server_handlers.go | 3 ++ weed/server/volume_server_handlers.go | 3 ++ weed/server/volume_server_handlers_admin.go | 2 + weed/server/volume_server_handlers_ui.go | 1 + weed/util/constants.go | 2 +- 11 files changed, 38 insertions(+), 29 deletions(-) diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 5d428fdec..5ef7439c8 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -293,14 +293,13 @@ func parseSignature(signElement string) (string, s3err.ErrorCode) { return signature, s3err.ErrNone } - // doesPolicySignatureMatch - Verify query headers with post policy // - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html // returns ErrNone if the signature matches. func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.Header) s3err.ErrorCode { // Parse credential tag. - credHeader, err := parseCredentialHeader("Credential="+formValues.Get("X-Amz-Credential")) + credHeader, err := parseCredentialHeader("Credential=" + formValues.Get("X-Amz-Credential")) if err != s3err.ErrNone { return s3err.ErrMissingFields } diff --git a/weed/s3api/policy/post-policy_test.go b/weed/s3api/policy/post-policy_test.go index 1bd0985f0..ce241b723 100644 --- a/weed/s3api/policy/post-policy_test.go +++ b/weed/s3api/policy/post-policy_test.go @@ -320,8 +320,6 @@ func makeTestTargetURL(endPoint, bucketName, objectName string, queryValues url. return urlStr } - - // if object matches reserved string, no need to encode them var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") diff --git a/weed/s3api/policy/postpolicyform.go b/weed/s3api/policy/postpolicyform.go index 798408aa4..3a6f3a882 100644 --- a/weed/s3api/policy/postpolicyform.go +++ b/weed/s3api/policy/postpolicyform.go @@ -100,7 +100,7 @@ type contentLengthRange struct { // PostPolicyForm provides strict static type conversion and validation for Amazon S3's POST policy JSON string. type PostPolicyForm struct { Expiration time.Time // Expiration date and time of the POST policy. - Conditions struct { // Conditional policy structure. + Conditions struct { // Conditional policy structure. Policies []struct { Operator string Key string diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go index f1605cc63..b2a83d48d 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy.go +++ b/weed/s3api/s3api_object_handlers_postpolicy.go @@ -147,7 +147,6 @@ func (s3a *S3ApiServer) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Re } - // Extract form fields and file data from a HTTP POST Policy func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser, fileName string, fileSize int64, formValues http.Header, err error) { /// HTML Form values @@ -215,7 +214,7 @@ func validateFormFieldSize(formValues http.Header) error { // Iterate over form values for k := range formValues { // Check if value's field exceeds S3 limit - if int64(len(formValues.Get(k))) > int64(1 * humanize.MiByte) { + if int64(len(formValues.Get(k))) > int64(1*humanize.MiByte) { return errors.New("Data size larger than expected") } } diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 4eaacd66c..c8211137b 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -49,49 +49,49 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { for _, bucket := range routers { // HeadObject - bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(stats(s3a.iam.Auth(s3a.HeadObjectHandler, ACTION_READ), "GET")) + bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.HeadObjectHandler, ACTION_READ), "GET")) // HeadBucket - bucket.Methods("HEAD").HandlerFunc(stats(s3a.iam.Auth(s3a.HeadBucketHandler, ACTION_ADMIN), "GET")) + bucket.Methods("HEAD").HandlerFunc(track(s3a.iam.Auth(s3a.HeadBucketHandler, ACTION_ADMIN), "GET")) // CopyObjectPart - bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(stats(s3a.iam.Auth(s3a.CopyObjectPartHandler, ACTION_WRITE), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.CopyObjectPartHandler, ACTION_WRITE), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") // PutObjectPart - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(stats(s3a.iam.Auth(s3a.PutObjectPartHandler, ACTION_WRITE), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.PutObjectPartHandler, ACTION_WRITE), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") // CompleteMultipartUpload - bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(stats(s3a.iam.Auth(s3a.CompleteMultipartUploadHandler, ACTION_WRITE), "POST")).Queries("uploadId", "{uploadId:.*}") + bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.CompleteMultipartUploadHandler, ACTION_WRITE), "POST")).Queries("uploadId", "{uploadId:.*}") // NewMultipartUpload - bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(stats(s3a.iam.Auth(s3a.NewMultipartUploadHandler, ACTION_WRITE), "POST")).Queries("uploads", "") + bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.NewMultipartUploadHandler, ACTION_WRITE), "POST")).Queries("uploads", "") // AbortMultipartUpload - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(stats(s3a.iam.Auth(s3a.AbortMultipartUploadHandler, ACTION_WRITE), "DELETE")).Queries("uploadId", "{uploadId:.*}") + bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.AbortMultipartUploadHandler, ACTION_WRITE), "DELETE")).Queries("uploadId", "{uploadId:.*}") // ListObjectParts - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(stats(s3a.iam.Auth(s3a.ListObjectPartsHandler, ACTION_WRITE), "GET")).Queries("uploadId", "{uploadId:.*}") + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectPartsHandler, ACTION_WRITE), "GET")).Queries("uploadId", "{uploadId:.*}") // ListMultipartUploads - bucket.Methods("GET").HandlerFunc(stats(s3a.iam.Auth(s3a.ListMultipartUploadsHandler, ACTION_WRITE), "GET")).Queries("uploads", "") + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListMultipartUploadsHandler, ACTION_WRITE), "GET")).Queries("uploads", "") // CopyObject - bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(stats(s3a.iam.Auth(s3a.CopyObjectHandler, ACTION_WRITE), "COPY")) + bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.CopyObjectHandler, ACTION_WRITE), "COPY")) // PutObject - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(stats(s3a.iam.Auth(s3a.PutObjectHandler, ACTION_WRITE), "PUT")) + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.PutObjectHandler, ACTION_WRITE), "PUT")) // PutBucket - bucket.Methods("PUT").HandlerFunc(stats(s3a.iam.Auth(s3a.PutBucketHandler, ACTION_ADMIN), "PUT")) + bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketHandler, ACTION_ADMIN), "PUT")) // DeleteObject - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(stats(s3a.iam.Auth(s3a.DeleteObjectHandler, ACTION_WRITE), "DELETE")) + bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteObjectHandler, ACTION_WRITE), "DELETE")) // DeleteBucket - bucket.Methods("DELETE").HandlerFunc(stats(s3a.iam.Auth(s3a.DeleteBucketHandler, ACTION_WRITE), "DELETE")) + bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketHandler, ACTION_WRITE), "DELETE")) // ListObjectsV2 - bucket.Methods("GET").HandlerFunc(stats(s3a.iam.Auth(s3a.ListObjectsV2Handler, ACTION_READ), "LIST")).Queries("list-type", "2") + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV2Handler, ACTION_READ), "LIST")).Queries("list-type", "2") // GetObject, but directory listing is not supported - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(stats(s3a.iam.Auth(s3a.GetObjectHandler, ACTION_READ), "GET")) + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.GetObjectHandler, ACTION_READ), "GET")) // ListObjectsV1 (Legacy) - bucket.Methods("GET").HandlerFunc(stats(s3a.iam.Auth(s3a.ListObjectsV1Handler, ACTION_READ), "LIST")) + bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.ListObjectsV1Handler, ACTION_READ), "LIST")) // PostPolicy - bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(stats(s3a.iam.Auth(s3a.PutBucketPolicyHandler, ACTION_WRITE), "POST")) + bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketPolicyHandler, ACTION_WRITE), "POST")) // DeleteMultipleObjects - bucket.Methods("POST").HandlerFunc(stats(s3a.iam.Auth(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE), "DELETE")).Queries("delete", "") + bucket.Methods("POST").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE), "DELETE")).Queries("delete", "") /* // not implemented @@ -114,7 +114,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { } // ListBuckets - apiRouter.Methods("GET").Path("/").HandlerFunc(stats(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_READ), "LIST")) + apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_READ), "LIST")) // NotFound apiRouter.NotFoundHandler = http.HandlerFunc(notFoundHandler) diff --git a/weed/s3api/stats.go b/weed/s3api/stats.go index 8776949d4..481cb1625 100644 --- a/weed/s3api/stats.go +++ b/weed/s3api/stats.go @@ -2,13 +2,17 @@ package s3api import ( stats_collect "github.com/chrislusf/seaweedfs/weed/stats" + "github.com/chrislusf/seaweedfs/weed/util" "net/http" "time" ) -func stats(f http.HandlerFunc, action string) http.HandlerFunc { +func track(f http.HandlerFunc, action string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + + w.Header().Set("Server", "Seaweed S3 "+util.VERSION) + start := time.Now() stats_collect.S3RequestCounter.WithLabelValues(action).Inc() f(w, r) diff --git a/weed/server/filer_server_handlers.go b/weed/server/filer_server_handlers.go index b6bfc3b04..18f78881c 100644 --- a/weed/server/filer_server_handlers.go +++ b/weed/server/filer_server_handlers.go @@ -1,6 +1,7 @@ package weed_server import ( + "github.com/chrislusf/seaweedfs/weed/util" "net/http" "time" @@ -8,6 +9,7 @@ import ( ) func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION) start := time.Now() switch r.Method { case "GET": @@ -34,6 +36,7 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { } func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION) start := time.Now() switch r.Method { case "GET": diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index 14ad27d42..5aa7ad60b 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -1,6 +1,7 @@ package weed_server import ( + "github.com/chrislusf/seaweedfs/weed/util" "net/http" "strings" @@ -25,6 +26,7 @@ security settings: */ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Server", "Seaweed Volume "+util.VERSION) switch r.Method { case "GET", "HEAD": stats.ReadRequest() @@ -39,6 +41,7 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque } func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Server", "Seaweed Volume "+util.VERSION) switch r.Method { case "GET": stats.ReadRequest() diff --git a/weed/server/volume_server_handlers_admin.go b/weed/server/volume_server_handlers_admin.go index 34655d833..4d84c9c4d 100644 --- a/weed/server/volume_server_handlers_admin.go +++ b/weed/server/volume_server_handlers_admin.go @@ -10,6 +10,7 @@ import ( ) func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION) m := make(map[string]interface{}) m["Version"] = util.Version() var ds []*volume_server_pb.DiskStatus @@ -24,6 +25,7 @@ func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { } func (vs *VolumeServer) statsDiskHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION) m := make(map[string]interface{}) m["Version"] = util.Version() var ds []*volume_server_pb.DiskStatus diff --git a/weed/server/volume_server_handlers_ui.go b/weed/server/volume_server_handlers_ui.go index 8b2027e7b..e535327e2 100644 --- a/weed/server/volume_server_handlers_ui.go +++ b/weed/server/volume_server_handlers_ui.go @@ -13,6 +13,7 @@ import ( ) func (vs *VolumeServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Server", "SeaweedFS Volume "+util.VERSION) infos := make(map[string]interface{}) infos["Up Time"] = time.Now().Sub(startTime).String() var ds []*volume_server_pb.DiskStatus diff --git a/weed/util/constants.go b/weed/util/constants.go index d29f9d2ce..6734af7d4 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -5,7 +5,7 @@ import ( ) var ( - VERSION = fmt.Sprintf("%s %d.%d", sizeLimit, 2, 00) + VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 00) COMMIT = "" )