From 596c3860cac83a75ae9ce728c8a043133c03d098 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 30 May 2022 22:47:26 -0700 Subject: [PATCH] use final destination to resolve fs configuration related to https://github.com/chrislusf/seaweedfs/issues/3075 --- weed/s3api/s3_constants/s3_actions.go | 2 ++ weed/s3api/s3api_object_copy_handlers.go | 8 +++++--- weed/s3api/s3api_object_handlers.go | 8 ++++++-- weed/s3api/s3api_object_handlers_postpolicy.go | 2 +- weed/s3api/s3api_object_multipart_handlers.go | 3 ++- weed/server/filer_server_handlers_write.go | 8 +++++++- 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/weed/s3api/s3_constants/s3_actions.go b/weed/s3api/s3_constants/s3_actions.go index 4e484ac98..0fbf134e3 100644 --- a/weed/s3api/s3_constants/s3_actions.go +++ b/weed/s3api/s3_constants/s3_actions.go @@ -6,4 +6,6 @@ const ( ACTION_ADMIN = "Admin" ACTION_TAGGING = "Tagging" ACTION_LIST = "List" + + SeaweedStorageDestinationHeader = "x-seaweedfs-destination" ) diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go index c44ca7ddf..23c278f7b 100644 --- a/weed/s3api/s3api_object_copy_handlers.go +++ b/weed/s3api/s3api_object_copy_handlers.go @@ -94,7 +94,8 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request return } glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl) - etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body) + destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject) + etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body, destination) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) @@ -129,7 +130,7 @@ type CopyPartResult struct { func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) { // https://docs.aws.amazon.com/AmazonS3/latest/dev/CopyingObjctsUsingRESTMPUapi.html // https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html - dstBucket, _ := xhttp.GetBucketAndObject(r) + dstBucket, dstObject := xhttp.GetBucketAndObject(r) // Copy source path. cpSrcPath, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source")) @@ -177,7 +178,8 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req defer dataReader.Close() glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl) - etag, errCode := s3a.putToFiler(r, dstUrl, dataReader) + destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject) + etag, errCode := s3a.putToFiler(r, dstUrl, dataReader, destination) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 91086fec8..c94284fc0 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -6,6 +6,7 @@ import ( "encoding/json" "encoding/xml" "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/util/mem" "golang.org/x/exp/slices" @@ -105,7 +106,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) dataReader = mimeDetect(r, dataReader) } - etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader) + etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader, "") if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) @@ -379,7 +380,7 @@ func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (s return statusCode } -func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader) (etag string, code s3err.ErrorCode) { +func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader, destination string) (etag string, code s3err.ErrorCode) { hash := md5.New() var body = io.TeeReader(dataReader, hash) @@ -392,6 +393,9 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader } proxyReq.Header.Set("X-Forwarded-For", r.RemoteAddr) + if destination != "" { + proxyReq.Header.Set(s3_constants.SeaweedStorageDestinationHeader, destination) + } for header, values := range r.Header { for _, value := range values { diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go index b0b71b1de..5704fcf38 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy.go +++ b/weed/s3api/s3api_object_handlers_postpolicy.go @@ -115,7 +115,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R uploadUrl := fmt.Sprintf("http://%s%s/%s%s", s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlPathEscape(object)) - etag, errCode := s3a.putToFiler(r, uploadUrl, fileBody) + etag, errCode := s3a.putToFiler(r, uploadUrl, fileBody, "") if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index d74867778..69d2fae72 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -250,8 +250,9 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ if partID == 1 && r.Header.Get("Content-Type") == "" { dataReader = mimeDetect(r, dataReader) } + destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object) - etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader) + etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader, destination) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index b0e0c5af0..acee23da7 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants" "net/http" "os" "strings" @@ -58,8 +59,13 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte ctx := context.Background() + destination := r.RequestURI + if finalDestination := r.Header.Get(s3_constants.SeaweedStorageDestinationHeader); finalDestination != "" { + destination = finalDestination + } + query := r.URL.Query() - so, err := fs.detectStorageOption0(r.RequestURI, + so, err := fs.detectStorageOption0(destination, query.Get("collection"), query.Get("replication"), query.Get("ttl"),