From c857cc7286ac8264c9ce6f2bfb8c77569b86c261 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 29 Oct 2021 12:39:19 -0700 Subject: [PATCH] cloud tier: remove tagging since not all s3 vendors support this --- weed/server/volume_grpc_tier_upload.go | 7 +------ weed/storage/backend/backend.go | 2 +- weed/storage/backend/s3_backend/s3_backend.go | 4 ++-- weed/storage/backend/s3_backend/s3_upload.go | 16 +--------------- 4 files changed, 5 insertions(+), 24 deletions(-) diff --git a/weed/server/volume_grpc_tier_upload.go b/weed/server/volume_grpc_tier_upload.go index e51de5f1d..fed15b6ab 100644 --- a/weed/server/volume_grpc_tier_upload.go +++ b/weed/server/volume_grpc_tier_upload.go @@ -62,13 +62,8 @@ func (vs *VolumeServer) VolumeTierMoveDatToRemote(req *volume_server_pb.VolumeTi }) } - // remember the file original source - attributes := make(map[string]string) - attributes["volumeId"] = v.Id.String() - attributes["collection"] = v.Collection - attributes["ext"] = ".dat" // copy the data file - key, size, err := backendStorage.CopyFile(diskFile.File, attributes, fn) + key, size, err := backendStorage.CopyFile(diskFile.File, fn) if err != nil { return fmt.Errorf("backend %s copy file %s: %v", req.DestinationBackendName, diskFile.Name(), err) } diff --git a/weed/storage/backend/backend.go b/weed/storage/backend/backend.go index 2dc61d02e..132c67636 100644 --- a/weed/storage/backend/backend.go +++ b/weed/storage/backend/backend.go @@ -25,7 +25,7 @@ type BackendStorageFile interface { type BackendStorage interface { ToProperties() map[string]string NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) BackendStorageFile - CopyFile(f *os.File, attributes map[string]string, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) + CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error) (size int64, err error) DeleteFile(key string) (err error) } diff --git a/weed/storage/backend/s3_backend/s3_backend.go b/weed/storage/backend/s3_backend/s3_backend.go index 4706c9334..58867b421 100644 --- a/weed/storage/backend/s3_backend/s3_backend.go +++ b/weed/storage/backend/s3_backend/s3_backend.go @@ -79,13 +79,13 @@ func (s *S3BackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb return f } -func (s *S3BackendStorage) CopyFile(f *os.File, attributes map[string]string, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) { +func (s *S3BackendStorage) CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) { randomUuid, _ := uuid.NewRandom() key = randomUuid.String() glog.V(1).Infof("copying dat file of %s to remote s3.%s as %s", f.Name(), s.id, key) - size, err = uploadToS3(s.conn, f.Name(), s.bucket, key, attributes, fn) + size, err = uploadToS3(s.conn, f.Name(), s.bucket, key, fn) return } diff --git a/weed/storage/backend/s3_backend/s3_upload.go b/weed/storage/backend/s3_backend/s3_upload.go index 500a85590..bc1c82730 100644 --- a/weed/storage/backend/s3_backend/s3_upload.go +++ b/weed/storage/backend/s3_backend/s3_upload.go @@ -12,9 +12,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" ) -func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string, - attributes map[string]string, - fn func(progressed int64, percentage float32) error) (fileSize int64, err error) { +func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string, fn func(progressed int64, percentage float32) error) (fileSize int64, err error) { //open the file f, err := os.Open(filename) @@ -48,25 +46,13 @@ func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey fn: fn, } - // process tagging - tags := "" - for k, v := range attributes { - if len(tags) > 0 { - tags = tags + "&" - } - tags = tags + k + "=" + v - } - // Upload the file to S3. var result *s3manager.UploadOutput result, err = uploader.Upload(&s3manager.UploadInput{ Bucket: aws.String(destBucket), Key: aws.String(destKey), Body: fileReader, - ACL: aws.String("private"), - ServerSideEncryption: aws.String("AES256"), StorageClass: aws.String("STANDARD_IA"), - Tagging: aws.String(tags), }) //in case it fails to upload