seaweedfs/weed/s3api/filer_multipart.go

219 lines
6.8 KiB
Go
Raw Normal View History

package s3api
import (
2019-03-16 00:20:24 +00:00
"context"
2019-02-27 08:21:37 +00:00
"encoding/xml"
2018-09-09 23:25:43 +00:00
"fmt"
"path/filepath"
2018-09-09 23:26:11 +00:00
"strconv"
"strings"
"time"
2018-09-09 23:25:43 +00:00
"github.com/aws/aws-sdk-go/aws"
2018-09-09 23:26:11 +00:00
"github.com/aws/aws-sdk-go/service/s3"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/google/uuid"
)
2018-09-12 07:46:12 +00:00
type InitiateMultipartUploadResult struct {
2019-02-27 08:21:37 +00:00
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"`
2018-09-12 07:46:12 +00:00
s3.CreateMultipartUploadOutput
}
2019-03-16 00:20:24 +00:00
func (s3a *S3ApiServer) createMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code ErrorCode) {
uploadId, _ := uuid.NewRandom()
uploadIdString := uploadId.String()
2019-03-16 00:20:24 +00:00
if err := s3a.mkdir(ctx, s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
if entry.Extended == nil {
2018-09-07 20:12:52 +00:00
entry.Extended = make(map[string][]byte)
}
2018-09-07 20:12:52 +00:00
entry.Extended["key"] = []byte(*input.Key)
}); err != nil {
glog.Errorf("NewMultipartUpload error: %v", err)
return nil, ErrInternalError
}
2018-09-12 07:46:12 +00:00
output = &InitiateMultipartUploadResult{
2019-02-27 08:21:37 +00:00
CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
2018-09-12 07:46:12 +00:00
Bucket: input.Bucket,
2019-07-08 19:37:20 +00:00
Key: objectKey(input.Key),
2018-09-12 07:46:12 +00:00
UploadId: aws.String(uploadIdString),
},
}
return
}
2018-09-12 07:46:12 +00:00
type CompleteMultipartUploadResult struct {
2019-02-27 08:21:37 +00:00
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult"`
2018-09-12 07:46:12 +00:00
s3.CompleteMultipartUploadOutput
}
2019-03-16 00:20:24 +00:00
func (s3a *S3ApiServer) completeMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (output *CompleteMultipartUploadResult, code ErrorCode) {
2018-09-09 23:25:43 +00:00
uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId
2019-03-16 00:20:24 +00:00
entries, err := s3a.list(ctx, uploadDirectory, "", "", false, 0)
2018-09-09 23:25:43 +00:00
if err != nil {
2018-09-11 20:01:51 +00:00
glog.Errorf("completeMultipartUpload %s %s error: %v", *input.Bucket, *input.UploadId, err)
2018-09-09 23:25:43 +00:00
return nil, ErrNoSuchUpload
}
var finalParts []*filer_pb.FileChunk
var offset int64
for _, entry := range entries {
if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
for _, chunk := range entry.Chunks {
2018-09-12 07:46:12 +00:00
p := &filer_pb.FileChunk{
2019-06-23 03:04:56 +00:00
FileId: chunk.GetFileIdString(),
2018-09-09 23:25:43 +00:00
Offset: offset,
Size: chunk.Size,
Mtime: chunk.Mtime,
ETag: chunk.ETag,
2018-09-12 07:46:12 +00:00
}
finalParts = append(finalParts, p)
2018-09-09 23:25:43 +00:00
offset += int64(chunk.Size)
}
}
}
entryName := filepath.Base(*input.Key)
dirName := filepath.Dir(*input.Key)
if dirName == "." {
dirName = ""
}
2018-09-12 07:46:12 +00:00
if strings.HasPrefix(dirName, "/") {
dirName = dirName[1:]
}
2018-09-09 23:25:43 +00:00
dirName = fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, *input.Bucket, dirName)
// remove suffix '/'
if strings.HasSuffix(dirName, "/") {
dirName = dirName[:len(dirName)-1]
}
2019-03-16 00:20:24 +00:00
err = s3a.mkFile(ctx, dirName, entryName, finalParts)
2018-09-09 23:25:43 +00:00
if err != nil {
glog.Errorf("completeMultipartUpload %s/%s error: %v", dirName, entryName, err)
return nil, ErrInternalError
}
2018-09-12 07:46:12 +00:00
output = &CompleteMultipartUploadResult{
2019-02-27 08:21:37 +00:00
CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer, dirName, entryName)),
2019-07-22 04:51:38 +00:00
Bucket: input.Bucket,
ETag: aws.String("\"" + filer2.ETag(finalParts) + "\""),
Key: objectKey(input.Key),
2018-09-12 07:46:12 +00:00
},
2018-09-09 23:25:43 +00:00
}
2019-03-16 00:20:24 +00:00
if err = s3a.rm(ctx, s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true, false, true); err != nil {
2018-09-12 08:00:51 +00:00
glog.V(1).Infof("completeMultipartUpload cleanup %s upload %s: %v", *input.Bucket, *input.UploadId, err)
}
return
}
2019-03-16 00:20:24 +00:00
func (s3a *S3ApiServer) abortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code ErrorCode) {
2019-03-16 00:20:24 +00:00
exists, err := s3a.exists(ctx, s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true)
if err != nil {
2018-09-09 23:25:43 +00:00
glog.V(1).Infof("bucket %s abort upload %s: %v", *input.Bucket, *input.UploadId, err)
return nil, ErrNoSuchUpload
}
2018-09-09 23:25:43 +00:00
if exists {
2019-03-16 00:20:24 +00:00
err = s3a.rm(ctx, s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true, true, true)
2018-09-09 23:25:43 +00:00
}
if err != nil {
glog.V(1).Infof("bucket %s remove upload %s: %v", *input.Bucket, *input.UploadId, err)
return nil, ErrInternalError
}
return &s3.AbortMultipartUploadOutput{}, ErrNone
}
2018-09-12 07:46:12 +00:00
type ListMultipartUploadsResult struct {
2019-02-27 08:21:37 +00:00
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult"`
2018-09-12 07:46:12 +00:00
s3.ListMultipartUploadsOutput
}
2019-03-16 00:20:24 +00:00
func (s3a *S3ApiServer) listMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code ErrorCode) {
2018-09-09 23:25:43 +00:00
2018-09-12 07:46:12 +00:00
output = &ListMultipartUploadsResult{
2019-02-27 08:21:37 +00:00
ListMultipartUploadsOutput: s3.ListMultipartUploadsOutput{
2018-09-12 07:46:12 +00:00
Bucket: input.Bucket,
Delimiter: input.Delimiter,
EncodingType: input.EncodingType,
KeyMarker: input.KeyMarker,
MaxUploads: input.MaxUploads,
Prefix: input.Prefix,
},
}
2018-09-09 23:25:43 +00:00
2019-03-16 00:20:24 +00:00
entries, err := s3a.list(ctx, s3a.genUploadsFolder(*input.Bucket), *input.Prefix, *input.KeyMarker, true, int(*input.MaxUploads))
2018-09-09 23:25:43 +00:00
if err != nil {
glog.Errorf("listMultipartUploads %s error: %v", *input.Bucket, err)
return
}
for _, entry := range entries {
if entry.Extended != nil {
key := entry.Extended["key"]
output.Uploads = append(output.Uploads, &s3.MultipartUpload{
2019-07-08 19:37:20 +00:00
Key: objectKey(aws.String(string(key))),
UploadId: aws.String(entry.Name),
})
}
}
2018-09-12 08:00:51 +00:00
return
}
2018-09-12 07:46:12 +00:00
type ListPartsResult struct {
2019-02-27 08:21:37 +00:00
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"`
2018-09-12 07:46:12 +00:00
s3.ListPartsOutput
}
2019-03-16 00:20:24 +00:00
func (s3a *S3ApiServer) listObjectParts(ctx context.Context, input *s3.ListPartsInput) (output *ListPartsResult, code ErrorCode) {
2018-09-12 07:46:12 +00:00
output = &ListPartsResult{
2019-02-27 08:21:37 +00:00
ListPartsOutput: s3.ListPartsOutput{
2018-09-12 07:46:12 +00:00
Bucket: input.Bucket,
2019-07-08 19:37:20 +00:00
Key: objectKey(input.Key),
2018-09-12 07:46:12 +00:00
UploadId: input.UploadId,
MaxParts: input.MaxParts, // the maximum number of parts to return.
PartNumberMarker: input.PartNumberMarker, // the part number starts after this, exclusive
},
2018-09-09 23:25:43 +00:00
}
2019-03-16 00:20:24 +00:00
entries, err := s3a.list(ctx, s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId,
2018-09-09 23:25:43 +00:00
"", fmt.Sprintf("%04d.part", *input.PartNumberMarker), false, int(*input.MaxParts))
if err != nil {
2018-09-11 20:01:51 +00:00
glog.Errorf("listObjectParts %s %s error: %v", *input.Bucket, *input.UploadId, err)
2018-09-09 23:25:43 +00:00
return nil, ErrNoSuchUpload
}
for _, entry := range entries {
if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
partNumberString := entry.Name[:len(entry.Name)-len(".part")]
partNumber, err := strconv.Atoi(partNumberString)
if err != nil {
2018-09-11 20:01:51 +00:00
glog.Errorf("listObjectParts %s %s parse %s: %v", *input.Bucket, *input.UploadId, entry.Name, err)
2018-09-09 23:25:43 +00:00
continue
}
output.Parts = append(output.Parts, &s3.Part{
PartNumber: aws.Int64(int64(partNumber)),
LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0)),
Size: aws.Int64(int64(filer2.TotalSize(entry.Chunks))),
ETag: aws.String("\"" + filer2.ETag(entry.Chunks) + "\""),
})
}
}
return
}