2018-09-04 07:42:44 +00:00
|
|
|
package s3api
|
|
|
|
|
|
|
|
import (
|
2019-02-27 08:21:37 +00:00
|
|
|
"encoding/xml"
|
2018-09-09 23:25:43 +00:00
|
|
|
"fmt"
|
|
|
|
"path/filepath"
|
2018-09-09 23:26:11 +00:00
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
"time"
|
2018-09-09 23:25:43 +00:00
|
|
|
|
2018-09-04 07:42:44 +00:00
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
2018-09-09 23:26:11 +00:00
|
|
|
"github.com/aws/aws-sdk-go/service/s3"
|
2020-02-26 06:23:59 +00:00
|
|
|
"github.com/google/uuid"
|
|
|
|
|
2020-09-01 07:21:19 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/filer"
|
2018-09-04 07:42:44 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
|
|
|
|
)
|
|
|
|
|
2018-09-12 07:46:12 +00:00
|
|
|
type InitiateMultipartUploadResult struct {
|
2019-02-27 08:21:37 +00:00
|
|
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"`
|
2018-09-12 07:46:12 +00:00
|
|
|
s3.CreateMultipartUploadOutput
|
|
|
|
}
|
|
|
|
|
2020-02-26 06:23:59 +00:00
|
|
|
func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code ErrorCode) {
|
2019-11-19 03:22:54 +00:00
|
|
|
uploadId, _ := uuid.NewRandom()
|
2018-09-04 07:42:44 +00:00
|
|
|
uploadIdString := uploadId.String()
|
|
|
|
|
2020-02-26 06:23:59 +00:00
|
|
|
if err := s3a.mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
|
2018-09-04 07:42:44 +00:00
|
|
|
if entry.Extended == nil {
|
2018-09-07 20:12:52 +00:00
|
|
|
entry.Extended = make(map[string][]byte)
|
2018-09-04 07:42:44 +00:00
|
|
|
}
|
2018-09-07 20:12:52 +00:00
|
|
|
entry.Extended["key"] = []byte(*input.Key)
|
2018-09-04 07:42:44 +00:00
|
|
|
}); err != nil {
|
|
|
|
glog.Errorf("NewMultipartUpload error: %v", err)
|
|
|
|
return nil, ErrInternalError
|
|
|
|
}
|
|
|
|
|
2018-09-12 07:46:12 +00:00
|
|
|
output = &InitiateMultipartUploadResult{
|
2019-02-27 08:21:37 +00:00
|
|
|
CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
|
2018-09-12 07:46:12 +00:00
|
|
|
Bucket: input.Bucket,
|
2019-07-08 19:37:20 +00:00
|
|
|
Key: objectKey(input.Key),
|
2018-09-12 07:46:12 +00:00
|
|
|
UploadId: aws.String(uploadIdString),
|
|
|
|
},
|
2018-09-04 07:42:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-09-12 07:46:12 +00:00
|
|
|
type CompleteMultipartUploadResult struct {
|
2019-02-27 08:21:37 +00:00
|
|
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult"`
|
2018-09-12 07:46:12 +00:00
|
|
|
s3.CompleteMultipartUploadOutput
|
|
|
|
}
|
|
|
|
|
2020-02-26 06:23:59 +00:00
|
|
|
func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput) (output *CompleteMultipartUploadResult, code ErrorCode) {
|
2018-09-09 23:25:43 +00:00
|
|
|
|
|
|
|
uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId
|
|
|
|
|
2020-09-11 21:53:50 +00:00
|
|
|
entries, _, err := s3a.list(uploadDirectory, "", "", false, 0)
|
2020-08-26 15:48:58 +00:00
|
|
|
if err != nil || len(entries) == 0 {
|
|
|
|
glog.Errorf("completeMultipartUpload %s %s error: %v, entries:%d", *input.Bucket, *input.UploadId, err, len(entries))
|
2018-09-09 23:25:43 +00:00
|
|
|
return nil, ErrNoSuchUpload
|
|
|
|
}
|
|
|
|
|
|
|
|
var finalParts []*filer_pb.FileChunk
|
|
|
|
var offset int64
|
|
|
|
|
|
|
|
for _, entry := range entries {
|
|
|
|
if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
|
|
|
|
for _, chunk := range entry.Chunks {
|
2018-09-12 07:46:12 +00:00
|
|
|
p := &filer_pb.FileChunk{
|
2020-04-29 09:41:47 +00:00
|
|
|
FileId: chunk.GetFileIdString(),
|
|
|
|
Offset: offset,
|
|
|
|
Size: chunk.Size,
|
|
|
|
Mtime: chunk.Mtime,
|
|
|
|
CipherKey: chunk.CipherKey,
|
|
|
|
ETag: chunk.ETag,
|
2018-09-12 07:46:12 +00:00
|
|
|
}
|
|
|
|
finalParts = append(finalParts, p)
|
2018-09-09 23:25:43 +00:00
|
|
|
offset += int64(chunk.Size)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
entryName := filepath.Base(*input.Key)
|
|
|
|
dirName := filepath.Dir(*input.Key)
|
|
|
|
if dirName == "." {
|
|
|
|
dirName = ""
|
|
|
|
}
|
2018-09-12 07:46:12 +00:00
|
|
|
if strings.HasPrefix(dirName, "/") {
|
|
|
|
dirName = dirName[1:]
|
|
|
|
}
|
2018-09-09 23:25:43 +00:00
|
|
|
dirName = fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, *input.Bucket, dirName)
|
|
|
|
|
2019-09-29 13:05:37 +00:00
|
|
|
// remove suffix '/'
|
|
|
|
if strings.HasSuffix(dirName, "/") {
|
|
|
|
dirName = dirName[:len(dirName)-1]
|
|
|
|
}
|
|
|
|
|
2020-02-26 06:23:59 +00:00
|
|
|
err = s3a.mkFile(dirName, entryName, finalParts)
|
2018-09-09 23:25:43 +00:00
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
glog.Errorf("completeMultipartUpload %s/%s error: %v", dirName, entryName, err)
|
|
|
|
return nil, ErrInternalError
|
|
|
|
}
|
|
|
|
|
2018-09-12 07:46:12 +00:00
|
|
|
output = &CompleteMultipartUploadResult{
|
2019-02-27 08:21:37 +00:00
|
|
|
CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
|
2019-07-08 19:37:45 +00:00
|
|
|
Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer, dirName, entryName)),
|
2019-07-22 04:51:38 +00:00
|
|
|
Bucket: input.Bucket,
|
2020-09-01 07:21:19 +00:00
|
|
|
ETag: aws.String("\"" + filer.ETagChunks(finalParts) + "\""),
|
2019-07-22 04:51:38 +00:00
|
|
|
Key: objectKey(input.Key),
|
2018-09-12 07:46:12 +00:00
|
|
|
},
|
2018-09-09 23:25:43 +00:00
|
|
|
}
|
|
|
|
|
2020-03-20 21:17:31 +00:00
|
|
|
if err = s3a.rm(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, false, true); err != nil {
|
2018-09-12 08:00:51 +00:00
|
|
|
glog.V(1).Infof("completeMultipartUpload cleanup %s upload %s: %v", *input.Bucket, *input.UploadId, err)
|
|
|
|
}
|
|
|
|
|
2018-09-04 07:42:44 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-02-26 06:23:59 +00:00
|
|
|
func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code ErrorCode) {
|
2018-09-04 07:42:44 +00:00
|
|
|
|
2020-02-26 06:23:59 +00:00
|
|
|
exists, err := s3a.exists(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true)
|
2018-09-04 07:42:44 +00:00
|
|
|
if err != nil {
|
2018-09-09 23:25:43 +00:00
|
|
|
glog.V(1).Infof("bucket %s abort upload %s: %v", *input.Bucket, *input.UploadId, err)
|
2018-09-04 07:42:44 +00:00
|
|
|
return nil, ErrNoSuchUpload
|
|
|
|
}
|
2018-09-09 23:25:43 +00:00
|
|
|
if exists {
|
2020-03-20 21:17:31 +00:00
|
|
|
err = s3a.rm(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true, true)
|
2018-09-09 23:25:43 +00:00
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
glog.V(1).Infof("bucket %s remove upload %s: %v", *input.Bucket, *input.UploadId, err)
|
|
|
|
return nil, ErrInternalError
|
|
|
|
}
|
|
|
|
|
|
|
|
return &s3.AbortMultipartUploadOutput{}, ErrNone
|
|
|
|
}
|
|
|
|
|
2018-09-12 07:46:12 +00:00
|
|
|
type ListMultipartUploadsResult struct {
|
2019-02-27 08:21:37 +00:00
|
|
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult"`
|
2018-09-12 07:46:12 +00:00
|
|
|
s3.ListMultipartUploadsOutput
|
|
|
|
}
|
|
|
|
|
2020-02-26 06:23:59 +00:00
|
|
|
func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code ErrorCode) {
|
2018-09-09 23:25:43 +00:00
|
|
|
|
2018-09-12 07:46:12 +00:00
|
|
|
output = &ListMultipartUploadsResult{
|
2019-02-27 08:21:37 +00:00
|
|
|
ListMultipartUploadsOutput: s3.ListMultipartUploadsOutput{
|
2018-09-12 07:46:12 +00:00
|
|
|
Bucket: input.Bucket,
|
|
|
|
Delimiter: input.Delimiter,
|
|
|
|
EncodingType: input.EncodingType,
|
|
|
|
KeyMarker: input.KeyMarker,
|
|
|
|
MaxUploads: input.MaxUploads,
|
|
|
|
Prefix: input.Prefix,
|
|
|
|
},
|
2018-09-04 07:42:44 +00:00
|
|
|
}
|
2018-09-09 23:25:43 +00:00
|
|
|
|
2020-09-11 21:53:50 +00:00
|
|
|
entries, _, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), *input.Prefix, *input.KeyMarker, true, uint32(*input.MaxUploads))
|
2018-09-09 23:25:43 +00:00
|
|
|
if err != nil {
|
|
|
|
glog.Errorf("listMultipartUploads %s error: %v", *input.Bucket, err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-09-04 07:42:44 +00:00
|
|
|
for _, entry := range entries {
|
|
|
|
if entry.Extended != nil {
|
|
|
|
key := entry.Extended["key"]
|
|
|
|
output.Uploads = append(output.Uploads, &s3.MultipartUpload{
|
2019-07-08 19:37:20 +00:00
|
|
|
Key: objectKey(aws.String(string(key))),
|
2018-09-04 07:42:44 +00:00
|
|
|
UploadId: aws.String(entry.Name),
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
2018-09-12 08:00:51 +00:00
|
|
|
|
2018-09-04 07:42:44 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-09-12 07:46:12 +00:00
|
|
|
type ListPartsResult struct {
|
2019-02-27 08:21:37 +00:00
|
|
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"`
|
2020-09-11 21:53:50 +00:00
|
|
|
|
|
|
|
// copied from s3.ListPartsOutput, the Parts is not converting to <Part></Part>
|
|
|
|
Bucket *string `type:"string"`
|
|
|
|
IsTruncated *bool `type:"boolean"`
|
|
|
|
Key *string `min:"1" type:"string"`
|
|
|
|
MaxParts *int64 `type:"integer"`
|
|
|
|
NextPartNumberMarker *int64 `type:"integer"`
|
|
|
|
PartNumberMarker *int64 `type:"integer"`
|
|
|
|
Part []*s3.Part `locationName:"Part" type:"list" flattened:"true"`
|
|
|
|
StorageClass *string `type:"string" enum:"StorageClass"`
|
|
|
|
UploadId *string `type:"string"`
|
2018-09-12 07:46:12 +00:00
|
|
|
}
|
|
|
|
|
2020-02-26 06:23:59 +00:00
|
|
|
func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListPartsResult, code ErrorCode) {
|
2018-09-12 07:46:12 +00:00
|
|
|
output = &ListPartsResult{
|
2020-09-11 21:53:50 +00:00
|
|
|
Bucket: input.Bucket,
|
|
|
|
Key: objectKey(input.Key),
|
|
|
|
UploadId: input.UploadId,
|
|
|
|
MaxParts: input.MaxParts, // the maximum number of parts to return.
|
|
|
|
PartNumberMarker: input.PartNumberMarker, // the part number starts after this, exclusive
|
|
|
|
StorageClass: aws.String("STANDARD"),
|
2018-09-09 23:25:43 +00:00
|
|
|
}
|
|
|
|
|
2020-09-11 21:53:50 +00:00
|
|
|
entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d.part", *input.PartNumberMarker), false, uint32(*input.MaxParts))
|
2018-09-09 23:25:43 +00:00
|
|
|
if err != nil {
|
2018-09-11 20:01:51 +00:00
|
|
|
glog.Errorf("listObjectParts %s %s error: %v", *input.Bucket, *input.UploadId, err)
|
2018-09-09 23:25:43 +00:00
|
|
|
return nil, ErrNoSuchUpload
|
|
|
|
}
|
|
|
|
|
2020-09-11 21:53:50 +00:00
|
|
|
output.IsTruncated = aws.Bool(!isLast)
|
|
|
|
|
2018-09-09 23:25:43 +00:00
|
|
|
for _, entry := range entries {
|
|
|
|
if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
|
|
|
|
partNumberString := entry.Name[:len(entry.Name)-len(".part")]
|
|
|
|
partNumber, err := strconv.Atoi(partNumberString)
|
|
|
|
if err != nil {
|
2018-09-11 20:01:51 +00:00
|
|
|
glog.Errorf("listObjectParts %s %s parse %s: %v", *input.Bucket, *input.UploadId, entry.Name, err)
|
2018-09-09 23:25:43 +00:00
|
|
|
continue
|
|
|
|
}
|
2020-09-11 21:53:50 +00:00
|
|
|
output.Part = append(output.Part, &s3.Part{
|
2018-09-09 23:25:43 +00:00
|
|
|
PartNumber: aws.Int64(int64(partNumber)),
|
2020-04-30 04:35:24 +00:00
|
|
|
LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0).UTC()),
|
2020-09-01 07:21:19 +00:00
|
|
|
Size: aws.Int64(int64(filer.FileSize(entry))),
|
|
|
|
ETag: aws.String("\"" + filer.ETag(entry) + "\""),
|
2018-09-09 23:25:43 +00:00
|
|
|
})
|
2020-09-11 21:53:50 +00:00
|
|
|
if !isLast {
|
|
|
|
output.NextPartNumberMarker = aws.Int64(int64(partNumber))
|
|
|
|
}
|
2018-09-09 23:25:43 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-09-04 07:42:44 +00:00
|
|
|
return
|
|
|
|
}
|