gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

add namespace for s3

Browse source
This commit is contained in:
Chris Lu 2019-02-27 00:21:37 -08:00
parent 70ac2f6ea8
commit cb07d15254
6 changed files with 121 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
weed/s3api/filer_multipart.go
View file

@ -1,6 +1,7 @@
package s3api
import (
"encoding/xml"
"fmt"
"path/filepath"
"strconv"
@ -16,6 +17,7 @@ import (
)
type InitiateMultipartUploadResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"`
s3.CreateMultipartUploadOutput
}
@ -34,7 +36,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
}
output = &InitiateMultipartUploadResult{
s3.CreateMultipartUploadOutput{
CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
Bucket: input.Bucket,
Key: input.Key,
UploadId: aws.String(uploadIdString),
@ -45,6 +47,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
}
type CompleteMultipartUploadResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult"`
s3.CompleteMultipartUploadOutput
}
@ -95,7 +98,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
}
output = &CompleteMultipartUploadResult{
s3.CompleteMultipartUploadOutput{
CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
Bucket: input.Bucket,
ETag: aws.String("\"" + filer2.ETag(finalParts) + "\""),
Key: input.Key,
@ -128,13 +131,14 @@ func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput
}
type ListMultipartUploadsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult"`
s3.ListMultipartUploadsOutput
}
func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code ErrorCode) {
output = &ListMultipartUploadsResult{
s3.ListMultipartUploadsOutput{
ListMultipartUploadsOutput: s3.ListMultipartUploadsOutput{
Bucket: input.Bucket,
Delimiter: input.Delimiter,
EncodingType: input.EncodingType,
@ -164,12 +168,13 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput
}
type ListPartsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"`
s3.ListPartsOutput
}
func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListPartsResult, code ErrorCode) {
output = &ListPartsResult{
s3.ListPartsOutput{
ListPartsOutput: s3.ListPartsOutput{
Bucket: input.Bucket,
Key: input.Key,
UploadId: input.UploadId,

26
weed/s3api/filer_multipart_test.go Normal file
View file

@ -0,0 +1,26 @@
package s3api
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"testing"
)
func TestInitiateMultipartUploadResult(t *testing.T) {
expected := `<?xml version="1.0" encoding="UTF-8"?>
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Bucket>example-bucket</Bucket><Key>example-object</Key><UploadId>VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA</UploadId></InitiateMultipartUploadResult>`
response := &InitiateMultipartUploadResult{
CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
Bucket: aws.String("example-bucket"),
Key: aws.String("example-object"),
UploadId: aws.String("VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"),
},
}
encoded := string(encodeResponse(response))
if encoded != expected {
t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
}
}

1
weed/s3api/s3api_bucket_handlers_test.go
View file

@ -33,7 +33,6 @@ func TestListBucketsHandler(t *testing.T) {
}
encoded := string(encodeResponse(response))
println(encoded)
if encoded != expected {
t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
}

46
weed/s3api/s3api_objects_list_handlers.go
View file

@ -9,8 +9,6 @@ import (
"strconv"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@ -85,7 +83,7 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
writeSuccessResponseXML(w, encodeResponse(response))
}
func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response *s3.ListObjectsOutput, err error) {
func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response ListBucketResult, err error) {
// convert full path prefix into directory name and prefix for entry name
dir, prefix := filepath.Split(originalPrefix)
@ -106,8 +104,8 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
return fmt.Errorf("list buckets: %v", err)
}
var contents []*s3.Object
var commonPrefixes []*s3.CommonPrefix
var contents []ListEntry
var commonPrefixes []PrefixEntry
var counter int
var lastEntryName string
var isTruncated bool
@ -119,32 +117,32 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
}
lastEntryName = entry.Name
if entry.IsDirectory {
commonPrefixes = append(commonPrefixes, &s3.CommonPrefix{
Prefix: aws.String(fmt.Sprintf("%s%s/", dir, entry.Name)),
commonPrefixes = append(commonPrefixes, PrefixEntry{
Prefix: fmt.Sprintf("%s%s/", dir, entry.Name),
})
} else {
contents = append(contents, &s3.Object{
Key: aws.String(fmt.Sprintf("%s%s", dir, entry.Name)),
LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0)),
ETag: aws.String("\"" + filer2.ETag(entry.Chunks) + "\""),
Size: aws.Int64(int64(filer2.TotalSize(entry.Chunks))),
Owner: &s3.Owner{
ID: aws.String("bcaf161ca5fb16fd081034f"),
DisplayName: aws.String("webfile"),
contents = append(contents, ListEntry{
Key: fmt.Sprintf("%s%s", dir, entry.Name),
LastModified: time.Unix(entry.Attributes.Mtime, 0),
ETag: "\"" + filer2.ETag(entry.Chunks) + "\"",
Size: int64(filer2.TotalSize(entry.Chunks)),
Owner: CanonicalUser{
ID: "bcaf161ca5fb16fd081034f",
DisplayName: "webfile",
},
StorageClass: aws.String("STANDARD"),
StorageClass: "STANDARD",
})
}
}
response = &s3.ListObjectsOutput{
Name: aws.String(bucket),
Prefix: aws.String(originalPrefix),
Marker: aws.String(marker),
NextMarker: aws.String(lastEntryName),
MaxKeys: aws.Int64(int64(maxKeys)),
Delimiter: aws.String("/"),
IsTruncated: aws.Bool(isTruncated),
response = ListBucketResult{
Name: bucket,
Prefix: originalPrefix,
Marker: marker,
NextMarker: lastEntryName,
MaxKeys: maxKeys,
Delimiter: "/",
IsTruncated: isTruncated,
Contents: contents,
CommonPrefixes: commonPrefixes,
}

38
weed/s3api/s3api_objects_list_handlers_test.go Normal file
View file

@ -0,0 +1,38 @@
package s3api
import (
"testing"
"time"
)
func TestListObjectsHandler(t *testing.T) {
// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
expected := `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>test_container</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>1.zip</Key><ETag>&#34;4397da7a7649e8085de9916c240e8166&#34;</ETag><Size>1234567</Size><Owner><ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID></Owner><StorageClass>STANDARD</StorageClass><LastModified>2011-04-09T12:34:49</LastModified></Contents></ListBucketResult>`
response := ListBucketResult{
Name: "test_container",
Prefix: "",
Marker: "",
NextMarker: "",
MaxKeys: 1000,
IsTruncated: false,
Contents: []ListEntry{{
Key: "1.zip",
LastModified: time.Date(2011, 4, 9, 12, 34, 49, 0, time.UTC),
ETag: "\"4397da7a7649e8085de9916c240e8166\"",
Size: 1234567,
Owner: CanonicalUser{
ID: "65a011niqo39cdf8ec533ec3d1ccaafsa932",
},
StorageClass: "STANDARD",
}},
}
encoded := string(encodeResponse(response))
if encoded != expected {
t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
}
}

51
weed/s3api/s3api_xsd_generated.go
View file

@ -25,8 +25,8 @@ type BucketLoggingStatus struct {
}
type CanonicalUser struct {
ID string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ID"`
DisplayName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DisplayName,omitempty"`
ID string `xml:"ID"`
DisplayName string `xml:"DisplayName,omitempty"`
}
type CopyObject struct {
@ -506,15 +506,15 @@ func (t *ListAllMyBuckets) UnmarshalXML(d *xml.Decoder, start xml.StartElement)
}
type ListAllMyBucketsEntry struct {
Name string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"`
CreationDate time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
Name string `xml:"Name"`
CreationDate time.Time `xml:"CreationDate"`
}
func (t *ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type T ListAllMyBucketsEntry
var layout struct {
*T
CreationDate *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
CreationDate *xsdDateTime `xml:"CreationDate"`
}
layout.T = (*T)(t)
layout.CreationDate = (*xsdDateTime)(&layout.T.CreationDate)
@ -524,7 +524,7 @@ func (t *ListAllMyBucketsEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElem
type T ListAllMyBucketsEntry
var overlay struct {
*T
CreationDate *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
CreationDate *xsdDateTime `xml:"CreationDate"`
}
overlay.T = (*T)(t)
overlay.CreationDate = (*xsdDateTime)(&overlay.T.CreationDate)
@ -532,7 +532,7 @@ func (t *ListAllMyBucketsEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElem
}
type ListAllMyBucketsList struct {
Bucket []ListAllMyBucketsEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket,omitempty"`
Bucket []ListAllMyBucketsEntry `xml:"Bucket,omitempty"`
}
type ListAllMyBucketsResponse struct {
@ -577,32 +577,33 @@ type ListBucketResponse struct {
}
type ListBucketResult struct {
Metadata []MetadataEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Metadata,omitempty"`
Name string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"`
Prefix string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Prefix"`
Marker string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Marker"`
NextMarker string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NextMarker,omitempty"`
MaxKeys int `xml:"http://s3.amazonaws.com/doc/2006-03-01/ MaxKeys"`
Delimiter string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Delimiter,omitempty"`
IsTruncated bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IsTruncated"`
Contents []ListEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Contents,omitempty"`
CommonPrefixes []PrefixEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CommonPrefixes,omitempty"`
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
Metadata []MetadataEntry `xml:"Metadata,omitempty"`
Name string `xml:"Name"`
Prefix string `xml:"Prefix"`
Marker string `xml:"Marker"`
NextMarker string `xml:"NextMarker,omitempty"`
MaxKeys int `xml:"MaxKeys"`
Delimiter string `xml:"Delimiter,omitempty"`
IsTruncated bool `xml:"IsTruncated"`
Contents []ListEntry `xml:"Contents,omitempty"`
CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
}
type ListEntry struct {
Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"`
LastModified time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
ETag string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ETag"`
Size int64 `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Size"`
Owner CanonicalUser `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Owner,omitempty"`
StorageClass StorageClass `xml:"http://s3.amazonaws.com/doc/2006-03-01/ StorageClass"`
Key string `xml:"Key"`
LastModified time.Time `xml:"LastModified"`
ETag string `xml:"ETag"`
Size int64 `xml:"Size"`
Owner CanonicalUser `xml:"Owner,omitempty"`
StorageClass StorageClass `xml:"StorageClass"`
}
func (t *ListEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type T ListEntry
var layout struct {
*T
LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
LastModified *xsdDateTime `xml:"LastModified"`
}
layout.T = (*T)(t)
layout.LastModified = (*xsdDateTime)(&layout.T.LastModified)
@ -612,7 +613,7 @@ func (t *ListEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
type T ListEntry
var overlay struct {
*T
LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
LastModified *xsdDateTime `xml:"LastModified"`
}
overlay.T = (*T)(t)
overlay.LastModified = (*xsdDateTime)(&overlay.T.LastModified)