gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

add namespace for s3

Browse source
This commit is contained in:
Chris Lu 2019-02-27 00:21:37 -08:00
parent 70ac2f6ea8
commit cb07d15254
6 changed files with 121 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
weed/s3api/filer_multipart.go
View file

@ -1,6 +1,7 @@
package s3api package s3api
import ( import (
"encoding/xml"
"fmt" "fmt"
"path/filepath" "path/filepath"
"strconv" "strconv"
@ -16,6 +17,7 @@ import (
) )
type InitiateMultipartUploadResult struct { type InitiateMultipartUploadResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"`
s3.CreateMultipartUploadOutput s3.CreateMultipartUploadOutput
} }
@ -34,7 +36,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
} }
output = &InitiateMultipartUploadResult{ output = &InitiateMultipartUploadResult{
s3.CreateMultipartUploadOutput{ CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
Bucket: input.Bucket, Bucket: input.Bucket,
Key: input.Key, Key: input.Key,
UploadId: aws.String(uploadIdString), UploadId: aws.String(uploadIdString),
@ -45,6 +47,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
} }
type CompleteMultipartUploadResult struct { type CompleteMultipartUploadResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult"`
s3.CompleteMultipartUploadOutput s3.CompleteMultipartUploadOutput
} }
@ -95,7 +98,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
} }
output = &CompleteMultipartUploadResult{ output = &CompleteMultipartUploadResult{
s3.CompleteMultipartUploadOutput{ CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
Bucket: input.Bucket, Bucket: input.Bucket,
ETag: aws.String("\"" + filer2.ETag(finalParts) + "\""), ETag: aws.String("\"" + filer2.ETag(finalParts) + "\""),
Key: input.Key, Key: input.Key,
@ -128,13 +131,14 @@ func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput
} }
type ListMultipartUploadsResult struct { type ListMultipartUploadsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult"`
s3.ListMultipartUploadsOutput s3.ListMultipartUploadsOutput
} }
func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code ErrorCode) { func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code ErrorCode) {
output = &ListMultipartUploadsResult{ output = &ListMultipartUploadsResult{
s3.ListMultipartUploadsOutput{ ListMultipartUploadsOutput: s3.ListMultipartUploadsOutput{
Bucket: input.Bucket, Bucket: input.Bucket,
Delimiter: input.Delimiter, Delimiter: input.Delimiter,
EncodingType: input.EncodingType, EncodingType: input.EncodingType,
@ -164,12 +168,13 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput
} }
type ListPartsResult struct { type ListPartsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"`
s3.ListPartsOutput s3.ListPartsOutput
} }
func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListPartsResult, code ErrorCode) { func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListPartsResult, code ErrorCode) {
output = &ListPartsResult{ output = &ListPartsResult{
s3.ListPartsOutput{ ListPartsOutput: s3.ListPartsOutput{
Bucket: input.Bucket, Bucket: input.Bucket,
Key: input.Key, Key: input.Key,
UploadId: input.UploadId, UploadId: input.UploadId,

26
weed/s3api/filer_multipart_test.go Normal file
View file

@ -0,0 +1,26 @@
package s3api
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"testing"
)
func TestInitiateMultipartUploadResult(t *testing.T) {
expected := `<?xml version="1.0" encoding="UTF-8"?>
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Bucket>example-bucket</Bucket><Key>example-object</Key><UploadId>VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA</UploadId></InitiateMultipartUploadResult>`
response := &InitiateMultipartUploadResult{
CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
Bucket: aws.String("example-bucket"),
Key: aws.String("example-object"),
UploadId: aws.String("VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"),
},
}
encoded := string(encodeResponse(response))
if encoded != expected {
t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
}
}

1
weed/s3api/s3api_bucket_handlers_test.go
View file

@ -33,7 +33,6 @@ func TestListBucketsHandler(t *testing.T) {
} }
encoded := string(encodeResponse(response)) encoded := string(encodeResponse(response))
println(encoded)
if encoded != expected { if encoded != expected {
t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected) t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
} }

46
weed/s3api/s3api_objects_list_handlers.go
View file

@ -9,8 +9,6 @@ import (
"strconv" "strconv"
"time" "time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@ -85,7 +83,7 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
writeSuccessResponseXML(w, encodeResponse(response)) writeSuccessResponseXML(w, encodeResponse(response))
} }
func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response *s3.ListObjectsOutput, err error) { func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response ListBucketResult, err error) {
// convert full path prefix into directory name and prefix for entry name // convert full path prefix into directory name and prefix for entry name
dir, prefix := filepath.Split(originalPrefix) dir, prefix := filepath.Split(originalPrefix)
@ -106,8 +104,8 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
return fmt.Errorf("list buckets: %v", err) return fmt.Errorf("list buckets: %v", err)
} }
var contents []*s3.Object var contents []ListEntry
var commonPrefixes []*s3.CommonPrefix var commonPrefixes []PrefixEntry
var counter int var counter int
var lastEntryName string var lastEntryName string
var isTruncated bool var isTruncated bool
@ -119,32 +117,32 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
} }
lastEntryName = entry.Name lastEntryName = entry.Name
if entry.IsDirectory { if entry.IsDirectory {
commonPrefixes = append(commonPrefixes, &s3.CommonPrefix{ commonPrefixes = append(commonPrefixes, PrefixEntry{
Prefix: aws.String(fmt.Sprintf("%s%s/", dir, entry.Name)), Prefix: fmt.Sprintf("%s%s/", dir, entry.Name),
}) })
} else { } else {
contents = append(contents, &s3.Object{ contents = append(contents, ListEntry{
Key: aws.String(fmt.Sprintf("%s%s", dir, entry.Name)), Key: fmt.Sprintf("%s%s", dir, entry.Name),
LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0)), LastModified: time.Unix(entry.Attributes.Mtime, 0),
ETag: aws.String("\"" + filer2.ETag(entry.Chunks) + "\""), ETag: "\"" + filer2.ETag(entry.Chunks) + "\"",
Size: aws.Int64(int64(filer2.TotalSize(entry.Chunks))), Size: int64(filer2.TotalSize(entry.Chunks)),
Owner: &s3.Owner{ Owner: CanonicalUser{
ID: aws.String("bcaf161ca5fb16fd081034f"), ID: "bcaf161ca5fb16fd081034f",
DisplayName: aws.String("webfile"), DisplayName: "webfile",
}, },
StorageClass: aws.String("STANDARD"), StorageClass: "STANDARD",
}) })
} }
} }
response = &s3.ListObjectsOutput{ response = ListBucketResult{
Name: aws.String(bucket), Name: bucket,
Prefix: aws.String(originalPrefix), Prefix: originalPrefix,
Marker: aws.String(marker), Marker: marker,
NextMarker: aws.String(lastEntryName), NextMarker: lastEntryName,
MaxKeys: aws.Int64(int64(maxKeys)), MaxKeys: maxKeys,
Delimiter: aws.String("/"), Delimiter: "/",
IsTruncated: aws.Bool(isTruncated), IsTruncated: isTruncated,
Contents: contents, Contents: contents,
CommonPrefixes: commonPrefixes, CommonPrefixes: commonPrefixes,
} }

38
weed/s3api/s3api_objects_list_handlers_test.go Normal file
View file

@ -0,0 +1,38 @@
package s3api
import (
"testing"
"time"
)
func TestListObjectsHandler(t *testing.T) {
// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
expected := `<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>test_container</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>1.zip</Key><ETag>&#34;4397da7a7649e8085de9916c240e8166&#34;</ETag><Size>1234567</Size><Owner><ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID></Owner><StorageClass>STANDARD</StorageClass><LastModified>2011-04-09T12:34:49</LastModified></Contents></ListBucketResult>`
response := ListBucketResult{
Name: "test_container",
Prefix: "",
Marker: "",
NextMarker: "",
MaxKeys: 1000,
IsTruncated: false,
Contents: []ListEntry{{
Key: "1.zip",
LastModified: time.Date(2011, 4, 9, 12, 34, 49, 0, time.UTC),
ETag: "\"4397da7a7649e8085de9916c240e8166\"",
Size: 1234567,
Owner: CanonicalUser{
ID: "65a011niqo39cdf8ec533ec3d1ccaafsa932",
},
StorageClass: "STANDARD",
}},
}
encoded := string(encodeResponse(response))
if encoded != expected {
t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
}
}

51
weed/s3api/s3api_xsd_generated.go
View file

@ -25,8 +25,8 @@ type BucketLoggingStatus struct {
} }
type CanonicalUser struct { type CanonicalUser struct {
ID string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ID"` ID string `xml:"ID"`
DisplayName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DisplayName,omitempty"` DisplayName string `xml:"DisplayName,omitempty"`
} }
type CopyObject struct { type CopyObject struct {
@ -506,15 +506,15 @@ func (t *ListAllMyBuckets) UnmarshalXML(d *xml.Decoder, start xml.StartElement)
} }
type ListAllMyBucketsEntry struct { type ListAllMyBucketsEntry struct {
Name string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"` Name string `xml:"Name"`
CreationDate time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"` CreationDate time.Time `xml:"CreationDate"`
} }
func (t *ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error { func (t *ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type T ListAllMyBucketsEntry type T ListAllMyBucketsEntry
var layout struct { var layout struct {
*T *T
CreationDate *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"` CreationDate *xsdDateTime `xml:"CreationDate"`
} }
layout.T = (*T)(t) layout.T = (*T)(t)
layout.CreationDate = (*xsdDateTime)(&layout.T.CreationDate) layout.CreationDate = (*xsdDateTime)(&layout.T.CreationDate)
@ -524,7 +524,7 @@ func (t *ListAllMyBucketsEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElem
type T ListAllMyBucketsEntry type T ListAllMyBucketsEntry
var overlay struct { var overlay struct {
*T *T
CreationDate *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"` CreationDate *xsdDateTime `xml:"CreationDate"`
} }
overlay.T = (*T)(t) overlay.T = (*T)(t)
overlay.CreationDate = (*xsdDateTime)(&overlay.T.CreationDate) overlay.CreationDate = (*xsdDateTime)(&overlay.T.CreationDate)
@ -532,7 +532,7 @@ func (t *ListAllMyBucketsEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElem
} }
type ListAllMyBucketsList struct { type ListAllMyBucketsList struct {
Bucket []ListAllMyBucketsEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket,omitempty"` Bucket []ListAllMyBucketsEntry `xml:"Bucket,omitempty"`
} }
type ListAllMyBucketsResponse struct { type ListAllMyBucketsResponse struct {
@ -577,32 +577,33 @@ type ListBucketResponse struct {
} }
type ListBucketResult struct { type ListBucketResult struct {
Metadata []MetadataEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Metadata,omitempty"` XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
Name string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"` Metadata []MetadataEntry `xml:"Metadata,omitempty"`
Prefix string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Prefix"` Name string `xml:"Name"`
Marker string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Marker"` Prefix string `xml:"Prefix"`
NextMarker string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NextMarker,omitempty"` Marker string `xml:"Marker"`
MaxKeys int `xml:"http://s3.amazonaws.com/doc/2006-03-01/ MaxKeys"` NextMarker string `xml:"NextMarker,omitempty"`
Delimiter string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Delimiter,omitempty"` MaxKeys int `xml:"MaxKeys"`
IsTruncated bool `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IsTruncated"` Delimiter string `xml:"Delimiter,omitempty"`
Contents []ListEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Contents,omitempty"` IsTruncated bool `xml:"IsTruncated"`
CommonPrefixes []PrefixEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CommonPrefixes,omitempty"` Contents []ListEntry `xml:"Contents,omitempty"`
CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
} }
type ListEntry struct { type ListEntry struct {
Key string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"` Key string `xml:"Key"`
LastModified time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` LastModified time.Time `xml:"LastModified"`
ETag string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ETag"` ETag string `xml:"ETag"`
Size int64 `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Size"` Size int64 `xml:"Size"`
Owner CanonicalUser `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Owner,omitempty"` Owner CanonicalUser `xml:"Owner,omitempty"`
StorageClass StorageClass `xml:"http://s3.amazonaws.com/doc/2006-03-01/ StorageClass"` StorageClass StorageClass `xml:"StorageClass"`
} }
func (t *ListEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error { func (t *ListEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
type T ListEntry type T ListEntry
var layout struct { var layout struct {
*T *T
LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` LastModified *xsdDateTime `xml:"LastModified"`
} }
layout.T = (*T)(t) layout.T = (*T)(t)
layout.LastModified = (*xsdDateTime)(&layout.T.LastModified) layout.LastModified = (*xsdDateTime)(&layout.T.LastModified)
@ -612,7 +613,7 @@ func (t *ListEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
type T ListEntry type T ListEntry
var overlay struct { var overlay struct {
*T *T
LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"` LastModified *xsdDateTime `xml:"LastModified"`
} }
overlay.T = (*T)(t) overlay.T = (*T)(t)
overlay.LastModified = (*xsdDateTime)(&overlay.T.LastModified) overlay.LastModified = (*xsdDateTime)(&overlay.T.LastModified)