add list all my buckets

This commit is contained in:
Chris Lu 2018-07-18 02:37:09 -07:00
parent 7abfab8e77
commit 834a25f084
11 changed files with 2139 additions and 17 deletions

View file

@ -16,6 +16,7 @@ var Commands = []*Command{
cmdServer, cmdServer,
cmdMaster, cmdMaster,
cmdFiler, cmdFiler,
cmdS3,
cmdUpload, cmdUpload,
cmdDownload, cmdDownload,
cmdShell, cmdShell,

View file

@ -53,27 +53,14 @@ func runMount(cmd *Command, args []string) bool {
c.Close() c.Close()
}) })
hostnameAndPort := strings.Split(*mountOptions.filer, ":") filerGrpcAddress, err := parseFilerGrpcAddress(*mountOptions.filer, *mountOptions.filerGrpcPort)
if len(hostnameAndPort) != 2 { if err != nil {
fmt.Printf("The filer should have hostname:port format: %v\n", hostnameAndPort) glog.Fatal(err)
return false return false
} }
filerPort, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64)
if parseErr != nil {
fmt.Printf("The filer filer port parse error: %v\n", parseErr)
return false
}
filerGrpcPort := filerPort + 10000
if *mountOptions.filerGrpcPort != 0 {
filerGrpcPort = uint64(*copy.filerGrpcPort)
}
filerAddress := fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort)
err = fs.Serve(c, filesys.NewSeaweedFileSystem( err = fs.Serve(c, filesys.NewSeaweedFileSystem(
filerAddress, *mountOptions.filerMountRootPath, *mountOptions.collection, *mountOptions.replication, int32(*mountOptions.ttlSec), filerGrpcAddress, *mountOptions.filerMountRootPath, *mountOptions.collection, *mountOptions.replication, int32(*mountOptions.ttlSec),
*mountOptions.chunkSizeLimitMB, *mountOptions.dataCenter)) *mountOptions.chunkSizeLimitMB, *mountOptions.dataCenter))
if err != nil { if err != nil {
fuse.Unmount(*mountOptions.dir) fuse.Unmount(*mountOptions.dir)
@ -87,3 +74,22 @@ func runMount(cmd *Command, args []string) bool {
return true return true
} }
func parseFilerGrpcAddress(filer string, optionalGrpcPort int) (filerGrpcAddress string, err error) {
hostnameAndPort := strings.Split(filer, ":")
if len(hostnameAndPort) != 2 {
return "", fmt.Errorf("The filer should have hostname:port format: %v", hostnameAndPort)
}
filerPort, parseErr := strconv.ParseUint(hostnameAndPort[1], 10, 64)
if parseErr != nil {
return "", fmt.Errorf("The filer filer port parse error: %v", parseErr)
}
filerGrpcPort := int(filerPort) + 10000
if optionalGrpcPort != 0 {
filerGrpcPort = optionalGrpcPort
}
return fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort), nil
}

73
weed/command/s3.go Normal file
View file

@ -0,0 +1,73 @@
package command
import (
"net/http"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/gorilla/mux"
"fmt"
"github.com/chrislusf/seaweedfs/weed/s3api"
)
var (
s3options S3Options
)
type S3Options struct {
filer *string
filerGrpcPort *int
port *int
domainName *string
}
func init() {
cmdS3.Run = runS3 // break init cycle
s3options.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
s3options.filerGrpcPort = cmdS3.Flag.Int("filer.grpcPort", 0, "filer server grpc port, default to filer http port plus 10000")
s3options.port = cmdS3.Flag.Int("port", 8333, "s3options server http listen port")
s3options.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name, {bucket}.{domainName}")
}
var cmdS3 = &Command{
UsageLine: "s3 -port=8333 -filer=<ip:port>",
Short: "start a s3 API compatible server that is backed by a filer",
Long: `start a s3 API compatible server that is backed by a filer.
`,
}
func runS3(cmd *Command, args []string) bool {
filerGrpcAddress, err := parseFilerGrpcAddress(*s3options.filer, *s3options.filerGrpcPort)
if err != nil {
glog.Fatal(err)
return false
}
router := mux.NewRouter().SkipClean(true)
_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
Filer: *s3options.filer,
FilerGrpcAddress: filerGrpcAddress,
DomainName: *s3options.domainName,
})
if s3ApiServer_err != nil {
glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
}
glog.V(0).Infof("Start Seaweed S3 API Server %s at port %d", util.VERSION, *s3options.port)
s3ApiListener, e := util.NewListener(fmt.Sprintf(":%d", *s3options.port), time.Duration(10)*time.Second)
if e != nil {
glog.Fatalf("S3 API Server listener error: %v", e)
}
httpS := &http.Server{Handler: router}
if err := httpS.Serve(s3ApiListener); err != nil {
glog.Fatalf("S3 API Server Fail to serve: %v", e)
}
return true
}

692
weed/s3api/AmazonS3.xsd Normal file
View file

@ -0,0 +1,692 @@
<?xml version="1.0" encoding="UTF-8"?>
<xsd:schema
xmlns:tns="http://s3.amazonaws.com/doc/2006-03-01/"
xmlns:xsd="http://www.w3.org/2001/XMLSchema"
elementFormDefault="qualified"
targetNamespace="http://s3.amazonaws.com/doc/2006-03-01/">
<xsd:element name="CreateBucket">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="MetadataEntry">
<xsd:sequence>
<xsd:element name="Name" type="xsd:string"/>
<xsd:element name="Value" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="CreateBucketResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="CreateBucketReturn" type="tns:CreateBucketResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="Status">
<xsd:sequence>
<xsd:element name="Code" type="xsd:int"/>
<xsd:element name="Description" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="Result">
<xsd:sequence>
<xsd:element name="Status" type="tns:Status"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="CreateBucketResult">
<xsd:sequence>
<xsd:element name="BucketName" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="DeleteBucket">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="DeleteBucketResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="DeleteBucketResponse" type="tns:Status"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="BucketLoggingStatus">
<xsd:sequence>
<xsd:element name="LoggingEnabled" type="tns:LoggingSettings" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="LoggingSettings">
<xsd:sequence>
<xsd:element name="TargetBucket" type="xsd:string"/>
<xsd:element name="TargetPrefix" type="xsd:string"/>
<xsd:element name="TargetGrants" type="tns:AccessControlList" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="GetBucketLoggingStatus">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetBucketLoggingStatusResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="GetBucketLoggingStatusResponse" type="tns:BucketLoggingStatus"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="SetBucketLoggingStatus">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
<xsd:element name="BucketLoggingStatus" type="tns:BucketLoggingStatus"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="SetBucketLoggingStatusResponse">
<xsd:complexType>
<xsd:sequence/>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetObjectAccessControlPolicy">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetObjectAccessControlPolicyResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="GetObjectAccessControlPolicyResponse" type="tns:AccessControlPolicy"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetBucketAccessControlPolicy">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetBucketAccessControlPolicyResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="GetBucketAccessControlPolicyResponse" type="tns:AccessControlPolicy"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType abstract="true" name="Grantee"/>
<xsd:complexType name="User" abstract="true">
<xsd:complexContent>
<xsd:extension base="tns:Grantee"/>
</xsd:complexContent>
</xsd:complexType>
<xsd:complexType name="AmazonCustomerByEmail">
<xsd:complexContent>
<xsd:extension base="tns:User">
<xsd:sequence>
<xsd:element name="EmailAddress" type="xsd:string"/>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:complexType name="CanonicalUser">
<xsd:complexContent>
<xsd:extension base="tns:User">
<xsd:sequence>
<xsd:element name="ID" type="xsd:string"/>
<xsd:element name="DisplayName" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:complexType name="Group">
<xsd:complexContent>
<xsd:extension base="tns:Grantee">
<xsd:sequence>
<xsd:element name="URI" type="xsd:string"/>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:simpleType name="Permission">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="READ"/>
<xsd:enumeration value="WRITE"/>
<xsd:enumeration value="READ_ACP"/>
<xsd:enumeration value="WRITE_ACP"/>
<xsd:enumeration value="FULL_CONTROL"/>
</xsd:restriction>
</xsd:simpleType>
<xsd:simpleType name="StorageClass">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="STANDARD"/>
<xsd:enumeration value="REDUCED_REDUNDANCY"/>
<xsd:enumeration value="GLACIER"/>
<xsd:enumeration value="UNKNOWN"/>
</xsd:restriction>
</xsd:simpleType>
<xsd:complexType name="Grant">
<xsd:sequence>
<xsd:element name="Grantee" type="tns:Grantee"/>
<xsd:element name="Permission" type="tns:Permission"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="AccessControlList">
<xsd:sequence>
<xsd:element name="Grant" type="tns:Grant" minOccurs="0" maxOccurs="100"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="CreateBucketConfiguration">
<xsd:sequence>
<xsd:element name="LocationConstraint" type="tns:LocationConstraint"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="LocationConstraint">
<xsd:simpleContent>
<xsd:extension base="xsd:string"/>
</xsd:simpleContent>
</xsd:complexType>
<xsd:complexType name="AccessControlPolicy">
<xsd:sequence>
<xsd:element name="Owner" type="tns:CanonicalUser"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="SetObjectAccessControlPolicy">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="SetObjectAccessControlPolicyResponse">
<xsd:complexType>
<xsd:sequence/>
</xsd:complexType>
</xsd:element>
<xsd:element name="SetBucketAccessControlPolicy">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="SetBucketAccessControlPolicyResponse">
<xsd:complexType>
<xsd:sequence/>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetObject">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="GetMetadata" type="xsd:boolean"/>
<xsd:element name="GetData" type="xsd:boolean"/>
<xsd:element name="InlineData" type="xsd:boolean"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetObjectResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="GetObjectResponse" type="tns:GetObjectResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="GetObjectResult">
<xsd:complexContent>
<xsd:extension base="tns:Result">
<xsd:sequence>
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="unbounded"/>
<xsd:element name="Data" type="xsd:base64Binary" nillable="true"/>
<xsd:element name="LastModified" type="xsd:dateTime"/>
<xsd:element name="ETag" type="xsd:string"/>
</xsd:sequence>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>
<xsd:element name="GetObjectExtended">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="GetMetadata" type="xsd:boolean"/>
<xsd:element name="GetData" type="xsd:boolean"/>
<xsd:element name="InlineData" type="xsd:boolean"/>
<xsd:element name="ByteRangeStart" type="xsd:long" minOccurs="0"/>
<xsd:element name="ByteRangeEnd" type="xsd:long" minOccurs="0"/>
<xsd:element name="IfModifiedSince" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="IfUnmodifiedSince" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="IfMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
<xsd:element name="IfNoneMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
<xsd:element name="ReturnCompleteObjectOnConditionFailure" type="xsd:boolean" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="GetObjectExtendedResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="GetObjectResponse" type="tns:GetObjectResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="PutObject">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="100"/>
<xsd:element name="ContentLength" type="xsd:long"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
<xsd:element name="StorageClass" type="tns:StorageClass" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="PutObjectResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="PutObjectResponse" type="tns:PutObjectResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="PutObjectResult">
<xsd:sequence>
<xsd:element name="ETag" type="xsd:string"/>
<xsd:element name="LastModified" type="xsd:dateTime"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="PutObjectInline">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element minOccurs="0" maxOccurs="100" name="Metadata" type="tns:MetadataEntry"/>
<xsd:element name="Data" type="xsd:base64Binary"/>
<xsd:element name="ContentLength" type="xsd:long"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
<xsd:element name="StorageClass" type="tns:StorageClass" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="PutObjectInlineResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="PutObjectInlineResponse" type="tns:PutObjectResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="DeleteObject">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="DeleteObjectResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="DeleteObjectResponse" type="tns:Status"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="ListBucket">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Prefix" type="xsd:string" minOccurs="0"/>
<xsd:element name="Marker" type="xsd:string" minOccurs="0"/>
<xsd:element name="MaxKeys" type="xsd:int" minOccurs="0"/>
<xsd:element name="Delimiter" type="xsd:string" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="ListBucketResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="ListBucketResponse" type="tns:ListBucketResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="ListVersionsResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="ListVersionsResponse" type="tns:ListVersionsResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="ListEntry">
<xsd:sequence>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="LastModified" type="xsd:dateTime"/>
<xsd:element name="ETag" type="xsd:string"/>
<xsd:element name="Size" type="xsd:long"/>
<xsd:element name="Owner" type="tns:CanonicalUser" minOccurs="0"/>
<xsd:element name="StorageClass" type="tns:StorageClass"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="VersionEntry">
<xsd:sequence>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="VersionId" type="xsd:string"/>
<xsd:element name="IsLatest" type="xsd:boolean"/>
<xsd:element name="LastModified" type="xsd:dateTime"/>
<xsd:element name="ETag" type="xsd:string"/>
<xsd:element name="Size" type="xsd:long"/>
<xsd:element name="Owner" type="tns:CanonicalUser" minOccurs="0"/>
<xsd:element name="StorageClass" type="tns:StorageClass"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="DeleteMarkerEntry">
<xsd:sequence>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="VersionId" type="xsd:string"/>
<xsd:element name="IsLatest" type="xsd:boolean"/>
<xsd:element name="LastModified" type="xsd:dateTime"/>
<xsd:element name="Owner" type="tns:CanonicalUser" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="PrefixEntry">
<xsd:sequence>
<xsd:element name="Prefix" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="ListBucketResult">
<xsd:sequence>
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="unbounded"/>
<xsd:element name="Name" type="xsd:string"/>
<xsd:element name="Prefix" type="xsd:string"/>
<xsd:element name="Marker" type="xsd:string"/>
<xsd:element name="NextMarker" type="xsd:string" minOccurs="0"/>
<xsd:element name="MaxKeys" type="xsd:int"/>
<xsd:element name="Delimiter" type="xsd:string" minOccurs="0"/>
<xsd:element name="IsTruncated" type="xsd:boolean"/>
<xsd:element name="Contents" type="tns:ListEntry" minOccurs="0" maxOccurs="unbounded"/>
<xsd:element name="CommonPrefixes" type="tns:PrefixEntry" minOccurs="0" maxOccurs="unbounded"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="ListVersionsResult">
<xsd:sequence>
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="unbounded"/>
<xsd:element name="Name" type="xsd:string"/>
<xsd:element name="Prefix" type="xsd:string"/>
<xsd:element name="KeyMarker" type="xsd:string"/>
<xsd:element name="VersionIdMarker" type="xsd:string"/>
<xsd:element name="NextKeyMarker" type="xsd:string" minOccurs="0"/>
<xsd:element name="NextVersionIdMarker" type="xsd:string" minOccurs="0"/>
<xsd:element name="MaxKeys" type="xsd:int"/>
<xsd:element name="Delimiter" type="xsd:string" minOccurs="0"/>
<xsd:element name="IsTruncated" type="xsd:boolean"/>
<xsd:choice minOccurs="0" maxOccurs="unbounded">
<xsd:element name="Version" type="tns:VersionEntry"/>
<xsd:element name="DeleteMarker" type="tns:DeleteMarkerEntry"/>
</xsd:choice>
<xsd:element name="CommonPrefixes" type="tns:PrefixEntry" minOccurs="0" maxOccurs="unbounded"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="ListAllMyBuckets">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="ListAllMyBucketsResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="ListAllMyBucketsResponse" type="tns:ListAllMyBucketsResult"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="ListAllMyBucketsEntry">
<xsd:sequence>
<xsd:element name="Name" type="xsd:string"/>
<xsd:element name="CreationDate" type="xsd:dateTime"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="ListAllMyBucketsResult">
<xsd:sequence>
<xsd:element name="Owner" type="tns:CanonicalUser"/>
<xsd:element name="Buckets" type="tns:ListAllMyBucketsList"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="ListAllMyBucketsList">
<xsd:sequence>
<xsd:element name="Bucket" type="tns:ListAllMyBucketsEntry" minOccurs="0" maxOccurs="unbounded"/>
</xsd:sequence>
</xsd:complexType>
<xsd:element name="PostResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="Location" type="xsd:anyURI"/>
<xsd:element name="Bucket" type="xsd:string"/>
<xsd:element name="Key" type="xsd:string"/>
<xsd:element name="ETag" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:simpleType name="MetadataDirective">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="COPY"/>
<xsd:enumeration value="REPLACE"/>
</xsd:restriction>
</xsd:simpleType>
<xsd:element name="CopyObject">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="SourceBucket" type="xsd:string"/>
<xsd:element name="SourceKey" type="xsd:string"/>
<xsd:element name="DestinationBucket" type="xsd:string"/>
<xsd:element name="DestinationKey" type="xsd:string"/>
<xsd:element name="MetadataDirective" type="tns:MetadataDirective" minOccurs="0"/>
<xsd:element name="Metadata" type="tns:MetadataEntry" minOccurs="0" maxOccurs="100"/>
<xsd:element name="AccessControlList" type="tns:AccessControlList" minOccurs="0"/>
<xsd:element name="CopySourceIfModifiedSince" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="CopySourceIfUnmodifiedSince" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="CopySourceIfMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
<xsd:element name="CopySourceIfNoneMatch" type="xsd:string" minOccurs="0" maxOccurs="100"/>
<xsd:element name="StorageClass" type="tns:StorageClass" minOccurs="0"/>
<xsd:element name="AWSAccessKeyId" type="xsd:string" minOccurs="0"/>
<xsd:element name="Timestamp" type="xsd:dateTime" minOccurs="0"/>
<xsd:element name="Signature" type="xsd:string" minOccurs="0"/>
<xsd:element name="Credential" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:element name="CopyObjectResponse">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="CopyObjectResult" type="tns:CopyObjectResult" />
</xsd:sequence>
</xsd:complexType>
</xsd:element>
<xsd:complexType name="CopyObjectResult">
<xsd:sequence>
<xsd:element name="LastModified" type="xsd:dateTime"/>
<xsd:element name="ETag" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="RequestPaymentConfiguration">
<xsd:sequence>
<xsd:element name="Payer" type="tns:Payer" minOccurs="1" maxOccurs="1"/>
</xsd:sequence>
</xsd:complexType>
<xsd:simpleType name="Payer">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="BucketOwner"/>
<xsd:enumeration value="Requester"/>
</xsd:restriction>
</xsd:simpleType>
<xsd:complexType name="VersioningConfiguration">
<xsd:sequence>
<xsd:element name="Status" type="tns:VersioningStatus" minOccurs="0"/>
<xsd:element name="MfaDelete" type="tns:MfaDeleteStatus" minOccurs="0"/>
</xsd:sequence>
</xsd:complexType>
<xsd:simpleType name="MfaDeleteStatus">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="Enabled"/>
<xsd:enumeration value="Disabled"/>
</xsd:restriction>
</xsd:simpleType>
<xsd:simpleType name="VersioningStatus">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="Enabled"/>
<xsd:enumeration value="Suspended"/>
</xsd:restriction>
</xsd:simpleType>
<xsd:complexType name="NotificationConfiguration">
<xsd:sequence>
<xsd:element name="TopicConfiguration" minOccurs="0" maxOccurs="unbounded" type="tns:TopicConfiguration"/>
</xsd:sequence>
</xsd:complexType>
<xsd:complexType name="TopicConfiguration">
<xsd:sequence>
<xsd:element name="Topic" minOccurs="1" maxOccurs="1" type="xsd:string"/>
<xsd:element name="Event" minOccurs="1" maxOccurs="unbounded" type="xsd:string"/>
</xsd:sequence>
</xsd:complexType>
</xsd:schema>

7
weed/s3api/README.txt Normal file
View file

@ -0,0 +1,7 @@
see https://blog.aqwari.net/xml-schema-go/
1. go get aqwari.net/xml/cmd/xsdgen
2. xsdgen -o s3api_xsd_generated.go -pkg s3api AmazonS3.xsd

View file

@ -0,0 +1,58 @@
package s3api
import (
"net/http"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/glog"
"time"
"context"
"fmt"
)
func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
var response ListAllMyBucketsResponse
err := s3a.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.ListEntriesRequest{
Directory: "/buckets",
}
glog.V(4).Infof("read directory: %v", request)
resp, err := client.ListEntries(context.Background(), request)
if err != nil {
return fmt.Errorf("list buckets: %v", err)
}
var buckets []ListAllMyBucketsEntry
for _, entry := range resp.Entries {
if entry.IsDirectory {
buckets = append(buckets, ListAllMyBucketsEntry{
Name: entry.Name,
CreationDate: time.Unix(entry.Attributes.Crtime, 0),
})
}
}
response = ListAllMyBucketsResponse{
ListAllMyBucketsResponse: ListAllMyBucketsResult{
Owner: CanonicalUser{
ID: "",
DisplayName: "",
},
Buckets: ListAllMyBucketsList{
Bucket: buckets,
},
},
}
return nil
})
if err != nil {
writeErrorResponse(w, ErrInternalError, r.URL)
return
}
writeSuccessResponseXML(w, encodeResponse(response))
}

View file

@ -0,0 +1,76 @@
package s3api
import (
"encoding/xml"
"net/http"
)
// APIError structure
type APIError struct {
Code string
Description string
HTTPStatusCode int
}
// RESTErrorResponse - error response format
type RESTErrorResponse struct {
XMLName xml.Name `xml:"Error" json:"-"`
Code string `xml:"Code" json:"Code"`
Message string `xml:"Message" json:"Message"`
Resource string `xml:"Resource" json:"Resource"`
RequestID string `xml:"RequestId" json:"RequestId"`
}
// ErrorCode type of error status.
type ErrorCode int
// Error codes, see full list at http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
const (
ErrNone ErrorCode = iota
ErrMethodNotAllowed
ErrBucketNotEmpty
ErrBucketAlreadyExists
ErrBucketAlreadyOwnedByYou
ErrInvalidBucketName
ErrInternalError
)
// error code to APIError structure, these fields carry respective
// descriptions for all the error responses.
var errorCodeResponse = map[ErrorCode]APIError{
ErrMethodNotAllowed: {
Code: "MethodNotAllowed",
Description: "The specified method is not allowed against this resource.",
HTTPStatusCode: http.StatusMethodNotAllowed,
},
ErrBucketNotEmpty: {
Code: "BucketNotEmpty",
Description: "The bucket you tried to delete is not empty",
HTTPStatusCode: http.StatusConflict,
},
ErrBucketAlreadyExists: {
Code: "BucketAlreadyExists",
Description: "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again.",
HTTPStatusCode: http.StatusConflict,
},
ErrBucketAlreadyOwnedByYou: {
Code: "BucketAlreadyOwnedByYou",
Description: "Your previous request to create the named bucket succeeded and you already own it.",
HTTPStatusCode: http.StatusConflict,
},
ErrInvalidBucketName: {
Code: "InvalidBucketName",
Description: "The specified bucket is not valid.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInternalError: {
Code: "InternalError",
Description: "We encountered an internal error, please try again.",
HTTPStatusCode: http.StatusInternalServerError,
},
}
// getAPIError provides API Error for input API error code.
func getAPIError(code ErrorCode) APIError {
return errorCodeResponse[code]
}

View file

@ -0,0 +1,67 @@
package s3api
import (
"net/http"
"net/url"
"fmt"
"time"
"github.com/gorilla/mux"
"context"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
)
func newContext(r *http.Request, api string) context.Context {
vars := mux.Vars(r)
return context.WithValue(context.Background(), "bucket", vars["bucket"])
}
func (s3a *S3ApiServer) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
grpcConnection, err := util.GrpcDial(s3a.option.FilerGrpcAddress)
if err != nil {
return fmt.Errorf("fail to dial %s: %v", s3a.option.FilerGrpcAddress, err)
}
defer grpcConnection.Close()
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}
// If none of the http routes match respond with MethodNotAllowed
func notFoundHandler(w http.ResponseWriter, r *http.Request) {
writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
}
func writeErrorResponse(w http.ResponseWriter, errorCode ErrorCode, reqURL *url.URL) {
apiError := getAPIError(errorCode)
errorResponse := getRESTErrorResponse(apiError, reqURL.Path)
encodedErrorResponse := encodeResponse(errorResponse)
writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML)
}
func getRESTErrorResponse(err APIError, resource string) RESTErrorResponse {
return RESTErrorResponse{
Code: err.Code,
Message: err.Description,
Resource: resource,
RequestID: fmt.Sprintf("%d", time.Now().UnixNano()),
}
}
func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
setCommonHeaders(w)
if mType != mimeNone {
w.Header().Set("Content-Type", string(mType))
}
w.WriteHeader(statusCode)
if response != nil {
w.Write(response)
w.(http.Flusher).Flush()
}
}
func writeSuccessResponseXML(w http.ResponseWriter, response []byte) {
writeResponse(w, http.StatusOK, response, mimeXML)
}

View file

@ -0,0 +1,31 @@
package s3api
import (
"bytes"
"encoding/xml"
"fmt"
"net/http"
"time"
)
type mimeType string
const (
mimeNone mimeType = ""
mimeJSON mimeType = "application/json"
mimeXML mimeType = "application/xml"
)
func setCommonHeaders(w http.ResponseWriter) {
w.Header().Set("x-amz-request-id", fmt.Sprintf("%d", time.Now().UnixNano()))
w.Header().Set("Accept-Ranges", "bytes")
}
// Encodes the response headers into XML format.
func encodeResponse(response interface{}) []byte {
var bytesBuffer bytes.Buffer
bytesBuffer.WriteString(xml.Header)
e := xml.NewEncoder(&bytesBuffer)
e.Encode(response)
return bytesBuffer.Bytes()
}

109
weed/s3api/s3api_server.go Normal file
View file

@ -0,0 +1,109 @@
package s3api
import (
_ "github.com/chrislusf/seaweedfs/weed/filer2/cassandra"
_ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb"
_ "github.com/chrislusf/seaweedfs/weed/filer2/memdb"
_ "github.com/chrislusf/seaweedfs/weed/filer2/mysql"
_ "github.com/chrislusf/seaweedfs/weed/filer2/postgres"
_ "github.com/chrislusf/seaweedfs/weed/filer2/redis"
"github.com/gorilla/mux"
"net/http"
)
type S3ApiServerOption struct {
Filer string
FilerGrpcAddress string
DomainName string
}
type S3ApiServer struct {
option *S3ApiServerOption
}
func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) {
s3ApiServer = &S3ApiServer{
option: option,
}
s3ApiServer.registerRouter(router)
return s3ApiServer, nil
}
func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
// API Router
apiRouter := router.PathPrefix("/").Subrouter()
var routers []*mux.Router
if s3a.option.DomainName != "" {
routers = append(routers, apiRouter.Host("{bucket:.+}." + s3a.option.DomainName).Subrouter())
}
routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
/*
for _, bucket := range routers {
// HeadObject
bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(s3a.HeadObjectHandler)
// GetObject
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.GetObjectHandler)
// CopyObject
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(s3a.CopyObjectHandler)
// PutObject
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(s3a.PutObjectHandler)
// DeleteObject
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(s3a.DeleteObjectHandler)
// CopyObjectPart
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(s3a.CopyObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
// PutObjectPart
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(s3a.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
// ListObjectPxarts
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
// CompleteMultipartUpload
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(s3a.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
// NewMultipartUpload
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(s3a.NewMultipartUploadHandler).Queries("uploads", "")
// AbortMultipartUpload
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(s3a.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
// ListMultipartUploads
bucket.Methods("GET").HandlerFunc(s3a.ListMultipartUploadsHandler).Queries("uploads", "")
// ListObjectsV2
bucket.Methods("GET").HandlerFunc(s3a.ListObjectsV2Handler).Queries("list-type", "2")
// ListObjectsV1 (Legacy)
bucket.Methods("GET").HandlerFunc(s3a.ListObjectsV1Handler)
// PutBucket
bucket.Methods("PUT").HandlerFunc(s3a.PutBucketHandler)
// HeadBucket
bucket.Methods("HEAD").HandlerFunc(s3a.HeadBucketHandler)
// DeleteMultipleObjects
bucket.Methods("POST").HandlerFunc(s3a.DeleteMultipleObjectsHandler).Queries("delete", "")
// DeleteBucket
bucket.Methods("DELETE").HandlerFunc(s3a.DeleteBucketHandler)
// not implemented
// GetBucketLocation
bucket.Methods("GET").HandlerFunc(s3a.GetBucketLocationHandler).Queries("location", "")
// GetBucketPolicy
bucket.Methods("GET").HandlerFunc(s3a.GetBucketPolicyHandler).Queries("policy", "")
// GetObjectACL
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(s3a.GetObjectACLHandler).Queries("acl", "")
// GetBucketACL
bucket.Methods("GET").HandlerFunc(s3a.GetBucketACLHandler).Queries("acl", "")
// PutBucketPolicy
bucket.Methods("PUT").HandlerFunc(s3a.PutBucketPolicyHandler).Queries("policy", "")
// DeleteBucketPolicy
bucket.Methods("DELETE").HandlerFunc(s3a.DeleteBucketPolicyHandler).Queries("policy", "")
// PostPolicy
bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(s3a.PostPolicyBucketHandler)
}
*/
// ListBuckets
apiRouter.Methods("GET").Path("/").HandlerFunc(s3a.ListBucketsHandler)
// NotFound
apiRouter.NotFoundHandler = http.HandlerFunc(notFoundHandler)
}

File diff suppressed because it is too large Load diff