seaweedfs/weed/s3api/s3api_bucket_handlers.go

536 lines
16 KiB
Go
Raw Normal View History

2018-07-18 09:37:09 +00:00
package s3api
import (
"context"
"encoding/xml"
2022-03-31 03:44:48 +00:00
"errors"
2018-07-18 09:37:09 +00:00
"fmt"
2022-10-02 02:18:00 +00:00
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
2022-10-02 02:18:00 +00:00
"github.com/seaweedfs/seaweedfs/weed/util"
"math"
2018-07-22 00:39:10 +00:00
"net/http"
"time"
2018-07-24 08:38:08 +00:00
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
2020-11-11 08:20:59 +00:00
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
2020-02-25 19:13:06 +00:00
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
type ListAllMyBucketsResult struct {
2019-03-15 22:55:34 +00:00
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult"`
Owner *s3.Owner
Buckets []*s3.Bucket `xml:"Buckets>Bucket"`
}
2018-07-18 09:37:09 +00:00
func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
2021-09-19 07:18:59 +00:00
glog.V(3).Infof("ListBucketsHandler")
2020-12-25 08:38:56 +00:00
var identity *Identity
var s3Err s3err.ErrorCode
if s3a.iam.isEnabled() {
identity, s3Err = s3a.iam.authUser(r)
if s3Err != s3err.ErrNone {
2021-11-01 01:05:34 +00:00
s3err.WriteErrorResponse(w, r, s3Err)
2020-12-25 08:38:56 +00:00
return
}
}
var response ListAllMyBucketsResult
2018-07-18 09:37:09 +00:00
entries, _, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32)
2018-07-18 09:37:09 +00:00
2018-09-03 20:03:16 +00:00
if err != nil {
2021-11-01 01:05:34 +00:00
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
2018-09-03 20:03:16 +00:00
return
}
2018-07-18 09:37:09 +00:00
identityId := r.Header.Get(s3_constants.AmzIdentityId)
2020-11-11 08:20:59 +00:00
var buckets []*s3.Bucket
2018-09-03 20:03:16 +00:00
for _, entry := range entries {
if entry.IsDirectory {
if identity != nil && !identity.canDo(s3_constants.ACTION_LIST, entry.Name, "") {
2020-11-12 21:57:54 +00:00
continue
2020-11-11 08:20:59 +00:00
}
buckets = append(buckets, &s3.Bucket{
Name: aws.String(entry.Name),
CreationDate: aws.Time(time.Unix(entry.Attributes.Crtime, 0).UTC()),
2018-09-03 20:03:16 +00:00
})
2018-07-18 09:37:09 +00:00
}
2018-09-03 20:03:16 +00:00
}
2018-07-18 09:37:09 +00:00
response = ListAllMyBucketsResult{
Owner: &s3.Owner{
2020-11-11 08:20:59 +00:00
ID: aws.String(identityId),
DisplayName: aws.String(identityId),
2018-09-03 20:03:16 +00:00
},
Buckets: buckets,
2018-07-18 09:37:09 +00:00
}
2021-11-01 01:02:08 +00:00
writeSuccessResponseXML(w, r, response)
2018-07-18 09:37:09 +00:00
}
func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
bucket, _ := s3_constants.GetBucketAndObject(r)
2021-09-19 07:18:59 +00:00
glog.V(3).Infof("PutBucketHandler %s", bucket)
// validate the bucket name
err := s3bucket.VerifyS3BucketName(bucket)
if err != nil {
glog.Errorf("put invalid bucket name: %v %v", bucket, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidBucketName)
return
}
2020-10-15 17:52:17 +00:00
// avoid duplicated buckets
errCode := s3err.ErrNone
if err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
2020-10-15 17:52:17 +00:00
if resp, err := client.CollectionList(context.Background(), &filer_pb.CollectionListRequest{
IncludeEcVolumes: true,
IncludeNormalVolumes: true,
}); err != nil {
glog.Errorf("list collection: %v", err)
return fmt.Errorf("list collections: %v", err)
2020-10-25 03:12:04 +00:00
} else {
2020-10-15 17:52:17 +00:00
for _, c := range resp.Collections {
if s3a.getCollectionName(bucket) == c.Name {
2020-10-15 17:52:17 +00:00
errCode = s3err.ErrBucketAlreadyExists
break
}
}
}
return nil
}); err != nil {
2021-11-01 01:05:34 +00:00
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
2020-10-15 17:52:17 +00:00
return
}
2020-11-12 06:11:03 +00:00
if exist, err := s3a.exists(s3a.option.BucketsPath, bucket, true); err == nil && exist {
errCode = s3err.ErrBucketAlreadyExists
}
2020-10-15 17:52:17 +00:00
if errCode != s3err.ErrNone {
2021-11-01 01:05:34 +00:00
s3err.WriteErrorResponse(w, r, errCode)
2020-10-15 17:52:17 +00:00
return
}
2021-11-03 19:11:36 +00:00
if s3a.iam.isEnabled() {
if _, errCode = s3a.iam.authRequest(r, s3_constants.ACTION_ADMIN); errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
}
2020-11-11 08:20:59 +00:00
fn := func(entry *filer_pb.Entry) {
if identityId := r.Header.Get(s3_constants.AmzIdentityId); identityId != "" {
2020-11-12 03:50:19 +00:00
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.AmzIdentityId] = []byte(identityId)
2020-11-11 08:20:59 +00:00
}
}
2018-09-03 20:03:16 +00:00
// create the folder for bucket, but lazily create actual collection
2020-11-11 08:20:59 +00:00
if err := s3a.mkdir(s3a.option.BucketsPath, bucket, fn); err != nil {
2020-10-15 17:52:17 +00:00
glog.Errorf("PutBucketHandler mkdir: %v", err)
2021-11-01 01:05:34 +00:00
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
2022-05-02 06:16:29 +00:00
w.Header().Set("Location", "/"+bucket)
2021-11-01 01:02:08 +00:00
writeSuccessResponseEmpty(w, r)
}
func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
bucket, _ := s3_constants.GetBucketAndObject(r)
2021-09-19 07:18:59 +00:00
glog.V(3).Infof("DeleteBucketHandler %s", bucket)
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
2021-11-01 01:05:34 +00:00
s3err.WriteErrorResponse(w, r, err)
return
}
2020-11-12 09:59:31 +00:00
err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
2022-03-30 17:46:13 +00:00
if !s3a.option.AllowDeleteBucketNotEmpty {
entries, _, err := s3a.list(s3a.option.BucketsPath+"/"+bucket, "", "", false, 2)
2022-03-30 17:46:13 +00:00
if err != nil {
return fmt.Errorf("failed to list bucket %s: %v", bucket, err)
}
for _, entry := range entries {
if entry.Name != s3_constants.MultipartUploadsFolder {
return errors.New(s3err.GetAPIError(s3err.ErrBucketNotEmpty).Code)
}
2022-03-30 17:46:13 +00:00
}
2021-05-24 14:13:04 +00:00
}
// delete collection
deleteCollectionRequest := &filer_pb.DeleteCollectionRequest{
Collection: s3a.getCollectionName(bucket),
}
glog.V(1).Infof("delete collection: %v", deleteCollectionRequest)
2020-02-26 06:23:59 +00:00
if _, err := client.DeleteCollection(context.Background(), deleteCollectionRequest); err != nil {
2018-07-20 07:58:10 +00:00
return fmt.Errorf("delete collection %s: %v", bucket, err)
}
return nil
})
2021-05-24 14:13:04 +00:00
if err != nil {
s3ErrorCode := s3err.ErrInternalError
2022-03-31 03:44:48 +00:00
if err.Error() == s3err.GetAPIError(s3err.ErrBucketNotEmpty).Code {
2021-05-24 14:13:04 +00:00
s3ErrorCode = s3err.ErrBucketNotEmpty
}
2022-03-30 17:46:13 +00:00
s3err.WriteErrorResponse(w, r, s3ErrorCode)
2021-05-24 14:13:04 +00:00
return
}
err = s3a.rm(s3a.option.BucketsPath, bucket, false, true)
2018-09-03 20:16:26 +00:00
if err != nil {
2021-11-01 01:05:34 +00:00
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
2021-11-01 01:02:08 +00:00
s3err.WriteEmptyResponse(w, r, http.StatusNoContent)
}
2018-07-19 08:43:27 +00:00
func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
bucket, _ := s3_constants.GetBucketAndObject(r)
2021-09-19 07:18:59 +00:00
glog.V(3).Infof("HeadBucketHandler %s", bucket)
2018-07-19 08:43:27 +00:00
if entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket); entry == nil || err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
2021-11-01 01:02:08 +00:00
writeSuccessResponseEmpty(w, r)
}
func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorCode {
2020-11-12 21:30:08 +00:00
entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
2020-11-12 21:30:46 +00:00
if entry == nil || err == filer_pb.ErrNotFound {
return s3err.ErrNoSuchBucket
2018-07-19 08:43:27 +00:00
}
2020-11-12 09:59:31 +00:00
2020-11-12 21:57:54 +00:00
if !s3a.hasAccess(r, entry) {
return s3err.ErrAccessDenied
}
return s3err.ErrNone
2018-07-19 08:43:27 +00:00
}
2020-11-12 21:57:54 +00:00
func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool {
isAdmin := r.Header.Get(s3_constants.AmzIsAdmin) != ""
2020-11-12 21:57:54 +00:00
if isAdmin {
return true
}
if entry.Extended == nil {
return true
}
identityId := r.Header.Get(s3_constants.AmzIdentityId)
if id, ok := entry.Extended[s3_constants.AmzIdentityId]; ok {
2020-11-12 21:57:54 +00:00
if identityId != string(id) {
return false
}
}
return true
}
2021-10-11 10:03:56 +00:00
// GetBucketAclHandler Get Bucket ACL
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html
func (s3a *S3ApiServer) GetBucketAclHandler(w http.ResponseWriter, r *http.Request) {
// collect parameters
bucket, _ := s3_constants.GetBucketAndObject(r)
2021-10-11 10:03:56 +00:00
glog.V(3).Infof("GetBucketAclHandler %s", bucket)
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
2021-11-01 01:05:34 +00:00
s3err.WriteErrorResponse(w, r, err)
2021-10-11 10:03:56 +00:00
return
}
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
amzDisplayName := s3a.iam.GetAccountNameById(amzAccountId)
response := AccessControlPolicy{
Owner: CanonicalUser{
ID: amzAccountId,
DisplayName: amzDisplayName,
},
}
response.AccessControlList.Grant = append(response.AccessControlList.Grant, Grant{
Grantee: Grantee{
ID: amzAccountId,
DisplayName: amzDisplayName,
Type: "CanonicalUser",
XMLXSI: "CanonicalUser",
XMLNS: "http://www.w3.org/2001/XMLSchema-instance"},
Permission: s3.PermissionFullControl,
})
writeSuccessResponseXML(w, r, response)
}
// PutBucketAclHandler Put bucket ACL only responds success if the ACL is private.
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html //
func (s3a *S3ApiServer) PutBucketAclHandler(w http.ResponseWriter, r *http.Request) {
// collect parameters
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("PutBucketAclHandler %s", bucket)
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, err)
return
}
cannedAcl := r.Header.Get(s3_constants.AmzCannedAcl)
switch {
case cannedAcl == "":
acl := &s3.AccessControlPolicy{}
if err := xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
glog.Errorf("PutBucketAclHandler: %s", err)
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
2021-10-11 10:03:56 +00:00
}
if len(acl.Grants) == 1 && acl.Grants[0].Permission != nil && *acl.Grants[0].Permission == s3_constants.PermissionFullControl {
writeSuccessResponseEmpty(w, r)
return
2021-10-11 10:03:56 +00:00
}
case cannedAcl == s3_constants.CannedAclPrivate:
writeSuccessResponseEmpty(w, r)
return
2021-10-11 10:03:56 +00:00
}
s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
2021-10-11 10:03:56 +00:00
}
// GetBucketLifecycleConfigurationHandler Get Bucket Lifecycle configuration
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLifecycleConfiguration.html
func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {
// collect parameters
bucket, _ := s3_constants.GetBucketAndObject(r)
2022-02-25 05:41:20 +00:00
glog.V(3).Infof("GetBucketLifecycleConfigurationHandler %s", bucket)
2021-10-11 10:03:56 +00:00
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
2021-11-01 01:05:34 +00:00
s3err.WriteErrorResponse(w, r, err)
2021-10-11 10:03:56 +00:00
return
}
fc, err := filer.ReadFilerConf(s3a.option.Filer, s3a.option.GrpcDialOption, nil)
if err != nil {
glog.Errorf("GetBucketLifecycleConfigurationHandler: %s", err)
2021-11-01 01:05:34 +00:00
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
2021-10-11 10:03:56 +00:00
return
}
ttls := fc.GetCollectionTtls(s3a.getCollectionName(bucket))
2021-10-13 20:35:33 +00:00
if len(ttls) == 0 {
2021-11-01 01:05:34 +00:00
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchLifecycleConfiguration)
return
2021-10-13 20:35:33 +00:00
}
response := Lifecycle{}
for prefix, internalTtl := range ttls {
ttl, _ := needle.ReadTTL(internalTtl)
days := int(ttl.Minutes() / 60 / 24)
if days == 0 {
continue
}
response.Rules = append(response.Rules, Rule{
Status: Enabled, Filter: Filter{
Prefix: Prefix{string: prefix, set: true},
set: true,
},
Expiration: Expiration{Days: days, set: true},
})
}
2021-11-01 01:02:08 +00:00
writeSuccessResponseXML(w, r, response)
2021-10-11 10:03:56 +00:00
}
// PutBucketLifecycleConfigurationHandler Put Bucket Lifecycle configuration
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html
func (s3a *S3ApiServer) PutBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {
2021-11-01 01:05:34 +00:00
s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
2021-10-11 10:03:56 +00:00
}
// DeleteBucketMetricsConfiguration Delete Bucket Lifecycle
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketLifecycle.html
func (s3a *S3ApiServer) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
2021-11-01 01:02:08 +00:00
s3err.WriteEmptyResponse(w, r, http.StatusNoContent)
2021-10-11 10:03:56 +00:00
}
// GetBucketLocationHandler Get bucket location
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html
func (s3a *S3ApiServer) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
writeSuccessResponseXML(w, r, LocationConstraint{})
}
// GetBucketRequestPaymentHandler Get bucket location
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketRequestPayment.html
func (s3a *S3ApiServer) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
writeSuccessResponseXML(w, r, RequestPaymentConfiguration{Payer: "BucketOwner"})
}
2022-10-02 02:18:00 +00:00
// PutBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketOwnershipControls.html
func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("PutBucketOwnershipControls %s", bucket)
errCode := s3a.checkAccessByOwnership(r, bucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
if r.Body == nil || r.Body == http.NoBody {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
var v s3.OwnershipControls
defer util.CloseRequest(r)
err := xmlutil.UnmarshalXML(&v, xml.NewDecoder(r.Body), "")
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
if len(v.Rules) != 1 {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
printOwnership := true
ownership := *v.Rules[0].ObjectOwnership
switch ownership {
case s3_constants.OwnershipObjectWriter:
case s3_constants.OwnershipBucketOwnerPreferred:
case s3_constants.OwnershipBucketOwnerEnforced:
printOwnership = false
default:
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
oldOwnership, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
if !ok || string(oldOwnership) != ownership {
if bucketEntry.Extended == nil {
bucketEntry.Extended = make(map[string][]byte)
}
bucketEntry.Extended[s3_constants.ExtOwnershipKey] = []byte(ownership)
err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
}
if printOwnership {
result := &s3.PutBucketOwnershipControlsInput{
OwnershipControls: &v,
}
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result)
} else {
writeSuccessResponseEmpty(w, r)
}
}
// GetBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketOwnershipControls.html
func (s3a *S3ApiServer) GetBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("GetBucketOwnershipControls %s", bucket)
errCode := s3a.checkAccessByOwnership(r, bucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
v, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
if !ok {
s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError)
return
}
ownership := string(v)
result := &s3.PutBucketOwnershipControlsInput{
OwnershipControls: &s3.OwnershipControls{
Rules: []*s3.OwnershipControlsRule{
{
ObjectOwnership: &ownership,
},
},
},
}
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result)
}
// DeleteBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketOwnershipControls.html
func (s3a *S3ApiServer) DeleteBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("PutBucketOwnershipControls %s", bucket)
errCode := s3a.checkAccessByOwnership(r, bucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
_, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
if !ok {
s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError)
return
}
delete(bucketEntry.Extended, s3_constants.ExtOwnershipKey)
err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
emptyOwnershipControls := &s3.OwnershipControls{
Rules: []*s3.OwnershipControlsRule{},
}
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, emptyOwnershipControls)
}