add ownership rest apis (#3765)

weed/s3api/auth_credentials.go

@@ -16,6 +16,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
 
+var IdentityAnonymous *Identity
+
 type Action string
 
 type Iam interface {
@@ -32,10 +34,15 @@ type IdentityAccessManagement struct {
 
 type Identity struct {
 	Name        string
+	AccountId   string
 	Credentials []*Credential
 	Actions     []Action
 }
 
+func (i *Identity) isAnonymous() bool {
+	return i.Name == AccountAnonymous.Name
+}
+
 type Credential struct {
 	AccessKey string
 	SecretKey string
@@ -125,9 +132,23 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 	for _, ident := range config.Identities {
 		t := &Identity{
 			Name:        ident.Name,
+			AccountId:   AccountAdmin.Id,
 			Credentials: nil,
 			Actions:     nil,
 		}
+
+		if ident.Name == AccountAnonymous.Name {
+			if ident.AccountId != "" && ident.AccountId != AccountAnonymous.Id {
+				glog.Warningf("anonymous identity is associated with a non-anonymous account ID, the association is invalid")
+			}
+			t.AccountId = AccountAnonymous.Id
+			IdentityAnonymous = t
+		} else {
+			if len(ident.AccountId) > 0 {
+				t.AccountId = ident.AccountId
+			}
+		}
+
 		for _, action := range ident.Actions {
 			t.Actions = append(t.Actions, Action(action))
 		}
@@ -139,6 +160,13 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 		}
 		identities = append(identities, t)
 	}
+
+	if IdentityAnonymous == nil {
+		IdentityAnonymous = &Identity{
+			Name:      AccountAnonymous.Name,
+			AccountId: AccountAnonymous.Id,
+		}
+	}
 	iam.m.Lock()
 	// atomically switch
 	iam.identities = identities
@@ -173,7 +201,7 @@ func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, foun
 	iam.m.RLock()
 	defer iam.m.RUnlock()
 	for _, ident := range iam.identities {
-		if ident.Name == "anonymous" {
+		if ident.isAnonymous() {
 			return ident, true
 		}
 	}
@@ -259,6 +287,9 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
 		return identity, s3err.ErrAccessDenied
 	}
 
+	if !identity.isAnonymous() {
+		r.Header.Set(s3_constants.AmzAccountId, identity.AccountId)
+	}
 	return identity, s3err.ErrNone
 
 }

weed/s3api/auth_credentials_test.go

@@ -3,6 +3,7 @@ package s3api
 import (
 	. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/stretchr/testify/assert"
+	"reflect"
 	"testing"
 
 	jsonpb "google.golang.org/protobuf/encoding/protojson"
@@ -124,5 +125,98 @@ func TestCanDo(t *testing.T) {
 	}
 	assert.Equal(t, true, ident5.canDo(ACTION_READ, "special_bucket", "/a/b/c/d.txt"))
 	assert.Equal(t, true, ident5.canDo(ACTION_WRITE, "special_bucket", "/a/b/c/d.txt"))
-
+}
+
+type LoadS3ApiConfigurationTestCase struct {
+	pbIdent     *iam_pb.Identity
+	expectIdent *Identity
+}
+
+func TestLoadS3ApiConfiguration(t *testing.T) {
+	testCases := map[string]*LoadS3ApiConfigurationTestCase{
+		"notSpecifyAccountId": {
+			pbIdent: &iam_pb.Identity{
+				Name: "notSpecifyAccountId",
+				Actions: []string{
+					"Read",
+					"Write",
+				},
+				Credentials: []*iam_pb.Credential{
+					{
+						AccessKey: "some_access_key1",
+						SecretKey: "some_secret_key2",
+					},
+				},
+			},
+			expectIdent: &Identity{
+				Name:      "notSpecifyAccountId",
+				AccountId: AccountAdmin.Id,
+				Actions: []Action{
+					"Read",
+					"Write",
+				},
+				Credentials: []*Credential{
+					{
+						AccessKey: "some_access_key1",
+						SecretKey: "some_secret_key2",
+					},
+				},
+			},
+		},
+		"specifiedAccountID": {
+			pbIdent: &iam_pb.Identity{
+				Name:      "specifiedAccountID",
+				AccountId: "specifiedAccountID",
+				Actions: []string{
+					"Read",
+					"Write",
+				},
+			},
+			expectIdent: &Identity{
+				Name:      "specifiedAccountID",
+				AccountId: "specifiedAccountID",
+				Actions: []Action{
+					"Read",
+					"Write",
+				},
+			},
+		},
+		"anonymous": {
+			pbIdent: &iam_pb.Identity{
+				Name: "anonymous",
+				Actions: []string{
+					"Read",
+					"Write",
+				},
+			},
+			expectIdent: &Identity{
+				Name:      "anonymous",
+				AccountId: "anonymous",
+				Actions: []Action{
+					"Read",
+					"Write",
+				},
+			},
+		},
+	}
+
+	config := &iam_pb.S3ApiConfiguration{
+		Identities: make([]*iam_pb.Identity, 0),
+	}
+	for _, v := range testCases {
+		config.Identities = append(config.Identities, v.pbIdent)
+	}
+
+	iam := IdentityAccessManagement{}
+	err := iam.loadS3ApiConfiguration(config)
+	if err != nil {
+		return
+	}
+
+	for _, ident := range iam.identities {
+		tc := testCases[ident.Name]
+		if !reflect.DeepEqual(ident, tc.expectIdent) {
+			t.Error("not expect")
+		}
+	}
 }

weed/s3api/filer_util.go

@@ -91,6 +91,22 @@ func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *
 	return filer_pb.GetEntry(s3a, fullPath)
 }
 
+func (s3a *S3ApiServer) updateEntry(parentDirectoryPath string, newEntry *filer_pb.Entry) error {
+	updateEntryRequest := &filer_pb.UpdateEntryRequest{
+		Directory: parentDirectoryPath,
+		Entry:     newEntry,
+	}
+
+	err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		err := filer_pb.UpdateEntry(client, updateEntryRequest)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	return err
+}
+
 func objectKey(key *string) *string {
 	if strings.HasPrefix(*key, "/") {
 		t := (*key)[1:]

weed/s3api/s3_constants/header.go

@@ -43,6 +43,7 @@ const (
 // Non-Standard S3 HTTP request constants
 const (
 	AmzIdentityId = "s3-identity-id"
+	AmzAccountId  = "s3-account-id"
 	AmzAuthType   = "s3-auth-type"
 	AmzIsAdmin    = "s3-is-admin" // only set to http request header as a context
 )

weed/s3api/s3api_acp.go

@@ -0,0 +1,28 @@
+package s3api
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"net/http"
+)
+
+func getAccountId(r *http.Request) string {
+	id := r.Header.Get(s3_constants.AmzAccountId)
+	if len(id) == 0 {
+		return AccountAnonymous.Id
+	} else {
+		return id
+	}
+}
+
+func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s3err.ErrorCode {
+	metadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
+	if errCode != s3err.ErrNone {
+		return errCode
+	}
+	accountId := getAccountId(r)
+	if accountId == AccountAdmin.Id || accountId == *metadata.Owner.ID {
+		return s3err.ErrNone
+	}
+	return s3err.ErrAccessDenied
+}

weed/s3api/s3api_bucket_handlers.go

@@ -5,6 +5,8 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
+	"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 	"math"
 	"net/http"
 	"time"
@@ -343,3 +345,159 @@ func (s3a *S3ApiServer) GetBucketLocationHandler(w http.ResponseWriter, r *http.
 func (s3a *S3ApiServer) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseXML(w, r, RequestPaymentConfiguration{Payer: "BucketOwner"})
 }
+
+// PutBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketOwnershipControls.html
+func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
+	bucket, _ := s3_constants.GetBucketAndObject(r)
+	glog.V(3).Infof("PutBucketOwnershipControls %s", bucket)
+
+	errCode := s3a.checkAccessByOwnership(r, bucket)
+	if errCode != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, errCode)
+		return
+	}
+
+	if r.Body == nil || r.Body == http.NoBody {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		return
+	}
+
+	var v s3.OwnershipControls
+	defer util.CloseRequest(r)
+
+	err := xmlutil.UnmarshalXML(&v, xml.NewDecoder(r.Body), "")
+	if err != nil {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		return
+	}
+
+	if len(v.Rules) != 1 {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		return
+	}
+
+	printOwnership := true
+	ownership := *v.Rules[0].ObjectOwnership
+	switch ownership {
+	case s3_constants.OwnershipObjectWriter:
+	case s3_constants.OwnershipBucketOwnerPreferred:
+	case s3_constants.OwnershipBucketOwnerEnforced:
+		printOwnership = false
+	default:
+		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+		return
+	}
+
+	bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			return
+		}
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	oldOwnership, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
+	if !ok || string(oldOwnership) != ownership {
+		if bucketEntry.Extended == nil {
+			bucketEntry.Extended = make(map[string][]byte)
+		}
+		bucketEntry.Extended[s3_constants.ExtOwnershipKey] = []byte(ownership)
+		err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
+		if err != nil {
+			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+			return
+		}
+	}
+
+	if printOwnership {
+		result := &s3.PutBucketOwnershipControlsInput{
+			OwnershipControls: &v,
+		}
+		s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result)
+	} else {
+		writeSuccessResponseEmpty(w, r)
+	}
+}
+
+// GetBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketOwnershipControls.html
+func (s3a *S3ApiServer) GetBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
+	bucket, _ := s3_constants.GetBucketAndObject(r)
+	glog.V(3).Infof("GetBucketOwnershipControls %s", bucket)
+
+	errCode := s3a.checkAccessByOwnership(r, bucket)
+	if errCode != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, errCode)
+		return
+	}
+
+	bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			return
+		}
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	v, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
+	if !ok {
+		s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError)
+		return
+	}
+	ownership := string(v)
+
+	result := &s3.PutBucketOwnershipControlsInput{
+		OwnershipControls: &s3.OwnershipControls{
+			Rules: []*s3.OwnershipControlsRule{
+				{
+					ObjectOwnership: &ownership,
+				},
+			},
+		},
+	}
+
+	s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result)
+}
+
+// DeleteBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketOwnershipControls.html
+func (s3a *S3ApiServer) DeleteBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
+	bucket, _ := s3_constants.GetBucketAndObject(r)
+	glog.V(3).Infof("PutBucketOwnershipControls %s", bucket)
+
+	errCode := s3a.checkAccessByOwnership(r, bucket)
+	if errCode != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, errCode)
+		return
+	}
+
+	bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+			return
+		}
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	_, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
+	if !ok {
+		s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError)
+		return
+	}
+
+	delete(bucketEntry.Extended, s3_constants.ExtOwnershipKey)
+	err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
+	if err != nil {
+		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
+		return
+	}
+
+	emptyOwnershipControls := &s3.OwnershipControls{
+		Rules: []*s3.OwnershipControlsRule{},
+	}
+	s3err.WriteAwsXMLResponse(w, r, http.StatusOK, emptyOwnershipControls)
+}

weed/s3api/s3api_server.go

@@ -216,6 +216,14 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2")
 
 		// buckets with query
+		// PutBucketOwnershipControls
+		bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN), "PUT")).Queries("ownershipControls", "")
+
+		//GetBucketOwnershipControls
+		bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketOwnershipControls, ACTION_READ), "GET")).Queries("ownershipControls", "")
+
+		//DeleteBucketOwnershipControls
+		bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN), "DELETE")).Queries("ownershipControls", "")
 
 		// raw buckets
 

weed/s3api/s3err/error_handler.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/xml"
 	"fmt"
+	"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
 	"github.com/gorilla/mux"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"net/http"
@@ -19,6 +20,16 @@ const (
 	MimeXML  mimeType = "application/xml"
 )
 
+func WriteAwsXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, result interface{}) {
+	var bytesBuffer bytes.Buffer
+	err := xmlutil.BuildXML(result, xml.NewEncoder(&bytesBuffer))
+	if err != nil {
+		WriteErrorResponse(w, r, ErrInternalError)
+		return
+	}
+	WriteResponse(w, r, statusCode, bytesBuffer.Bytes(), MimeXML)
+}
+
 func WriteXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, response interface{}) {
 	WriteResponse(w, r, statusCode, EncodeXMLResponse(response), MimeXML)
 }

weed/s3api/s3err/s3api_errors.go

@@ -107,6 +107,8 @@ const (
 
 	ErrTooManyRequest
 	ErrRequestBytesExceed
+
+	OwnershipControlsNotFoundError
 )
 
 // error code to APIError structure, these fields carry respective
@@ -414,6 +416,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "Simultaneous request bytes exceed limitations",
 		HTTPStatusCode: http.StatusTooManyRequests,
 	},
+
+	OwnershipControlsNotFoundError: {
+		Code:           "OwnershipControlsNotFoundError",
+		Description:    "The bucket ownership controls were not found",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 }
 
 // GetAPIError provides API Error for input API error code.