Merge branch 'master' of https://github.com/seaweedfs/seaweedfs

docker/compose/s3tests.conf

@@ -18,10 +18,10 @@ bucket prefix = yournamehere-{random}-
 
 [s3 main]
 # main display_name set in vstart.sh
-display_name = M. Tester
+display_name = s3_tests
 
 # main user_idname set in vstart.sh
-user_id = testid
+user_id = s3_tests
 
 # main email set in vstart.sh
 email = tester@ceph.com

weed/s3api/auth_credentials.go

@@ -18,8 +18,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
 
-var IdentityAnonymous *Identity
-
 type Action string
 
 type Iam interface {
@@ -29,12 +27,14 @@ type Iam interface {
 type IdentityAccessManagement struct {
 	m sync.RWMutex
 
-	identities    []*Identity
+	identities        []*Identity
-	isAuthEnabled bool
+	accessKeyIdent    map[string]*Identity
-	domain        string
+	hashes            map[string]*sync.Pool
-	hashes        map[string]*sync.Pool
+	hashCounters      map[string]*int32
-	hashCounters  map[string]*int32
+	identityAnonymous *Identity
-	hashMu        sync.RWMutex
+	hashMu            sync.RWMutex
+	domain            string
+	isAuthEnabled     bool
 }
 
 type Identity struct {
@@ -136,6 +136,8 @@ func (iam *IdentityAccessManagement) LoadS3ApiConfigurationFromBytes(content []b
 
 func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error {
 	var identities []*Identity
+	var identityAnonymous *Identity
+	accessKeyIdent := make(map[string]*Identity)
 	for _, ident := range config.Identities {
 		t := &Identity{
 			Name:        ident.Name,
@@ -149,7 +151,7 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 				glog.Warningf("anonymous identity is associated with a non-anonymous account ID, the association is invalid")
 			}
 			t.AccountId = s3account.AccountAnonymous.Id
-			IdentityAnonymous = t
+			identityAnonymous = t
 		} else {
 			if len(ident.AccountId) > 0 {
 				t.AccountId = ident.AccountId
@@ -164,19 +166,15 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 				AccessKey: cred.AccessKey,
 				SecretKey: cred.SecretKey,
 			})
+			accessKeyIdent[cred.AccessKey] = t
 		}
 		identities = append(identities, t)
 	}
-
-	if IdentityAnonymous == nil {
-		IdentityAnonymous = &Identity{
-			Name:      s3account.AccountAnonymous.Name,
-			AccountId: s3account.AccountAnonymous.Id,
-		}
-	}
 	iam.m.Lock()
 	// atomically switch
 	iam.identities = identities
+	iam.identityAnonymous = identityAnonymous
+	iam.accessKeyIdent = accessKeyIdent
 	if !iam.isAuthEnabled { // one-directional, no toggling
 		iam.isAuthEnabled = len(identities) > 0
 	}
@@ -189,14 +187,12 @@ func (iam *IdentityAccessManagement) isEnabled() bool {
 }
 
 func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) {
-
 	iam.m.RLock()
 	defer iam.m.RUnlock()
-	for _, ident := range iam.identities {
+	if ident, ok := iam.accessKeyIdent[accessKey]; ok {
-		for _, cred := range ident.Credentials {
+		for _, credential := range ident.Credentials {
-			// println("checking", ident.Name, cred.AccessKey)
+			if credential.AccessKey == accessKey {
-			if cred.AccessKey == accessKey {
+				return ident, credential, true
-				return ident, cred, true
 			}
 		}
 	}
@@ -207,10 +203,8 @@ func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identi
 func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, found bool) {
 	iam.m.RLock()
 	defer iam.m.RUnlock()
-	for _, ident := range iam.identities {
+	if iam.identityAnonymous != nil {
-		if ident.isAnonymous() {
+		return iam.identityAnonymous, true
-			return ident, true
-		}
 	}
 	return nil, false
 }
@@ -270,8 +264,7 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
 		return identity, s3err.ErrNotImplemented
 	case authTypeAnonymous:
 		authType = "Anonymous"
-		identity, found = iam.lookupAnonymous()
+		if identity, found = iam.lookupAnonymous(); !found {
-		if !found {
 			r.Header.Set(s3_constants.AmzAuthType, authType)
 			return identity, s3err.ErrAccessDenied
 		}

weed/s3api/auth_credentials_test.go

@@ -126,6 +126,15 @@ func TestCanDo(t *testing.T) {
 	}
 	assert.Equal(t, true, ident5.canDo(ACTION_READ, "special_bucket", "/a/b/c/d.txt"))
 	assert.Equal(t, true, ident5.canDo(ACTION_WRITE, "special_bucket", "/a/b/c/d.txt"))
+
+	// anonymous buckets
+	ident6 := &Identity{
+		Name: "anonymous",
+		Actions: []Action{
+			"Read",
+		},
+	}
+	assert.Equal(t, true, ident6.canDo(ACTION_READ, "anything_bucket", "/a/b/c/d.txt"))
 }
 
 type LoadS3ApiConfigurationTestCase struct {

weed/s3api/auto_signature_v4_test.go

@@ -8,8 +8,8 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"google.golang.org/grpc"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
-	"google.golang.org/grpc/credentials/insecure"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"io"
 	"net/http"
 	"net/url"
@@ -60,22 +60,24 @@ func TestIsRequestPresignedSignatureV4(t *testing.T) {
 
 // Tests is requested authenticated function, tests replies for s3 errors.
 func TestIsReqAuthenticated(t *testing.T) {
-	option := S3ApiServerOption{
+	iam := &IdentityAccessManagement{
-		GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+		hashes:       make(map[string]*sync.Pool),
+		hashCounters: make(map[string]*int32),
 	}
-	iam := NewIdentityAccessManagement(&option)
+	_ = iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
-	iam.identities = []*Identity{
+		Identities: []*iam_pb.Identity{
-		{
+			{
-			Name: "someone",
+				Name: "someone",
-			Credentials: []*Credential{
+				Credentials: []*iam_pb.Credential{
-				{
+					{
-					AccessKey: "access_key_1",
+						AccessKey: "access_key_1",
-					SecretKey: "secret_key_1",
+						SecretKey: "secret_key_1",
+					},
 				},
+				Actions: []string{},
 			},
-			Actions: nil,
 		},
-	}
+	})
 
 	// List of test cases for validating http request authentication.
 	testCases := []struct {
@@ -97,24 +99,58 @@ func TestIsReqAuthenticated(t *testing.T) {
 	}
 }
 
-func TestCheckAdminRequestAuthType(t *testing.T) {
+func TestCheckaAnonymousRequestAuthType(t *testing.T) {
-	option := S3ApiServerOption{
+	iam := &IdentityAccessManagement{
-		GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+		hashes:       make(map[string]*sync.Pool),
+		hashCounters: make(map[string]*int32),
 	}
-	iam := NewIdentityAccessManagement(&option)
+	_ = iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
-	iam.identities = []*Identity{
+		Identities: []*iam_pb.Identity{
-		{
+			{
-			Name: "someone",
+				Name:    "anonymous",
-			Credentials: []*Credential{
+				Actions: []string{s3_constants.ACTION_READ},
-				{
-					AccessKey: "access_key_1",
-					SecretKey: "secret_key_1",
-				},
 			},
-			Actions: nil,
 		},
+	})
+	testCases := []struct {
+		Request *http.Request
+		ErrCode s3err.ErrorCode
+		Action  Action
+	}{
+		{Request: mustNewRequest("GET", "http://127.0.0.1:9000/bucket", 0, nil, t), ErrCode: s3err.ErrNone, Action: s3_constants.ACTION_READ},
+		{Request: mustNewRequest("PUT", "http://127.0.0.1:9000/bucket", 0, nil, t), ErrCode: s3err.ErrAccessDenied, Action: s3_constants.ACTION_WRITE},
+	}
+	for i, testCase := range testCases {
+		_, s3Error := iam.authRequest(testCase.Request, testCase.Action)
+		if s3Error != testCase.ErrCode {
+			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
+		}
+		if testCase.Request.Header.Get(s3_constants.AmzAuthType) != "Anonymous" {
+			t.Errorf("Test %d: Unexpected AuthType returned wanted %s, got %s", i, "Anonymous", testCase.Request.Header.Get(s3_constants.AmzAuthType))
+		}
 	}
 
+}
+
+func TestCheckAdminRequestAuthType(t *testing.T) {
+	iam := &IdentityAccessManagement{
+		hashes:       make(map[string]*sync.Pool),
+		hashCounters: make(map[string]*int32),
+	}
+	_ = iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
+		Identities: []*iam_pb.Identity{
+			{
+				Name: "someone",
+				Credentials: []*iam_pb.Credential{
+					{
+						AccessKey: "access_key_1",
+						SecretKey: "secret_key_1",
+					},
+				},
+				Actions: []string{},
+			},
+		},
+	})
 	testCases := []struct {
 		Request *http.Request
 		ErrCode s3err.ErrorCode

weed/s3api/s3api_bucket_handlers.go

@@ -259,34 +259,56 @@ func (s3a *S3ApiServer) GetBucketAclHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	response := AccessControlPolicy{}
+	identityId := r.Header.Get(s3_constants.AmzIdentityId)
-	for _, ident := range s3a.iam.identities {
+	response := AccessControlPolicy{
-		if len(ident.Credentials) == 0 {
+		Owner: CanonicalUser{
-			continue
+			ID:          identityId,
-		}
+			DisplayName: identityId,
-		for _, action := range ident.Actions {
+		},
-			if !action.overBucket(bucket) || action.getPermission() == "" {
-				continue
-			}
-			id := ident.Credentials[0].AccessKey
-			if response.Owner.DisplayName == "" && action.isOwner(bucket) && len(ident.Credentials) > 0 {
-				response.Owner.DisplayName = ident.Name
-				response.Owner.ID = id
-			}
-			response.AccessControlList.Grant = append(response.AccessControlList.Grant, Grant{
-				Grantee: Grantee{
-					ID:          id,
-					DisplayName: ident.Name,
-					Type:        "CanonicalUser",
-					XMLXSI:      "CanonicalUser",
-					XMLNS:       "http://www.w3.org/2001/XMLSchema-instance"},
-				Permission: action.getPermission(),
-			})
-		}
 	}
+	response.AccessControlList.Grant = append(response.AccessControlList.Grant, Grant{
+		Grantee: Grantee{
+			ID:          identityId,
+			DisplayName: identityId,
+			Type:        "CanonicalUser",
+			XMLXSI:      "CanonicalUser",
+			XMLNS:       "http://www.w3.org/2001/XMLSchema-instance"},
+		Permission: s3.PermissionFullControl,
+	})
 	writeSuccessResponseXML(w, r, response)
 }
 
+// PutBucketAclHandler Put bucket ACL only responds success if the ACL is private.
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html //
+func (s3a *S3ApiServer) PutBucketAclHandler(w http.ResponseWriter, r *http.Request) {
+	// collect parameters
+	bucket, _ := s3_constants.GetBucketAndObject(r)
+	glog.V(3).Infof("PutBucketAclHandler %s", bucket)
+
+	if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
+		s3err.WriteErrorResponse(w, r, err)
+		return
+	}
+	cannedAcl := r.Header.Get(s3_constants.AmzCannedAcl)
+	switch {
+	case cannedAcl == "":
+		acl := &s3.AccessControlPolicy{}
+		if err := xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
+			glog.Errorf("PutBucketAclHandler: %s", err)
+			s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
+			return
+		}
+		if len(acl.Grants) == 1 && acl.Grants[0].Permission != nil && *acl.Grants[0].Permission == s3_constants.PermissionFullControl {
+			writeSuccessResponseEmpty(w, r)
+			return
+		}
+	case cannedAcl == s3_constants.CannedAclPrivate:
+		writeSuccessResponseEmpty(w, r)
+		return
+	}
+	s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
+}
+
 // GetBucketLifecycleConfigurationHandler Get Bucket Lifecycle configuration
 // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLifecycleConfiguration.html
 func (s3a *S3ApiServer) GetBucketLifecycleConfigurationHandler(w http.ResponseWriter, r *http.Request) {

weed/s3api/s3api_bucket_skip_handlers.go

@@ -41,9 +41,3 @@ func (s3a *S3ApiServer) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Re
 func (s3a *S3ApiServer) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	s3err.WriteErrorResponse(w, r, http.StatusNoContent)
 }
-
-// PutBucketAclHandler Put bucket ACL
-// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html
-func (s3a *S3ApiServer) PutBucketAclHandler(w http.ResponseWriter, r *http.Request) {
-	s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
-}