mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
split ExtAcpKey
to ExtAmzOwnerKey
and ExtAmzAclKey
to avoid unn… (#3824)
split `ExtAcpKey` to `ExtAmzOwnerKey` and `ExtAmzAclKey` to avoid unnecessary `json.Unmarshal()` call Signed-off-by: changlin.shi <changlin.shi@ly.com> Signed-off-by: changlin.shi <changlin.shi@ly.com>
This commit is contained in:
parent
c34f04b35b
commit
d21e2f523d
|
@ -1,16 +1,12 @@
|
||||||
package s3api
|
package s3api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
|
|
||||||
"github.com/aws/aws-sdk-go/service/s3"
|
"github.com/aws/aws-sdk-go/service/s3"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
|
||||||
|
|
||||||
//"github.com/seaweedfs/seaweedfs/weed/s3api"
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
"math"
|
"math"
|
||||||
|
@ -23,7 +19,7 @@ var loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*B
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return buildBucketMetadata(entry), nil
|
return buildBucketMetadata(r.s3a.accountManager, entry), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type BucketMetaData struct {
|
type BucketMetaData struct {
|
||||||
|
@ -77,13 +73,13 @@ func (r *BucketRegistry) init() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *BucketRegistry) LoadBucketMetadata(entry *filer_pb.Entry) {
|
func (r *BucketRegistry) LoadBucketMetadata(entry *filer_pb.Entry) {
|
||||||
bucketMetadata := buildBucketMetadata(entry)
|
bucketMetadata := buildBucketMetadata(r.s3a.accountManager, entry)
|
||||||
r.metadataCacheLock.Lock()
|
r.metadataCacheLock.Lock()
|
||||||
defer r.metadataCacheLock.Unlock()
|
defer r.metadataCacheLock.Unlock()
|
||||||
r.metadataCache[entry.Name] = bucketMetadata
|
r.metadataCache[entry.Name] = bucketMetadata
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildBucketMetadata(entry *filer_pb.Entry) *BucketMetaData {
|
func buildBucketMetadata(accountManager *s3account.AccountManager, entry *filer_pb.Entry) *BucketMetaData {
|
||||||
entryJson, _ := json.Marshal(entry)
|
entryJson, _ := json.Marshal(entry)
|
||||||
glog.V(3).Infof("build bucket metadata,entry=%s", entryJson)
|
glog.V(3).Infof("build bucket metadata,entry=%s", entryJson)
|
||||||
bucketMetadata := &BucketMetaData{
|
bucketMetadata := &BucketMetaData{
|
||||||
|
@ -112,22 +108,29 @@ func buildBucketMetadata(entry *filer_pb.Entry) *BucketMetaData {
|
||||||
}
|
}
|
||||||
|
|
||||||
//access control policy
|
//access control policy
|
||||||
acpBytes, ok := entry.Extended[s3_constants.ExtAcpKey]
|
//owner
|
||||||
if ok {
|
acpOwnerBytes, ok := entry.Extended[s3_constants.ExtAmzOwnerKey]
|
||||||
var acp s3.AccessControlPolicy
|
if ok && len(acpOwnerBytes) > 0 {
|
||||||
err := jsonutil.UnmarshalJSON(&acp, bytes.NewReader(acpBytes))
|
ownerAccountId := string(acpOwnerBytes)
|
||||||
if err == nil {
|
ownerAccountName, exists := accountManager.IdNameMapping[ownerAccountId]
|
||||||
//validate owner
|
if !exists {
|
||||||
if acp.Owner != nil && acp.Owner.ID != nil {
|
glog.Warningf("owner[id=%s] is invalid, bucket: %s", ownerAccountId, bucketMetadata.Name)
|
||||||
bucketMetadata.Owner = acp.Owner
|
|
||||||
} else {
|
|
||||||
glog.Warningf("bucket ownerId is empty! bucket: %s", bucketMetadata.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
//acl
|
|
||||||
bucketMetadata.Acl = acp.Grants
|
|
||||||
} else {
|
} else {
|
||||||
glog.Warningf("Unmarshal ACP: %s(%v), bucket: %s", string(acpBytes), err, bucketMetadata.Name)
|
bucketMetadata.Owner = &s3.Owner{
|
||||||
|
ID: &ownerAccountId,
|
||||||
|
DisplayName: &ownerAccountName,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//grants
|
||||||
|
acpGrantsBytes, ok := entry.Extended[s3_constants.ExtAmzAclKey]
|
||||||
|
if ok && len(acpGrantsBytes) > 0 {
|
||||||
|
var grants []*s3.Grant
|
||||||
|
err := json.Unmarshal(acpGrantsBytes, &grants)
|
||||||
|
if err == nil {
|
||||||
|
bucketMetadata.Acl = grants
|
||||||
|
} else {
|
||||||
|
glog.Warningf("Unmarshal ACP grants: %s(%v), bucket: %s", string(acpGrantsBytes), err, bucketMetadata.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
package s3api
|
package s3api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
|
|
||||||
"github.com/aws/aws-sdk-go/service/s3"
|
"github.com/aws/aws-sdk-go/service/s3"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||||
|
@ -26,18 +26,13 @@ var (
|
||||||
}
|
}
|
||||||
|
|
||||||
//good entry
|
//good entry
|
||||||
goodEntryAcp, _ = jsonutil.BuildJSON(&s3.AccessControlPolicy{
|
goodEntryAcl, _ = json.Marshal(s3_constants.PublicRead)
|
||||||
Owner: &s3.Owner{
|
goodEntry = &filer_pb.Entry{
|
||||||
DisplayName: &s3account.AccountAdmin.Name,
|
|
||||||
ID: &s3account.AccountAdmin.Id,
|
|
||||||
},
|
|
||||||
Grants: s3_constants.PublicRead,
|
|
||||||
})
|
|
||||||
goodEntry = &filer_pb.Entry{
|
|
||||||
Name: "entryWithValidAcp",
|
Name: "entryWithValidAcp",
|
||||||
Extended: map[string][]byte{
|
Extended: map[string][]byte{
|
||||||
s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
|
s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
|
||||||
s3_constants.ExtAcpKey: goodEntryAcp,
|
s3_constants.ExtAmzOwnerKey: []byte(s3account.AccountAdmin.Name),
|
||||||
|
s3_constants.ExtAmzAclKey: goodEntryAcl,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -57,35 +52,28 @@ var (
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
//acp is ""
|
//owner is ""
|
||||||
acpEmptyStr = &filer_pb.Entry{
|
acpEmptyStr = &filer_pb.Entry{
|
||||||
Name: "acpEmptyStr",
|
Name: "acpEmptyStr",
|
||||||
Extended: map[string][]byte{
|
Extended: map[string][]byte{
|
||||||
s3_constants.ExtAcpKey: []byte(""),
|
s3_constants.ExtAmzOwnerKey: []byte(""),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
//acp is empty object
|
//owner not exists
|
||||||
acpEmptyObjectAcp, _ = jsonutil.BuildJSON(&s3.AccessControlPolicy{
|
|
||||||
Owner: nil,
|
|
||||||
Grants: nil,
|
|
||||||
})
|
|
||||||
acpEmptyObject = &filer_pb.Entry{
|
acpEmptyObject = &filer_pb.Entry{
|
||||||
Name: "acpEmptyObject",
|
Name: "acpEmptyObject",
|
||||||
Extended: map[string][]byte{
|
Extended: map[string][]byte{
|
||||||
s3_constants.ExtAcpKey: acpEmptyObjectAcp,
|
s3_constants.ExtAmzOwnerKey: []byte("xxxxx"),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
//acp owner is nil
|
//grants is nil
|
||||||
acpOwnerNilAcp, _ = jsonutil.BuildJSON(&s3.AccessControlPolicy{
|
acpOwnerNilAcp, _ = json.Marshal(make([]*s3.Grant, 0))
|
||||||
Owner: nil,
|
acpOwnerNil = &filer_pb.Entry{
|
||||||
Grants: make([]*s3.Grant, 1),
|
|
||||||
})
|
|
||||||
acpOwnerNil = &filer_pb.Entry{
|
|
||||||
Name: "acpOwnerNil",
|
Name: "acpOwnerNil",
|
||||||
Extended: map[string][]byte{
|
Extended: map[string][]byte{
|
||||||
s3_constants.ExtAcpKey: acpOwnerNilAcp,
|
s3_constants.ExtAmzAclKey: acpOwnerNilAcp,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -175,8 +163,14 @@ var tcs = []*BucketMetadataTestCase{
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildBucketMetadata(t *testing.T) {
|
func TestBuildBucketMetadata(t *testing.T) {
|
||||||
|
accountManager := &s3account.AccountManager{
|
||||||
|
IdNameMapping: map[string]string{
|
||||||
|
s3account.AccountAdmin.Id: s3account.AccountAdmin.Name,
|
||||||
|
s3account.AccountAnonymous.Id: s3account.AccountAnonymous.Name,
|
||||||
|
},
|
||||||
|
}
|
||||||
for _, tc := range tcs {
|
for _, tc := range tcs {
|
||||||
resultBucketMetadata := buildBucketMetadata(tc.filerEntry)
|
resultBucketMetadata := buildBucketMetadata(accountManager, tc.filerEntry)
|
||||||
if !reflect.DeepEqual(resultBucketMetadata, tc.expectBucketMetadata) {
|
if !reflect.DeepEqual(resultBucketMetadata, tc.expectBucketMetadata) {
|
||||||
t.Fatalf("result is unexpect: \nresult: %v, \nexpect: %v", resultBucketMetadata, tc.expectBucketMetadata)
|
t.Fatalf("result is unexpect: \nresult: %v, \nexpect: %v", resultBucketMetadata, tc.expectBucketMetadata)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package s3_constants
|
package s3_constants
|
||||||
|
|
||||||
const (
|
const (
|
||||||
ExtAcpKey = "Seaweed-X-Amz-Acp"
|
ExtAmzOwnerKey = "Seaweed-X-Amz-Owner"
|
||||||
|
ExtAmzAclKey = "Seaweed-X-Amz-Acl"
|
||||||
ExtOwnershipKey = "Seaweed-X-Amz-Ownership"
|
ExtOwnershipKey = "Seaweed-X-Amz-Ownership"
|
||||||
)
|
)
|
||||||
|
|
|
@ -375,10 +375,16 @@ func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//acp
|
//acp-owner
|
||||||
acp := r.Header.Get(s3_constants.ExtAcpKey)
|
acpOwner := r.Header.Get(s3_constants.ExtAmzOwnerKey)
|
||||||
if len(acp) > 0 {
|
if len(acpOwner) > 0 {
|
||||||
metadata[s3_constants.ExtAcpKey] = []byte(acp)
|
metadata[s3_constants.ExtAmzOwnerKey] = []byte(acpOwner)
|
||||||
|
}
|
||||||
|
|
||||||
|
//acp-grants
|
||||||
|
acpGrants := r.Header.Get(s3_constants.ExtAmzAclKey)
|
||||||
|
if len(acpOwner) > 0 {
|
||||||
|
metadata[s3_constants.ExtAmzAclKey] = []byte(acpGrants)
|
||||||
}
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
|
|
Loading…
Reference in a new issue