new pkg s3iam

weed/s3api/auth_credentials.go

@@ -3,11 +3,15 @@ package s3api
 import (
 	"bytes"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"google.golang.org/grpc"
 	"io/ioutil"
 	"net/http"
 
 	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
+	"github.com/chrislusf/seaweedfs/weed/s3iam"
 	"github.com/golang/protobuf/jsonpb"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -48,19 +52,33 @@ func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManag
 	iam := &IdentityAccessManagement{
 		domain: option.DomainName,
 	}
-	if err := loadS3config(iam, option); err != nil {
+	if err := iam.loadS3ApiConfigurationFromFiler(option); err != nil {
 		glog.Warningf("fail to load config %v", err)
 	}
 	if len(iam.identities) == 0 && option.Config != "" {
-		if err := iam.loadS3ApiConfiguration(option.Config); err != nil {
+		if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil {
 			glog.Fatalf("fail to load config file %s: %v", option.Config, err)
 		}
 	}
 	return iam
 }
 
-func (iam *IdentityAccessManagement) loadS3ApiConfiguration(fileName string) error {
+func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) error {
+	s3ApiConfiguration := &iam_pb.S3ApiConfiguration{}
+	return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		store := s3iam.NewIAMFilerStore(&client)
+		if err := store.LoadIAMConfig(s3ApiConfiguration); err != nil {
+			return nil
+		}
+		if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil {
+			return err
+		}
+		return nil
+	}, option.FilerGrpcAddress, option.GrpcDialOption)
+}
 
+func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName string) error {
 	s3ApiConfiguration := &iam_pb.S3ApiConfiguration{}
 	rawData, readErr := ioutil.ReadFile(fileName)
 	if readErr != nil {
@@ -73,8 +91,14 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(fileName string) err
 		glog.Warningf("unmarshal error: %v", err)
 		return fmt.Errorf("unmarshal %s error: %v", fileName, err)
 	}
+	if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil {
+		return err
+	}
+	return nil
+}
 
-	for _, ident := range s3ApiConfiguration.Identities {
+func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error {
+	for _, ident := range config.Identities {
 		t := &Identity{
 			Name:        ident.Name,
 			Credentials: nil,
@@ -91,7 +115,6 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(fileName string) err
 		}
 		iam.identities = append(iam.identities, t)
 	}
-
 	return nil
 }
 

weed/s3api/filer_util.go

@@ -3,20 +3,12 @@ package s3api
 import (
 	"context"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"strings"
-
-	proto "github.com/golang/protobuf/proto"
-	"google.golang.org/grpc"
 )
 
-const S3ConfName = "s3.conf"
-
 func (s3a *S3ApiServer) mkdir(parentDirectoryPath string, dirName string, fn func(entry *filer_pb.Entry)) error {
 
 	return filer_pb.Mkdir(s3a, parentDirectoryPath, dirName, fn)
@@ -88,135 +80,6 @@ func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *
 	return filer_pb.GetEntry(s3a, fullPath)
 }
 
-func LoadS3configFromEntryExtended(extended *map[string][]byte, identities *[]*Identity) (err error) {
-	for name, ident := range *extended {
-		t := &Identity{
-			Name:        name,
-			Credentials: nil,
-			Actions:     nil,
-		}
-		identity := &iam_pb.Identity{}
-		if err := proto.Unmarshal(ident, identity); err != nil {
-			return err
-		}
-		for _, action := range identity.Actions {
-			t.Actions = append(t.Actions, Action(action))
-		}
-		for _, cred := range identity.Credentials {
-			t.Credentials = append(t.Credentials, &Credential{
-				AccessKey: cred.AccessKey,
-				SecretKey: cred.SecretKey,
-			})
-		}
-		*identities = append(*identities, t)
-	}
-	return nil
-}
-
-func SaveS3configToEntryExtended(extended *map[string][]byte, identities *[]*Identity) (err error) {
-	for _, identity := range *identities {
-		i := &iam_pb.Identity{
-			Name:        identity.Name,
-			Credentials: []*iam_pb.Credential{},
-			Actions:     []string{},
-		}
-		for _, cred := range identity.Credentials {
-			i.Credentials = append(i.Credentials, &iam_pb.Credential{
-				AccessKey: cred.AccessKey,
-				SecretKey: cred.SecretKey,
-			})
-		}
-		for _, action := range identity.Actions {
-			i.Actions = append(i.Actions, string(action))
-		}
-		ident, err := proto.Marshal(i)
-		if err != nil {
-			return err
-		}
-		(*extended)[identity.Name] = ident
-	}
-	return nil
-}
-
-func loadS3config(iam *IdentityAccessManagement, option *S3ApiServerOption) error {
-	return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
-		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-		resp, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{
-			Directory: filer.DirectoryEtc,
-			Name:      S3ConfName,
-		})
-		if err != nil {
-			return err
-		}
-		if err = LoadS3configFromEntryExtended(&resp.Entry.Extended, &iam.identities); err != nil {
-			return err
-		}
-		return nil
-	}, option.FilerGrpcAddress, option.GrpcDialOption)
-}
-
-/* testing save
-func saveS3config(iam *IdentityAccessManagement, option *S3ApiServerOption) (error) {
-	return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
-		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-		entry := &filer_pb.Entry{
-			Name:        "s3identities",
-			IsDirectory: false,
-			Attributes: &filer_pb.FuseAttributes{
-				Mtime:		time.Now().Unix(),
-				Crtime:      time.Now().Unix(),
-				FileMode:    uint32(0644),
-				Collection:  "",
-				Replication: "",
-			},
-			Extended: make(map[string][]byte),
-		}
-		for _, identity := range iam.identities {
-			glog.V(0).Infof("get iam identities %s", identity.Name)
-			i := &iam_pb.Identity{
-				Name:        identity.Name,
-				Credentials: []*iam_pb.Credential{},
-				Actions:    []string{},
-			}
-			for _, cred := range identity.Credentials {
-				i.Credentials = append(i.Credentials, &iam_pb.Credential{
-					AccessKey: cred.AccessKey,
-					SecretKey: cred.SecretKey,
-				})
-			}
-			for _, action := range identity.Actions {
-				i.Actions = append(i.Actions, string(action))
-			}
-			ident, err := proto.Marshal(i)
-			if err != nil {
-				return err
-			}
-			entry.Extended[identity.Name] = ident
-		}
-		_, err := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{
-			Directory: "/.configs",
-			Name:      "s3identities",
-		})
-		if err == filer_pb.ErrNotFound {
-			err = filer_pb.CreateEntry(client, &filer_pb.CreateEntryRequest{
-				Directory:          "/.configs",
-				Entry:              entry,
-				IsFromOtherCluster: false,
-				Signatures:         nil,
-			})
-		} else {
-			err = filer_pb.UpdateEntry(client, &filer_pb.UpdateEntryRequest{
-				Directory:          "/.configs",
-				Entry:              entry,
-				IsFromOtherCluster: false,
-				Signatures:         nil,
-			})
-		}
-		return err
-	},option.FilerGrpcAddress, option.GrpcDialOption)
-}
-*/
-
 func objectKey(key *string) *string {
 	if strings.HasPrefix(*key, "/") {
 		t := (*key)[1:]

weed/s3iam/s3iam_filer_store.go

@@ -0,0 +1,104 @@
+package s3iam
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
+	"time"
+
+	proto "github.com/golang/protobuf/proto"
+)
+
+const (
+	iamConfigPrefix = "/etc/iam"
+	iamIdentityFile = "identity.json"
+)
+
+type IAMFilerStore struct {
+	client *filer_pb.SeaweedFilerClient
+}
+
+func NewIAMFilerStore(client *filer_pb.SeaweedFilerClient) *IAMFilerStore {
+	return &IAMFilerStore{client: client}
+}
+
+func (ifs *IAMFilerStore) getIAMConfigRequest() *filer_pb.LookupDirectoryEntryRequest {
+	return &filer_pb.LookupDirectoryEntryRequest{
+		Directory: iamConfigPrefix,
+		Name:      iamIdentityFile,
+	}
+}
+
+func (ifs *IAMFilerStore) LoadIAMConfig(config *iam_pb.S3ApiConfiguration) error {
+	resp, err := filer_pb.LookupEntry(*ifs.client, ifs.getIAMConfigRequest())
+	if err != nil {
+		return err
+	}
+	err = ifs.loadIAMConfigFromEntryExtended(&resp.Entry.Extended, config)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ifs *IAMFilerStore) SaveIAMConfig(config *iam_pb.S3ApiConfiguration) error {
+	entry := &filer_pb.Entry{
+		Name:        iamIdentityFile,
+		IsDirectory: false,
+		Attributes: &filer_pb.FuseAttributes{
+			Mtime:       time.Now().Unix(),
+			Crtime:      time.Now().Unix(),
+			FileMode:    uint32(0644),
+			Collection:  "",
+			Replication: "",
+		},
+		Extended: make(map[string][]byte),
+	}
+
+	err := ifs.saveIAMConfigToEntryExtended(&entry.Extended, config)
+	if err != nil {
+		return err
+	}
+
+	_, err = filer_pb.LookupEntry(*ifs.client, ifs.getIAMConfigRequest())
+	if err == filer_pb.ErrNotFound {
+		err = filer_pb.CreateEntry(*ifs.client, &filer_pb.CreateEntryRequest{
+			Directory:          iamConfigPrefix,
+			Entry:              entry,
+			IsFromOtherCluster: false,
+			Signatures:         nil,
+		})
+	} else {
+		err = filer_pb.UpdateEntry(*ifs.client, &filer_pb.UpdateEntryRequest{
+			Directory:          iamConfigPrefix,
+			Entry:              entry,
+			IsFromOtherCluster: false,
+			Signatures:         nil,
+		})
+	}
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ifs *IAMFilerStore) loadIAMConfigFromEntryExtended(extended *map[string][]byte, config *iam_pb.S3ApiConfiguration) error {
+	for _, ident := range *extended {
+		identity := &iam_pb.Identity{}
+		if err := proto.Unmarshal(ident, identity); err != nil {
+			return err
+		}
+		config.Identities = append(config.Identities, identity)
+	}
+	return nil
+}
+
+func (ifs *IAMFilerStore) saveIAMConfigToEntryExtended(extended *map[string][]byte, config *iam_pb.S3ApiConfiguration) error {
+	for _, identity := range config.Identities {
+		ident, err := proto.Marshal(identity)
+		if err != nil {
+			return err
+		}
+		(*extended)[identity.Name] = ident
+	}
+	return nil
+}

weed/shell/command_s3_configure.go

@@ -3,13 +3,13 @@ package shell
 import (
 	"flag"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/s3api"
 	"io"
 	"sort"
 	"strings"
 
-	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
+	"github.com/chrislusf/seaweedfs/weed/s3iam"
 )
 
 func init() {
@@ -44,18 +44,12 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
 		return nil
 	}
 
-	var identities []*s3api.Identity
+	s3cfg := &iam_pb.S3ApiConfiguration{}
+	ifs := &s3iam.IAMFilerStore{}
 	if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		request := &filer_pb.LookupDirectoryEntryRequest{
+		ifs = s3iam.NewIAMFilerStore(&client)
-			Directory: filer.DirectoryEtc,
+		if err := ifs.LoadIAMConfig(s3cfg); err != nil {
-			Name:      s3api.S3ConfName,
+			return nil
-		}
-		respLookupEntry, err := filer_pb.LookupEntry(client, request)
-		if err != nil {
-			return err
-		}
-		if err = s3api.LoadS3configFromEntryExtended(&respLookupEntry.Entry.Extended, &identities); err != nil {
-			return err
 		}
 		return nil
 	}); err != nil {
@@ -65,7 +59,7 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
 	idx := 0
 	changed := false
 	if *user != "" && *buckets != "" {
-		for i, identity := range identities {
+		for i, identity := range s3cfg.Identities {
 			if *user == identity.Name {
 				idx = i
 				changed = true
@@ -73,22 +67,17 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
 			}
 		}
 	}
-	cmdActions := []s3api.Action{}
+	var cmdActions []string
 	for _, bucket := range strings.Split(*buckets, ",") {
 		for _, action := range strings.Split(*actions, ",") {
-			cmdActions = append(cmdActions, s3api.Action(fmt.Sprintf("%s:%s", action, bucket)))
+			cmdActions = append(cmdActions, fmt.Sprintf("%s:%s", action, bucket))
 		}
 	}
-	cmdCredential := &s3api.Credential{
-		AccessKey: *accessKey,
-		SecretKey: *secretKey,
-	}
-
 	if changed {
 		if *isDelete {
-			exists := []int{}
+			var exists []int
 			for _, cmdAction := range cmdActions {
-				for i, currentAction := range identities[idx].Actions {
+				for i, currentAction := range s3cfg.Identities[idx].Actions {
 					if cmdAction == currentAction {
 						exists = append(exists, i)
 					}
@@ -96,65 +85,57 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
 			}
 			sort.Sort(sort.Reverse(sort.IntSlice(exists)))
 			for _, i := range exists {
-				identities[idx].Actions = append(identities[idx].Actions[:i], identities[idx].Actions[i+1:]...)
+				s3cfg.Identities[idx].Actions = append(
+					s3cfg.Identities[idx].Actions[:i],
+					s3cfg.Identities[idx].Actions[i+1:]...,
+				)
 			}
 			if *accessKey != "" {
 				exists = []int{}
-				for i, credential := range identities[idx].Credentials {
+				for i, credential := range s3cfg.Identities[idx].Credentials {
 					if credential.AccessKey == *accessKey {
 						exists = append(exists, i)
 					}
 				}
 				sort.Sort(sort.Reverse(sort.IntSlice(exists)))
 				for _, i := range exists {
-					identities[idx].Credentials = append(identities[idx].Credentials[:i], identities[idx].Credentials[:i+1]...)
+					s3cfg.Identities[idx].Credentials = append(
+						s3cfg.Identities[idx].Credentials[:i],
+						s3cfg.Identities[idx].Credentials[:i+1]...,
+					)
 				}
 
 			}
 			if *actions == "" && *accessKey == "" {
-				identities = append(identities[:idx], identities[idx+1:]...)
+				s3cfg.Identities = append(s3cfg.Identities[:idx], s3cfg.Identities[idx+1:]...)
 			}
 		} else {
-			identities[idx].Actions = append(identities[idx].Actions, cmdActions...)
+			s3cfg.Identities[idx].Actions = append(s3cfg.Identities[idx].Actions, cmdActions...)
-			identities[idx].Credentials = append(identities[idx].Credentials, &s3api.Credential{
+			s3cfg.Identities[idx].Credentials = append(s3cfg.Identities[idx].Credentials, &iam_pb.Credential{
 				AccessKey: *accessKey,
 				SecretKey: *secretKey,
 			})
 		}
 	} else {
-		identity := s3api.Identity{
+		identity := iam_pb.Identity{
 			Name:    *user,
 			Actions: cmdActions,
 		}
-		identity.Credentials = append(identity.Credentials, &s3api.Credential{
+		identity.Credentials = append(identity.Credentials, &iam_pb.Credential{
 			AccessKey: *accessKey,
 			SecretKey: *secretKey,
 		})
-		identities = append(identities, &identity)
+		s3cfg.Identities = append(s3cfg.Identities, &identity)
 	}
 
-	fmt.Fprintf(writer, fmt.Sprintf("%+v\n", identities))
+	fmt.Fprintf(writer, fmt.Sprintf("%+v\n", s3cfg.Identities))
 	fmt.Fprintln(writer)
 
-	if !*apply {
+	if *apply {
-		return nil
+		if err := ifs.SaveIAMConfig(s3cfg); err != nil {
+			return err
+		}
 	}
 
-	if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		request := &filer_pb.LookupDirectoryEntryRequest{
-			Directory: filer.DirectoryEtc,
-			Name:      s3api.S3ConfName,
-		}
-		respLookupEntry, err := filer_pb.LookupEntry(client, request)
-		if err != nil {
-			return err
-		}
-		if err = s3api.SaveS3configToEntryExtended(&respLookupEntry.Entry.Extended, &identities); err != nil {
-			return err
-		}
-		return nil
-	}); err != nil {
-		return err
-	}
 	return nil
 }