refactoring

weed/filer/filer_conf.go

@@ -15,6 +15,8 @@ import (
 const (
 	DirectoryEtc  = "/etc"
 	FilerConfName = "filer.conf"
+	IamConfigDirecotry = "/etc/iam"
+	IamIdentityFile = "identity.json"
 )
 
 type FilerConf struct {

weed/filer/read.go

@@ -1,27 +0,0 @@
-package filer
-
-import (
-	"bytes"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/wdclient"
-	"math"
-)
-
-func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.SeaweedFilerClient, dir, name string, byteBuffer *bytes.Buffer) error {
-
-	request := &filer_pb.LookupDirectoryEntryRequest{
-		Directory: dir,
-		Name:      name,
-	}
-	respLookupEntry, err := filer_pb.LookupEntry(filerClient, request)
-	if err != nil {
-		return err
-	}
-	if len(respLookupEntry.Entry.Content) > 0 {
-		_, err = byteBuffer.Write(respLookupEntry.Entry.Content)
-		return err
-	}
-
-	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
-
-}

weed/filer/read_write.go

@@ -0,0 +1,63 @@
+package filer
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
+	"math"
+	"net/http"
+)
+
+func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.SeaweedFilerClient, dir, name string, byteBuffer *bytes.Buffer) error {
+
+	request := &filer_pb.LookupDirectoryEntryRequest{
+		Directory: dir,
+		Name:      name,
+	}
+	respLookupEntry, err := filer_pb.LookupEntry(filerClient, request)
+	if err != nil {
+		return err
+	}
+	if len(respLookupEntry.Entry.Content) > 0 {
+		_, err = byteBuffer.Write(respLookupEntry.Entry.Content)
+		return err
+	}
+
+	return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
+
+}
+
+func ReadContent(filerAddress string, dir, name string) ([]byte, error) {
+
+	target := fmt.Sprintf("http://%s%s/%s", filerAddress, dir, name)
+
+	data, _, err := util.Get(target)
+
+	return data, err
+}
+
+func SaveAs(host string, port int, dir, name string, contentType string, byteBuffer *bytes.Buffer) error {
+
+	target := fmt.Sprintf("http://%s:%d%s/%s", host, port, dir, name)
+
+	// set the HTTP method, url, and request body
+	req, err := http.NewRequest(http.MethodPut, target, byteBuffer)
+	if err != nil {
+		return err
+	}
+
+	// set the request header Content-Type for json
+	if contentType != "" {
+		req.Header.Set("Content-Type", contentType)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	util.CloseResponse(resp)
+
+	return nil
+
+}

weed/filer/s3iam_conf.go

@@ -0,0 +1,25 @@
+package filer
+
+import (
+	"bytes"
+	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
+	"github.com/golang/protobuf/jsonpb"
+	"io"
+)
+
+func ParseS3ConfigurationFromBytes(content []byte, config *iam_pb.S3ApiConfiguration) error {
+	if err := jsonpb.Unmarshal(bytes.NewBuffer(content), config); err != nil {
+		return err
+	}
+	return nil
+}
+
+func S3ConfigurationToText(writer io.Writer, config *iam_pb.S3ApiConfiguration) error {
+
+	m := jsonpb.Marshaler{
+		EmitDefaults: false,
+		Indent:       "  ",
+	}
+
+	return m.Marshal(writer, config)
+}

weed/filer/s3iam_conf_test.go

@@ -1,7 +1,8 @@
-package s3iam
+package filer
 
 import (
-	"github.com/golang/protobuf/proto"
+	"bytes"
+	"github.com/chrislusf/seaweedfs/weed/s3api"
 	"testing"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
@@ -9,16 +10,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-const (
-	ACTION_READ    = "Read"
-	ACTION_WRITE   = "Write"
-	ACTION_ADMIN   = "Admin"
-	ACTION_TAGGING = "Tagging"
-	ACTION_LIST    = "List"
-)
-
 func TestS3Conf(t *testing.T) {
-	ifs := &IAMFilerStore{}
 	s3Conf := &iam_pb.S3ApiConfiguration{
 		Identities: []*iam_pb.Identity{
 			{
@@ -30,9 +22,9 @@ func TestS3Conf(t *testing.T) {
 					},
 				},
 				Actions: []string{
-					ACTION_ADMIN,
+					s3api.ACTION_ADMIN,
-					ACTION_READ,
+					s3api.ACTION_READ,
-					ACTION_WRITE,
+					s3api.ACTION_WRITE,
 				},
 			},
 			{
@@ -44,17 +36,18 @@ func TestS3Conf(t *testing.T) {
 					},
 				},
 				Actions: []string{
-					ACTION_READ,
+					s3api.ACTION_READ,
-					ACTION_TAGGING,
+					s3api.ACTION_TAGGING,
-					ACTION_LIST,
+					s3api.ACTION_LIST,
 				},
 			},
 		},
 	}
-	content, err := proto.Marshal(s3Conf)
+	var buf bytes.Buffer
+	err := S3ConfigurationToText(&buf, s3Conf)
 	assert.Equal(t, err, nil)
 	s3ConfSaved := &iam_pb.S3ApiConfiguration{}
-	err = ifs.loadIAMConfigFromBytes(content, s3ConfSaved)
+	err = ParseS3ConfigurationFromBytes(buf.Bytes(), s3ConfSaved)
 	assert.Equal(t, err, nil)
 
 	assert.Equal(t, "some_name", s3ConfSaved.Identities[0].Name)

weed/s3api/auth_credentials.go

@@ -3,15 +3,12 @@ package s3api
 import (
 	"bytes"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/filer"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"google.golang.org/grpc"
 	"io/ioutil"
 	"net/http"
 
 	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
 	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
-	"github.com/chrislusf/seaweedfs/weed/s3iam"
 	"github.com/golang/protobuf/jsonpb"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -65,17 +62,17 @@ func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManag
 
 func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) error {
 	s3ApiConfiguration := &iam_pb.S3ApiConfiguration{}
-	return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+	content, err := filer.ReadContent(option.Filer, filer.IamConfigDirecotry, filer.IamIdentityFile)
-		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+	if err != nil {
-		store := s3iam.NewIAMFilerStore(&client)
+		return fmt.Errorf("read S3 config: %v", err)
-		if err := store.LoadIAMConfig(s3ApiConfiguration); err != nil {
+	}
-			return nil
+	if err = filer.ParseS3ConfigurationFromBytes(content, s3ApiConfiguration); err != nil {
-		}
+		return fmt.Errorf("parse S3 config: %v", err)
-		if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil {
+	}
-			return err
+	if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil {
-		}
+		return fmt.Errorf("laod S3 config: %v", err)
-		return nil
+	}
-	}, option.FilerGrpcAddress, option.GrpcDialOption)
+	return nil
 }
 
 func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName string) error {

weed/s3iam/s3iam_filer_store.go

@@ -1,95 +0,0 @@
-package s3iam
-
-import (
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
-	"time"
-
-	proto "github.com/golang/protobuf/proto"
-)
-
-const (
-	iamConfigPrefix = "/etc/iam"
-	iamIdentityFile = "identity.json"
-)
-
-type IAMFilerStore struct {
-	client *filer_pb.SeaweedFilerClient
-}
-
-func NewIAMFilerStore(client *filer_pb.SeaweedFilerClient) *IAMFilerStore {
-	return &IAMFilerStore{client: client}
-}
-
-func (ifs *IAMFilerStore) getIAMConfigRequest() *filer_pb.LookupDirectoryEntryRequest {
-	return &filer_pb.LookupDirectoryEntryRequest{
-		Directory: iamConfigPrefix,
-		Name:      iamIdentityFile,
-	}
-}
-
-func (ifs *IAMFilerStore) LoadIAMConfig(config *iam_pb.S3ApiConfiguration) error {
-	resp, err := filer_pb.LookupEntry(*ifs.client, ifs.getIAMConfigRequest())
-	if err != nil {
-		return err
-	}
-	err = ifs.loadIAMConfigFromBytes(resp.Entry.Content, config)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (ifs *IAMFilerStore) SaveIAMConfig(config *iam_pb.S3ApiConfiguration) error {
-	entry := &filer_pb.Entry{
-		Name:        iamIdentityFile,
-		IsDirectory: false,
-		Attributes: &filer_pb.FuseAttributes{
-			Mtime:       time.Now().Unix(),
-			Crtime:      time.Now().Unix(),
-			FileMode:    uint32(0644),
-			Collection:  "",
-			Replication: "",
-		},
-		Content: []byte{},
-	}
-	err := ifs.saveIAMConfigToEntry(entry, config)
-	if err != nil {
-		return err
-	}
-	_, err = filer_pb.LookupEntry(*ifs.client, ifs.getIAMConfigRequest())
-	if err == filer_pb.ErrNotFound {
-		err = filer_pb.CreateEntry(*ifs.client, &filer_pb.CreateEntryRequest{
-			Directory:          iamConfigPrefix,
-			Entry:              entry,
-			IsFromOtherCluster: false,
-			Signatures:         nil,
-		})
-	} else {
-		err = filer_pb.UpdateEntry(*ifs.client, &filer_pb.UpdateEntryRequest{
-			Directory:          iamConfigPrefix,
-			Entry:              entry,
-			IsFromOtherCluster: false,
-			Signatures:         nil,
-		})
-	}
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (ifs *IAMFilerStore) loadIAMConfigFromBytes(content []byte, config *iam_pb.S3ApiConfiguration) error {
-	if err := proto.Unmarshal(content, config); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (ifs *IAMFilerStore) saveIAMConfigToEntry(entry *filer_pb.Entry, config *iam_pb.S3ApiConfiguration) (err error) {
-	entry.Content, err = proto.Marshal(config)
-	if err != nil {
-		return err
-	}
-	return nil
-}

weed/shell/command_fs_configure.go

@@ -5,13 +5,11 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"net/http"
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/storage/super_block"
-	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 func init() {
@@ -118,22 +116,10 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io
 
 	if *apply {
 
-		target := fmt.Sprintf("http://%s:%d%s/%s", commandEnv.option.FilerHost, commandEnv.option.FilerPort, filer.DirectoryEtc, filer.FilerConfName)
+		if err := filer.SaveAs(commandEnv.option.FilerHost, int(commandEnv.option.FilerPort), filer.DirectoryEtc, filer.FilerConfName, "text/plain; charset=utf-8", &buf); err != nil {
-
-		// set the HTTP method, url, and request body
-		req, err := http.NewRequest(http.MethodPut, target, &buf)
-		if err != nil {
 			return err
 		}
 
-		// set the request header Content-Type for json
-		req.Header.Set("Content-Type", "text/plain; charset=utf-8")
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			return err
-		}
-		util.CloseResponse(resp)
-
 	}
 
 	return nil

weed/shell/command_s3_configure.go

@@ -1,15 +1,16 @@
 package shell
 
 import (
+	"bytes"
 	"flag"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/filer"
 	"io"
 	"sort"
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
-	"github.com/chrislusf/seaweedfs/weed/s3iam"
 )
 
 func init() {
@@ -32,6 +33,7 @@ func (c *commandS3Configure) Help() string {
 }
 
 func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
 	s3ConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
 	actions := s3ConfigureCommand.String("actions", "", "comma separated actions names: Read,Write,List,Tagging,Admin")
 	user := s3ConfigureCommand.String("user", "", "user name")
@@ -45,18 +47,20 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
 		return nil
 	}
 
-	s3cfg := &iam_pb.S3ApiConfiguration{}
+	var buf bytes.Buffer
-	ifs := &s3iam.IAMFilerStore{}
 	if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		ifs = s3iam.NewIAMFilerStore(&client)
+		return filer.ReadEntry(commandEnv.MasterClient, client, filer.IamConfigDirecotry, filer.IamIdentityFile, &buf)
-		if err := ifs.LoadIAMConfig(s3cfg); err != nil {
+	}); err != nil && err != filer_pb.ErrNotFound {
-			return nil
-		}
-		return nil
-	}); err != nil {
 		return err
 	}
 
+	s3cfg := &iam_pb.S3ApiConfiguration{}
+	if buf.Len() > 0 {
+		if err = filer.ParseS3ConfigurationFromBytes(buf.Bytes(), s3cfg); err != nil {
+			return err
+		}
+	}
+
 	idx := 0
 	changed := false
 	if *user != "" {
@@ -159,16 +163,19 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
 		s3cfg.Identities = append(s3cfg.Identities, &identity)
 	}
 
-	for _, identity := range s3cfg.Identities {
+	buf.Reset()
-		fmt.Fprintf(writer, fmt.Sprintf("%+v\n", identity))
+	filer.S3ConfigurationToText(&buf, s3cfg)
-	}
 
+	fmt.Fprintf(writer, string(buf.Bytes()))
 	fmt.Fprintln(writer)
 
+
 	if *apply {
-		if err := ifs.SaveIAMConfig(s3cfg); err != nil {
+
+		if err := filer.SaveAs(commandEnv.option.FilerHost, int(commandEnv.option.FilerPort), filer.IamConfigDirecotry, filer.IamIdentityFile, "text/plain; charset=utf-8", &buf); err != nil {
 			return err
 		}
+
 	}
 
 	return nil