From 01e2da5782f7fb14fb4ba1350e0c1fa097557dcf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 6 Dec 2020 23:16:20 -0800 Subject: [PATCH] refactoring --- weed/filer/filer_conf.go | 2 + weed/filer/read.go | 27 ------ weed/filer/read_write.go | 63 ++++++++++++ weed/filer/s3iam_conf.go | 25 +++++ .../s3iam_conf_test.go} | 31 +++--- weed/s3api/auth_credentials.go | 27 +++--- weed/s3iam/s3iam_filer_store.go | 95 ------------------- weed/shell/command_fs_configure.go | 16 +--- weed/shell/command_s3_configure.go | 33 ++++--- 9 files changed, 135 insertions(+), 184 deletions(-) delete mode 100644 weed/filer/read.go create mode 100644 weed/filer/read_write.go create mode 100644 weed/filer/s3iam_conf.go rename weed/{s3iam/s3iam_filer_store_test.go => filer/s3iam_conf_test.go} (71%) delete mode 100644 weed/s3iam/s3iam_filer_store.go diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index d56db3b8e..6a746f7ba 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -15,6 +15,8 @@ import ( const ( DirectoryEtc = "/etc" FilerConfName = "filer.conf" + IamConfigDirecotry = "/etc/iam" + IamIdentityFile = "identity.json" ) type FilerConf struct { diff --git a/weed/filer/read.go b/weed/filer/read.go deleted file mode 100644 index 13f282dc7..000000000 --- a/weed/filer/read.go +++ /dev/null @@ -1,27 +0,0 @@ -package filer - -import ( - "bytes" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/wdclient" - "math" -) - -func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.SeaweedFilerClient, dir, name string, byteBuffer *bytes.Buffer) error { - - request := &filer_pb.LookupDirectoryEntryRequest{ - Directory: dir, - Name: name, - } - respLookupEntry, err := filer_pb.LookupEntry(filerClient, request) - if err != nil { - return err - } - if len(respLookupEntry.Entry.Content) > 0 { - _, err = byteBuffer.Write(respLookupEntry.Entry.Content) - return err - } - - return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) - -} diff --git a/weed/filer/read_write.go b/weed/filer/read_write.go new file mode 100644 index 000000000..8561d848b --- /dev/null +++ b/weed/filer/read_write.go @@ -0,0 +1,63 @@ +package filer + +import ( + "bytes" + "fmt" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/wdclient" + "math" + "net/http" +) + +func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.SeaweedFilerClient, dir, name string, byteBuffer *bytes.Buffer) error { + + request := &filer_pb.LookupDirectoryEntryRequest{ + Directory: dir, + Name: name, + } + respLookupEntry, err := filer_pb.LookupEntry(filerClient, request) + if err != nil { + return err + } + if len(respLookupEntry.Entry.Content) > 0 { + _, err = byteBuffer.Write(respLookupEntry.Entry.Content) + return err + } + + return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64) + +} + +func ReadContent(filerAddress string, dir, name string) ([]byte, error) { + + target := fmt.Sprintf("http://%s%s/%s", filerAddress, dir, name) + + data, _, err := util.Get(target) + + return data, err +} + +func SaveAs(host string, port int, dir, name string, contentType string, byteBuffer *bytes.Buffer) error { + + target := fmt.Sprintf("http://%s:%d%s/%s", host, port, dir, name) + + // set the HTTP method, url, and request body + req, err := http.NewRequest(http.MethodPut, target, byteBuffer) + if err != nil { + return err + } + + // set the request header Content-Type for json + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + util.CloseResponse(resp) + + return nil + +} diff --git a/weed/filer/s3iam_conf.go b/weed/filer/s3iam_conf.go new file mode 100644 index 000000000..92387fb09 --- /dev/null +++ b/weed/filer/s3iam_conf.go @@ -0,0 +1,25 @@ +package filer + +import ( + "bytes" + "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" + "github.com/golang/protobuf/jsonpb" + "io" +) + +func ParseS3ConfigurationFromBytes(content []byte, config *iam_pb.S3ApiConfiguration) error { + if err := jsonpb.Unmarshal(bytes.NewBuffer(content), config); err != nil { + return err + } + return nil +} + +func S3ConfigurationToText(writer io.Writer, config *iam_pb.S3ApiConfiguration) error { + + m := jsonpb.Marshaler{ + EmitDefaults: false, + Indent: " ", + } + + return m.Marshal(writer, config) +} diff --git a/weed/s3iam/s3iam_filer_store_test.go b/weed/filer/s3iam_conf_test.go similarity index 71% rename from weed/s3iam/s3iam_filer_store_test.go rename to weed/filer/s3iam_conf_test.go index 5845f1383..88f446151 100644 --- a/weed/s3iam/s3iam_filer_store_test.go +++ b/weed/filer/s3iam_conf_test.go @@ -1,7 +1,8 @@ -package s3iam +package filer import ( - "github.com/golang/protobuf/proto" + "bytes" + "github.com/chrislusf/seaweedfs/weed/s3api" "testing" "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" @@ -9,16 +10,7 @@ import ( "github.com/stretchr/testify/assert" ) -const ( - ACTION_READ = "Read" - ACTION_WRITE = "Write" - ACTION_ADMIN = "Admin" - ACTION_TAGGING = "Tagging" - ACTION_LIST = "List" -) - func TestS3Conf(t *testing.T) { - ifs := &IAMFilerStore{} s3Conf := &iam_pb.S3ApiConfiguration{ Identities: []*iam_pb.Identity{ { @@ -30,9 +22,9 @@ func TestS3Conf(t *testing.T) { }, }, Actions: []string{ - ACTION_ADMIN, - ACTION_READ, - ACTION_WRITE, + s3api.ACTION_ADMIN, + s3api.ACTION_READ, + s3api.ACTION_WRITE, }, }, { @@ -44,17 +36,18 @@ func TestS3Conf(t *testing.T) { }, }, Actions: []string{ - ACTION_READ, - ACTION_TAGGING, - ACTION_LIST, + s3api.ACTION_READ, + s3api.ACTION_TAGGING, + s3api.ACTION_LIST, }, }, }, } - content, err := proto.Marshal(s3Conf) + var buf bytes.Buffer + err := S3ConfigurationToText(&buf, s3Conf) assert.Equal(t, err, nil) s3ConfSaved := &iam_pb.S3ApiConfiguration{} - err = ifs.loadIAMConfigFromBytes(content, s3ConfSaved) + err = ParseS3ConfigurationFromBytes(buf.Bytes(), s3ConfSaved) assert.Equal(t, err, nil) assert.Equal(t, "some_name", s3ConfSaved.Identities[0].Name) diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index de1a0e3a1..9fcd016e2 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -3,15 +3,12 @@ package s3api import ( "bytes" "fmt" - "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "google.golang.org/grpc" + "github.com/chrislusf/seaweedfs/weed/filer" "io/ioutil" "net/http" xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" "github.com/chrislusf/seaweedfs/weed/s3api/s3err" - "github.com/chrislusf/seaweedfs/weed/s3iam" "github.com/golang/protobuf/jsonpb" "github.com/chrislusf/seaweedfs/weed/glog" @@ -65,17 +62,17 @@ func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManag func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) error { s3ApiConfiguration := &iam_pb.S3ApiConfiguration{} - return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error { - client := filer_pb.NewSeaweedFilerClient(grpcConnection) - store := s3iam.NewIAMFilerStore(&client) - if err := store.LoadIAMConfig(s3ApiConfiguration); err != nil { - return nil - } - if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil { - return err - } - return nil - }, option.FilerGrpcAddress, option.GrpcDialOption) + content, err := filer.ReadContent(option.Filer, filer.IamConfigDirecotry, filer.IamIdentityFile) + if err != nil { + return fmt.Errorf("read S3 config: %v", err) + } + if err = filer.ParseS3ConfigurationFromBytes(content, s3ApiConfiguration); err != nil { + return fmt.Errorf("parse S3 config: %v", err) + } + if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil { + return fmt.Errorf("laod S3 config: %v", err) + } + return nil } func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName string) error { diff --git a/weed/s3iam/s3iam_filer_store.go b/weed/s3iam/s3iam_filer_store.go deleted file mode 100644 index 4171ef143..000000000 --- a/weed/s3iam/s3iam_filer_store.go +++ /dev/null @@ -1,95 +0,0 @@ -package s3iam - -import ( - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" - "time" - - proto "github.com/golang/protobuf/proto" -) - -const ( - iamConfigPrefix = "/etc/iam" - iamIdentityFile = "identity.json" -) - -type IAMFilerStore struct { - client *filer_pb.SeaweedFilerClient -} - -func NewIAMFilerStore(client *filer_pb.SeaweedFilerClient) *IAMFilerStore { - return &IAMFilerStore{client: client} -} - -func (ifs *IAMFilerStore) getIAMConfigRequest() *filer_pb.LookupDirectoryEntryRequest { - return &filer_pb.LookupDirectoryEntryRequest{ - Directory: iamConfigPrefix, - Name: iamIdentityFile, - } -} - -func (ifs *IAMFilerStore) LoadIAMConfig(config *iam_pb.S3ApiConfiguration) error { - resp, err := filer_pb.LookupEntry(*ifs.client, ifs.getIAMConfigRequest()) - if err != nil { - return err - } - err = ifs.loadIAMConfigFromBytes(resp.Entry.Content, config) - if err != nil { - return err - } - return nil -} - -func (ifs *IAMFilerStore) SaveIAMConfig(config *iam_pb.S3ApiConfiguration) error { - entry := &filer_pb.Entry{ - Name: iamIdentityFile, - IsDirectory: false, - Attributes: &filer_pb.FuseAttributes{ - Mtime: time.Now().Unix(), - Crtime: time.Now().Unix(), - FileMode: uint32(0644), - Collection: "", - Replication: "", - }, - Content: []byte{}, - } - err := ifs.saveIAMConfigToEntry(entry, config) - if err != nil { - return err - } - _, err = filer_pb.LookupEntry(*ifs.client, ifs.getIAMConfigRequest()) - if err == filer_pb.ErrNotFound { - err = filer_pb.CreateEntry(*ifs.client, &filer_pb.CreateEntryRequest{ - Directory: iamConfigPrefix, - Entry: entry, - IsFromOtherCluster: false, - Signatures: nil, - }) - } else { - err = filer_pb.UpdateEntry(*ifs.client, &filer_pb.UpdateEntryRequest{ - Directory: iamConfigPrefix, - Entry: entry, - IsFromOtherCluster: false, - Signatures: nil, - }) - } - if err != nil { - return err - } - return nil -} - -func (ifs *IAMFilerStore) loadIAMConfigFromBytes(content []byte, config *iam_pb.S3ApiConfiguration) error { - if err := proto.Unmarshal(content, config); err != nil { - return err - } - return nil -} - -func (ifs *IAMFilerStore) saveIAMConfigToEntry(entry *filer_pb.Entry, config *iam_pb.S3ApiConfiguration) (err error) { - entry.Content, err = proto.Marshal(config) - if err != nil { - return err - } - return nil -} diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go index b67a6e672..565e6c66e 100644 --- a/weed/shell/command_fs_configure.go +++ b/weed/shell/command_fs_configure.go @@ -5,13 +5,11 @@ import ( "flag" "fmt" "io" - "net/http" "strings" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/storage/super_block" - "github.com/chrislusf/seaweedfs/weed/util" ) func init() { @@ -118,22 +116,10 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io if *apply { - target := fmt.Sprintf("http://%s:%d%s/%s", commandEnv.option.FilerHost, commandEnv.option.FilerPort, filer.DirectoryEtc, filer.FilerConfName) - - // set the HTTP method, url, and request body - req, err := http.NewRequest(http.MethodPut, target, &buf) - if err != nil { + if err := filer.SaveAs(commandEnv.option.FilerHost, int(commandEnv.option.FilerPort), filer.DirectoryEtc, filer.FilerConfName, "text/plain; charset=utf-8", &buf); err != nil { return err } - // set the request header Content-Type for json - req.Header.Set("Content-Type", "text/plain; charset=utf-8") - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - util.CloseResponse(resp) - } return nil diff --git a/weed/shell/command_s3_configure.go b/weed/shell/command_s3_configure.go index f081eb303..42d1048a3 100644 --- a/weed/shell/command_s3_configure.go +++ b/weed/shell/command_s3_configure.go @@ -1,15 +1,16 @@ package shell import ( + "bytes" "flag" "fmt" + "github.com/chrislusf/seaweedfs/weed/filer" "io" "sort" "strings" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/iam_pb" - "github.com/chrislusf/seaweedfs/weed/s3iam" ) func init() { @@ -32,6 +33,7 @@ func (c *commandS3Configure) Help() string { } func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + s3ConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) actions := s3ConfigureCommand.String("actions", "", "comma separated actions names: Read,Write,List,Tagging,Admin") user := s3ConfigureCommand.String("user", "", "user name") @@ -45,18 +47,20 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io return nil } - s3cfg := &iam_pb.S3ApiConfiguration{} - ifs := &s3iam.IAMFilerStore{} + var buf bytes.Buffer if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { - ifs = s3iam.NewIAMFilerStore(&client) - if err := ifs.LoadIAMConfig(s3cfg); err != nil { - return nil - } - return nil - }); err != nil { + return filer.ReadEntry(commandEnv.MasterClient, client, filer.IamConfigDirecotry, filer.IamIdentityFile, &buf) + }); err != nil && err != filer_pb.ErrNotFound { return err } + s3cfg := &iam_pb.S3ApiConfiguration{} + if buf.Len() > 0 { + if err = filer.ParseS3ConfigurationFromBytes(buf.Bytes(), s3cfg); err != nil { + return err + } + } + idx := 0 changed := false if *user != "" { @@ -159,16 +163,19 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io s3cfg.Identities = append(s3cfg.Identities, &identity) } - for _, identity := range s3cfg.Identities { - fmt.Fprintf(writer, fmt.Sprintf("%+v\n", identity)) - } + buf.Reset() + filer.S3ConfigurationToText(&buf, s3cfg) + fmt.Fprintf(writer, string(buf.Bytes())) fmt.Fprintln(writer) + if *apply { - if err := ifs.SaveIAMConfig(s3cfg); err != nil { + + if err := filer.SaveAs(commandEnv.option.FilerHost, int(commandEnv.option.FilerPort), filer.IamConfigDirecotry, filer.IamIdentityFile, "text/plain; charset=utf-8", &buf); err != nil { return err } + } return nil