gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

refactoring

Browse source
This commit is contained in:
Chris Lu 2020-12-06 23:16:20 -08:00
parent 77286f8bea
commit 01e2da5782
9 changed files with 135 additions and 184 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
weed/filer/filer_conf.go
View file

@ -15,6 +15,8 @@ import (
const (
DirectoryEtc = "/etc"
FilerConfName = "filer.conf"
IamConfigDirecotry = "/etc/iam"
IamIdentityFile = "identity.json"
)
type FilerConf struct {

27
weed/filer/read.go
View file

@ -1,27 +0,0 @@
package filer
import (
"bytes"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"math"
)
func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.SeaweedFilerClient, dir, name string, byteBuffer *bytes.Buffer) error {
request := &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
}
respLookupEntry, err := filer_pb.LookupEntry(filerClient, request)
if err != nil {
return err
}
if len(respLookupEntry.Entry.Content) > 0 {
_, err = byteBuffer.Write(respLookupEntry.Entry.Content)
return err
}
return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
}

63
weed/filer/read_write.go Normal file
View file

@ -0,0 +1,63 @@
package filer
import (
"bytes"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"math"
"net/http"
)
func ReadEntry(masterClient *wdclient.MasterClient, filerClient filer_pb.SeaweedFilerClient, dir, name string, byteBuffer *bytes.Buffer) error {
request := &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
}
respLookupEntry, err := filer_pb.LookupEntry(filerClient, request)
if err != nil {
return err
}
if len(respLookupEntry.Entry.Content) > 0 {
_, err = byteBuffer.Write(respLookupEntry.Entry.Content)
return err
}
return StreamContent(masterClient, byteBuffer, respLookupEntry.Entry.Chunks, 0, math.MaxInt64)
}
func ReadContent(filerAddress string, dir, name string) ([]byte, error) {
target := fmt.Sprintf("http://%s%s/%s", filerAddress, dir, name)
data, _, err := util.Get(target)
return data, err
}
func SaveAs(host string, port int, dir, name string, contentType string, byteBuffer *bytes.Buffer) error {
target := fmt.Sprintf("http://%s:%d%s/%s", host, port, dir, name)
// set the HTTP method, url, and request body
req, err := http.NewRequest(http.MethodPut, target, byteBuffer)
if err != nil {
return err
}
// set the request header Content-Type for json
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
util.CloseResponse(resp)
return nil
}

25
weed/filer/s3iam_conf.go Normal file
View file

@ -0,0 +1,25 @@
package filer
import (
"bytes"
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
"github.com/golang/protobuf/jsonpb"
"io"
)
func ParseS3ConfigurationFromBytes(content []byte, config *iam_pb.S3ApiConfiguration) error {
if err := jsonpb.Unmarshal(bytes.NewBuffer(content), config); err != nil {
return err
}
return nil
}
func S3ConfigurationToText(writer io.Writer, config *iam_pb.S3ApiConfiguration) error {
m := jsonpb.Marshaler{
EmitDefaults: false,
Indent: " ",
}
return m.Marshal(writer, config)
}

31
weed/s3iam/s3iam_filer_store_test.go → weed/filer/s3iam_conf_test.go
View file

@ -1,7 +1,8 @@
package s3iam
package filer
import (
"github.com/golang/protobuf/proto"
"bytes"
"github.com/chrislusf/seaweedfs/weed/s3api"
"testing"
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
@ -9,16 +10,7 @@ import (
"github.com/stretchr/testify/assert"
)
const (
ACTION_READ = "Read"
ACTION_WRITE = "Write"
ACTION_ADMIN = "Admin"
ACTION_TAGGING = "Tagging"
ACTION_LIST = "List"
)
func TestS3Conf(t *testing.T) {
ifs := &IAMFilerStore{}
s3Conf := &iam_pb.S3ApiConfiguration{
Identities: []*iam_pb.Identity{
{
@ -30,9 +22,9 @@ func TestS3Conf(t *testing.T) {
},
},
Actions: []string{
ACTION_ADMIN,
ACTION_READ,
ACTION_WRITE,
s3api.ACTION_ADMIN,
s3api.ACTION_READ,
s3api.ACTION_WRITE,
},
},
{
@ -44,17 +36,18 @@ func TestS3Conf(t *testing.T) {
},
},
Actions: []string{
ACTION_READ,
ACTION_TAGGING,
ACTION_LIST,
s3api.ACTION_READ,
s3api.ACTION_TAGGING,
s3api.ACTION_LIST,
},
},
},
}
content, err := proto.Marshal(s3Conf)
var buf bytes.Buffer
err := S3ConfigurationToText(&buf, s3Conf)
assert.Equal(t, err, nil)
s3ConfSaved := &iam_pb.S3ApiConfiguration{}
err = ifs.loadIAMConfigFromBytes(content, s3ConfSaved)
err = ParseS3ConfigurationFromBytes(buf.Bytes(), s3ConfSaved)
assert.Equal(t, err, nil)
assert.Equal(t, "some_name", s3ConfSaved.Identities[0].Name)

19
weed/s3api/auth_credentials.go
View file

@ -3,15 +3,12 @@ package s3api
import (
"bytes"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/filer"
"io/ioutil"
"net/http"
xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"github.com/chrislusf/seaweedfs/weed/s3iam"
"github.com/golang/protobuf/jsonpb"
"github.com/chrislusf/seaweedfs/weed/glog"
@ -65,17 +62,17 @@ func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManag
func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFiler(option *S3ApiServerOption) error {
s3ApiConfiguration := &iam_pb.S3ApiConfiguration{}
return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
store := s3iam.NewIAMFilerStore(&client)
if err := store.LoadIAMConfig(s3ApiConfiguration); err != nil {
return nil
content, err := filer.ReadContent(option.Filer, filer.IamConfigDirecotry, filer.IamIdentityFile)
if err != nil {
return fmt.Errorf("read S3 config: %v", err)
}
if err = filer.ParseS3ConfigurationFromBytes(content, s3ApiConfiguration); err != nil {
return fmt.Errorf("parse S3 config: %v", err)
}
if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil {
return err
return fmt.Errorf("laod S3 config: %v", err)
}
return nil
}, option.FilerGrpcAddress, option.GrpcDialOption)
}
func (iam *IdentityAccessManagement) loadS3ApiConfigurationFromFile(fileName string) error {

95
weed/s3iam/s3iam_filer_store.go
View file

@ -1,95 +0,0 @@
package s3iam
import (
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
"time"
proto "github.com/golang/protobuf/proto"
)
const (
iamConfigPrefix = "/etc/iam"
iamIdentityFile = "identity.json"
)
type IAMFilerStore struct {
client *filer_pb.SeaweedFilerClient
}
func NewIAMFilerStore(client *filer_pb.SeaweedFilerClient) *IAMFilerStore {
return &IAMFilerStore{client: client}
}
func (ifs *IAMFilerStore) getIAMConfigRequest() *filer_pb.LookupDirectoryEntryRequest {
return &filer_pb.LookupDirectoryEntryRequest{
Directory: iamConfigPrefix,
Name: iamIdentityFile,
}
}
func (ifs *IAMFilerStore) LoadIAMConfig(config *iam_pb.S3ApiConfiguration) error {
resp, err := filer_pb.LookupEntry(*ifs.client, ifs.getIAMConfigRequest())
if err != nil {
return err
}
err = ifs.loadIAMConfigFromBytes(resp.Entry.Content, config)
if err != nil {
return err
}
return nil
}
func (ifs *IAMFilerStore) SaveIAMConfig(config *iam_pb.S3ApiConfiguration) error {
entry := &filer_pb.Entry{
Name: iamIdentityFile,
IsDirectory: false,
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
Crtime: time.Now().Unix(),
FileMode: uint32(0644),
Collection: "",
Replication: "",
},
Content: []byte{},
}
err := ifs.saveIAMConfigToEntry(entry, config)
if err != nil {
return err
}
_, err = filer_pb.LookupEntry(*ifs.client, ifs.getIAMConfigRequest())
if err == filer_pb.ErrNotFound {
err = filer_pb.CreateEntry(*ifs.client, &filer_pb.CreateEntryRequest{
Directory: iamConfigPrefix,
Entry: entry,
IsFromOtherCluster: false,
Signatures: nil,
})
} else {
err = filer_pb.UpdateEntry(*ifs.client, &filer_pb.UpdateEntryRequest{
Directory: iamConfigPrefix,
Entry: entry,
IsFromOtherCluster: false,
Signatures: nil,
})
}
if err != nil {
return err
}
return nil
}
func (ifs *IAMFilerStore) loadIAMConfigFromBytes(content []byte, config *iam_pb.S3ApiConfiguration) error {
if err := proto.Unmarshal(content, config); err != nil {
return err
}
return nil
}
func (ifs *IAMFilerStore) saveIAMConfigToEntry(entry *filer_pb.Entry, config *iam_pb.S3ApiConfiguration) (err error) {
entry.Content, err = proto.Marshal(config)
if err != nil {
return err
}
return nil
}

16
weed/shell/command_fs_configure.go
View file

@ -5,13 +5,11 @@ import (
"flag"
"fmt"
"io"
"net/http"
"strings"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
"github.com/chrislusf/seaweedfs/weed/util"
)
func init() {
@ -118,22 +116,10 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io
if *apply {
target := fmt.Sprintf("http://%s:%d%s/%s", commandEnv.option.FilerHost, commandEnv.option.FilerPort, filer.DirectoryEtc, filer.FilerConfName)
// set the HTTP method, url, and request body
req, err := http.NewRequest(http.MethodPut, target, &buf)
if err != nil {
if err := filer.SaveAs(commandEnv.option.FilerHost, int(commandEnv.option.FilerPort), filer.DirectoryEtc, filer.FilerConfName, "text/plain; charset=utf-8", &buf); err != nil {
return err
}
// set the request header Content-Type for json
req.Header.Set("Content-Type", "text/plain; charset=utf-8")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
util.CloseResponse(resp)
}
return nil

33
weed/shell/command_s3_configure.go
View file

@ -1,15 +1,16 @@
package shell
import (
"bytes"
"flag"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"io"
"sort"
"strings"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/iam_pb"
"github.com/chrislusf/seaweedfs/weed/s3iam"
)
func init() {
@ -32,6 +33,7 @@ func (c *commandS3Configure) Help() string {
}
func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
s3ConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
actions := s3ConfigureCommand.String("actions", "", "comma separated actions names: Read,Write,List,Tagging,Admin")
user := s3ConfigureCommand.String("user", "", "user name")
@ -45,18 +47,20 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
return nil
}
s3cfg := &iam_pb.S3ApiConfiguration{}
ifs := &s3iam.IAMFilerStore{}
var buf bytes.Buffer
if err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
ifs = s3iam.NewIAMFilerStore(&client)
if err := ifs.LoadIAMConfig(s3cfg); err != nil {
return nil
}
return nil
}); err != nil {
return filer.ReadEntry(commandEnv.MasterClient, client, filer.IamConfigDirecotry, filer.IamIdentityFile, &buf)
}); err != nil && err != filer_pb.ErrNotFound {
return err
}
s3cfg := &iam_pb.S3ApiConfiguration{}
if buf.Len() > 0 {
if err = filer.ParseS3ConfigurationFromBytes(buf.Bytes(), s3cfg); err != nil {
return err
}
}
idx := 0
changed := false
if *user != "" {
@ -159,16 +163,19 @@ func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io
s3cfg.Identities = append(s3cfg.Identities, &identity)
}
for _, identity := range s3cfg.Identities {
fmt.Fprintf(writer, fmt.Sprintf("%+v\n", identity))
}
buf.Reset()
filer.S3ConfigurationToText(&buf, s3cfg)
fmt.Fprintf(writer, string(buf.Bytes()))
fmt.Fprintln(writer)
if *apply {
if err := ifs.SaveIAMConfig(s3cfg); err != nil {
if err := filer.SaveAs(commandEnv.option.FilerHost, int(commandEnv.option.FilerPort), filer.IamConfigDirecotry, filer.IamIdentityFile, "text/plain; charset=utf-8", &buf); err != nil {
return err
}
}
return nil