gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

simplify parsing filer host and port

Browse source
This commit is contained in:
Chris Lu 2020-03-23 21:26:15 -07:00
parent d151185b7e
commit e666aeece2
14 changed files with 44 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
weed/shell/command_bucket_create.go
View file

@ -43,16 +43,11 @@ func (c *commandBucketCreate) Do(args []string, commandEnv *CommandEnv, writer i
return fmt.Errorf("empty bucket name") return fmt.Errorf("empty bucket name")
} }
filerServer, filerPort, _, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args())) err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
if parseErr != nil {
return parseErr
}
err = commandEnv.withFilerClient(filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil { if err != nil {
return fmt.Errorf("get filer %s:%d configuration: %v", filerServer, filerPort, err) return fmt.Errorf("get filer configuration: %v", err)
} }
filerBucketsPath := resp.DirBuckets filerBucketsPath := resp.DirBuckets

8
weed/shell/command_bucket_delete.go
View file

@ -38,19 +38,17 @@ func (c *commandBucketDelete) Do(args []string, commandEnv *CommandEnv, writer i
return fmt.Errorf("empty bucket name") return fmt.Errorf("empty bucket name")
} }
filerServer, filerPort, _, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args())) _, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args()))
if parseErr != nil { if parseErr != nil {
return parseErr return parseErr
} }
filerClient := commandEnv.getFilerClient(filerServer, filerPort)
var filerBucketsPath string var filerBucketsPath string
filerBucketsPath, err = readFilerBucketsPath(filerClient) filerBucketsPath, err = readFilerBucketsPath(commandEnv)
if err != nil { if err != nil {
return fmt.Errorf("read buckets: %v", err) return fmt.Errorf("read buckets: %v", err)
} }
return filer_pb.Remove(filerClient, filerBucketsPath, *bucketName, false, true, true) return filer_pb.Remove(commandEnv, filerBucketsPath, *bucketName, false, true, true)
} }

8
weed/shell/command_bucket_list.go
View file

@ -34,20 +34,18 @@ func (c *commandBucketList) Do(args []string, commandEnv *CommandEnv, writer io.
return nil return nil
} }
filerServer, filerPort, _, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args())) _, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args()))
if parseErr != nil { if parseErr != nil {
return parseErr return parseErr
} }
filerClient := commandEnv.getFilerClient(filerServer, filerPort)
var filerBucketsPath string var filerBucketsPath string
filerBucketsPath, err = readFilerBucketsPath(filerClient) filerBucketsPath, err = readFilerBucketsPath(commandEnv)
if err != nil { if err != nil {
return fmt.Errorf("read buckets: %v", err) return fmt.Errorf("read buckets: %v", err)
} }
err = filer_pb.List(filerClient, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) { err = filer_pb.List(commandEnv, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) {
if entry.Attributes.Replication == "" || entry.Attributes.Replication == "000" { if entry.Attributes.Replication == "" || entry.Attributes.Replication == "000" {
fmt.Fprintf(writer, " %s\n", entry.Name) fmt.Fprintf(writer, " %s\n", entry.Name)
} else { } else {

8
weed/shell/command_fs_cat.go
View file

@ -30,20 +30,18 @@ func (c *commandFsCat) Help() string {
func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
input := findInputDirectory(args) path, err := commandEnv.parseUrl(findInputDirectory(args))
filerServer, filerPort, path, err := commandEnv.parseUrl(input)
if err != nil { if err != nil {
return err return err
} }
if commandEnv.isDirectory(filerServer, filerPort, path) { if commandEnv.isDirectory(path) {
return fmt.Errorf("%s is a directory", path) return fmt.Errorf("%s is a directory", path)
} }
dir, name := util.FullPath(path).DirAndName() dir, name := util.FullPath(path).DirAndName()
return commandEnv.withFilerClient(filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error { return commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.LookupDirectoryEntryRequest{ request := &filer_pb.LookupDirectoryEntryRequest{
Name: name, Name: name,

10
weed/shell/command_fs_cd.go
View file

@ -30,25 +30,19 @@ func (c *commandFsCd) Help() string {
func (c *commandFsCd) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { func (c *commandFsCd) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
input := findInputDirectory(args) path, err := commandEnv.parseUrl(findInputDirectory(args))
filerServer, filerPort, path, err := commandEnv.parseUrl(input)
if err != nil { if err != nil {
return err return err
} }
if path == "/" { if path == "/" {
commandEnv.option.FilerHost = filerServer
commandEnv.option.FilerPort = filerPort
commandEnv.option.Directory = "/" commandEnv.option.Directory = "/"
return nil return nil
} }
err = commandEnv.checkDirectory(filerServer, filerPort, path) err = commandEnv.checkDirectory(path)
if err == nil { if err == nil {
commandEnv.option.FilerHost = filerServer
commandEnv.option.FilerPort = filerPort
commandEnv.option.Directory = path commandEnv.option.Directory = path
} }

28
weed/shell/command_fs_du.go
View file

@ -32,18 +32,18 @@ func (c *commandFsDu) Help() string {
func (c *commandFsDu) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { func (c *commandFsDu) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
filerServer, filerPort, path, err := commandEnv.parseUrl(findInputDirectory(args)) path, err := commandEnv.parseUrl(findInputDirectory(args))
if err != nil { if err != nil {
return err return err
} }
if commandEnv.isDirectory(filerServer, filerPort, path) { if commandEnv.isDirectory(path) {
path = path + "/" path = path + "/"
} }
var blockCount, byteCount uint64 var blockCount, byteCount uint64
dir, name := util.FullPath(path).DirAndName() dir, name := util.FullPath(path).DirAndName()
blockCount, byteCount, err = duTraverseDirectory(writer, commandEnv.getFilerClient(filerServer, filerPort), dir, name) blockCount, byteCount, err = duTraverseDirectory(writer, commandEnv, dir, name)
if name == "" && err == nil { if name == "" && err == nil {
fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s\n", blockCount, byteCount, dir) fmt.Fprintf(writer, "block:%4d\tbyte:%10d\t%s\n", blockCount, byteCount, dir)
@ -78,29 +78,13 @@ func duTraverseDirectory(writer io.Writer, filerClient filer_pb.FilerClient, dir
return return
} }
func (env *CommandEnv) withFilerClient(filerServer string, filerPort int64, fn func(filer_pb.SeaweedFilerClient) error) error { func (env *CommandEnv) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
filerGrpcAddress := fmt.Sprintf("%s:%d", filerServer, filerPort+10000) filerGrpcAddress := fmt.Sprintf("%s:%d", env.option.FilerHost, env.option.FilerPort+10000)
return pb.WithGrpcFilerClient(filerGrpcAddress, env.option.GrpcDialOption, fn) return pb.WithGrpcFilerClient(filerGrpcAddress, env.option.GrpcDialOption, fn)
} }
type commandFilerClient struct { func (env *CommandEnv) AdjustedUrl(hostAndPort string) string {
env *CommandEnv
filerServer string
filerPort int64
}
func (env *CommandEnv) getFilerClient(filerServer string, filerPort int64) *commandFilerClient {
return &commandFilerClient{
env: env,
filerServer: filerServer,
filerPort: filerPort,
}
}
func (c *commandFilerClient) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
return c.env.withFilerClient(c.filerServer, c.filerPort, fn)
}
func (c *commandFilerClient) AdjustedUrl(hostAndPort string) string {
return hostAndPort return hostAndPort
} }

8
weed/shell/command_fs_ls.go
View file

@ -50,21 +50,19 @@ func (c *commandFsLs) Do(args []string, commandEnv *CommandEnv, writer io.Writer
} }
} }
input := findInputDirectory(args) path, err := commandEnv.parseUrl(findInputDirectory(args))
filerServer, filerPort, path, err := commandEnv.parseUrl(input)
if err != nil { if err != nil {
return err return err
} }
if commandEnv.isDirectory(filerServer, filerPort, path) { if commandEnv.isDirectory(path) {
path = path + "/" path = path + "/"
} }
dir, name := util.FullPath(path).DirAndName() dir, name := util.FullPath(path).DirAndName()
entryCount := 0 entryCount := 0
err = filer_pb.ReadDirAllEntries(commandEnv.getFilerClient(filerServer, filerPort), util.FullPath(dir), name, func(entry *filer_pb.Entry, isLast bool) { err = filer_pb.ReadDirAllEntries(commandEnv, util.FullPath(dir), name, func(entry *filer_pb.Entry, isLast bool) {
if !showHidden && strings.HasPrefix(entry.Name, ".") { if !showHidden && strings.HasPrefix(entry.Name, ".") {
return return

6
weed/shell/command_fs_meta_cat.go
View file

@ -31,16 +31,14 @@ func (c *commandFsMetaCat) Help() string {
func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
input := findInputDirectory(args) path, err := commandEnv.parseUrl(findInputDirectory(args))
filerServer, filerPort, path, err := commandEnv.parseUrl(input)
if err != nil { if err != nil {
return err return err
} }
dir, name := util.FullPath(path).DirAndName() dir, name := util.FullPath(path).DirAndName()
return commandEnv.withFilerClient(filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error { return commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.LookupDirectoryEntryRequest{ request := &filer_pb.LookupDirectoryEntryRequest{
Name: name, Name: name,

9
weed/shell/command_fs_meta_load.go
View file

@ -37,11 +37,6 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io.
return nil return nil
} }
filerServer, filerPort, path, err := commandEnv.parseUrl(findInputDirectory(nil))
if err != nil {
return err
}
fileName := args[len(args)-1] fileName := args[len(args)-1]
dst, err := os.OpenFile(fileName, os.O_RDONLY, 0644) dst, err := os.OpenFile(fileName, os.O_RDONLY, 0644)
@ -52,7 +47,7 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io.
var dirCount, fileCount uint64 var dirCount, fileCount uint64
err = commandEnv.withFilerClient(filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error { err = commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
sizeBuf := make([]byte, 4) sizeBuf := make([]byte, 4)
@ -98,7 +93,7 @@ func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io.
if err == nil { if err == nil {
fmt.Fprintf(writer, "\ntotal %d directories, %d files", dirCount, fileCount) fmt.Fprintf(writer, "\ntotal %d directories, %d files", dirCount, fileCount)
fmt.Fprintf(writer, "\n%s is loaded to http://%s:%d%s\n", fileName, filerServer, filerPort, path) fmt.Fprintf(writer, "\n%s is loaded.\n", fileName)
} }
return err return err

4
weed/shell/command_fs_meta_notify.go
View file

@ -32,7 +32,7 @@ func (c *commandFsMetaNotify) Help() string {
func (c *commandFsMetaNotify) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { func (c *commandFsMetaNotify) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
filerServer, filerPort, path, err := commandEnv.parseUrl(findInputDirectory(args)) path, err := commandEnv.parseUrl(findInputDirectory(args))
if err != nil { if err != nil {
return err return err
} }
@ -43,7 +43,7 @@ func (c *commandFsMetaNotify) Do(args []string, commandEnv *CommandEnv, writer i
var dirCount, fileCount uint64 var dirCount, fileCount uint64
err = doTraverseBFS(writer, commandEnv.getFilerClient(filerServer, filerPort), util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) { err = doTraverseBFS(writer, commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) {
if entry.IsDirectory { if entry.IsDirectory {
dirCount++ dirCount++

8
weed/shell/command_fs_meta_save.go
View file

@ -52,7 +52,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.
return nil return nil
} }
filerServer, filerPort, path, parseErr := commandEnv.parseUrl(findInputDirectory(fsMetaSaveCommand.Args())) path, parseErr := commandEnv.parseUrl(findInputDirectory(fsMetaSaveCommand.Args()))
if parseErr != nil { if parseErr != nil {
return parseErr return parseErr
} }
@ -61,7 +61,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.
fileName := *outputFileName fileName := *outputFileName
if fileName == "" { if fileName == "" {
fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta", fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta",
filerServer, filerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) commandEnv.option.FilerHost, commandEnv.option.FilerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
} }
dst, openErr := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) dst, openErr := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
@ -85,7 +85,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.
var dirCount, fileCount uint64 var dirCount, fileCount uint64
err = doTraverseBFS(writer, commandEnv.getFilerClient(filerServer, filerPort), util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) { err = doTraverseBFS(writer, commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) {
protoMessage := &filer_pb.FullEntry{ protoMessage := &filer_pb.FullEntry{
Dir: string(parentPath), Dir: string(parentPath),
@ -118,7 +118,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.
if err == nil { if err == nil {
fmt.Fprintf(writer, "total %d directories, %d files\n", dirCount, fileCount) fmt.Fprintf(writer, "total %d directories, %d files\n", dirCount, fileCount)
fmt.Fprintf(writer, "meta data for http://%s:%d%s is saved to %s\n", filerServer, filerPort, path, fileName) fmt.Fprintf(writer, "meta data for %s is saved to %s\n", path, fileName)
} }
return err return err

6
weed/shell/command_fs_mv.go
View file

@ -37,12 +37,12 @@ func (c *commandFsMv) Help() string {
func (c *commandFsMv) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { func (c *commandFsMv) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
filerServer, filerPort, sourcePath, err := commandEnv.parseUrl(args[0]) sourcePath, err := commandEnv.parseUrl(args[0])
if err != nil { if err != nil {
return err return err
} }
_, _, destinationPath, err := commandEnv.parseUrl(args[1]) destinationPath, err := commandEnv.parseUrl(args[1])
if err != nil { if err != nil {
return err return err
} }
@ -51,7 +51,7 @@ func (c *commandFsMv) Do(args []string, commandEnv *CommandEnv, writer io.Writer
destinationDir, destinationName := util.FullPath(destinationPath).DirAndName() destinationDir, destinationName := util.FullPath(destinationPath).DirAndName()
return commandEnv.withFilerClient(filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error { return commandEnv.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// collect destination entry info // collect destination entry info
destinationRequest := &filer_pb.LookupDirectoryEntryRequest{ destinationRequest := &filer_pb.LookupDirectoryEntryRequest{

4
weed/shell/command_fs_tree.go
View file

@ -30,14 +30,14 @@ func (c *commandFsTree) Help() string {
func (c *commandFsTree) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { func (c *commandFsTree) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
filerServer, filerPort, path, err := commandEnv.parseUrl(findInputDirectory(args)) path, err := commandEnv.parseUrl(findInputDirectory(args))
if err != nil { if err != nil {
return err return err
} }
dir, name := util.FullPath(path).DirAndName() dir, name := util.FullPath(path).DirAndName()
dirCount, fCount, terr := treeTraverseDirectory(writer, commandEnv.getFilerClient(filerServer, filerPort), util.FullPath(dir), name, newPrefix(), -1) dirCount, fCount, terr := treeTraverseDirectory(writer, commandEnv, util.FullPath(dir), name, newPrefix(), -1)
if terr == nil { if terr == nil {
fmt.Fprintf(writer, "%d directories, %d files\n", dirCount, fCount) fmt.Fprintf(writer, "%d directories, %d files\n", dirCount, fCount)

27
weed/shell/commands.go
View file

@ -48,7 +48,7 @@ func NewCommandEnv(options ShellOptions) *CommandEnv {
} }
} }
func (ce *CommandEnv) parseUrl(input string) (filerServer string, filerPort int64, path string, err error) { func (ce *CommandEnv) parseUrl(input string) (path string, err error) {
if strings.HasPrefix(input, "http") { if strings.HasPrefix(input, "http") {
err = fmt.Errorf("http://<filer>:<port> prefix is not supported any more") err = fmt.Errorf("http://<filer>:<port> prefix is not supported any more")
return return
@ -56,35 +56,22 @@ func (ce *CommandEnv) parseUrl(input string) (filerServer string, filerPort int6
if !strings.HasPrefix(input, "/") { if !strings.HasPrefix(input, "/") {
input = filepath.ToSlash(filepath.Join(ce.option.Directory, input)) input = filepath.ToSlash(filepath.Join(ce.option.Directory, input))
} }
return ce.option.FilerHost, ce.option.FilerPort, input, err return input, err
} }
func (ce *CommandEnv) isDirectory(filerServer string, filerPort int64, path string) bool { func (ce *CommandEnv) isDirectory(path string) bool {
return ce.checkDirectory(filerServer, filerPort, path) == nil return ce.checkDirectory(path) == nil
} }
func (ce *CommandEnv) checkDirectory(filerServer string, filerPort int64, path string) error { func (ce *CommandEnv) checkDirectory(path string) error {
dir, name := util.FullPath(path).DirAndName() dir, name := util.FullPath(path).DirAndName()
return ce.withFilerClient(filerServer, filerPort, func(client filer_pb.SeaweedFilerClient) error { _, err := filer_pb.Exists(ce, dir, name, true)
resp, lookupErr := filer_pb.LookupEntry(client, &filer_pb.LookupDirectoryEntryRequest{ return err
Directory: dir,
Name: name,
})
if lookupErr != nil {
return lookupErr
}
if !resp.Entry.IsDirectory {
return fmt.Errorf("not a directory")
}
return nil
})
} }