gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

option to map remote bucket to trimmed bucket name

Browse source
This commit is contained in:
Chris Lu 2021-09-05 11:55:52 -07:00
parent 2348e8d8da
commit 60573fd3e2
3 changed files with 20 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
weed/command/filer_remote_sync_buckets.go
View file

@ -47,6 +47,10 @@ func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.
if !entry.IsDirectory { if !entry.IsDirectory {
return nil return nil
} }
if entry.RemoteEntry != nil {
// this directory is imported from "remote.mount.buckets" or "remote.mount"
return nil
}
remoteConf, found := option.remoteConfs[*option.createBucketAt] remoteConf, found := option.remoteConfs[*option.createBucketAt]
if !found { if !found {
return fmt.Errorf("un-configured remote storage %s", *option.createBucketAt) return fmt.Errorf("un-configured remote storage %s", *option.createBucketAt)

6
weed/shell/command_remote_mount.go
View file

@ -124,7 +124,7 @@ func syncMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty
Name: name, Name: name,
}) })
if lookupErr != nil { if lookupErr != nil {
if !strings.Contains(lookupErr.Error(), filer_pb.ErrNotFound.Error()) { if strings.Contains(lookupErr.Error(), filer_pb.ErrNotFound.Error()) {
_, createErr := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{ _, createErr := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
Directory: parent, Directory: parent,
Entry: &filer_pb.Entry{ Entry: &filer_pb.Entry{
@ -135,6 +135,9 @@ func syncMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty
Crtime: time.Now().Unix(), Crtime: time.Now().Unix(),
FileMode: uint32(0644 | os.ModeDir), FileMode: uint32(0644 | os.ModeDir),
}, },
RemoteEntry: &filer_pb.RemoteEntry{
StorageName: remoteConf.Name,
},
}, },
}) })
return createErr return createErr
@ -171,7 +174,6 @@ func syncMetadata(commandEnv *CommandEnv, writer io.Writer, dir string, nonEmpty
return nil return nil
} }
// if an entry has synchronized metadata but has not synchronized content // if an entry has synchronized metadata but has not synchronized content
// entry.Attributes.FileSize == entry.RemoteEntry.RemoteSize // entry.Attributes.FileSize == entry.RemoteEntry.RemoteSize
// entry.Attributes.Mtime == entry.RemoteEntry.RemoteMtime // entry.Attributes.Mtime == entry.RemoteEntry.RemoteMtime

13
weed/shell/command_remote_mount_buckets.go
View file

@ -9,6 +9,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
"io" "io"
"path/filepath" "path/filepath"
"regexp"
) )
func init() { func init() {
@ -43,6 +44,7 @@ func (c *commandRemoteMountBuckets) Do(args []string, commandEnv *CommandEnv, wr
remote := remoteMountBucketsCommand.String("remote", "", "a already configured storage name") remote := remoteMountBucketsCommand.String("remote", "", "a already configured storage name")
bucketPattern := remoteMountBucketsCommand.String("bucketPattern", "", "match existing bucket name with wildcard characters '*' and '?'") bucketPattern := remoteMountBucketsCommand.String("bucketPattern", "", "match existing bucket name with wildcard characters '*' and '?'")
trimBucketSuffix := remoteMountBucketsCommand.Bool("trimBucketSuffix", false, "remote suffix auto generated by 'weed filer.remote.sync'")
apply := remoteMountBucketsCommand.Bool("apply", false, "apply the mount for listed buckets") apply := remoteMountBucketsCommand.Bool("apply", false, "apply the mount for listed buckets")
if err = remoteMountBucketsCommand.Parse(args); err != nil { if err = remoteMountBucketsCommand.Parse(args); err != nil {
@ -76,6 +78,8 @@ func (c *commandRemoteMountBuckets) Do(args []string, commandEnv *CommandEnv, wr
return fmt.Errorf("read filer buckets path: %v", err) return fmt.Errorf("read filer buckets path: %v", err)
} }
hasSuffixPattern, _ := regexp.Compile(".+-[0-9][0-9][0-9][0-9]")
for _, bucket := range buckets { for _, bucket := range buckets {
if *bucketPattern != "" { if *bucketPattern != "" {
if matched, _ := filepath.Match(*bucketPattern, bucket.Name); !matched { if matched, _ := filepath.Match(*bucketPattern, bucket.Name); !matched {
@ -84,9 +88,16 @@ func (c *commandRemoteMountBuckets) Do(args []string, commandEnv *CommandEnv, wr
} }
fmt.Fprintf(writer, "bucket %s\n", bucket.Name) fmt.Fprintf(writer, "bucket %s\n", bucket.Name)
localBucketName := bucket.Name
if *trimBucketSuffix {
if hasSuffixPattern.MatchString(localBucketName) {
localBucketName = localBucketName[:len(localBucketName)-5]
fmt.Fprintf(writer, " mount bucket %s as %s\n", bucket.Name, localBucketName)
}
}
if *apply { if *apply {
dir := util.FullPath(fillerBucketsPath).Child(bucket.Name) dir := util.FullPath(fillerBucketsPath).Child(localBucketName)
remoteStorageLocation := &remote_pb.RemoteStorageLocation{ remoteStorageLocation := &remote_pb.RemoteStorageLocation{
Name: *remote, Name: *remote,
Bucket: bucket.Name, Bucket: bucket.Name,