seaweedfs/weed/command/filer_remote_sync.go

101 lines
3.3 KiB
Go
Raw Normal View History

2021-08-08 08:21:42 +00:00
package command
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/grpc"
"time"
)
type RemoteSyncOptions struct {
2021-09-19 19:06:15 +00:00
filerAddress *string
grpcDialOption grpc.DialOption
readChunkFromFiler *bool
timeAgo *time.Duration
dir *string
clientId int32
clientEpoch int32
2021-08-08 08:21:42 +00:00
}
var _ = filer_pb.FilerClient(&RemoteSyncOptions{})
func (option *RemoteSyncOptions) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
return pb.WithFilerClient(streamingMode, pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
2021-08-08 08:21:42 +00:00
return fn(client)
})
}
func (option *RemoteSyncOptions) AdjustedUrl(location *filer_pb.Location) string {
return location.Url
}
var (
remoteSyncOptions RemoteSyncOptions
)
func init() {
cmdFilerRemoteSynchronize.Run = runFilerRemoteSynchronize // break init cycle
remoteSyncOptions.filerAddress = cmdFilerRemoteSynchronize.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster")
remoteSyncOptions.dir = cmdFilerRemoteSynchronize.Flag.String("dir", "", "a mounted directory on filer")
2021-08-08 08:21:42 +00:00
remoteSyncOptions.readChunkFromFiler = cmdFilerRemoteSynchronize.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers")
2021-12-13 21:14:36 +00:00
remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now, skipping previous metadata changes. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
remoteSyncOptions.clientId = util.RandomInt32()
2021-08-08 08:21:42 +00:00
}
var cmdFilerRemoteSynchronize = &Command{
2021-09-06 23:16:22 +00:00
UsageLine: "filer.remote.sync",
Short: "resumable continuously write back updates to remote storage",
Long: `resumable continuously write back updates to remote storage
2021-08-08 08:21:42 +00:00
filer.remote.sync listens on filer update events.
If any mounted remote file is updated, it will fetch the updated content,
and write to the remote storage.
2021-09-06 23:16:22 +00:00
weed filer.remote.sync -dir=/mount/s3_on_cloud
2021-12-13 21:14:36 +00:00
The metadata sync starting time is determined with the following priority order:
1. specified by timeAgo
2. last sync timestamp for this directory
3. directory creation time
2021-08-08 08:21:42 +00:00
`,
}
func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
remoteSyncOptions.grpcDialOption = grpcDialOption
dir := *remoteSyncOptions.dir
filerAddress := pb.ServerAddress(*remoteSyncOptions.filerAddress)
2021-08-08 08:21:42 +00:00
filerSource := &source.FilerSource{}
filerSource.DoInitialize(
filerAddress.ToHttpAddress(),
filerAddress.ToGrpcAddress(),
2021-08-08 08:21:42 +00:00
"/", // does not matter
*remoteSyncOptions.readChunkFromFiler,
)
if dir != "" {
2021-09-04 20:46:44 +00:00
fmt.Printf("synchronize %s to remote storage...\n", dir)
util.RetryForever("filer.remote.sync "+dir, func() error {
return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir)
}, func(err error) bool {
if err != nil {
glog.Errorf("synchronize %s: %v", dir, err)
}
return true
})
return true
2021-09-04 20:46:44 +00:00
}
return true
2021-08-09 05:30:36 +00:00
}