retry forever with filer.remote.sync, and some refactoring

weed/command/filer_remote_sync.go

@@ -70,43 +70,33 @@ func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 	remoteSyncOptions.grpcDialOption = grpcDialOption
 
+	dir := *remoteSyncOptions.dir
+	filerAddress := *remoteSyncOptions.filerAddress
+
 	// read filer remote storage mount mappings
-	mappings, readErr := filer.ReadMountMappings(grpcDialOption, *remoteSyncOptions.filerAddress)
+	_, _, remoteStorageMountLocation, storageConf, detectErr := filer.DetectMountInfo(grpcDialOption, filerAddress, dir)
-	if readErr != nil {
+	if detectErr != nil {
-		fmt.Printf("read mount mapping: %v", readErr)
+		fmt.Printf("read mount info: %v", detectErr)
 		return false
 	}
 
 	filerSource := &source.FilerSource{}
 	filerSource.DoInitialize(
-		*remoteSyncOptions.filerAddress,
+		filerAddress,
-		pb.ServerToGrpcAddress(*remoteSyncOptions.filerAddress),
+		pb.ServerToGrpcAddress(filerAddress),
 		"/", // does not matter
 		*remoteSyncOptions.readChunkFromFiler,
 	)
 
-	var found bool
+	fmt.Printf("synchronize %s to remote storage...\n", dir)
-	for dir, remoteStorageMountLocation := range mappings.Mappings {
+	util.RetryForever("filer.remote.sync "+dir, func() error {
-		if *remoteSyncOptions.dir == dir {
+		return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir, storageConf, remoteStorageMountLocation)
-			found = true
+	}, func(err error) bool {
-			storageConf, readErr := filer.ReadRemoteStorageConf(grpcDialOption, *remoteSyncOptions.filerAddress, remoteStorageMountLocation.Name)
+		if err != nil {
-			if readErr != nil {
+			fmt.Printf("synchronize %s: %v\n", dir, err)
-				fmt.Printf("read remote storage configuration for %s: %v", dir, readErr)
-				continue
-			}
-			fmt.Printf("synchronize %s to remote storage...\n", *remoteSyncOptions.dir)
-			if err := util.Retry("filer.remote.sync "+dir, func() error {
-				return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir, storageConf, remoteStorageMountLocation)
-			}); err != nil {
-				fmt.Printf("synchronize %s: %v\n", *remoteSyncOptions.dir, err)
-			}
-			break
 		}
-	}
+		return true
-	if !found {
+	})
-		fmt.Printf("directory %s is not mounted to any remote storage\n", *remoteSyncOptions.dir)
-		return false
-	}
 
 	return true
 }

weed/filer/filer_remote_storage.go

@@ -197,3 +197,33 @@ func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress string,
 
 	return
 }
+
+func DetectMountInfo(grpcDialOption grpc.DialOption, filerAddress string, dir string) (*filer_pb.RemoteStorageMapping, string, *filer_pb.RemoteStorageLocation, *filer_pb.RemoteConf, error) {
+
+	mappings, listErr := ReadMountMappings(grpcDialOption, filerAddress)
+	if listErr != nil {
+		return nil, "", nil, nil, listErr
+	}
+	if dir == "" {
+		return mappings, "", nil, nil, fmt.Errorf("need to specify '-dir' option")
+	}
+
+	var localMountedDir string
+	var remoteStorageMountedLocation *filer_pb.RemoteStorageLocation
+	for k, loc := range mappings.Mappings {
+		if strings.HasPrefix(dir, k) {
+			localMountedDir, remoteStorageMountedLocation = k, loc
+		}
+	}
+	if localMountedDir == "" {
+		return mappings, localMountedDir, remoteStorageMountedLocation, nil, fmt.Errorf("%s is not mounted", dir)
+	}
+
+	// find remote storage configuration
+	remoteStorageConf, err := ReadRemoteStorageConf(grpcDialOption, filerAddress, remoteStorageMountedLocation.Name)
+	if err != nil {
+		return mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, err
+	}
+
+	return mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, nil
+}

weed/shell/command_remote_meta_sync.go

@@ -9,7 +9,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/remote_storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"io"
-	"strings"
 )
 
 func init() {
@@ -69,32 +68,7 @@ func (c *commandRemoteMetaSync) Do(args []string, commandEnv *CommandEnv, writer
 }
 
 func detectMountInfo(commandEnv *CommandEnv, writer io.Writer, dir string) (*filer_pb.RemoteStorageMapping, string, *filer_pb.RemoteStorageLocation, *filer_pb.RemoteConf, error) {
-	mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress)
+	return filer.DetectMountInfo(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, dir)
-	if listErr != nil {
-		return nil, "", nil, nil, listErr
-	}
-	if dir == "" {
-		return mappings, "", nil, nil, fmt.Errorf("need to specify '-dir' option")
-	}
-
-	var localMountedDir string
-	var remoteStorageMountedLocation *filer_pb.RemoteStorageLocation
-	for k, loc := range mappings.Mappings {
-		if strings.HasPrefix(dir, k) {
-			localMountedDir, remoteStorageMountedLocation = k, loc
-		}
-	}
-	if localMountedDir == "" {
-		return mappings, localMountedDir, remoteStorageMountedLocation, nil, fmt.Errorf("%s is not mounted", dir)
-	}
-
-	// find remote storage configuration
-	remoteStorageConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remoteStorageMountedLocation.Name)
-	if err != nil {
-		return mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, err
-	}
-
-	return mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, nil
 }
 
 /*