gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

add additionalPathPrefixes

Browse source
This commit is contained in:
Chris Lu 2021-08-31 23:26:26 -07:00
parent 43fd11278e
commit 68ea99190f
7 changed files with 24 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go
View file

@ -72,8 +72,7 @@ func startGenerateMetadata() {
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) { func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
tailErr := pb.FollowMetadata(*tailFiler, grpc.WithInsecure(), "tail", tailErr := pb.FollowMetadata(*tailFiler, grpc.WithInsecure(), "tail", *dir, nil, 0, 0, eachEntryFunc, false)
*dir, 0, 0, eachEntryFunc, false)
if tailErr != nil { if tailErr != nil {
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr) fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)

3
weed/command/filer_backup.go
View file

@ -112,7 +112,6 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
return setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), lastTsNs) return setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, int32(sinkId), lastTsNs)
}) })
return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), sourcePath, nil, startFrom.UnixNano(), 0, processEventFnWithOffset, false)
sourcePath, startFrom.UnixNano(), 0, processEventFnWithOffset, false)
} }

3
weed/command/filer_meta_backup.go
View file

@ -195,8 +195,7 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
return metaBackup.setOffset(lastTime) return metaBackup.setOffset(lastTime)
}) })
return pb.FollowMetadata(*metaBackup.filerAddress, metaBackup.grpcDialOption, "meta_backup", return pb.FollowMetadata(*metaBackup.filerAddress, metaBackup.grpcDialOption, "meta_backup", *metaBackup.filerDirectory, nil, startTime.UnixNano(), 0, processEventFnWithOffset, false)
*metaBackup.filerDirectory, startTime.UnixNano(), 0, processEventFnWithOffset, false)
} }

18
weed/command/filer_meta_tail.go
View file

@ -103,17 +103,15 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
} }
} }
tailErr := pb.FollowMetadata(*tailFiler, grpcDialOption, "tail", tailErr := pb.FollowMetadata(*tailFiler, grpcDialOption, "tail", *tailTarget, nil, time.Now().Add(-*tailStart).UnixNano(), 0, func(resp *filer_pb.SubscribeMetadataResponse) error {
*tailTarget, time.Now().Add(-*tailStart).UnixNano(), 0, if !shouldPrint(resp) {
func(resp *filer_pb.SubscribeMetadataResponse) error {
if !shouldPrint(resp) {
return nil
}
if err := eachEntryFunc(resp); err != nil {
return err
}
return nil return nil
}, false) }
if err := eachEntryFunc(resp); err != nil {
return err
}
return nil
}, false)
if tailErr != nil { if tailErr != nil {
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr) fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)

3
weed/command/filer_remote_sync.go
View file

@ -210,8 +210,7 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
return setOffset(option.grpcDialOption, *option.filerAddress, RemoteSyncKeyPrefix, int32(dirHash), lastTsNs) return setOffset(option.grpcDialOption, *option.filerAddress, RemoteSyncKeyPrefix, int32(dirHash), lastTsNs)
}) })
return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, "filer.remote.sync", mountedDir, nil, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
"filer.remote.sync", mountedDir, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
} }
func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation { func toRemoteStorageLocation(mountDir, sourcePath util.FullPath, remoteMountLocation *remote_pb.RemoteStorageLocation) *remote_pb.RemoteStorageLocation {

3
weed/command/filer_sync.go
View file

@ -170,8 +170,7 @@ func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler, so
return setOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature, lastTsNs) return setOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature, lastTsNs)
}) })
return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+targetFiler, return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+targetFiler, sourcePath, nil, sourceFilerOffsetTsNs, targetFilerSignature, processEventFnWithOffset, false)
sourcePath, sourceFilerOffsetTsNs, targetFilerSignature, processEventFnWithOffset, false)
} }

23
weed/pb/filer_pb_tail.go
View file

@ -12,12 +12,12 @@ import (
type ProcessMetadataFunc func(resp *filer_pb.SubscribeMetadataResponse) error type ProcessMetadataFunc func(resp *filer_pb.SubscribeMetadataResponse) error
func FollowMetadata(filerAddress string, grpcDialOption grpc.DialOption, func FollowMetadata(filerAddress string, grpcDialOption grpc.DialOption, clientName string,
clientName string, pathPrefix string, lastTsNs int64, selfSignature int32, pathPrefix string, additionalPathPrefixes []string, lastTsNs int64, selfSignature int32,
processEventFn ProcessMetadataFunc, fatalOnError bool) error { processEventFn ProcessMetadataFunc, fatalOnError bool) error {
err := WithFilerClient(filerAddress, grpcDialOption, makeFunc( err := WithFilerClient(filerAddress, grpcDialOption, makeFunc(clientName,
clientName, pathPrefix, lastTsNs, selfSignature, processEventFn, fatalOnError)) pathPrefix, additionalPathPrefixes, lastTsNs, selfSignature, processEventFn, fatalOnError))
if err != nil { if err != nil {
return fmt.Errorf("subscribing filer meta change: %v", err) return fmt.Errorf("subscribing filer meta change: %v", err)
} }
@ -28,8 +28,7 @@ func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient,
clientName string, pathPrefix string, lastTsNs int64, selfSignature int32, clientName string, pathPrefix string, lastTsNs int64, selfSignature int32,
processEventFn ProcessMetadataFunc, fatalOnError bool) error { processEventFn ProcessMetadataFunc, fatalOnError bool) error {
err := filerClient.WithFilerClient(makeFunc( err := filerClient.WithFilerClient(makeFunc(clientName, pathPrefix, nil, lastTsNs, selfSignature, processEventFn, fatalOnError))
clientName, pathPrefix, lastTsNs, selfSignature, processEventFn, fatalOnError))
if err != nil { if err != nil {
return fmt.Errorf("subscribing filer meta change: %v", err) return fmt.Errorf("subscribing filer meta change: %v", err)
} }
@ -37,16 +36,16 @@ func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient,
return nil return nil
} }
func makeFunc(clientName string, pathPrefix string, lastTsNs int64, selfSignature int32, func makeFunc(clientName string, pathPrefix string, additionalPathPrefixes []string, lastTsNs int64, selfSignature int32, processEventFn ProcessMetadataFunc, fatalOnError bool) func(client filer_pb.SeaweedFilerClient) error {
processEventFn ProcessMetadataFunc, fatalOnError bool) func(client filer_pb.SeaweedFilerClient) error {
return func(client filer_pb.SeaweedFilerClient) error { return func(client filer_pb.SeaweedFilerClient) error {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{ stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
ClientName: clientName, ClientName: clientName,
PathPrefix: pathPrefix, PathPrefix: pathPrefix,
SinceNs: lastTsNs, PathPrefixes: additionalPathPrefixes,
Signature: selfSignature, SinceNs: lastTsNs,
Signature: selfSignature,
}) })
if err != nil { if err != nil {
return fmt.Errorf("subscribe: %v", err) return fmt.Errorf("subscribe: %v", err)