seaweedfs/weed/replication/replicator.go

107 lines
3.3 KiB
Go
Raw Normal View History

2018-09-17 07:27:56 +00:00
package replication
import (
2019-03-16 00:20:24 +00:00
"context"
"fmt"
2020-09-09 18:21:23 +00:00
"github.com/chrislusf/seaweedfs/weed/pb"
"google.golang.org/grpc"
2018-09-17 08:37:24 +00:00
"strings"
"time"
2018-09-21 08:56:43 +00:00
2018-10-11 07:08:13 +00:00
"github.com/chrislusf/seaweedfs/weed/glog"
2018-09-21 08:56:43 +00:00
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/replication/sink"
2018-10-11 07:08:13 +00:00
"github.com/chrislusf/seaweedfs/weed/replication/source"
2018-09-21 08:56:43 +00:00
"github.com/chrislusf/seaweedfs/weed/util"
2018-09-17 07:27:56 +00:00
)
type Replicator struct {
2022-07-27 14:22:57 +00:00
sink sink.ReplicationSink
source *source.FilerSource
excludeDirs []string
2018-09-17 07:27:56 +00:00
}
func NewReplicator(sourceConfig util.Configuration, configPrefix string, dataSink sink.ReplicationSink) *Replicator {
2018-09-17 08:37:24 +00:00
source := &source.FilerSource{}
source.Initialize(sourceConfig, configPrefix)
2018-09-17 08:37:24 +00:00
2018-10-04 06:36:52 +00:00
dataSink.SetSourceFiler(source)
2018-09-21 08:54:29 +00:00
2018-09-17 07:27:56 +00:00
return &Replicator{
2022-07-27 14:22:57 +00:00
sink: dataSink,
source: source,
excludeDirs: sourceConfig.GetStringSlice(configPrefix + "excludeDirectories"),
2018-09-17 07:27:56 +00:00
}
}
2019-03-16 00:20:24 +00:00
func (r *Replicator) Replicate(ctx context.Context, key string, message *filer_pb.EventNotification) error {
if message.IsFromOtherCluster && r.sink.GetName() == "filer" {
return nil
}
2018-09-17 08:37:24 +00:00
if !strings.HasPrefix(key, r.source.Dir) {
2018-10-06 20:01:29 +00:00
glog.V(4).Infof("skipping %v outside of %v", key, r.source.Dir)
2018-09-17 08:37:24 +00:00
return nil
}
2022-07-27 14:22:57 +00:00
for _, excludeDir := range r.excludeDirs {
if strings.HasPrefix(key, excludeDir) {
glog.V(4).Infof("skipping %v of exclude dir %v", key, excludeDir)
return nil
}
}
var dateKey string
2021-03-01 00:19:03 +00:00
if r.sink.IsIncremental() {
var mTime int64
if message.NewEntry != nil {
mTime = message.NewEntry.Attributes.Mtime
} else if message.OldEntry != nil {
mTime = message.OldEntry.Attributes.Mtime
}
dateKey = time.Unix(mTime, 0).Format("2006-01-02")
}
newKey := util.Join(r.sink.GetSinkToDirectory(), dateKey, key[len(r.source.Dir):])
2018-11-04 19:58:59 +00:00
glog.V(3).Infof("replicate %s => %s", key, newKey)
key = newKey
2018-09-17 07:27:56 +00:00
if message.OldEntry != nil && message.NewEntry == nil {
2018-10-06 20:00:33 +00:00
glog.V(4).Infof("deleting %v", key)
2020-09-09 18:21:23 +00:00
return r.sink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
2018-09-17 07:27:56 +00:00
}
if message.OldEntry == nil && message.NewEntry != nil {
2018-10-06 20:00:33 +00:00
glog.V(4).Infof("creating %v", key)
2020-09-09 18:21:23 +00:00
return r.sink.CreateEntry(key, message.NewEntry, message.Signatures)
2018-09-17 07:27:56 +00:00
}
if message.OldEntry == nil && message.NewEntry == nil {
2018-10-04 06:36:52 +00:00
glog.V(0).Infof("weird message %+v", message)
return nil
}
2020-09-09 18:21:23 +00:00
foundExisting, err := r.sink.UpdateEntry(key, message.OldEntry, message.NewParentPath, message.NewEntry, message.DeleteChunks, message.Signatures)
2018-10-04 06:36:52 +00:00
if foundExisting {
2018-10-06 20:00:33 +00:00
glog.V(4).Infof("updated %v", key)
2018-10-04 06:36:52 +00:00
return err
}
2020-09-09 18:21:23 +00:00
err = r.sink.DeleteEntry(key, message.OldEntry.IsDirectory, false, message.Signatures)
if err != nil {
return fmt.Errorf("delete old entry %v: %v", key, err)
}
2018-10-06 20:00:33 +00:00
glog.V(4).Infof("creating missing %v", key)
2020-09-09 18:21:23 +00:00
return r.sink.CreateEntry(key, message.NewEntry, message.Signatures)
2018-09-17 07:27:56 +00:00
}
2020-09-09 18:21:23 +00:00
func ReadFilerSignature(grpcDialOption grpc.DialOption, filer pb.ServerAddress) (filerSignature int32, readErr error) {
if readErr = pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
2020-09-09 18:21:23 +00:00
if resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}); err != nil {
return fmt.Errorf("GetFilerConfiguration %s: %v", filer, err)
} else {
filerSignature = resp.Signature
}
return nil
}); readErr != nil {
return 0, readErr
}
return filerSignature, nil
}