seaweedfs/weed/filer/meta_aggregator.go

216 lines
6.1 KiB
Go
Raw Normal View History

2020-09-01 07:21:19 +00:00
package filer
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/util"
"io"
"sync"
"time"
"github.com/golang/protobuf/proto"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
)
type MetaAggregator struct {
filers []pb.ServerAddress
grpcDialOption grpc.DialOption
MetaLogBuffer *log_buffer.LogBuffer
// notifying clients
ListenersLock sync.Mutex
ListenersCond *sync.Cond
}
// MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
// The old data comes from what each LocalMetadata persisted on disk.
func NewMetaAggregator(filers []pb.ServerAddress, grpcDialOption grpc.DialOption) *MetaAggregator {
t := &MetaAggregator{
filers: filers,
grpcDialOption: grpcDialOption,
}
t.ListenersCond = sync.NewCond(&t.ListenersLock)
2021-06-27 12:51:28 +00:00
t.MetaLogBuffer = log_buffer.NewLogBuffer("aggr", LogFlushInterval, nil, func() {
t.ListenersCond.Broadcast()
})
return t
}
func (ma *MetaAggregator) StartLoopSubscribe(f *Filer, self pb.ServerAddress) {
for _, filer := range ma.filers {
2020-07-13 07:05:20 +00:00
go ma.subscribeToOneFiler(f, self, filer)
}
}
func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress) {
2020-09-06 05:52:15 +00:00
/*
Each filer reads the "filer.store.id", which is the store's signature when filer starts.
2020-09-06 05:52:15 +00:00
When reading from other filers' local meta changes:
* if the received change does not contain signature from self, apply the change to current filer store.
2020-09-06 05:52:15 +00:00
Upon connecting to other filers, need to remember their signature and their offsets.
2020-09-06 05:52:15 +00:00
*/
2020-09-06 05:52:15 +00:00
var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse)
lastPersistTime := time.Now()
lastTsNs := time.Now().Add(-LogFlushInterval).UnixNano()
2020-09-06 07:29:16 +00:00
peerSignature, err := ma.readFilerStoreSignature(peer)
for err != nil {
2020-09-06 07:29:16 +00:00
glog.V(0).Infof("connecting to peer filer %s: %v", peer, err)
time.Sleep(1357 * time.Millisecond)
2020-09-06 07:29:16 +00:00
peerSignature, err = ma.readFilerStoreSignature(peer)
}
2020-07-13 07:05:20 +00:00
2021-03-09 22:13:48 +00:00
// when filer store is not shared by multiple filers
2020-09-06 07:29:16 +00:00
if peerSignature != f.Signature {
lastTsNs = 0
2020-09-06 07:50:38 +00:00
if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil {
lastTsNs = prevTsNs
}
2020-07-13 07:05:20 +00:00
2020-09-06 07:29:16 +00:00
glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
2020-09-06 07:50:51 +00:00
var counter int64
2020-09-06 19:10:37 +00:00
var synced bool
maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) {
if err := Replay(f.Store, event); err != nil {
2020-09-06 07:29:16 +00:00
glog.Errorf("failed to reply metadata change from %v: %v", peer, err)
return
}
2020-09-06 07:50:51 +00:00
counter++
2020-09-06 07:12:41 +00:00
if lastPersistTime.Add(time.Minute).Before(time.Now()) {
2020-09-06 07:29:16 +00:00
if err := ma.updateOffset(f, peer, peerSignature, event.TsNs); err == nil {
2020-09-06 07:12:41 +00:00
if event.TsNs < time.Now().Add(-2*time.Minute).UnixNano() {
2020-09-06 07:50:51 +00:00
glog.V(0).Infof("sync with %s progressed to: %v %0.2f/sec", peer, time.Unix(0, event.TsNs), float64(counter)/60.0)
2020-09-12 11:08:03 +00:00
} else if !synced {
2020-09-06 19:10:37 +00:00
synced = true
glog.V(0).Infof("synced with %s", peer)
2020-09-06 07:12:41 +00:00
}
lastPersistTime = time.Now()
2020-09-06 07:50:51 +00:00
counter = 0
} else {
2020-09-06 07:29:16 +00:00
glog.V(0).Infof("failed to update offset for %v: %v", peer, err)
}
}
}
}
processEventFn := func(event *filer_pb.SubscribeMetadataResponse) error {
data, err := proto.Marshal(event)
if err != nil {
glog.Errorf("failed to marshal subscribed filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
return err
}
dir := event.Directory
2020-07-08 06:17:17 +00:00
// println("received meta change", dir, "size", len(data))
2021-09-26 18:54:13 +00:00
ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
if maybeReplicateMetadataChange != nil {
maybeReplicateMetadataChange(event)
}
return nil
}
for {
2021-06-27 12:51:28 +00:00
glog.V(4).Infof("subscribing remote %s meta change: %v", peer, time.Unix(0, lastTsNs))
2020-09-06 07:29:16 +00:00
err := pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
ClientName: "filer:" + string(self),
PathPrefix: "/",
SinceNs: lastTsNs,
})
if err != nil {
return fmt.Errorf("subscribe: %v", err)
}
for {
resp, listenErr := stream.Recv()
if listenErr == io.EOF {
return nil
}
if listenErr != nil {
return listenErr
}
if err := processEventFn(resp); err != nil {
return fmt.Errorf("process %v: %v", resp, err)
}
lastTsNs = resp.TsNs
f.onMetadataChangeEvent(resp)
}
})
if err != nil {
2020-09-06 07:29:16 +00:00
glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err)
time.Sleep(1733 * time.Millisecond)
}
}
}
func (ma *MetaAggregator) readFilerStoreSignature(peer pb.ServerAddress) (sig int32, err error) {
err = pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil {
return err
}
2020-09-06 07:29:16 +00:00
sig = resp.Signature
return nil
})
return
}
2020-09-12 11:08:03 +00:00
const (
2020-09-06 07:29:16 +00:00
MetaOffsetPrefix = "Meta"
)
func (ma *MetaAggregator) readOffset(f *Filer, peer pb.ServerAddress, peerSignature int32) (lastTsNs int64, err error) {
2020-09-12 11:08:03 +00:00
key := []byte(MetaOffsetPrefix + "xxxx")
2020-09-06 07:29:16 +00:00
util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))
value, err := f.Store.KvGet(context.Background(), key)
2020-09-06 07:12:41 +00:00
if err == ErrKvNotFound {
glog.Warningf("readOffset %s not found", peer)
return 0, nil
}
if err != nil {
return 0, fmt.Errorf("readOffset %s : %v", peer, err)
}
lastTsNs = int64(util.BytesToUint64(value))
glog.V(0).Infof("readOffset %s : %d", peer, lastTsNs)
return
}
func (ma *MetaAggregator) updateOffset(f *Filer, peer pb.ServerAddress, peerSignature int32, lastTsNs int64) (err error) {
2020-09-06 07:29:16 +00:00
2020-09-12 11:08:03 +00:00
key := []byte(MetaOffsetPrefix + "xxxx")
2020-09-06 07:29:16 +00:00
util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))
value := make([]byte, 8)
util.Uint64toBytes(value, uint64(lastTsNs))
2020-09-06 07:29:16 +00:00
err = f.Store.KvPut(context.Background(), key, value)
if err != nil {
return fmt.Errorf("updateOffset %s : %v", peer, err)
}
glog.V(4).Infof("updateOffset %s : %d", peer, lastTsNs)
return
}