seaweedfs/weed/pb/filer_pb_tail.go

117 lines
3.6 KiB
Go
Raw Normal View History

2021-08-04 23:25:46 +00:00
package pb
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
2021-08-04 23:25:46 +00:00
"google.golang.org/grpc"
"io"
"time"
)
type EventErrorType int
const (
TrivialOnError EventErrorType = iota
FatalOnError
RetryForeverOnError
)
2021-08-04 23:25:46 +00:00
type ProcessMetadataFunc func(resp *filer_pb.SubscribeMetadataResponse) error
func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption, clientName string, clientId int32, clientEpoch int32,
2022-05-30 22:04:19 +00:00
pathPrefix string, additionalPathPrefixes []string, lastTsNs int64, untilTsNs int64, selfSignature int32,
processEventFn ProcessMetadataFunc, eventErrorType EventErrorType) error {
2021-08-04 23:25:46 +00:00
err := WithFilerClient(true, filerAddress, grpcDialOption, makeSubscribeMetadataFunc(
clientName, clientId, clientEpoch,
pathPrefix, additionalPathPrefixes, &lastTsNs, untilTsNs, selfSignature, processEventFn, eventErrorType))
2021-08-04 23:25:46 +00:00
if err != nil {
return fmt.Errorf("subscribing filer meta change: %v", err)
}
return err
}
func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient,
clientName string, clientId int32, clientEpoch int32, pathPrefix string, lastTsNs *int64, untilTsNs int64, selfSignature int32,
processEventFn ProcessMetadataFunc, eventErrorType EventErrorType) error {
2021-08-04 23:25:46 +00:00
err := filerClient.WithFilerClient(true, makeSubscribeMetadataFunc(clientName, clientId, clientEpoch, pathPrefix, nil, lastTsNs, untilTsNs, selfSignature, processEventFn, eventErrorType))
2021-08-04 23:25:46 +00:00
if err != nil {
return fmt.Errorf("subscribing filer meta change: %v", err)
}
return nil
}
func makeSubscribeMetadataFunc(clientName string, clientId int32, clientEpoch int32, pathPrefix string, additionalPathPrefixes []string, lastTsNs *int64, untilTsNs int64, selfSignature int32, processEventFn ProcessMetadataFunc, eventErrorType EventErrorType) func(client filer_pb.SeaweedFilerClient) error {
2021-08-04 23:25:46 +00:00
return func(client filer_pb.SeaweedFilerClient) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
2021-09-01 06:26:26 +00:00
ClientName: clientName,
PathPrefix: pathPrefix,
PathPrefixes: additionalPathPrefixes,
2021-10-17 09:01:20 +00:00
SinceNs: *lastTsNs,
2021-09-01 06:26:26 +00:00
Signature: selfSignature,
ClientId: clientId,
ClientEpoch: clientEpoch,
2022-05-30 22:04:19 +00:00
UntilNs: untilTsNs,
2021-08-04 23:25:46 +00:00
})
if err != nil {
return fmt.Errorf("subscribe: %v", err)
}
for {
resp, listenErr := stream.Recv()
if listenErr == io.EOF {
return nil
}
if listenErr != nil {
return listenErr
}
if err := processEventFn(resp); err != nil {
switch eventErrorType {
case TrivialOnError:
glog.Errorf("process %v: %v", resp, err)
case FatalOnError:
2021-08-04 23:25:46 +00:00
glog.Fatalf("process %v: %v", resp, err)
case RetryForeverOnError:
util.RetryForever("followMetaUpdates", func() error {
return processEventFn(resp)
}, func(err error) bool {
glog.Errorf("process %v: %v", resp, err)
return true
})
default:
2021-08-04 23:25:46 +00:00
glog.Errorf("process %v: %v", resp, err)
}
}
2021-10-17 09:01:20 +00:00
*lastTsNs = resp.TsNs
2021-08-04 23:25:46 +00:00
}
}
}
func AddOffsetFunc(processEventFn ProcessMetadataFunc, offsetInterval time.Duration, offsetFunc func(counter int64, offset int64) error) ProcessMetadataFunc {
var counter int64
var lastWriteTime time.Time
return func(resp *filer_pb.SubscribeMetadataResponse) error {
if err := processEventFn(resp); err != nil {
return err
}
counter++
if lastWriteTime.Add(offsetInterval).Before(time.Now()) {
lastWriteTime = time.Now()
if err := offsetFunc(counter, resp.TsNs); err != nil {
return err
}
2022-01-10 09:00:11 +00:00
counter = 0
2021-08-04 23:25:46 +00:00
}
return nil
}
}