gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

notification add ack and nack

Browse source
This commit is contained in:
Chris Lu 2021-01-26 11:08:44 -08:00
parent 3a1d3d3413
commit ad2a20c8a5
6 changed files with 38 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
weed/command/filer_replication.go
View file

@ -97,13 +97,19 @@ func runFilerReplicate(cmd *Command, args []string) bool {
replicator := replication.NewReplicator(config, "source.filer.", dataSink)
for {
key, m, err := notificationInput.ReceiveMessage()
key, m, onSuccessFn, onFailureFn, err := notificationInput.ReceiveMessage()
if err != nil {
glog.Errorf("receive %s: %+v", key, err)
if onFailureFn != nil {
onFailureFn()
}
continue
}
if key == "" {
// long poll received no messages
if onSuccessFn != nil {
onSuccessFn()
}
continue
}
if m.OldEntry != nil && m.NewEntry == nil {
@ -115,8 +121,14 @@ func runFilerReplicate(cmd *Command, args []string) bool {
}
if err = replicator.Replicate(context.Background(), key, m); err != nil {
glog.Errorf("replicate %s: %+v", key, err)
if onFailureFn != nil {
onFailureFn()
}
} else {
glog.V(1).Infof("replicated %s", key)
if onSuccessFn != nil {
onSuccessFn()
}
}
}

2
weed/replication/sub/notification_aws_sqs.go
View file

@ -68,7 +68,7 @@ func (k *AwsSqsInput) initialize(awsAccessKeyId, awsSecretAccessKey, region, que
return nil
}
func (k *AwsSqsInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) {
func (k *AwsSqsInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) {
// receive message
result, err := k.svc.ReceiveMessage(&sqs.ReceiveMessageInput{

17
weed/replication/sub/notification_gocdk_pub_sub.go
View file

@ -38,13 +38,24 @@ func (k *GoCDKPubSubInput) Initialize(configuration util.Configuration, prefix s
return nil
}
func (k *GoCDKPubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) {
func (k *GoCDKPubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) {
msg, err := k.sub.Receive(context.Background())
if err != nil {
return
}
onSuccessFn = func() {
msg.Ack()
}
onFailureFn = func() {
if msg.Nackable() {
msg.Nack()
}
}
key = msg.Metadata["key"]
message = &filer_pb.EventNotification{}
err = proto.Unmarshal(msg.Body, message)
if err != nil {
return "", nil, err
return "", nil, onSuccessFn, onFailureFn, err
}
return key, message, nil
return key, message, onSuccessFn, onFailureFn, nil
}

10
weed/replication/sub/notification_google_pub_sub.go
View file

@ -85,16 +85,22 @@ func (k *GooglePubSubInput) initialize(google_application_credentials, projectId
go k.sub.Receive(ctx, func(ctx context.Context, m *pubsub.Message) {
k.messageChan <- m
m.Ack()
})
return err
}
func (k *GooglePubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) {
func (k *GooglePubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) {
m := <-k.messageChan
onSuccessFn = func() {
m.Ack()
}
onFailureFn = func() {
m.Nack()
}
// process the message
key = m.Attributes["key"]
message = &filer_pb.EventNotification{}

2
weed/replication/sub/notification_kafka.go
View file

@ -97,7 +97,7 @@ func (k *KafkaInput) initialize(hosts []string, topic string, offsetFile string,
return nil
}
func (k *KafkaInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) {
func (k *KafkaInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) {
msg := <-k.messageChan

2
weed/replication/sub/notifications.go
View file

@ -10,7 +10,7 @@ type NotificationInput interface {
GetName() string
// Initialize initializes the file store
Initialize(configuration util.Configuration, prefix string) error
ReceiveMessage() (key string, message *filer_pb.EventNotification, err error)
ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error)
}
var (