add RetryForever

This commit is contained in:
Chris Lu 2021-08-15 12:37:35 -07:00
parent a539d64896
commit fda2fc47b1
2 changed files with 24 additions and 1 deletions

View file

@ -59,8 +59,11 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
}
return util.Retry("followMetaUpdates", func() error {
util.RetryForever("followMetaUpdates", func() error {
return pb.WithFilerClientFollowMetadata(client, "mount", dir, lastTsNs, selfSignature, processEventFn, true)
}, func(err error) bool {
glog.Errorf("follow metadata updates: %v", err)
return true
})
}

View file

@ -32,6 +32,26 @@ func Retry(name string, job func() error) (err error) {
return err
}
func RetryForever(name string, job func() error, onErrFn func(err error) bool) {
waitTime := time.Second
for {
err = job()
if err == nil {
break
}
if onErrFn(err) {
if strings.Contains(err.Error(), "transport") {
glog.V(0).Infof("retry %s: err: %v", name, err)
time.Sleep(waitTime)
if waitTime < RetryWaitTime {
waitTime += waitTime / 2
}
}
continue
}
}
}
// return the first non empty string
func Nvl(values ...string) string {
for _, s := range values {