gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

add retry to master clients also

Browse source
This commit is contained in:
Chris Lu 2020-11-01 01:40:16 -07:00
parent 004aa9ca5d
commit 7192a378cc
2 changed files with 16 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
weed/util/retry.go
View file

@ -9,13 +9,18 @@ import (
func Retry(name string, waitTimeLimit time.Duration, job func() error) (err error) {
waitTime := time.Second
hasErr := false
for waitTime < waitTimeLimit {
err = job()
if err == nil {
if hasErr {
glog.V(0).Infof("retry %s successfully", name)
}
break
}
if strings.Contains(err.Error(), "transport: ") {
glog.V(1).Infof("retry %s", name)
if strings.Contains(err.Error(), "transport") {
hasErr = true
glog.V(0).Infof("retry %s", name)
time.Sleep(waitTime)
waitTime += waitTime / 2
}

4
weed/wdclient/masterclient.go
View file

@ -5,6 +5,8 @@ import (
"math/rand"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog"
@ -150,10 +152,12 @@ func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader stri
}
func (mc *MasterClient) WithClient(fn func(client master_pb.SeaweedClient) error) error {
return util.Retry("master grpc", filer.ReadWaitTime, func() error {
for mc.currentMaster == "" {
time.Sleep(3 * time.Second)
}
return pb.WithMasterClient(mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
return fn(client)
})
})
}