gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

recreate grpc connections if too many errors

Browse source 
address https://github.com/chrislusf/seaweedfs/issues/2098
This commit is contained in:
Chris Lu 2021-05-30 00:07:43 -07:00
parent fb8036385a
commit 1456616a77
1 changed files with 15 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
weed/pb/grpc_client_server.go
View file

@ -31,7 +31,8 @@ var (
type versionedGrpcClient struct { type versionedGrpcClient struct {
*grpc.ClientConn *grpc.ClientConn
version int version int
errCount int
} }
func init() { func init() {
@ -103,6 +104,7 @@ func getOrCreateConnection(address string, opts ...grpc.DialOption) (*versionedG
vgc := &versionedGrpcClient{ vgc := &versionedGrpcClient{
grpcConnection, grpcConnection,
rand.Int(), rand.Int(),
0,
} }
grpcClients[address] = vgc grpcClients[address] = vgc
@ -116,15 +118,20 @@ func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts
return fmt.Errorf("getOrCreateConnection %s: %v", address, err) return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
} }
executionErr := fn(vgc.ClientConn) executionErr := fn(vgc.ClientConn)
if executionErr != nil && strings.Contains(executionErr.Error(), "transport") { if executionErr != nil {
grpcClientsLock.Lock() vgc.errCount++
if t, ok := grpcClients[address]; ok { if vgc.errCount > 3 ||
if t.version == vgc.version { strings.Contains(executionErr.Error(), "transport") ||
vgc.Close() strings.Contains(executionErr.Error(), "connection closed") {
delete(grpcClients, address) grpcClientsLock.Lock()
if t, ok := grpcClients[address]; ok {
if t.version == vgc.version {
vgc.Close()
delete(grpcClients, address)
}
} }
grpcClientsLock.Unlock()
} }
grpcClientsLock.Unlock()
} }
return executionErr return executionErr