lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-09-21 09:28:01 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

recreate grpc connections if too many errors

Browse Source 
address https://github.com/chrislusf/seaweedfs/issues/2098
This commit is contained in:
Chris Lu
2021-05-30 00:07:43 -07:00
parent fb8036385a
commit 1456616a77
1 changed files with 15 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
weed/pb/grpc_client_server.go
View File

@@ -32,6 +32,7 @@ var (
type versionedGrpcClient struct { type versionedGrpcClient struct {
*grpc.ClientConn *grpc.ClientConn
version int version int
errCount int
} }
func init() { func init() {
@@ -103,6 +104,7 @@ func getOrCreateConnection(address string, opts ...grpc.DialOption) (*versionedG
vgc := &versionedGrpcClient{ vgc := &versionedGrpcClient{
grpcConnection, grpcConnection,
rand.Int(), rand.Int(),
0,
} }
grpcClients[address] = vgc grpcClients[address] = vgc
@@ -116,7 +118,11 @@ func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts
return fmt.Errorf("getOrCreateConnection %s: %v", address, err) return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
} }
executionErr := fn(vgc.ClientConn) executionErr := fn(vgc.ClientConn)
if executionErr != nil && strings.Contains(executionErr.Error(), "transport") { if executionErr != nil {
vgc.errCount++
if vgc.errCount > 3 ||
strings.Contains(executionErr.Error(), "transport") ||
strings.Contains(executionErr.Error(), "connection closed") {
grpcClientsLock.Lock() grpcClientsLock.Lock()
if t, ok := grpcClients[address]; ok { if t, ok := grpcClients[address]; ok {
if t.version == vgc.version { if t.version == vgc.version {
@@ -126,6 +132,7 @@ func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts
} }
grpcClientsLock.Unlock() grpcClientsLock.Unlock()
} }
}
return executionErr return executionErr
} }