lsm/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2025-09-20 04:29:23 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

recreate grpc connections if too many errors

Browse Source 
address https://github.com/chrislusf/seaweedfs/issues/2098
This commit is contained in:
Chris Lu
2021-05-30 00:07:43 -07:00
parent fb8036385a
commit 1456616a77
1 changed files with 15 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
weed/pb/grpc_client_server.go
View File

@@ -32,6 +32,7 @@ var (
type versionedGrpcClient struct {
*grpc.ClientConn
version int
errCount int
}
func init() {
@@ -103,6 +104,7 @@ func getOrCreateConnection(address string, opts ...grpc.DialOption) (*versionedG
vgc := &versionedGrpcClient{
grpcConnection,
rand.Int(),
0,
}
grpcClients[address] = vgc
@@ -116,7 +118,11 @@ func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts
return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
}
executionErr := fn(vgc.ClientConn)
if executionErr != nil && strings.Contains(executionErr.Error(), "transport") {
if executionErr != nil {
vgc.errCount++
if vgc.errCount > 3 ||
strings.Contains(executionErr.Error(), "transport") ||
strings.Contains(executionErr.Error(), "connection closed") {
grpcClientsLock.Lock()
if t, ok := grpcClients[address]; ok {
if t.version == vgc.version {
@@ -126,6 +132,7 @@ func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts
}
grpcClientsLock.Unlock()
}
}
return executionErr
}