recreate grpc connections if too many errors

address https://github.com/chrislusf/seaweedfs/issues/2098

weed/pb/grpc_client_server.go

@@ -31,7 +31,8 @@ var (
 
 type versionedGrpcClient struct {
 	*grpc.ClientConn
-	version int
+	version  int
+	errCount int
 }
 
 func init() {
@@ -103,6 +104,7 @@ func getOrCreateConnection(address string, opts ...grpc.DialOption) (*versionedG
 	vgc := &versionedGrpcClient{
 		grpcConnection,
 		rand.Int(),
+		0,
 	}
 	grpcClients[address] = vgc
 
@@ -116,15 +118,20 @@ func WithCachedGrpcClient(fn func(*grpc.ClientConn) error, address string, opts
 		return fmt.Errorf("getOrCreateConnection %s: %v", address, err)
 	}
 	executionErr := fn(vgc.ClientConn)
-	if executionErr != nil && strings.Contains(executionErr.Error(), "transport") {
+	if executionErr != nil {
-		grpcClientsLock.Lock()
+		vgc.errCount++
-		if t, ok := grpcClients[address]; ok {
+		if vgc.errCount > 3 ||
-			if t.version == vgc.version {
+			strings.Contains(executionErr.Error(), "transport") ||
-				vgc.Close()
+			strings.Contains(executionErr.Error(), "connection closed") {
-				delete(grpcClients, address)
+			grpcClientsLock.Lock()
+			if t, ok := grpcClients[address]; ok {
+				if t.version == vgc.version {
+					vgc.Close()
+					delete(grpcClients, address)
+				}
 			}
+			grpcClientsLock.Unlock()
 		}
-		grpcClientsLock.Unlock()
 	}
 
 	return executionErr