mount: add retry for all operations with filer

fix https://github.com/chrislusf/seaweedfs/issues/1589

weed/filesys/wfs_filer_client.go

@@ -1,6 +1,8 @@
 package filesys
 
 import (
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"google.golang.org/grpc"
 
 	"github.com/chrislusf/seaweedfs/weed/pb"
@@ -11,10 +13,12 @@ var _ = filer_pb.FilerClient(&WFS{})
 
 func (wfs *WFS) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	err := pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+	err := util.Retry("filer grpc", filer.ReadWaitTime, func() error {
-		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
-		return fn(client)
+			client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-	}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)
+			return fn(client)
+		}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)
+	})
 
 	if err == nil {
 		return nil

weed/util/retry.go

@@ -0,0 +1,21 @@
+package util
+
+import (
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+func Retry(name string, waitTimeLimit time.Duration, job func() error) (err error) {
+	waitTime := time.Second
+	for waitTime < waitTimeLimit {
+		err = job()
+		if err == nil {
+			break
+		}
+		glog.V(1).Infof("retry %s", name)
+		time.Sleep(waitTime)
+		waitTime += waitTime / 2
+	}
+	return err
+}