mount: add retry for read only case (#4416)

* mount: add retry for read only case

Signed-off-by: Wusong Wang <wangwusong@virtaitech.com>

* add new util retry function for mount

Signed-off-by: Wusong Wang <wangwusong@virtaitech.com>

* change error list param

Signed-off-by: Wusong Wang <wangwusong@virtaitech.com>

---------

Signed-off-by: Wusong Wang <wangwusong@virtaitech.com>
Co-authored-by: Wusong Wang <wangwusong@virtaitech.com>

weed/util/retry.go

@@ -32,6 +32,29 @@ func Retry(name string, job func() error) (err error) {
 	return err
 }
 
+func MultiRetry(name string, errList []string, job func() error) (err error) {
+	waitTime := time.Second
+	hasErr := false
+	for waitTime < RetryWaitTime {
+		err = job()
+		if err == nil {
+			if hasErr {
+				glog.V(0).Infof("retry %s successfully", name)
+			}
+			break
+		}
+		if containErr(err.Error(), errList) {
+			hasErr = true
+			glog.V(0).Infof("retry %s: err: %v", name, err)
+		} else {
+			break
+		}
+		time.Sleep(waitTime)
+		waitTime += waitTime / 2
+	}
+	return err
+}
+
 func RetryForever(name string, job func() error, onErrFn func(err error) (shouldContinue bool)) {
 	waitTime := time.Second
 	for {
@@ -62,3 +85,12 @@ func Nvl(values ...string) string {
 	}
 	return ""
 }
+
+func containErr(err string, errList []string) bool {
+	for _, e := range errList {
+		if strings.Contains(err, e) {
+			return true
+		}
+	}
+	return false
+}