mount: add retry for read only case (#4416)

* mount: add retry for read only case

Signed-off-by: Wusong Wang <wangwusong@virtaitech.com>

* add new util retry function for mount

Signed-off-by: Wusong Wang <wangwusong@virtaitech.com>

* change error list param

Signed-off-by: Wusong Wang <wangwusong@virtaitech.com>

---------

Signed-off-by: Wusong Wang <wangwusong@virtaitech.com>
Co-authored-by: Wusong Wang <wangwusong@virtaitech.com>

weed/operation/upload_content.go

@@ -5,11 +5,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"github.com/seaweedfs/seaweedfs/weed/security"
-	"github.com/seaweedfs/seaweedfs/weed/stats"
-	"github.com/seaweedfs/seaweedfs/weed/util"
 	"io"
 	"mime"
 	"mime/multipart"
@@ -19,6 +14,12 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 type UploadOption struct {
@@ -120,7 +121,8 @@ func UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.A
 			return true
 		})
 	} else {
-		err = util.Retry("uploadWithRetry", doUploadFunc)
+		uploadErrList := []string{"transport", "is read only"}
+		err = util.MultiRetry("uploadWithRetry", uploadErrList, doUploadFunc)
 	}
 
 	return

weed/util/retry.go

@@ -32,6 +32,29 @@ func Retry(name string, job func() error) (err error) {
 	return err
 }
 
+func MultiRetry(name string, errList []string, job func() error) (err error) {
+	waitTime := time.Second
+	hasErr := false
+	for waitTime < RetryWaitTime {
+		err = job()
+		if err == nil {
+			if hasErr {
+				glog.V(0).Infof("retry %s successfully", name)
+			}
+			break
+		}
+		if containErr(err.Error(), errList) {
+			hasErr = true
+			glog.V(0).Infof("retry %s: err: %v", name, err)
+		} else {
+			break
+		}
+		time.Sleep(waitTime)
+		waitTime += waitTime / 2
+	}
+	return err
+}
+
 func RetryForever(name string, job func() error, onErrFn func(err error) (shouldContinue bool)) {
 	waitTime := time.Second
 	for {
@@ -62,3 +85,12 @@ func Nvl(values ...string) string {
 	}
 	return ""
 }
+
+func containErr(err string, errList []string) bool {
+	for _, e := range errList {
+		if strings.Contains(err, e) {
+			return true
+		}
+	}
+	return false
+}