add retry to master clients also

weed/util/retry.go

@@ -9,13 +9,18 @@ import (
 
 func Retry(name string, waitTimeLimit time.Duration, job func() error) (err error) {
 	waitTime := time.Second
+	hasErr := false
 	for waitTime < waitTimeLimit {
 		err = job()
 		if err == nil {
+			if hasErr {
+				glog.V(0).Infof("retry %s successfully", name)
+			}
 			break
 		}
-		if strings.Contains(err.Error(), "transport: ") {
-			glog.V(1).Infof("retry %s", name)
+		if strings.Contains(err.Error(), "transport") {
+			hasErr = true
+			glog.V(0).Infof("retry %s", name)
 			time.Sleep(waitTime)
 			waitTime += waitTime / 2
 		}

weed/wdclient/masterclient.go

@@ -5,6 +5,8 @@ import (
 	"math/rand"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"google.golang.org/grpc"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -150,10 +152,12 @@ func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader stri
 }
 
 func (mc *MasterClient) WithClient(fn func(client master_pb.SeaweedClient) error) error {
+	return util.Retry("master grpc", filer.ReadWaitTime, func() error {
 		for mc.currentMaster == "" {
 			time.Sleep(3 * time.Second)
 		}
 		return pb.WithMasterClient(mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
 			return fn(client)
 		})
+	})
 }