Browse Source

add retry to master clients also

pull/1596/head
Chris Lu 4 years ago
parent
commit
7192a378cc
  1. 9
      weed/util/retry.go
  2. 14
      weed/wdclient/masterclient.go

9
weed/util/retry.go

@ -9,13 +9,18 @@ import (
func Retry(name string, waitTimeLimit time.Duration, job func() error) (err error) {
waitTime := time.Second
hasErr := false
for waitTime < waitTimeLimit {
err = job()
if err == nil {
if hasErr {
glog.V(0).Infof("retry %s successfully", name)
}
break
}
if strings.Contains(err.Error(), "transport: ") {
glog.V(1).Infof("retry %s", name)
if strings.Contains(err.Error(), "transport") {
hasErr = true
glog.V(0).Infof("retry %s", name)
time.Sleep(waitTime)
waitTime += waitTime / 2
}

14
weed/wdclient/masterclient.go

@ -5,6 +5,8 @@ import (
"math/rand"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog"
@ -150,10 +152,12 @@ func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader stri
}
func (mc *MasterClient) WithClient(fn func(client master_pb.SeaweedClient) error) error {
for mc.currentMaster == "" {
time.Sleep(3 * time.Second)
}
return pb.WithMasterClient(mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
return fn(client)
return util.Retry("master grpc", filer.ReadWaitTime, func() error {
for mc.currentMaster == "" {
time.Sleep(3 * time.Second)
}
return pb.WithMasterClient(mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
return fn(client)
})
})
}
Loading…
Cancel
Save