Browse Source

mount: add retry for all operations with filer

fix https://github.com/chrislusf/seaweedfs/issues/1589
pull/1596/head
Chris Lu 4 years ago
parent
commit
f7b21973ac
  1. 6
      weed/filesys/wfs_filer_client.go
  2. 21
      weed/util/retry.go

6
weed/filesys/wfs_filer_client.go

@ -1,6 +1,8 @@
package filesys
import (
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/util"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/pb"
@ -11,10 +13,12 @@ var _ = filer_pb.FilerClient(&WFS{})
func (wfs *WFS) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
err := pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
err := util.Retry("filer grpc", filer.ReadWaitTime, func() error {
return pb.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)
})
if err == nil {
return nil

21
weed/util/retry.go

@ -0,0 +1,21 @@
package util
import (
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
)
func Retry(name string, waitTimeLimit time.Duration, job func() error) (err error) {
waitTime := time.Second
for waitTime < waitTimeLimit {
err = job()
if err == nil {
break
}
glog.V(1).Infof("retry %s", name)
time.Sleep(waitTime)
waitTime += waitTime / 2
}
return err
}
Loading…
Cancel
Save