retry context canceled request

weed/filesys/wfs.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -89,11 +90,24 @@ func (wfs *WFS) Root() (fs.Node, error) {
 
 func (wfs *WFS) WithFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
+	err := util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
 		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 		return fn(client)
 	}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)
 
+	if err == nil {
+		return nil
+	}
+	if strings.Contains(err.Error(), "context canceled") {
+		time.Sleep(1337 * time.Millisecond)
+		glog.V(2).Infoln("retry context canceled request...")
+		return util.WithCachedGrpcClient(context.Background(), func(grpcConnection *grpc.ClientConn) error {
+			client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+			return fn(client)
+		}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)
+	}
+	return err
+
 }
 
 func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHandle) {

weed/util/grpc_client_server.go

@@ -64,7 +64,14 @@ func WithCachedGrpcClient(ctx context.Context, fn func(*grpc.ClientConn) error,
 	existingConnection, found := grpcClients[address]
 	if found {
 		grpcClientsLock.Unlock()
-		return fn(existingConnection)
+		err := fn(existingConnection)
+		if err != nil {
+			grpcClientsLock.Lock()
+			delete(grpcClients, address)
+			grpcClientsLock.Unlock()
+			existingConnection.Close()
+		}
+		return err
 	}
 
 	grpcConnection, err := GrpcDial(ctx, address, opts...)