add retry to assign volume

fix https://github.com/chrislusf/seaweedfs/issues/2056

weed/command/filer_copy.go

@@ -308,7 +308,8 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 		}
 
 		// assign a volume
-		err = pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err = util.Retry("assignVolume", func() error {
+			return pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 				request := &filer_pb.AssignVolumeRequest{
 					Count:       1,
@@ -328,6 +329,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 				}
 				return nil
 			})
+		})
 		if err != nil {
 			return fmt.Errorf("Failed to assign from %v: %v\n", worker.options.masters, err)
 		}
@@ -404,7 +406,8 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 			// assign a volume
 			var assignResult *filer_pb.AssignVolumeResponse
 			var assignError error
-			err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			err := util.Retry("assignVolume", func() error {
+				return pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 					request := &filer_pb.AssignVolumeRequest{
 						Count:       1,
 						Replication: *worker.options.replication,
@@ -423,6 +426,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 					}
 					return nil
 				})
+			})
 			if err != nil {
 				fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
 			}

weed/filesys/wfs_write.go

@@ -20,7 +20,7 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath, writeOnly bool) filer.Sa
 		var auth security.EncodedJwt
 
 		if err := wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-
+			return util.Retry("assignVolume", func() error {
 				request := &filer_pb.AssignVolumeRequest{
 					Count:       1,
 					Replication: wfs.option.Replication,
@@ -49,6 +49,7 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath, writeOnly bool) filer.Sa
 				collection, replication = resp.Collection, resp.Replication
 
 				return nil
+			})
 		}); err != nil {
 			return nil, "", "", fmt.Errorf("filerGrpcAddress assign volume: %v", err)
 		}

weed/messaging/broker/broker_append.go

@@ -3,6 +3,7 @@ package broker
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/security"
 	"io"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -10,7 +11,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
-	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -53,6 +53,7 @@ func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConf
 	// assign a volume location
 	if err := broker.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
+		assignErr := util.Retry("assignVolume", func() error {
 			request := &filer_pb.AssignVolumeRequest{
 				Count:       1,
 				Replication: topicConfig.Replication,
@@ -74,6 +75,12 @@ func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConf
 			assignResult.PublicUrl = resp.PublicUrl
 			assignResult.Count = uint64(resp.Count)
 
+			return nil
+		})
+		if assignErr != nil {
+			return assignErr
+		}
+
 		return nil
 	}); err != nil {
 		return nil, nil, err

weed/replication/sink/filersink/fetch_write.go

@@ -71,7 +71,7 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string)
 	var auth security.EncodedJwt
 
 	if err := fs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-
+		return util.Retry("assignVolume", func() error {
 			request := &filer_pb.AssignVolumeRequest{
 				Count:       1,
 				Replication: fs.replication,
@@ -94,6 +94,7 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string)
 			fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)
 
 			return nil
+		})
 	}); err != nil {
 		return "", fmt.Errorf("filerGrpcAddress assign volume: %v", err)
 	}

weed/server/webdav_server.go

@@ -380,6 +380,7 @@ func (f *WebDavFile) saveDataAsChunk(reader io.Reader, name string, offset int64
 
 		ctx := context.Background()
 
+		assignErr := util.Retry("assignVolume", func() error {
 			request := &filer_pb.AssignVolumeRequest{
 				Count:       1,
 				Replication: f.fs.option.Replication,
@@ -400,6 +401,12 @@ func (f *WebDavFile) saveDataAsChunk(reader io.Reader, name string, offset int64
 			fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)
 			f.collection, f.replication = resp.Collection, resp.Replication
 
+			return nil
+		})
+		if assignErr != nil {
+			return assignErr
+		}
+
 		return nil
 	}); flushErr != nil {
 		return nil, f.collection, f.replication, fmt.Errorf("filerGrpcAddress assign volume: %v", flushErr)