filer: retryable data chunk upload

weed/server/filer_server_handlers_write_autochunk.go

@@ -256,12 +256,18 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
 func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAsChunkFunctionType {
 
 	return func(reader io.Reader, name string, offset int64) (*filer_pb.FileChunk, error) {
+		var fileId string
+		var uploadResult *operation.UploadResult
+
+		err := util.Retry("saveAsChunk", func() error {
 			// assign one file id for one chunk
-		fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so)
+			assignedFileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(so)
 			if assignErr != nil {
-			return nil, assignErr
+				return assignErr
 			}
 
+			fileId = assignedFileId
+
 			// upload the chunk to the volume server
 			uploadOption := &operation.UploadOption{
 				UploadUrl:         urlLocation,
@@ -272,9 +278,15 @@ func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAs
 				PairMap:           nil,
 				Jwt:               auth,
 			}
-		uploadResult, uploadErr, _ := operation.Upload(reader, uploadOption)
+			var uploadErr error
+			uploadResult, uploadErr, _ = operation.Upload(reader, uploadOption)
 			if uploadErr != nil {
-			return nil, uploadErr
+				return uploadErr
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
 		}
 
 		return uploadResult.ToPbFileChunk(fileId, offset), nil