parallelize remote content fetching

weed/server/filer_grpc_server_remote.go

@@ -78,25 +78,32 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo
 	dest := util.FullPath(remoteStorageMountedLocation.Path).Child(string(util.FullPath(req.Directory).Child(req.Name))[len(localMountedDir):])
 
 	var chunks []*filer_pb.FileChunk
+	var fetchAndWriteErr error
 
-	// FIXME limit on parallel
+	limitedConcurrentExecutor := util.NewLimitedConcurrentExecutor(8)
 	for offset := int64(0); offset < entry.Remote.RemoteSize; offset += chunkSize {
+		localOffset := offset
+
+		limitedConcurrentExecutor.Execute(func() {
 			size := chunkSize
-		if offset+chunkSize > entry.Remote.RemoteSize {
-			size = entry.Remote.RemoteSize - offset
+			if localOffset+chunkSize > entry.Remote.RemoteSize {
+				size = entry.Remote.RemoteSize - localOffset
 			}
 
 			// assign one volume server
 			assignResult, err := operation.Assign(fs.filer.GetMaster, fs.grpcDialOption, assignRequest, altRequest)
 			if err != nil {
-			return resp, err
+				fetchAndWriteErr = err
+				return
 			}
 			if assignResult.Error != "" {
-			return resp, fmt.Errorf("assign: %v", assignResult.Error)
+				fetchAndWriteErr = fmt.Errorf("assign: %v", assignResult.Error)
+				return
 			}
 			fileId, parseErr := needle.ParseFileIdFromString(assignResult.Fid)
 			if assignResult.Error != "" {
-			return resp, fmt.Errorf("unrecognized file id %s: %v", assignResult.Fid, parseErr)
+				fetchAndWriteErr = fmt.Errorf("unrecognized file id %s: %v", assignResult.Fid, parseErr)
+				return
 			}
 
 			// tell filer to tell volume server to download into needles
@@ -105,7 +112,7 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo
 					VolumeId:     uint32(fileId.VolumeId),
 					NeedleId:     uint64(fileId.Key),
 					Cookie:       uint32(fileId.Cookie),
-				Offset:       offset,
+					Offset:       localOffset,
 					Size:         size,
 					RemoteType:   storageConf.Type,
 					RemoteName:   storageConf.Name,
@@ -123,12 +130,13 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo
 			})
 
 			if err != nil {
-			return nil, err
+				fetchAndWriteErr = err
+				return
 			}
 
 			chunks = append(chunks, &filer_pb.FileChunk{
 				FileId: assignResult.Fid,
-			Offset: offset,
+				Offset: localOffset,
 				Size:   uint64(size),
 				Mtime:  time.Now().Unix(),
 				Fid: &filer_pb.FileId{
@@ -137,7 +145,7 @@ func (fs *FilerServer) DownloadToLocal(ctx context.Context, req *filer_pb.Downlo
 					Cookie:   uint32(fileId.Cookie),
 				},
 			})
-
+		})
 	}
 
 	garbage := entry.Chunks