You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

651 lines
19 KiB

4 years ago
3 years ago
7 years ago
4 years ago
4 years ago
4 years ago
3 years ago
3 years ago
3 years ago
5 years ago
3 years ago
  1. package command
  2. import (
  3. "context"
  4. "fmt"
  5. "io"
  6. "net/http"
  7. "os"
  8. "path/filepath"
  9. "strconv"
  10. "strings"
  11. "sync"
  12. "time"
  13. "google.golang.org/grpc"
  14. "github.com/seaweedfs/seaweedfs/weed/filer"
  15. "github.com/seaweedfs/seaweedfs/weed/operation"
  16. "github.com/seaweedfs/seaweedfs/weed/pb"
  17. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  18. "github.com/seaweedfs/seaweedfs/weed/security"
  19. "github.com/seaweedfs/seaweedfs/weed/storage/needle"
  20. "github.com/seaweedfs/seaweedfs/weed/util"
  21. "github.com/seaweedfs/seaweedfs/weed/util/grace"
  22. "github.com/seaweedfs/seaweedfs/weed/wdclient"
  23. )
  24. var (
  25. copy CopyOptions
  26. waitGroup sync.WaitGroup
  27. )
  28. type CopyOptions struct {
  29. include *string
  30. replication *string
  31. collection *string
  32. ttl *string
  33. diskType *string
  34. maxMB *int
  35. masterClient *wdclient.MasterClient
  36. concurrenctFiles *int
  37. concurrenctChunks *int
  38. grpcDialOption grpc.DialOption
  39. masters []string
  40. cipher bool
  41. ttlSec int32
  42. checkSize *bool
  43. verbose *bool
  44. }
  45. func init() {
  46. cmdFilerCopy.Run = runCopy // break init cycle
  47. cmdFilerCopy.IsDebug = cmdFilerCopy.Flag.Bool("debug", false, "verbose debug information")
  48. copy.include = cmdFilerCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
  49. copy.replication = cmdFilerCopy.Flag.String("replication", "", "replication type")
  50. copy.collection = cmdFilerCopy.Flag.String("collection", "", "optional collection name")
  51. copy.ttl = cmdFilerCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
  52. copy.diskType = cmdFilerCopy.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
  53. copy.maxMB = cmdFilerCopy.Flag.Int("maxMB", 4, "split files larger than the limit")
  54. copy.concurrenctFiles = cmdFilerCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
  55. copy.concurrenctChunks = cmdFilerCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
  56. copy.checkSize = cmdFilerCopy.Flag.Bool("check.size", false, "copy when the target file size is different from the source file")
  57. copy.verbose = cmdFilerCopy.Flag.Bool("verbose", false, "print out details during copying")
  58. }
  59. var cmdFilerCopy = &Command{
  60. UsageLine: "filer.copy file_or_dir1 [file_or_dir2 file_or_dir3] http://localhost:8888/path/to/a/folder/",
  61. Short: "copy one or a list of files to a filer folder",
  62. Long: `copy one or a list of files, or batch copy one whole folder recursively, to a filer folder
  63. It can copy one or a list of files or folders.
  64. If copying a whole folder recursively:
  65. All files under the folder and sub folders will be copied.
  66. Optional parameter "-include" allows you to specify the file name patterns.
  67. If "maxMB" is set to a positive number, files larger than it would be split into chunks.
  68. `,
  69. }
  70. func runCopy(cmd *Command, args []string) bool {
  71. util.LoadConfiguration("security", false)
  72. if len(args) <= 1 {
  73. return false
  74. }
  75. filerDestination := args[len(args)-1]
  76. fileOrDirs := args[0 : len(args)-1]
  77. filerAddress, urlPath, err := pb.ParseUrl(filerDestination)
  78. if err != nil {
  79. fmt.Printf("The last argument should be a URL on filer: %v\n", err)
  80. return false
  81. }
  82. if !strings.HasSuffix(urlPath, "/") {
  83. fmt.Printf("The last argument should be a folder and end with \"/\"\n")
  84. return false
  85. }
  86. copy.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
  87. masters, collection, replication, dirBuckets, maxMB, cipher, err := readFilerConfiguration(copy.grpcDialOption, filerAddress)
  88. if err != nil {
  89. fmt.Printf("read from filer %s: %v\n", filerAddress, err)
  90. return false
  91. }
  92. if strings.HasPrefix(urlPath, dirBuckets+"/") {
  93. restPath := urlPath[len(dirBuckets)+1:]
  94. if strings.Index(restPath, "/") > 0 {
  95. expectedBucket := restPath[:strings.Index(restPath, "/")]
  96. if *copy.collection == "" {
  97. *copy.collection = expectedBucket
  98. } else if *copy.collection != expectedBucket {
  99. fmt.Printf("destination %s uses collection \"%s\": unexpected collection \"%v\"\n", urlPath, expectedBucket, *copy.collection)
  100. return true
  101. }
  102. }
  103. }
  104. if *copy.collection == "" {
  105. *copy.collection = collection
  106. }
  107. if *copy.replication == "" {
  108. *copy.replication = replication
  109. }
  110. if *copy.maxMB == 0 {
  111. *copy.maxMB = int(maxMB)
  112. }
  113. copy.masters = masters
  114. copy.cipher = cipher
  115. ttl, err := needle.ReadTTL(*copy.ttl)
  116. if err != nil {
  117. fmt.Printf("parsing ttl %s: %v\n", *copy.ttl, err)
  118. return false
  119. }
  120. copy.ttlSec = int32(ttl.Minutes()) * 60
  121. if *cmdFilerCopy.IsDebug {
  122. grace.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof")
  123. }
  124. fileCopyTaskChan := make(chan FileCopyTask, *copy.concurrenctFiles)
  125. go func() {
  126. defer close(fileCopyTaskChan)
  127. for _, fileOrDir := range fileOrDirs {
  128. if err := genFileCopyTask(fileOrDir, urlPath, fileCopyTaskChan); err != nil {
  129. fmt.Fprintf(os.Stderr, "genFileCopyTask : %v\n", err)
  130. break
  131. }
  132. }
  133. }()
  134. for i := 0; i < *copy.concurrenctFiles; i++ {
  135. waitGroup.Add(1)
  136. go func() {
  137. defer waitGroup.Done()
  138. worker := FileCopyWorker{
  139. options: &copy,
  140. filerAddress: filerAddress,
  141. }
  142. if err := worker.copyFiles(fileCopyTaskChan); err != nil {
  143. fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
  144. return
  145. }
  146. }()
  147. }
  148. waitGroup.Wait()
  149. return true
  150. }
  151. func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress pb.ServerAddress) (masters []string, collection, replication string, dirBuckets string, maxMB uint32, cipher bool, err error) {
  152. err = pb.WithGrpcFilerClient(false, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
  153. resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
  154. if err != nil {
  155. return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
  156. }
  157. masters, collection, replication, maxMB = resp.Masters, resp.Collection, resp.Replication, resp.MaxMb
  158. dirBuckets = resp.DirBuckets
  159. cipher = resp.Cipher
  160. return nil
  161. })
  162. return
  163. }
  164. func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan FileCopyTask) error {
  165. fi, err := os.Stat(fileOrDir)
  166. if err != nil {
  167. fmt.Fprintf(os.Stderr, "Error: read file %s: %v\n", fileOrDir, err)
  168. return nil
  169. }
  170. mode := fi.Mode()
  171. uid, gid := util.GetFileUidGid(fi)
  172. fileSize := fi.Size()
  173. if mode.IsDir() {
  174. fileSize = 0
  175. }
  176. fileCopyTaskChan <- FileCopyTask{
  177. sourceLocation: fileOrDir,
  178. destinationUrlPath: destPath,
  179. fileSize: fileSize,
  180. fileMode: fi.Mode(),
  181. uid: uid,
  182. gid: gid,
  183. }
  184. if mode.IsDir() {
  185. files, _ := os.ReadDir(fileOrDir)
  186. for _, subFileOrDir := range files {
  187. cleanedDestDirectory := destPath + fi.Name()
  188. if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), cleanedDestDirectory+"/", fileCopyTaskChan); err != nil {
  189. return err
  190. }
  191. }
  192. }
  193. return nil
  194. }
  195. type FileCopyWorker struct {
  196. options *CopyOptions
  197. filerAddress pb.ServerAddress
  198. }
  199. func (worker *FileCopyWorker) copyFiles(fileCopyTaskChan chan FileCopyTask) error {
  200. for task := range fileCopyTaskChan {
  201. if err := worker.doEachCopy(task); err != nil {
  202. return err
  203. }
  204. }
  205. return nil
  206. }
  207. type FileCopyTask struct {
  208. sourceLocation string
  209. destinationUrlPath string
  210. fileSize int64
  211. fileMode os.FileMode
  212. uid uint32
  213. gid uint32
  214. }
  215. func (worker *FileCopyWorker) doEachCopy(task FileCopyTask) error {
  216. f, err := os.Open(task.sourceLocation)
  217. if err != nil {
  218. fmt.Printf("Failed to open file %s: %v\n", task.sourceLocation, err)
  219. if _, ok := err.(*os.PathError); ok {
  220. fmt.Printf("skipping %s\n", task.sourceLocation)
  221. return nil
  222. }
  223. return err
  224. }
  225. defer f.Close()
  226. // this is a regular file
  227. if *worker.options.include != "" {
  228. if ok, _ := filepath.Match(*worker.options.include, filepath.Base(task.sourceLocation)); !ok {
  229. return nil
  230. }
  231. }
  232. if shouldCopy, err := worker.checkExistingFileFirst(task, f); err != nil {
  233. return fmt.Errorf("check existing file: %v", err)
  234. } else if !shouldCopy {
  235. if *worker.options.verbose {
  236. fmt.Printf("skipping copied file: %v\n", f.Name())
  237. }
  238. return nil
  239. }
  240. // find the chunk count
  241. chunkSize := int64(*worker.options.maxMB * 1024 * 1024)
  242. chunkCount := 1
  243. if chunkSize > 0 && task.fileSize > chunkSize {
  244. chunkCount = int(task.fileSize/chunkSize) + 1
  245. }
  246. if chunkCount == 1 {
  247. return worker.uploadFileAsOne(task, f)
  248. }
  249. return worker.uploadFileInChunks(task, f, chunkCount, chunkSize)
  250. }
  251. func (worker *FileCopyWorker) checkExistingFileFirst(task FileCopyTask, f *os.File) (shouldCopy bool, err error) {
  252. shouldCopy = true
  253. if !*worker.options.checkSize {
  254. return
  255. }
  256. fileStat, err := f.Stat()
  257. if err != nil {
  258. shouldCopy = false
  259. return
  260. }
  261. err = pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
  262. request := &filer_pb.LookupDirectoryEntryRequest{
  263. Directory: task.destinationUrlPath,
  264. Name: filepath.Base(f.Name()),
  265. }
  266. resp, lookupErr := client.LookupDirectoryEntry(context.Background(), request)
  267. if lookupErr != nil {
  268. // mostly not found error
  269. return nil
  270. }
  271. if fileStat.Size() == int64(filer.FileSize(resp.Entry)) {
  272. shouldCopy = false
  273. }
  274. return nil
  275. })
  276. return
  277. }
  278. func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) error {
  279. // upload the file content
  280. fileName := filepath.Base(f.Name())
  281. var mimeType string
  282. var chunks []*filer_pb.FileChunk
  283. var assignResult *filer_pb.AssignVolumeResponse
  284. var assignError error
  285. if task.fileMode&os.ModeDir == 0 && task.fileSize > 0 {
  286. mimeType = detectMimeType(f)
  287. data, err := io.ReadAll(f)
  288. if err != nil {
  289. return err
  290. }
  291. err = util.Retry("upload", func() error {
  292. // assign a volume
  293. assignErr := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
  294. request := &filer_pb.AssignVolumeRequest{
  295. Count: 1,
  296. Replication: *worker.options.replication,
  297. Collection: *worker.options.collection,
  298. TtlSec: worker.options.ttlSec,
  299. DiskType: *worker.options.diskType,
  300. Path: task.destinationUrlPath,
  301. }
  302. assignResult, assignError = client.AssignVolume(context.Background(), request)
  303. if assignError != nil {
  304. return fmt.Errorf("assign volume failure %v: %v", request, assignError)
  305. }
  306. if assignResult.Error != "" {
  307. return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
  308. }
  309. if assignResult.Location.Url == "" {
  310. return fmt.Errorf("assign volume failure %v: %v", request, assignResult)
  311. }
  312. return nil
  313. })
  314. if assignErr != nil {
  315. return assignErr
  316. }
  317. // upload data
  318. targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId
  319. uploadOption := &operation.UploadOption{
  320. UploadUrl: targetUrl,
  321. Filename: fileName,
  322. Cipher: worker.options.cipher,
  323. IsInputCompressed: false,
  324. MimeType: mimeType,
  325. PairMap: nil,
  326. Jwt: security.EncodedJwt(assignResult.Auth),
  327. }
  328. uploadResult, err := operation.UploadData(data, uploadOption)
  329. if err != nil {
  330. return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
  331. }
  332. if uploadResult.Error != "" {
  333. return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
  334. }
  335. if *worker.options.verbose {
  336. fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
  337. }
  338. fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName)
  339. chunks = append(chunks, uploadResult.ToPbFileChunk(assignResult.FileId, 0))
  340. return nil
  341. })
  342. if err != nil {
  343. return fmt.Errorf("upload %v: %v\n", fileName, err)
  344. }
  345. }
  346. if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
  347. request := &filer_pb.CreateEntryRequest{
  348. Directory: task.destinationUrlPath,
  349. Entry: &filer_pb.Entry{
  350. Name: fileName,
  351. Attributes: &filer_pb.FuseAttributes{
  352. Crtime: time.Now().Unix(),
  353. Mtime: time.Now().Unix(),
  354. Gid: task.gid,
  355. Uid: task.uid,
  356. FileSize: uint64(task.fileSize),
  357. FileMode: uint32(task.fileMode),
  358. Mime: mimeType,
  359. TtlSec: worker.options.ttlSec,
  360. },
  361. Chunks: chunks,
  362. },
  363. }
  364. if err := filer_pb.CreateEntry(client, request); err != nil {
  365. return fmt.Errorf("update fh: %v", err)
  366. }
  367. return nil
  368. }); err != nil {
  369. return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName, err)
  370. }
  371. return nil
  372. }
  373. func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, chunkCount int, chunkSize int64) error {
  374. fileName := filepath.Base(f.Name())
  375. mimeType := detectMimeType(f)
  376. chunksChan := make(chan *filer_pb.FileChunk, chunkCount)
  377. concurrentChunks := make(chan struct{}, *worker.options.concurrenctChunks)
  378. var wg sync.WaitGroup
  379. var uploadError error
  380. var collection, replication string
  381. fmt.Printf("uploading %s in %d chunks ...\n", fileName, chunkCount)
  382. for i := int64(0); i < int64(chunkCount) && uploadError == nil; i++ {
  383. wg.Add(1)
  384. concurrentChunks <- struct{}{}
  385. go func(i int64) {
  386. defer func() {
  387. wg.Done()
  388. <-concurrentChunks
  389. }()
  390. // assign a volume
  391. var assignResult *filer_pb.AssignVolumeResponse
  392. var assignError error
  393. err := util.Retry("assignVolume", func() error {
  394. return pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
  395. request := &filer_pb.AssignVolumeRequest{
  396. Count: 1,
  397. Replication: *worker.options.replication,
  398. Collection: *worker.options.collection,
  399. TtlSec: worker.options.ttlSec,
  400. DiskType: *worker.options.diskType,
  401. Path: task.destinationUrlPath + fileName,
  402. }
  403. assignResult, assignError = client.AssignVolume(context.Background(), request)
  404. if assignError != nil {
  405. return fmt.Errorf("assign volume failure %v: %v", request, assignError)
  406. }
  407. if assignResult.Error != "" {
  408. return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
  409. }
  410. return nil
  411. })
  412. })
  413. if err != nil {
  414. uploadError = fmt.Errorf("Failed to assign from %v: %v\n", worker.options.masters, err)
  415. return
  416. }
  417. targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId
  418. if collection == "" {
  419. collection = assignResult.Collection
  420. }
  421. if replication == "" {
  422. replication = assignResult.Replication
  423. }
  424. uploadOption := &operation.UploadOption{
  425. UploadUrl: targetUrl,
  426. Filename: fileName + "-" + strconv.FormatInt(i+1, 10),
  427. Cipher: worker.options.cipher,
  428. IsInputCompressed: false,
  429. MimeType: "",
  430. PairMap: nil,
  431. Jwt: security.EncodedJwt(assignResult.Auth),
  432. }
  433. uploadResult, err, _ := operation.Upload(io.NewSectionReader(f, i*chunkSize, chunkSize), uploadOption)
  434. if err != nil {
  435. uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
  436. return
  437. }
  438. if uploadResult.Error != "" {
  439. uploadError = fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
  440. return
  441. }
  442. chunksChan <- uploadResult.ToPbFileChunk(assignResult.FileId, i*chunkSize)
  443. fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
  444. }(i)
  445. }
  446. wg.Wait()
  447. close(chunksChan)
  448. var chunks []*filer_pb.FileChunk
  449. for chunk := range chunksChan {
  450. chunks = append(chunks, chunk)
  451. }
  452. if uploadError != nil {
  453. var fileIds []string
  454. for _, chunk := range chunks {
  455. fileIds = append(fileIds, chunk.FileId)
  456. }
  457. operation.DeleteFiles(func() pb.ServerAddress {
  458. return pb.ServerAddress(copy.masters[0])
  459. }, false, worker.options.grpcDialOption, fileIds)
  460. return uploadError
  461. }
  462. manifestedChunks, manifestErr := filer.MaybeManifestize(worker.saveDataAsChunk, chunks)
  463. if manifestErr != nil {
  464. return fmt.Errorf("create manifest: %v", manifestErr)
  465. }
  466. if err := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
  467. request := &filer_pb.CreateEntryRequest{
  468. Directory: task.destinationUrlPath,
  469. Entry: &filer_pb.Entry{
  470. Name: fileName,
  471. Attributes: &filer_pb.FuseAttributes{
  472. Crtime: time.Now().Unix(),
  473. Mtime: time.Now().Unix(),
  474. Gid: task.gid,
  475. Uid: task.uid,
  476. FileSize: uint64(task.fileSize),
  477. FileMode: uint32(task.fileMode),
  478. Mime: mimeType,
  479. TtlSec: worker.options.ttlSec,
  480. },
  481. Chunks: manifestedChunks,
  482. },
  483. }
  484. if err := filer_pb.CreateEntry(client, request); err != nil {
  485. return fmt.Errorf("update fh: %v", err)
  486. }
  487. return nil
  488. }); err != nil {
  489. return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName, err)
  490. }
  491. fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName)
  492. return nil
  493. }
  494. func detectMimeType(f *os.File) string {
  495. head := make([]byte, 512)
  496. f.Seek(0, io.SeekStart)
  497. n, err := f.Read(head)
  498. if err == io.EOF {
  499. return ""
  500. }
  501. if err != nil {
  502. fmt.Printf("read head of %v: %v\n", f.Name(), err)
  503. return ""
  504. }
  505. f.Seek(0, io.SeekStart)
  506. mimeType := http.DetectContentType(head[:n])
  507. if mimeType == "application/octet-stream" {
  508. return ""
  509. }
  510. return mimeType
  511. }
  512. func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, collection, replication string, err error) {
  513. var fileId, host string
  514. var auth security.EncodedJwt
  515. if flushErr := pb.WithGrpcFilerClient(false, worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
  516. ctx := context.Background()
  517. assignErr := util.Retry("assignVolume", func() error {
  518. request := &filer_pb.AssignVolumeRequest{
  519. Count: 1,
  520. Replication: *worker.options.replication,
  521. Collection: *worker.options.collection,
  522. TtlSec: worker.options.ttlSec,
  523. DiskType: *worker.options.diskType,
  524. Path: name,
  525. }
  526. resp, err := client.AssignVolume(ctx, request)
  527. if err != nil {
  528. return fmt.Errorf("assign volume failure %v: %v", request, err)
  529. }
  530. if resp.Error != "" {
  531. return fmt.Errorf("assign volume failure %v: %v", request, resp.Error)
  532. }
  533. fileId, host, auth = resp.FileId, resp.Location.Url, security.EncodedJwt(resp.Auth)
  534. collection, replication = resp.Collection, resp.Replication
  535. return nil
  536. })
  537. if assignErr != nil {
  538. return assignErr
  539. }
  540. return nil
  541. }); flushErr != nil {
  542. return nil, collection, replication, fmt.Errorf("filerGrpcAddress assign volume: %v", flushErr)
  543. }
  544. uploadOption := &operation.UploadOption{
  545. UploadUrl: fmt.Sprintf("http://%s/%s", host, fileId),
  546. Filename: name,
  547. Cipher: worker.options.cipher,
  548. IsInputCompressed: false,
  549. MimeType: "",
  550. PairMap: nil,
  551. Jwt: auth,
  552. }
  553. uploadResult, flushErr, _ := operation.Upload(reader, uploadOption)
  554. if flushErr != nil {
  555. return nil, collection, replication, fmt.Errorf("upload data: %v", flushErr)
  556. }
  557. if uploadResult.Error != "" {
  558. return nil, collection, replication, fmt.Errorf("upload result: %v", uploadResult.Error)
  559. }
  560. return uploadResult.ToPbFileChunk(fileId, offset), collection, replication, nil
  561. }