You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

147 lines
4.3 KiB

  1. package command
  2. import (
  3. "fmt"
  4. "io/ioutil"
  5. "net/url"
  6. "os"
  7. "path/filepath"
  8. "strings"
  9. "github.com/chrislusf/seaweedfs/weed/operation"
  10. filer_operation "github.com/chrislusf/seaweedfs/weed/operation/filer"
  11. "github.com/chrislusf/seaweedfs/weed/security"
  12. )
  13. var (
  14. copy CopyOptions
  15. )
  16. type CopyOptions struct {
  17. master *string
  18. include *string
  19. replication *string
  20. collection *string
  21. ttl *string
  22. maxMB *int
  23. secretKey *string
  24. secret security.Secret
  25. }
  26. func init() {
  27. cmdCopy.Run = runCopy // break init cycle
  28. cmdCopy.IsDebug = cmdCopy.Flag.Bool("debug", false, "verbose debug information")
  29. copy.master = cmdCopy.Flag.String("master", "localhost:9333", "SeaweedFS master location")
  30. copy.include = cmdCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
  31. copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
  32. copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
  33. copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
  34. copy.maxMB = cmdCopy.Flag.Int("maxMB", 0, "split files larger than the limit")
  35. copy.secretKey = cmdCopy.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
  36. }
  37. var cmdCopy = &Command{
  38. UsageLine: "filer.copy file_or_dir1 [file_or_dir2 file_or_dir3] http://localhost:8888/path/to/a/folder/",
  39. Short: "copy one or a list of files to a filer folder",
  40. Long: `copy one or a list of files, or batch copy one whole folder recursively, to a filer folder
  41. It can copy one or a list of files or folders.
  42. If copying a whole folder recursively:
  43. All files under the folder and subfolders will be copyed.
  44. Optional parameter "-include" allows you to specify the file name patterns.
  45. If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
  46. This can save volume server's gzipped processing and allow customizable gzip compression level.
  47. The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
  48. If "maxMB" is set to a positive number, files larger than it would be split into chunks and copyed separatedly.
  49. The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
  50. `,
  51. }
  52. func runCopy(cmd *Command, args []string) bool {
  53. copy.secret = security.Secret(*copy.secretKey)
  54. if len(args) <= 1 {
  55. return false
  56. }
  57. filerDestination := args[len(args)-1]
  58. fileOrDirs := args[0 : len(args)-1]
  59. filerUrl, err := url.Parse(filerDestination)
  60. if err != nil {
  61. fmt.Printf("The last argument should be a URL on filer: %v\n", err)
  62. return false
  63. }
  64. path := filerUrl.Path
  65. if !strings.HasSuffix(path, "/") {
  66. path = path + "/"
  67. }
  68. for _, fileOrDir := range fileOrDirs {
  69. if !doEachCopy(fileOrDir, filerUrl.Host, path) {
  70. return false
  71. }
  72. }
  73. return true
  74. }
  75. func doEachCopy(fileOrDir string, host string, path string) bool {
  76. f, err := os.Open(fileOrDir)
  77. if err != nil {
  78. fmt.Printf("Failed to open file %s: %v", fileOrDir, err)
  79. return false
  80. }
  81. defer f.Close()
  82. fi, err := f.Stat()
  83. if err != nil {
  84. fmt.Printf("Failed to get stat for file %s: %v", fileOrDir, err)
  85. return false
  86. }
  87. mode := fi.Mode()
  88. if mode.IsDir() {
  89. files, _ := ioutil.ReadDir(fileOrDir)
  90. for _, subFileOrDir := range files {
  91. if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), host, path+fi.Name()+"/") {
  92. return false
  93. }
  94. }
  95. return true
  96. }
  97. // this is a regular file
  98. if *copy.include != "" {
  99. if ok, _ := filepath.Match(*copy.include, filepath.Base(fileOrDir)); !ok {
  100. return true
  101. }
  102. }
  103. parts, err := operation.NewFileParts([]string{fileOrDir})
  104. if err != nil {
  105. fmt.Printf("Failed to read file %s: %v", fileOrDir, err)
  106. }
  107. results, err := operation.SubmitFiles(*copy.master, parts,
  108. *copy.replication, *copy.collection,
  109. *copy.ttl, *copy.maxMB, copy.secret)
  110. if err != nil {
  111. fmt.Printf("Failed to submit file %s: %v", fileOrDir, err)
  112. }
  113. if strings.HasSuffix(path, "/") {
  114. path = path + fi.Name()
  115. }
  116. if err = filer_operation.RegisterFile(host, path, results[0].Fid, copy.secret); err != nil {
  117. fmt.Printf("Failed to register file %s on %s: %v", fileOrDir, host, err)
  118. return false
  119. }
  120. fmt.Printf("Copy %s => http://%s%s\n", fileOrDir, host, path)
  121. return true
  122. }