214 lines
6.0 KiB

12 years ago
12 years ago
12 years ago
11 years ago
12 years ago
12 years ago
  1. package command
  2. import (
  3. "archive/tar"
  4. "bytes"
  5. "fmt"
  6. "os"
  7. "path"
  8. "path/filepath"
  9. "strconv"
  10. "strings"
  11. "text/template"
  12. "time"
  13. "github.com/chrislusf/seaweedfs/weed/glog"
  14. "github.com/chrislusf/seaweedfs/weed/storage"
  15. "github.com/chrislusf/seaweedfs/weed/storage/types"
  16. )
  17. const (
  18. defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}`
  19. timeFormat = "2006-01-02T15:04:05"
  20. )
  21. var (
  22. export ExportOptions
  23. )
  24. type ExportOptions struct {
  25. dir *string
  26. collection *string
  27. volumeId *int
  28. }
  29. var cmdExport = &Command{
  30. UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}} -newer='" + timeFormat + "'",
  31. Short: "list or export files from one volume data file",
  32. Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified.
  33. The format of file name in the tar file can be customized. Default is {{.Mime}}/{{.Id}}:{{.Name}}. Also available is {{.Key}}.
  34. `,
  35. }
  36. func init() {
  37. cmdExport.Run = runExport // break init cycle
  38. export.dir = cmdExport.Flag.String("dir", ".", "input data directory to store volume data files")
  39. export.collection = cmdExport.Flag.String("collection", "", "the volume collection name")
  40. export.volumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
  41. }
  42. var (
  43. output = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
  44. format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename formatted with {{.Mime}} {{.Id}} {{.Name}} {{.Ext}}")
  45. newer = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone, e.g. 2006-01-02T15:04:05")
  46. tarOutputFile *tar.Writer
  47. tarHeader tar.Header
  48. fileNameTemplate *template.Template
  49. fileNameTemplateBuffer = bytes.NewBuffer(nil)
  50. newerThan time.Time
  51. newerThanUnix int64 = -1
  52. localLocation, _ = time.LoadLocation("Local")
  53. )
  54. func runExport(cmd *Command, args []string) bool {
  55. var err error
  56. if *newer != "" {
  57. if newerThan, err = time.ParseInLocation(timeFormat, *newer, localLocation); err != nil {
  58. fmt.Println("cannot parse 'newer' argument: " + err.Error())
  59. return false
  60. }
  61. newerThanUnix = newerThan.Unix()
  62. }
  63. if *export.volumeId == -1 {
  64. return false
  65. }
  66. if *output != "" {
  67. if *output != "-" && !strings.HasSuffix(*output, ".tar") {
  68. fmt.Println("the output file", *output, "should be '-' or end with .tar")
  69. return false
  70. }
  71. if fileNameTemplate, err = template.New("name").Parse(*format); err != nil {
  72. fmt.Println("cannot parse format " + *format + ": " + err.Error())
  73. return false
  74. }
  75. var outputFile *os.File
  76. if *output == "-" {
  77. outputFile = os.Stdout
  78. } else {
  79. if outputFile, err = os.Create(*output); err != nil {
  80. glog.Fatalf("cannot open output tar %s: %s", *output, err)
  81. }
  82. }
  83. defer outputFile.Close()
  84. tarOutputFile = tar.NewWriter(outputFile)
  85. defer tarOutputFile.Close()
  86. t := time.Now()
  87. tarHeader = tar.Header{Mode: 0644,
  88. ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
  89. Typeflag: tar.TypeReg,
  90. AccessTime: t, ChangeTime: t}
  91. }
  92. fileName := strconv.Itoa(*export.volumeId)
  93. if *export.collection != "" {
  94. fileName = *export.collection + "_" + fileName
  95. }
  96. vid := storage.VolumeId(*export.volumeId)
  97. indexFile, err := os.OpenFile(path.Join(*export.dir, fileName+".idx"), os.O_RDONLY, 0644)
  98. if err != nil {
  99. glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
  100. }
  101. defer indexFile.Close()
  102. needleMap, err := storage.LoadBtreeNeedleMap(indexFile)
  103. if err != nil {
  104. glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
  105. }
  106. var version storage.Version
  107. err = storage.ScanVolumeFile(*export.dir, *export.collection, vid,
  108. storage.NeedleMapInMemory,
  109. func(superBlock storage.SuperBlock) error {
  110. version = superBlock.Version()
  111. return nil
  112. }, true, func(n *storage.Needle, offset int64) error {
  113. nv, ok := needleMap.Get(n.Id)
  114. glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
  115. n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv)
  116. if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset {
  117. if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
  118. glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
  119. n.LastModified, newerThanUnix)
  120. return nil
  121. }
  122. return walker(vid, n, version)
  123. }
  124. if !ok {
  125. glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
  126. } else {
  127. glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
  128. }
  129. return nil
  130. })
  131. if err != nil {
  132. glog.Fatalf("Export Volume File [ERROR] %s\n", err)
  133. }
  134. return true
  135. }
  136. type nameParams struct {
  137. Name string
  138. Id types.NeedleId
  139. Mime string
  140. Key string
  141. Ext string
  142. }
  143. func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) {
  144. key := storage.NewFileIdFromNeedle(vid, n).String()
  145. if tarOutputFile != nil {
  146. fileNameTemplateBuffer.Reset()
  147. if err = fileNameTemplate.Execute(fileNameTemplateBuffer,
  148. nameParams{
  149. Name: string(n.Name),
  150. Id: n.Id,
  151. Mime: string(n.Mime),
  152. Key: key,
  153. Ext: filepath.Ext(string(n.Name)),
  154. },
  155. ); err != nil {
  156. return err
  157. }
  158. fileName := fileNameTemplateBuffer.String()
  159. if n.IsGzipped() && path.Ext(fileName) != ".gz" {
  160. fileName = fileName + ".gz"
  161. }
  162. tarHeader.Name, tarHeader.Size = fileName, int64(len(n.Data))
  163. if n.HasLastModifiedDate() {
  164. tarHeader.ModTime = time.Unix(int64(n.LastModified), 0)
  165. } else {
  166. tarHeader.ModTime = time.Unix(0, 0)
  167. }
  168. tarHeader.ChangeTime = tarHeader.ModTime
  169. if err = tarOutputFile.WriteHeader(&tarHeader); err != nil {
  170. return err
  171. }
  172. _, err = tarOutputFile.Write(n.Data)
  173. } else {
  174. size := n.DataSize
  175. if version == storage.Version1 {
  176. size = n.Size
  177. }
  178. fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
  179. key,
  180. n.Name,
  181. size,
  182. n.IsGzipped(),
  183. n.Mime,
  184. )
  185. }
  186. return
  187. }