You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

170 lines
4.5 KiB

12 years ago
12 years ago
12 years ago
12 years ago
11 years ago
11 years ago
11 years ago
12 years ago
12 years ago
  1. package main
  2. import (
  3. "archive/tar"
  4. "bytes"
  5. "code.google.com/p/weed-fs/go/glog"
  6. "code.google.com/p/weed-fs/go/storage"
  7. "fmt"
  8. "os"
  9. "path"
  10. "strconv"
  11. "strings"
  12. "text/template"
  13. "time"
  14. )
  15. func init() {
  16. cmdExport.Run = runExport // break init cycle
  17. cmdExport.IsDebug = cmdExport.Flag.Bool("debug", false, "enable debug mode")
  18. }
  19. const (
  20. defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}`
  21. )
  22. var cmdExport = &Command{
  23. UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}}",
  24. Short: "list or export files from one volume data file",
  25. Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified.
  26. The format of file name in the tar file can be customized. Default is {{.Mime}}/{{.Id}}:{{.Name}}. Also available is {{.Key}}.
  27. `,
  28. }
  29. var (
  30. exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "input data directory to store volume data files")
  31. exportCollection = cmdExport.Flag.String("collection", "", "the volume collection name")
  32. exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
  33. dest = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
  34. format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}")
  35. tarFh *tar.Writer
  36. tarHeader tar.Header
  37. fnTmpl *template.Template
  38. fnTmplBuf = bytes.NewBuffer(nil)
  39. )
  40. func runExport(cmd *Command, args []string) bool {
  41. if *exportVolumeId == -1 {
  42. return false
  43. }
  44. var err error
  45. if *dest != "" {
  46. if *dest != "-" && !strings.HasSuffix(*dest, ".tar") {
  47. fmt.Println("the output file", *dest, "should be '-' or end with .tar")
  48. return false
  49. }
  50. if fnTmpl, err = template.New("name").Parse(*format); err != nil {
  51. fmt.Println("cannot parse format " + *format + ": " + err.Error())
  52. return false
  53. }
  54. var fh *os.File
  55. if *dest == "-" {
  56. fh = os.Stdout
  57. } else {
  58. if fh, err = os.Create(*dest); err != nil {
  59. glog.Fatalf("cannot open output tar %s: %s", *dest, err)
  60. }
  61. }
  62. defer fh.Close()
  63. tarFh = tar.NewWriter(fh)
  64. defer tarFh.Close()
  65. t := time.Now()
  66. tarHeader = tar.Header{Mode: 0644,
  67. ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
  68. Typeflag: tar.TypeReg,
  69. AccessTime: t, ChangeTime: t}
  70. }
  71. fileName := strconv.Itoa(*exportVolumeId)
  72. if *exportCollection != "" {
  73. fileName = *exportCollection + "_" + fileName
  74. }
  75. vid := storage.VolumeId(*exportVolumeId)
  76. indexFile, err := os.OpenFile(path.Join(*exportVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
  77. if err != nil {
  78. glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
  79. }
  80. defer indexFile.Close()
  81. nm, err := storage.LoadNeedleMap(indexFile)
  82. if err != nil {
  83. glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
  84. }
  85. var version storage.Version
  86. err = storage.ScanVolumeFile(*exportVolumePath, *exportCollection, vid, func(superBlock storage.SuperBlock) error {
  87. version = superBlock.Version
  88. return nil
  89. }, true, func(n *storage.Needle, offset int64) error {
  90. nv, ok := nm.Get(n.Id)
  91. glog.V(3).Infoln("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped(), "ok", ok, "nv", nv)
  92. if ok && nv.Size > 0 {
  93. return walker(vid, n, version)
  94. } else {
  95. if !ok {
  96. debug("This seems deleted", n.Id, "size", n.Size)
  97. } else {
  98. debug("Id", n.Id, "size", n.Size)
  99. }
  100. }
  101. return nil
  102. })
  103. if err != nil {
  104. glog.Fatalf("Export Volume File [ERROR] %s\n", err)
  105. }
  106. return true
  107. }
  108. type nameParams struct {
  109. Name string
  110. Id uint64
  111. Mime string
  112. Key string
  113. }
  114. func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) {
  115. key := storage.NewFileIdFromNeedle(vid, n).String()
  116. if tarFh != nil {
  117. fnTmplBuf.Reset()
  118. if err = fnTmpl.Execute(fnTmplBuf,
  119. nameParams{Name: string(n.Name),
  120. Id: n.Id,
  121. Mime: string(n.Mime),
  122. Key: key,
  123. },
  124. ); err != nil {
  125. return err
  126. }
  127. nm := fnTmplBuf.String()
  128. if n.IsGzipped() && path.Ext(nm) != ".gz" {
  129. nm = nm + ".gz"
  130. }
  131. tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
  132. if err = tarFh.WriteHeader(&tarHeader); err != nil {
  133. return err
  134. }
  135. _, err = tarFh.Write(n.Data)
  136. } else {
  137. size := n.DataSize
  138. if version == storage.Version1 {
  139. size = n.Size
  140. }
  141. fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
  142. key,
  143. n.Name,
  144. size,
  145. n.IsGzipped(),
  146. n.Mime,
  147. )
  148. }
  149. return
  150. }