You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

166 lines
4.2 KiB

12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
12 years ago
  1. package main
  2. import (
  3. "archive/tar"
  4. "bytes"
  5. "code.google.com/p/weed-fs/go/glog"
  6. "code.google.com/p/weed-fs/go/storage"
  7. "fmt"
  8. "os"
  9. "path"
  10. "strconv"
  11. "strings"
  12. "text/template"
  13. "time"
  14. )
  15. func init() {
  16. cmdExport.Run = runExport // break init cycle
  17. cmdExport.IsDebug = cmdExport.Flag.Bool("debug", false, "enable debug mode")
  18. }
  19. const (
  20. defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}`
  21. )
  22. var cmdExport = &Command{
  23. UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}}",
  24. Short: "list or export files from one volume data file",
  25. Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified.
  26. The format of file name in the tar file can be customized. Default is {{.Mime}}/{{.Id}}:{{.Name}}. Also available is {{.Key}}.
  27. `,
  28. }
  29. var (
  30. exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "input data directory to store volume data files")
  31. exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
  32. dest = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
  33. format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}")
  34. tarFh *tar.Writer
  35. tarHeader tar.Header
  36. fnTmpl *template.Template
  37. fnTmplBuf = bytes.NewBuffer(nil)
  38. )
  39. func runExport(cmd *Command, args []string) bool {
  40. if *exportVolumeId == -1 {
  41. return false
  42. }
  43. var err error
  44. if *dest != "" {
  45. if *dest != "-" && !strings.HasSuffix(*dest, ".tar") {
  46. fmt.Println("the output file", *dest, "should be '-' or end with .tar")
  47. return false
  48. }
  49. if fnTmpl, err = template.New("name").Parse(*format); err != nil {
  50. fmt.Println("cannot parse format " + *format + ": " + err.Error())
  51. return false
  52. }
  53. var fh *os.File
  54. if *dest == "-" {
  55. fh = os.Stdout
  56. } else {
  57. if fh, err = os.Create(*dest); err != nil {
  58. glog.Fatalf("cannot open output tar %s: %s", *dest, err)
  59. }
  60. }
  61. defer fh.Close()
  62. tarFh = tar.NewWriter(fh)
  63. defer tarFh.Close()
  64. t := time.Now()
  65. tarHeader = tar.Header{Mode: 0644,
  66. ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
  67. Typeflag: tar.TypeReg,
  68. AccessTime: t, ChangeTime: t}
  69. }
  70. fileName := strconv.Itoa(*exportVolumeId)
  71. vid := storage.VolumeId(*exportVolumeId)
  72. indexFile, err := os.OpenFile(path.Join(*exportVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
  73. if err != nil {
  74. glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
  75. }
  76. defer indexFile.Close()
  77. nm, err := storage.LoadNeedleMap(indexFile)
  78. if err != nil {
  79. glog.Fatalf("cannot load needle map from %s: %s", indexFile, err)
  80. }
  81. var version storage.Version
  82. err = storage.ScanVolumeFile(*exportVolumePath, vid, func(superBlock storage.SuperBlock) error {
  83. version = superBlock.Version
  84. return nil
  85. }, func(n *storage.Needle, offset int64) error {
  86. debug("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped())
  87. nv, ok := nm.Get(n.Id)
  88. if ok && nv.Size > 0 {
  89. return walker(vid, n, version)
  90. } else {
  91. if !ok {
  92. debug("This seems deleted", n.Id)
  93. } else {
  94. debug("Id", n.Id, "size", n.Size)
  95. }
  96. }
  97. return nil
  98. })
  99. if err != nil {
  100. glog.Fatalf("Export Volume File [ERROR] %s\n", err)
  101. }
  102. return true
  103. }
  104. type nameParams struct {
  105. Name string
  106. Id uint64
  107. Mime string
  108. Key string
  109. }
  110. func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) {
  111. key := storage.NewFileId(vid, n.Id, n.Cookie).String()
  112. if tarFh != nil {
  113. fnTmplBuf.Reset()
  114. if err = fnTmpl.Execute(fnTmplBuf,
  115. nameParams{Name: string(n.Name),
  116. Id: n.Id,
  117. Mime: string(n.Mime),
  118. Key: key,
  119. },
  120. ); err != nil {
  121. return err
  122. }
  123. nm := fnTmplBuf.String()
  124. if n.IsGzipped() && path.Ext(nm) != ".gz" {
  125. nm = nm + ".gz"
  126. }
  127. tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
  128. if err = tarFh.WriteHeader(&tarHeader); err != nil {
  129. return err
  130. }
  131. _, err = tarFh.Write(n.Data)
  132. } else {
  133. size := n.DataSize
  134. if version == storage.Version1 {
  135. size = n.Size
  136. }
  137. fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
  138. key,
  139. n.Name,
  140. size,
  141. n.IsGzipped(),
  142. n.Mime,
  143. )
  144. }
  145. return
  146. }