You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

170 lines
4.5 KiB

12 years ago
12 years ago
12 years ago
12 years ago
11 years ago
12 years ago
12 years ago
  1. package main
  2. import (
  3. "archive/tar"
  4. "bytes"
  5. "fmt"
  6. "os"
  7. "path"
  8. "strconv"
  9. "strings"
  10. "text/template"
  11. "time"
  12. "github.com/chrislusf/weed-fs/go/glog"
  13. "github.com/chrislusf/weed-fs/go/storage"
  14. )
  15. func init() {
  16. cmdExport.Run = runExport // break init cycle
  17. }
  18. const (
  19. defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}`
  20. )
  21. var cmdExport = &Command{
  22. UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}}",
  23. Short: "list or export files from one volume data file",
  24. Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified.
  25. The format of file name in the tar file can be customized. Default is {{.Mime}}/{{.Id}}:{{.Name}}. Also available is {{.Key}}.
  26. `,
  27. }
  28. var (
  29. exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "input data directory to store volume data files")
  30. exportCollection = cmdExport.Flag.String("collection", "", "the volume collection name")
  31. exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
  32. dest = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
  33. format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}")
  34. tarFh *tar.Writer
  35. tarHeader tar.Header
  36. fnTmpl *template.Template
  37. fnTmplBuf = bytes.NewBuffer(nil)
  38. )
  39. func runExport(cmd *Command, args []string) bool {
  40. if *exportVolumeId == -1 {
  41. return false
  42. }
  43. var err error
  44. if *dest != "" {
  45. if *dest != "-" && !strings.HasSuffix(*dest, ".tar") {
  46. fmt.Println("the output file", *dest, "should be '-' or end with .tar")
  47. return false
  48. }
  49. if fnTmpl, err = template.New("name").Parse(*format); err != nil {
  50. fmt.Println("cannot parse format " + *format + ": " + err.Error())
  51. return false
  52. }
  53. var fh *os.File
  54. if *dest == "-" {
  55. fh = os.Stdout
  56. } else {
  57. if fh, err = os.Create(*dest); err != nil {
  58. glog.Fatalf("cannot open output tar %s: %s", *dest, err)
  59. }
  60. }
  61. defer fh.Close()
  62. tarFh = tar.NewWriter(fh)
  63. defer tarFh.Close()
  64. t := time.Now()
  65. tarHeader = tar.Header{Mode: 0644,
  66. ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
  67. Typeflag: tar.TypeReg,
  68. AccessTime: t, ChangeTime: t}
  69. }
  70. fileName := strconv.Itoa(*exportVolumeId)
  71. if *exportCollection != "" {
  72. fileName = *exportCollection + "_" + fileName
  73. }
  74. vid := storage.VolumeId(*exportVolumeId)
  75. indexFile, err := os.OpenFile(path.Join(*exportVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
  76. if err != nil {
  77. glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
  78. }
  79. defer indexFile.Close()
  80. nm, err := storage.LoadNeedleMap(indexFile)
  81. if err != nil {
  82. glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
  83. }
  84. var version storage.Version
  85. err = storage.ScanVolumeFile(*exportVolumePath, *exportCollection, vid, func(superBlock storage.SuperBlock) error {
  86. version = superBlock.Version()
  87. return nil
  88. }, true, func(n *storage.Needle, offset int64) error {
  89. nv, ok := nm.Get(n.Id)
  90. glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
  91. n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv)
  92. if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset {
  93. return walker(vid, n, version)
  94. }
  95. if !ok {
  96. glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
  97. } else {
  98. glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
  99. }
  100. return nil
  101. })
  102. if err != nil {
  103. glog.Fatalf("Export Volume File [ERROR] %s\n", err)
  104. }
  105. return true
  106. }
  107. type nameParams struct {
  108. Name string
  109. Id uint64
  110. Mime string
  111. Key string
  112. }
  113. func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) {
  114. key := storage.NewFileIdFromNeedle(vid, n).String()
  115. if tarFh != nil {
  116. fnTmplBuf.Reset()
  117. if err = fnTmpl.Execute(fnTmplBuf,
  118. nameParams{Name: string(n.Name),
  119. Id: n.Id,
  120. Mime: string(n.Mime),
  121. Key: key,
  122. },
  123. ); err != nil {
  124. return err
  125. }
  126. nm := fnTmplBuf.String()
  127. if n.IsGzipped() && path.Ext(nm) != ".gz" {
  128. nm = nm + ".gz"
  129. }
  130. tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
  131. if err = tarFh.WriteHeader(&tarHeader); err != nil {
  132. return err
  133. }
  134. _, err = tarFh.Write(n.Data)
  135. } else {
  136. size := n.DataSize
  137. if version == storage.Version1 {
  138. size = n.Size
  139. }
  140. fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
  141. key,
  142. n.Name,
  143. size,
  144. n.IsGzipped(),
  145. n.Mime,
  146. )
  147. }
  148. return
  149. }