You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

197 lines
5.5 KiB

12 years ago
12 years ago
12 years ago
12 years ago
11 years ago
12 years ago
12 years ago
  1. package main
  2. import (
  3. "archive/tar"
  4. "bytes"
  5. "fmt"
  6. "os"
  7. "path"
  8. "strconv"
  9. "strings"
  10. "text/template"
  11. "time"
  12. "github.com/chrislusf/seaweedfs/go/glog"
  13. "github.com/chrislusf/seaweedfs/go/storage"
  14. )
  15. func init() {
  16. cmdExport.Run = runExport // break init cycle
  17. }
  18. const (
  19. defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}`
  20. timeFormat = "2006-01-02T15:04:05"
  21. )
  22. var cmdExport = &Command{
  23. UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}} -newer='"+timeFormat+"'",
  24. Short: "list or export files from one volume data file",
  25. Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified.
  26. The format of file name in the tar file can be customized. Default is {{.Mime}}/{{.Id}}:{{.Name}}. Also available is {{.Key}}.
  27. `,
  28. }
  29. var (
  30. exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "input data directory to store volume data files")
  31. exportCollection = cmdExport.Flag.String("collection", "", "the volume collection name")
  32. exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
  33. dest = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
  34. format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}")
  35. newer = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone")
  36. tarFh *tar.Writer
  37. tarHeader tar.Header
  38. fnTmpl *template.Template
  39. fnTmplBuf = bytes.NewBuffer(nil)
  40. newerThan time.Time
  41. newerThanUnix int64 = -1
  42. localLocation, _ = time.LoadLocation("Local")
  43. )
  44. func runExport(cmd *Command, args []string) bool {
  45. var err error
  46. if *newer != "" {
  47. if newerThan, err = time.ParseInLocation(timeFormat, *newer, localLocation); err != nil {
  48. fmt.Println("cannot parse 'newer' argument: " + err.Error())
  49. return false
  50. }
  51. newerThanUnix = newerThan.Unix()
  52. }
  53. if *exportVolumeId == -1 {
  54. return false
  55. }
  56. if *dest != "" {
  57. if *dest != "-" && !strings.HasSuffix(*dest, ".tar") {
  58. fmt.Println("the output file", *dest, "should be '-' or end with .tar")
  59. return false
  60. }
  61. if fnTmpl, err = template.New("name").Parse(*format); err != nil {
  62. fmt.Println("cannot parse format " + *format + ": " + err.Error())
  63. return false
  64. }
  65. var fh *os.File
  66. if *dest == "-" {
  67. fh = os.Stdout
  68. } else {
  69. if fh, err = os.Create(*dest); err != nil {
  70. glog.Fatalf("cannot open output tar %s: %s", *dest, err)
  71. }
  72. }
  73. defer fh.Close()
  74. tarFh = tar.NewWriter(fh)
  75. defer tarFh.Close()
  76. t := time.Now()
  77. tarHeader = tar.Header{Mode: 0644,
  78. ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
  79. Typeflag: tar.TypeReg,
  80. AccessTime: t, ChangeTime: t}
  81. }
  82. fileName := strconv.Itoa(*exportVolumeId)
  83. if *exportCollection != "" {
  84. fileName = *exportCollection + "_" + fileName
  85. }
  86. vid := storage.VolumeId(*exportVolumeId)
  87. indexFile, err := os.OpenFile(path.Join(*exportVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
  88. if err != nil {
  89. glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
  90. }
  91. defer indexFile.Close()
  92. nm, err := storage.LoadNeedleMap(indexFile)
  93. if err != nil {
  94. glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
  95. }
  96. var version storage.Version
  97. err = storage.ScanVolumeFile(*exportVolumePath, *exportCollection, vid,
  98. storage.NeedleMapInMemory,
  99. func(superBlock storage.SuperBlock) error {
  100. version = superBlock.Version()
  101. return nil
  102. }, true, func(n *storage.Needle, offset int64) error {
  103. nv, ok := nm.Get(n.Id)
  104. glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
  105. n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv)
  106. if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset {
  107. if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
  108. glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
  109. n.LastModified, newerThanUnix)
  110. return nil
  111. }
  112. return walker(vid, n, version)
  113. }
  114. if !ok {
  115. glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
  116. } else {
  117. glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
  118. }
  119. return nil
  120. })
  121. if err != nil {
  122. glog.Fatalf("Export Volume File [ERROR] %s\n", err)
  123. }
  124. return true
  125. }
  126. type nameParams struct {
  127. Name string
  128. Id uint64
  129. Mime string
  130. Key string
  131. }
  132. func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) {
  133. key := storage.NewFileIdFromNeedle(vid, n).String()
  134. if tarFh != nil {
  135. fnTmplBuf.Reset()
  136. if err = fnTmpl.Execute(fnTmplBuf,
  137. nameParams{Name: string(n.Name),
  138. Id: n.Id,
  139. Mime: string(n.Mime),
  140. Key: key,
  141. },
  142. ); err != nil {
  143. return err
  144. }
  145. nm := fnTmplBuf.String()
  146. if n.IsGzipped() && path.Ext(nm) != ".gz" {
  147. nm = nm + ".gz"
  148. }
  149. tarHeader.Name, tarHeader.Size = nm, int64(len(n.Data))
  150. if n.HasLastModifiedDate() {
  151. tarHeader.ModTime = time.Unix(int64(n.LastModified), 0)
  152. } else {
  153. tarHeader.ModTime = time.Unix(0,0)
  154. }
  155. tarHeader.ChangeTime = tarHeader.ModTime
  156. if err = tarFh.WriteHeader(&tarHeader); err != nil {
  157. return err
  158. }
  159. _, err = tarFh.Write(n.Data)
  160. } else {
  161. size := n.DataSize
  162. if version == storage.Version1 {
  163. size = n.Size
  164. }
  165. fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
  166. key,
  167. n.Name,
  168. size,
  169. n.IsGzipped(),
  170. n.Mime,
  171. )
  172. }
  173. return
  174. }