You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

209 lines
5.8 KiB

12 years ago
12 years ago
12 years ago
11 years ago
12 years ago
12 years ago
  1. package main
  2. import (
  3. "archive/tar"
  4. "bytes"
  5. "fmt"
  6. "os"
  7. "path"
  8. "strconv"
  9. "strings"
  10. "text/template"
  11. "time"
  12. "github.com/chrislusf/seaweedfs/go/glog"
  13. "github.com/chrislusf/seaweedfs/go/storage"
  14. )
  15. const (
  16. defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}`
  17. timeFormat = "2006-01-02T15:04:05"
  18. )
  19. var (
  20. export ExportOptions
  21. )
  22. type ExportOptions struct {
  23. dir *string
  24. collection *string
  25. volumeId *int
  26. }
  27. var cmdExport = &Command{
  28. UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}} -newer='" + timeFormat + "'",
  29. Short: "list or export files from one volume data file",
  30. Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified.
  31. The format of file name in the tar file can be customized. Default is {{.Mime}}/{{.Id}}:{{.Name}}. Also available is {{.Key}}.
  32. `,
  33. }
  34. func init() {
  35. cmdExport.Run = runExport // break init cycle
  36. export.dir = cmdExport.Flag.String("dir", ".", "input data directory to store volume data files")
  37. export.collection = cmdExport.Flag.String("collection", "", "the volume collection name")
  38. export.volumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
  39. }
  40. var (
  41. output = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
  42. format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}")
  43. newer = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone")
  44. tarOutputFile *tar.Writer
  45. tarHeader tar.Header
  46. fileNameTemplate *template.Template
  47. fileNameTemplateBuffer = bytes.NewBuffer(nil)
  48. newerThan time.Time
  49. newerThanUnix int64 = -1
  50. localLocation, _ = time.LoadLocation("Local")
  51. )
  52. func runExport(cmd *Command, args []string) bool {
  53. var err error
  54. if *newer != "" {
  55. if newerThan, err = time.ParseInLocation(timeFormat, *newer, localLocation); err != nil {
  56. fmt.Println("cannot parse 'newer' argument: " + err.Error())
  57. return false
  58. }
  59. newerThanUnix = newerThan.Unix()
  60. }
  61. if *export.volumeId == -1 {
  62. return false
  63. }
  64. if *output != "" {
  65. if *output != "-" && !strings.HasSuffix(*output, ".tar") {
  66. fmt.Println("the output file", *output, "should be '-' or end with .tar")
  67. return false
  68. }
  69. if fileNameTemplate, err = template.New("name").Parse(*format); err != nil {
  70. fmt.Println("cannot parse format " + *format + ": " + err.Error())
  71. return false
  72. }
  73. var outputFile *os.File
  74. if *output == "-" {
  75. outputFile = os.Stdout
  76. } else {
  77. if outputFile, err = os.Create(*output); err != nil {
  78. glog.Fatalf("cannot open output tar %s: %s", *output, err)
  79. }
  80. }
  81. defer outputFile.Close()
  82. tarOutputFile = tar.NewWriter(outputFile)
  83. defer tarOutputFile.Close()
  84. t := time.Now()
  85. tarHeader = tar.Header{Mode: 0644,
  86. ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(),
  87. Typeflag: tar.TypeReg,
  88. AccessTime: t, ChangeTime: t}
  89. }
  90. fileName := strconv.Itoa(*export.volumeId)
  91. if *export.collection != "" {
  92. fileName = *export.collection + "_" + fileName
  93. }
  94. vid := storage.VolumeId(*export.volumeId)
  95. indexFile, err := os.OpenFile(path.Join(*export.dir, fileName+".idx"), os.O_RDONLY, 0644)
  96. if err != nil {
  97. glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
  98. }
  99. defer indexFile.Close()
  100. needleMap, err := storage.LoadNeedleMap(indexFile)
  101. if err != nil {
  102. glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
  103. }
  104. var version storage.Version
  105. err = storage.ScanVolumeFile(*export.dir, *export.collection, vid,
  106. storage.NeedleMapInMemory,
  107. func(superBlock storage.SuperBlock) error {
  108. version = superBlock.Version()
  109. return nil
  110. }, true, func(n *storage.Needle, offset int64) error {
  111. nv, ok := needleMap.Get(n.Id)
  112. glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
  113. n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv)
  114. if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset {
  115. if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
  116. glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
  117. n.LastModified, newerThanUnix)
  118. return nil
  119. }
  120. return walker(vid, n, version)
  121. }
  122. if !ok {
  123. glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
  124. } else {
  125. glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
  126. }
  127. return nil
  128. })
  129. if err != nil {
  130. glog.Fatalf("Export Volume File [ERROR] %s\n", err)
  131. }
  132. return true
  133. }
  134. type nameParams struct {
  135. Name string
  136. Id uint64
  137. Mime string
  138. Key string
  139. }
  140. func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) {
  141. key := storage.NewFileIdFromNeedle(vid, n).String()
  142. if tarOutputFile != nil {
  143. fileNameTemplateBuffer.Reset()
  144. if err = fileNameTemplate.Execute(fileNameTemplateBuffer,
  145. nameParams{Name: string(n.Name),
  146. Id: n.Id,
  147. Mime: string(n.Mime),
  148. Key: key,
  149. },
  150. ); err != nil {
  151. return err
  152. }
  153. fileName := fileNameTemplateBuffer.String()
  154. if n.IsGzipped() && path.Ext(fileName) != ".gz" {
  155. fileName = fileName + ".gz"
  156. }
  157. tarHeader.Name, tarHeader.Size = fileName, int64(len(n.Data))
  158. if n.HasLastModifiedDate() {
  159. tarHeader.ModTime = time.Unix(int64(n.LastModified), 0)
  160. } else {
  161. tarHeader.ModTime = time.Unix(0, 0)
  162. }
  163. tarHeader.ChangeTime = tarHeader.ModTime
  164. if err = tarOutputFile.WriteHeader(&tarHeader); err != nil {
  165. return err
  166. }
  167. _, err = tarOutputFile.Write(n.Data)
  168. } else {
  169. size := n.DataSize
  170. if version == storage.Version1 {
  171. size = n.Size
  172. }
  173. fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n",
  174. key,
  175. n.Name,
  176. size,
  177. n.IsGzipped(),
  178. n.Mime,
  179. )
  180. }
  181. return
  182. }