You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

179 lines
4.5 KiB

  1. package shell
  2. import (
  3. "flag"
  4. "fmt"
  5. "io"
  6. "os"
  7. "sync"
  8. "sync/atomic"
  9. "time"
  10. "github.com/golang/protobuf/proto"
  11. "github.com/chrislusf/seaweedfs/weed/filer2"
  12. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  13. "github.com/chrislusf/seaweedfs/weed/util"
  14. )
  15. func init() {
  16. Commands = append(Commands, &commandFsMetaSave{})
  17. }
  18. type commandFsMetaSave struct {
  19. }
  20. func (c *commandFsMetaSave) Name() string {
  21. return "fs.meta.save"
  22. }
  23. func (c *commandFsMetaSave) Help() string {
  24. return `save all directory and file meta data to a local file for metadata backup.
  25. fs.meta.save / # save from the root
  26. fs.meta.save -v -o t.meta / # save from the root, output to t.meta file.
  27. fs.meta.save /path/to/save # save from the directory /path/to/save
  28. fs.meta.save . # save from current directory
  29. fs.meta.save # save from current directory
  30. The meta data will be saved into a local <filer_host>-<port>-<time>.meta file.
  31. These meta data can be later loaded by fs.meta.load command,
  32. This assumes there are no deletions, so this is different from taking a snapshot.
  33. `
  34. }
  35. func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
  36. fsMetaSaveCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
  37. verbose := fsMetaSaveCommand.Bool("v", false, "print out each processed files")
  38. outputFileName := fsMetaSaveCommand.String("o", "", "output the meta data to this file")
  39. if err = fsMetaSaveCommand.Parse(args); err != nil {
  40. return nil
  41. }
  42. filerServer, filerPort, path, parseErr := commandEnv.parseUrl(findInputDirectory(fsMetaSaveCommand.Args()))
  43. if parseErr != nil {
  44. return parseErr
  45. }
  46. t := time.Now()
  47. fileName := *outputFileName
  48. if fileName == "" {
  49. fileName = fmt.Sprintf("%s-%d-%4d%02d%02d-%02d%02d%02d.meta",
  50. filerServer, filerPort, t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
  51. }
  52. dst, openErr := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
  53. if openErr != nil {
  54. return fmt.Errorf("failed to create file %s: %v", fileName, openErr)
  55. }
  56. defer dst.Close()
  57. var wg sync.WaitGroup
  58. wg.Add(1)
  59. outputChan := make(chan []byte, 1024)
  60. go func() {
  61. sizeBuf := make([]byte, 4)
  62. for b := range outputChan {
  63. util.Uint32toBytes(sizeBuf, uint32(len(b)))
  64. dst.Write(sizeBuf)
  65. dst.Write(b)
  66. }
  67. wg.Done()
  68. }()
  69. var dirCount, fileCount uint64
  70. err = doTraverseBFS(writer, commandEnv.getFilerClient(filerServer, filerPort), filer2.FullPath(path), func(parentPath filer2.FullPath, entry *filer_pb.Entry) {
  71. protoMessage := &filer_pb.FullEntry{
  72. Dir: string(parentPath),
  73. Entry: entry,
  74. }
  75. bytes, err := proto.Marshal(protoMessage)
  76. if err != nil {
  77. fmt.Fprintf(writer, "marshall error: %v\n", err)
  78. return
  79. }
  80. outputChan <- bytes
  81. if entry.IsDirectory {
  82. atomic.AddUint64(&dirCount, 1)
  83. } else {
  84. atomic.AddUint64(&fileCount, 1)
  85. }
  86. if *verbose {
  87. println(parentPath.Child(entry.Name))
  88. }
  89. })
  90. close(outputChan)
  91. wg.Wait()
  92. if err == nil {
  93. fmt.Fprintf(writer, "total %d directories, %d files\n", dirCount, fileCount)
  94. fmt.Fprintf(writer, "meta data for http://%s:%d%s is saved to %s\n", filerServer, filerPort, path, fileName)
  95. }
  96. return err
  97. }
  98. func doTraverseBFS(writer io.Writer, filerClient filer2.FilerClient, parentPath filer2.FullPath, fn func(parentPath filer2.FullPath, entry *filer_pb.Entry)) (err error) {
  99. K := 5
  100. var jobQueueWg sync.WaitGroup
  101. queue := util.NewQueue()
  102. jobQueueWg.Add(1)
  103. queue.Enqueue(parentPath)
  104. var isTerminating bool
  105. for i := 0; i < K; i++ {
  106. go func() {
  107. for {
  108. if isTerminating {
  109. break
  110. }
  111. t := queue.Dequeue()
  112. if t == nil {
  113. time.Sleep(329 * time.Millisecond)
  114. continue
  115. }
  116. dir := t.(filer2.FullPath)
  117. processErr := processOneDirectory(writer, filerClient, dir, queue, &jobQueueWg, fn)
  118. if processErr != nil {
  119. err = processErr
  120. }
  121. jobQueueWg.Done()
  122. }
  123. }()
  124. }
  125. jobQueueWg.Wait()
  126. isTerminating = true
  127. return
  128. }
  129. func processOneDirectory(writer io.Writer, filerClient filer2.FilerClient, parentPath filer2.FullPath, queue *util.Queue, jobQueueWg *sync.WaitGroup, fn func(parentPath filer2.FullPath, entry *filer_pb.Entry)) (err error) {
  130. return filer2.ReadDirAllEntries(filerClient, parentPath, "", func(entry *filer_pb.Entry, isLast bool) {
  131. fn(parentPath, entry)
  132. if entry.IsDirectory {
  133. subDir := fmt.Sprintf("%s/%s", parentPath, entry.Name)
  134. if parentPath == "/" {
  135. subDir = "/" + entry.Name
  136. }
  137. jobQueueWg.Add(1)
  138. queue.Enqueue(filer2.FullPath(subDir))
  139. }
  140. })
  141. }