You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

187 lines
5.2 KiB

6 years ago
6 years ago
6 years ago
6 years ago
  1. package command
  2. import (
  3. "github.com/chrislusf/seaweedfs/weed/filer2"
  4. "github.com/chrislusf/seaweedfs/weed/glog"
  5. "github.com/chrislusf/seaweedfs/weed/notification"
  6. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  7. "github.com/chrislusf/seaweedfs/weed/server"
  8. "github.com/spf13/viper"
  9. )
  10. func init() {
  11. cmdFilerExport.Run = runFilerExport // break init cycle
  12. }
  13. var cmdFilerExport = &Command{
  14. UsageLine: "filer.export -sourceStore=mysql -targetStore=cassandra",
  15. Short: "export meta data in filer store",
  16. Long: `Iterate the file tree and export all metadata out
  17. Both source and target store:
  18. * should be a store name already specified in filer.toml
  19. * do not need to be enabled state
  20. If target store is empty, only the directory tree will be listed.
  21. If target store is "notification", the list of entries will be sent to notification.
  22. This is usually used to bootstrap filer replication to a remote system.
  23. `,
  24. }
  25. var (
  26. // filerExportOutputFile = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree")
  27. filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml, default to currently enabled store")
  28. filerExportTargetStore = cmdFilerExport.Flag.String("targetStore", "", "the target store name in filer.toml, or \"notification\" to export all files to message queue")
  29. dir = cmdFilerExport.Flag.String("dir", "/", "only process files under this directory")
  30. dirListLimit = cmdFilerExport.Flag.Int("dirListLimit", 100000, "limit directory list size")
  31. dryRun = cmdFilerExport.Flag.Bool("dryRun", false, "not actually moving data")
  32. verboseFilerExport = cmdFilerExport.Flag.Bool("v", false, "verbose entry details")
  33. )
  34. type statistics struct {
  35. directoryCount int
  36. fileCount int
  37. }
  38. func runFilerExport(cmd *Command, args []string) bool {
  39. weed_server.LoadConfiguration("filer", true)
  40. config := viper.GetViper()
  41. var sourceStore, targetStore filer2.FilerStore
  42. for _, store := range filer2.Stores {
  43. if store.GetName() == *filerExportSourceStore || *filerExportSourceStore == "" && config.GetBool(store.GetName()+".enabled") {
  44. viperSub := config.Sub(store.GetName())
  45. if err := store.Initialize(viperSub); err != nil {
  46. glog.Fatalf("Failed to initialize source store for %s: %+v",
  47. store.GetName(), err)
  48. } else {
  49. sourceStore = store
  50. }
  51. break
  52. }
  53. }
  54. for _, store := range filer2.Stores {
  55. if store.GetName() == *filerExportTargetStore {
  56. viperSub := config.Sub(store.GetName())
  57. if err := store.Initialize(viperSub); err != nil {
  58. glog.Fatalf("Failed to initialize target store for %s: %+v",
  59. store.GetName(), err)
  60. } else {
  61. targetStore = store
  62. }
  63. break
  64. }
  65. }
  66. if sourceStore == nil {
  67. glog.Errorf("Failed to find source store %s", *filerExportSourceStore)
  68. println("existing data sources are:")
  69. for _, store := range filer2.Stores {
  70. println(" " + store.GetName())
  71. }
  72. return false
  73. }
  74. if targetStore == nil && *filerExportTargetStore != "" && *filerExportTargetStore != "notification" {
  75. glog.Errorf("Failed to find target store %s", *filerExportTargetStore)
  76. println("existing data sources are:")
  77. for _, store := range filer2.Stores {
  78. println(" " + store.GetName())
  79. }
  80. return false
  81. }
  82. stat := statistics{}
  83. var fn func(level int, entry *filer2.Entry) error
  84. if *filerExportTargetStore == "notification" {
  85. weed_server.LoadConfiguration("notification", false)
  86. v := viper.GetViper()
  87. notification.LoadConfiguration(v.Sub("notification"))
  88. fn = func(level int, entry *filer2.Entry) error {
  89. printout(level, entry)
  90. if *dryRun {
  91. return nil
  92. }
  93. return notification.Queue.SendMessage(
  94. string(entry.FullPath),
  95. &filer_pb.EventNotification{
  96. NewEntry: entry.ToProtoEntry(),
  97. },
  98. )
  99. }
  100. } else if targetStore == nil {
  101. fn = printout
  102. } else {
  103. fn = func(level int, entry *filer2.Entry) error {
  104. printout(level, entry)
  105. if *dryRun {
  106. return nil
  107. }
  108. return targetStore.InsertEntry(entry)
  109. }
  110. }
  111. doTraverse(&stat, sourceStore, filer2.FullPath(*dir), 0, fn)
  112. glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount)
  113. return true
  114. }
  115. func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {
  116. limit := *dirListLimit
  117. lastEntryName := ""
  118. for {
  119. entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit)
  120. if err != nil {
  121. break
  122. }
  123. for _, entry := range entries {
  124. if fnErr := fn(level, entry); fnErr != nil {
  125. glog.Errorf("failed to process entry: %s", entry.FullPath)
  126. }
  127. if entry.IsDirectory() {
  128. stat.directoryCount++
  129. doTraverse(stat, filerStore, entry.FullPath, level+1, fn)
  130. } else {
  131. stat.fileCount++
  132. }
  133. }
  134. if len(entries) < limit {
  135. break
  136. }
  137. }
  138. }
  139. func printout(level int, entry *filer2.Entry) error {
  140. for i := 0; i < level; i++ {
  141. if i == level-1 {
  142. print("+-")
  143. } else {
  144. print("| ")
  145. }
  146. }
  147. print(entry.FullPath.Name())
  148. if *verboseFilerExport {
  149. for _, chunk := range entry.Chunks {
  150. print("[")
  151. print(chunk.FileId)
  152. print(",")
  153. print(chunk.Offset)
  154. print(",")
  155. print(chunk.Size)
  156. print(")")
  157. }
  158. }
  159. println()
  160. return nil
  161. }