You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
132 lines
3.3 KiB
132 lines
3.3 KiB
package command
|
|
|
|
import (
|
|
"github.com/chrislusf/seaweedfs/weed/filer2"
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
|
"github.com/chrislusf/seaweedfs/weed/server"
|
|
"github.com/spf13/viper"
|
|
)
|
|
|
|
func init() {
|
|
cmdFilerExport.Run = runFilerExport // break init cycle
|
|
}
|
|
|
|
var cmdFilerExport = &Command{
|
|
UsageLine: "filer.export -sourceStore=mysql -targetStroe=cassandra",
|
|
Short: "export meta data in filer store",
|
|
Long: `Iterate the file tree and export all metadata out
|
|
|
|
Both source and target store:
|
|
* should be a store name already specified in filer.toml
|
|
* do not need to be enabled state
|
|
|
|
If target store is empty, only the directory tree will be listed.
|
|
|
|
`,
|
|
}
|
|
|
|
var (
|
|
// filerExportOutputFile = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree")
|
|
filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml")
|
|
filerExportTargetStore = cmdFilerExport.Flag.String("targetStore", "", "the target store name in filer.toml")
|
|
)
|
|
|
|
type statistics struct {
|
|
directoryCount int
|
|
fileCount int
|
|
}
|
|
|
|
func runFilerExport(cmd *Command, args []string) bool {
|
|
|
|
weed_server.LoadConfiguration("filer", true)
|
|
config := viper.GetViper()
|
|
|
|
var sourceStore, targetStore filer2.FilerStore
|
|
|
|
for _, store := range filer2.Stores {
|
|
if store.GetName() == *filerExportSourceStore {
|
|
viperSub := config.Sub(store.GetName())
|
|
if err := store.Initialize(viperSub); err != nil {
|
|
glog.Fatalf("Failed to initialize store for %s: %+v",
|
|
store.GetName(), err)
|
|
} else {
|
|
sourceStore = store
|
|
}
|
|
break
|
|
}
|
|
}
|
|
|
|
for _, store := range filer2.Stores {
|
|
if store.GetName() == *filerExportTargetStore {
|
|
viperSub := config.Sub(store.GetName())
|
|
if err := store.Initialize(viperSub); err != nil {
|
|
glog.Fatalf("Failed to initialize store for %s: %+v",
|
|
store.GetName(), err)
|
|
} else {
|
|
targetStore = store
|
|
}
|
|
break
|
|
}
|
|
}
|
|
|
|
if sourceStore == nil {
|
|
glog.Errorf("Failed to find source store %s", *filerExportSourceStore)
|
|
println("existing data sources are:")
|
|
for _, store := range filer2.Stores {
|
|
println(" " + store.GetName())
|
|
}
|
|
return false
|
|
}
|
|
|
|
stat := statistics{}
|
|
|
|
var fn func(level int, entry *filer2.Entry) error
|
|
|
|
if targetStore == nil {
|
|
fn = printout
|
|
} else {
|
|
fn = func(level int, entry *filer2.Entry) error {
|
|
return targetStore.InsertEntry(entry)
|
|
}
|
|
}
|
|
|
|
doTraverse(&stat, sourceStore, filer2.FullPath("/"), 0, fn)
|
|
|
|
glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount)
|
|
|
|
return true
|
|
}
|
|
|
|
func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {
|
|
|
|
limit := 1000
|
|
lastEntryName := ""
|
|
for {
|
|
entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit)
|
|
if err != nil {
|
|
break
|
|
}
|
|
for _, entry := range entries {
|
|
if fnErr := fn(level, entry); fnErr != nil {
|
|
glog.Errorf("failed to process entry: %s", entry.FullPath)
|
|
}
|
|
if entry.IsDirectory() {
|
|
stat.directoryCount++
|
|
doTraverse(stat, filerStore, entry.FullPath, level+1, fn)
|
|
} else {
|
|
stat.fileCount++
|
|
}
|
|
}
|
|
if len(entries) < limit {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
func printout(level int, entry *filer2.Entry) error {
|
|
for i := 0; i < level; i++ {
|
|
print(" ")
|
|
}
|
|
println(entry.FullPath.Name())
|
|
return nil
|
|
}
|