adding filer.export prototype

This commit is contained in:
Chris Lu 2018-08-22 23:54:22 -07:00
parent 2b4277ec18
commit 98b8f8649d
3 changed files with 110 additions and 2 deletions

View file

@ -13,6 +13,7 @@ var Commands = []*Command{
cmdCompact,
cmdCopy,
cmdFix,
cmdFilerExport,
cmdServer,
cmdMaster,
cmdFiler,

View file

@ -0,0 +1,107 @@
package command
import (
"github.com/chrislusf/seaweedfs/weed/server"
"github.com/spf13/viper"
"github.com/chrislusf/seaweedfs/weed/filer2"
"github.com/chrislusf/seaweedfs/weed/glog"
)
func init() {
cmdFilerExport.Run = runFilerExport // break init cycle
}
var cmdFilerExport = &Command{
UsageLine: "filer.export -sourceStore=mysql -targetStroe=cassandra",
Short: "export meta data in filer store",
Long: `Iterate the file tree and export all metadata out
Both source and target store:
* should be a store name already specified in filer.toml
* do not need to be enabled state
If target store is empty, only the directory tree will be listed.
`,
}
var (
// filerExportOutputFile = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree")
filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml")
)
type statistics struct {
directoryCount int
fileCount int
}
func runFilerExport(cmd *Command, args []string) bool {
weed_server.LoadConfiguration("filer", true)
config := viper.GetViper()
var sourceStore filer2.FilerStore
for _, store := range filer2.Stores {
if store.GetName() == *filerExportSourceStore {
viperSub := config.Sub(store.GetName())
if err := store.Initialize(viperSub); err != nil {
glog.Fatalf("Failed to initialize store for %s: %+v",
store.GetName(), err)
} else {
sourceStore = store
}
break
}
}
if sourceStore == nil {
glog.Errorf("Failed to find source store %s", *filerExportSourceStore)
println("existing data sources are:")
for _, store := range filer2.Stores {
println(" " + store.GetName())
}
return false
}
stat := statistics{}
doTraverse(&stat, sourceStore, filer2.FullPath("/"), 0, printout)
glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount)
return true
}
func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {
limit := 1000
lastEntryName := ""
for {
entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit)
if err != nil {
break
}
for _, entry := range entries {
if fnErr := fn(level, entry); fnErr != nil {
glog.Errorf("failed to process entry: %s", entry.FullPath)
}
if entry.IsDirectory() {
stat.directoryCount++
doTraverse(stat, filerStore, entry.FullPath, level+1, fn)
} else {
stat.fileCount++
}
}
if len(entries) < limit {
break
}
}
}
func printout(level int, entry *filer2.Entry) error {
for i := 0; i < level; i++ {
print(" ")
}
println(entry.FullPath.Name())
return nil
}

View file

@ -49,7 +49,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
go fs.filer.KeepConnectedToMaster()
loadConfiguration("filer", true)
LoadConfiguration("filer", true)
v := viper.GetViper()
fs.filer.LoadConfiguration(v)
@ -69,7 +69,7 @@ func (fs *FilerServer) jwt(fileId string) security.EncodedJwt {
return security.GenJwt(fs.secret, fileId)
}
func loadConfiguration(configFileName string, required bool) {
func LoadConfiguration(configFileName string, required bool) {
// find a filer store
viper.SetConfigName(configFileName) // name of config file (without extension)