diff --git a/weed/Makefile b/weed/Makefile index 33886215a..fd0843c22 100644 --- a/weed/Makefile +++ b/weed/Makefile @@ -20,7 +20,7 @@ debug_mount: debug_server: go build -gcflags="all=-N -l" - dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=/Volumes/mobile_disk/99 -filer -volume.port=8343 -s3 -volume.max=0 + dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=/Volumes/mobile_disk/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 debug_volume: go build -gcflags="all=-N -l" @@ -32,4 +32,4 @@ debug_webdav: debug_s3: go build -gcflags="all=-N -l" - dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 s3 \ No newline at end of file + dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 s3 diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 82410f6d9..1ab763004 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -185,6 +185,28 @@ sniff_enabled = false healthcheck_enabled = false # increase the value is recommend, be sure the value in Elastic is greater or equal here index.max_result_window = 10000 + + + +########################## +########################## +# To add path-specific filer store: +# +# 1. Add a name following the store type separated by a dot ".". E.g., cassandra.tmp +# 2. Add a location configuraiton. E.g., location = "/tmp/" +# 3. Copy and customize all other configurations. +# Make sure they are not the same if using the same store type! +# 4. Set enabled to true +# +# The following is just using cassandra as an example +########################## +[redis2.tmp] +enabled = false +location = "/tmp/" +address = "localhost:6379" +password = "" +database = 1 + ` NOTIFICATION_TOML_EXAMPLE = ` diff --git a/weed/filer/configuration.go b/weed/filer/configuration.go index d04ab06cb..5f5dfed25 100644 --- a/weed/filer/configuration.go +++ b/weed/filer/configuration.go @@ -1,10 +1,11 @@ package filer import ( - "os" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/spf13/viper" + "os" + "reflect" + "strings" ) var ( @@ -15,28 +16,64 @@ func (f *Filer) LoadConfiguration(config *viper.Viper) { validateOneEnabledStore(config) + // load configuration for default filer store + hasDefaultStoreConfigured := false for _, store := range Stores { if config.GetBool(store.GetName() + ".enabled") { + store = reflect.New(reflect.ValueOf(store).Elem().Type()).Interface().(FilerStore) if err := store.Initialize(config, store.GetName()+"."); err != nil { - glog.Fatalf("Failed to initialize store for %s: %+v", - store.GetName(), err) + glog.Fatalf("failed to initialize store for %s: %+v", store.GetName(), err) } f.SetStore(store) - glog.V(0).Infof("Configure filer for %s", store.GetName()) - return + glog.V(0).Infof("configured filer for %s", store.GetName()) + hasDefaultStoreConfigured = true + break } } - // TODO load path-specific filer store here - // f.Store.AddPathSpecificStore(path, store) + if !hasDefaultStoreConfigured { + println() + println("Supported filer stores are:") + for _, store := range Stores { + println(" " + store.GetName()) + } + os.Exit(-1) + } - println() - println("Supported filer stores are:") + // load path-specific filer store here + // f.Store.AddPathSpecificStore(path, store) + storeNames := make(map[string]FilerStore) for _, store := range Stores { - println(" " + store.GetName()) + storeNames[store.GetName()] = store + } + allKeys := config.AllKeys() + for _, key := range allKeys { + if !strings.HasSuffix(key, ".enabled") { + continue + } + key = key[:len(key)-len(".enabled")] + if !strings.Contains(key, ".") { + continue + } + + parts := strings.Split(key, ".") + storeName, storeId := parts[0], parts[1] + + store := storeNames[storeName] + store = reflect.New(reflect.ValueOf(store).Elem().Type()).Interface().(FilerStore) + if err := store.Initialize(config, key+"."); err != nil { + glog.Fatalf("Failed to initialize store for %s: %+v", key, err) + } + location := config.GetString(key+".location") + if location == "" { + glog.Errorf("path-specific filer store needs %s", key+".location") + os.Exit(-1) + } + f.Store.AddPathSpecificStore(location, storeId, store) + + glog.V(0).Infof("configure filer %s for %s", store.GetName(), location) } - os.Exit(-1) } func validateOneEnabledStore(config *viper.Viper) { diff --git a/weed/filer/filerstore.go b/weed/filer/filerstore.go index 5e10bb0a6..10aee75df 100644 --- a/weed/filer/filerstore.go +++ b/weed/filer/filerstore.go @@ -50,12 +50,13 @@ type VirtualFilerStore interface { FilerStore DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error DeleteOneEntry(ctx context.Context, entry *Entry) error - AddPathSpecificStore(path string, store FilerStore) + AddPathSpecificStore(path string, storeId string, store FilerStore) } type FilerStoreWrapper struct { - defaultStore FilerStore - pathToStore ptrie.Trie + defaultStore FilerStore + pathToStore ptrie.Trie + storeIdToStore map[string]FilerStore } func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper { @@ -63,23 +64,34 @@ func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper { return innerStore } return &FilerStoreWrapper{ - defaultStore: store, - pathToStore: ptrie.New(), + defaultStore: store, + pathToStore: ptrie.New(), + storeIdToStore: make(map[string]FilerStore), } } -func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, store FilerStore) { - fsw.pathToStore.Put([]byte(path), store) +func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, storeId string, store FilerStore) { + fsw.storeIdToStore[storeId] = store + err := fsw.pathToStore.Put([]byte(path), storeId) + if err != nil { + glog.Fatalf("put path specific store: %v", err) + } } -func (fsw *FilerStoreWrapper) getActualStore(path util.FullPath) FilerStore { +func (fsw *FilerStoreWrapper) getActualStore(path util.FullPath) (store FilerStore) { + store = fsw.defaultStore if path == "" { - return fsw.defaultStore + return } - if store, found := fsw.pathToStore.Get([]byte(path)); found { - return store.(FilerStore) + var storeId string + fsw.pathToStore.MatchPrefix([]byte(path), func(key []byte, value interface{}) bool { + storeId = value.(string) + return false + }) + if storeId != "" { + store = fsw.storeIdToStore[storeId] } - return fsw.defaultStore + return } func (fsw *FilerStoreWrapper) GetName() string {