Browse Source

filer: support path-specific filer store

pull/1690/head
Chris Lu 4 years ago
parent
commit
41c0f3ad24
  1. 4
      weed/Makefile
  2. 22
      weed/command/scaffold.go
  3. 61
      weed/filer/configuration.go
  4. 36
      weed/filer/filerstore.go

4
weed/Makefile

@ -20,7 +20,7 @@ debug_mount:
debug_server: debug_server:
go build -gcflags="all=-N -l" go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=/Volumes/mobile_disk/99 -filer -volume.port=8343 -s3 -volume.max=0
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- server -dir=/Volumes/mobile_disk/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1
debug_volume: debug_volume:
go build -gcflags="all=-N -l" go build -gcflags="all=-N -l"
@ -32,4 +32,4 @@ debug_webdav:
debug_s3: debug_s3:
go build -gcflags="all=-N -l" go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 s3
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec weed -- -v=4 s3

22
weed/command/scaffold.go

@ -185,6 +185,28 @@ sniff_enabled = false
healthcheck_enabled = false healthcheck_enabled = false
# increase the value is recommend, be sure the value in Elastic is greater or equal here # increase the value is recommend, be sure the value in Elastic is greater or equal here
index.max_result_window = 10000 index.max_result_window = 10000
##########################
##########################
# To add path-specific filer store:
#
# 1. Add a name following the store type separated by a dot ".". E.g., cassandra.tmp
# 2. Add a location configuraiton. E.g., location = "/tmp/"
# 3. Copy and customize all other configurations.
# Make sure they are not the same if using the same store type!
# 4. Set enabled to true
#
# The following is just using cassandra as an example
##########################
[redis2.tmp]
enabled = false
location = "/tmp/"
address = "localhost:6379"
password = ""
database = 1
` `
NOTIFICATION_TOML_EXAMPLE = ` NOTIFICATION_TOML_EXAMPLE = `

61
weed/filer/configuration.go

@ -1,10 +1,11 @@
package filer package filer
import ( import (
"os"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/spf13/viper" "github.com/spf13/viper"
"os"
"reflect"
"strings"
) )
var ( var (
@ -15,28 +16,64 @@ func (f *Filer) LoadConfiguration(config *viper.Viper) {
validateOneEnabledStore(config) validateOneEnabledStore(config)
// load configuration for default filer store
hasDefaultStoreConfigured := false
for _, store := range Stores { for _, store := range Stores {
if config.GetBool(store.GetName() + ".enabled") { if config.GetBool(store.GetName() + ".enabled") {
store = reflect.New(reflect.ValueOf(store).Elem().Type()).Interface().(FilerStore)
if err := store.Initialize(config, store.GetName()+"."); err != nil { if err := store.Initialize(config, store.GetName()+"."); err != nil {
glog.Fatalf("Failed to initialize store for %s: %+v",
store.GetName(), err)
glog.Fatalf("failed to initialize store for %s: %+v", store.GetName(), err)
} }
f.SetStore(store) f.SetStore(store)
glog.V(0).Infof("Configure filer for %s", store.GetName())
return
glog.V(0).Infof("configured filer for %s", store.GetName())
hasDefaultStoreConfigured = true
break
} }
} }
// TODO load path-specific filer store here
// f.Store.AddPathSpecificStore(path, store)
if !hasDefaultStoreConfigured {
println()
println("Supported filer stores are:")
for _, store := range Stores {
println(" " + store.GetName())
}
os.Exit(-1)
}
println()
println("Supported filer stores are:")
// load path-specific filer store here
// f.Store.AddPathSpecificStore(path, store)
storeNames := make(map[string]FilerStore)
for _, store := range Stores { for _, store := range Stores {
println(" " + store.GetName())
storeNames[store.GetName()] = store
}
allKeys := config.AllKeys()
for _, key := range allKeys {
if !strings.HasSuffix(key, ".enabled") {
continue
}
key = key[:len(key)-len(".enabled")]
if !strings.Contains(key, ".") {
continue
}
parts := strings.Split(key, ".")
storeName, storeId := parts[0], parts[1]
store := storeNames[storeName]
store = reflect.New(reflect.ValueOf(store).Elem().Type()).Interface().(FilerStore)
if err := store.Initialize(config, key+"."); err != nil {
glog.Fatalf("Failed to initialize store for %s: %+v", key, err)
}
location := config.GetString(key+".location")
if location == "" {
glog.Errorf("path-specific filer store needs %s", key+".location")
os.Exit(-1)
}
f.Store.AddPathSpecificStore(location, storeId, store)
glog.V(0).Infof("configure filer %s for %s", store.GetName(), location)
} }
os.Exit(-1)
} }
func validateOneEnabledStore(config *viper.Viper) { func validateOneEnabledStore(config *viper.Viper) {

36
weed/filer/filerstore.go

@ -50,12 +50,13 @@ type VirtualFilerStore interface {
FilerStore FilerStore
DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error
DeleteOneEntry(ctx context.Context, entry *Entry) error DeleteOneEntry(ctx context.Context, entry *Entry) error
AddPathSpecificStore(path string, store FilerStore)
AddPathSpecificStore(path string, storeId string, store FilerStore)
} }
type FilerStoreWrapper struct { type FilerStoreWrapper struct {
defaultStore FilerStore
pathToStore ptrie.Trie
defaultStore FilerStore
pathToStore ptrie.Trie
storeIdToStore map[string]FilerStore
} }
func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper { func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
@ -63,23 +64,34 @@ func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
return innerStore return innerStore
} }
return &FilerStoreWrapper{ return &FilerStoreWrapper{
defaultStore: store,
pathToStore: ptrie.New(),
defaultStore: store,
pathToStore: ptrie.New(),
storeIdToStore: make(map[string]FilerStore),
} }
} }
func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, store FilerStore) {
fsw.pathToStore.Put([]byte(path), store)
func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, storeId string, store FilerStore) {
fsw.storeIdToStore[storeId] = store
err := fsw.pathToStore.Put([]byte(path), storeId)
if err != nil {
glog.Fatalf("put path specific store: %v", err)
}
} }
func (fsw *FilerStoreWrapper) getActualStore(path util.FullPath) FilerStore {
func (fsw *FilerStoreWrapper) getActualStore(path util.FullPath) (store FilerStore) {
store = fsw.defaultStore
if path == "" { if path == "" {
return fsw.defaultStore
return
} }
if store, found := fsw.pathToStore.Get([]byte(path)); found {
return store.(FilerStore)
var storeId string
fsw.pathToStore.MatchPrefix([]byte(path), func(key []byte, value interface{}) bool {
storeId = value.(string)
return false
})
if storeId != "" {
store = fsw.storeIdToStore[storeId]
} }
return fsw.defaultStore
return
} }
func (fsw *FilerStoreWrapper) GetName() string { func (fsw *FilerStoreWrapper) GetName() string {

Loading…
Cancel
Save