Browse Source
add leveldb store
add leveldb store
1. switch to viper for filer store configuration 2. simplify FindEntry() return values, removing “found” 3. add leveldb storepull/664/head
Chris Lu
7 years ago
17 changed files with 382 additions and 193 deletions
-
35weed/command/filer.go
-
16weed/command/server.go
-
87weed/filer2/configuration.go
-
38weed/filer2/embedded/embedded_store.go
-
5weed/filer2/entry.go
-
4weed/filer2/entry_codec.go
-
12weed/filer2/filer.go
-
9weed/filer2/filerstore.go
-
171weed/filer2/leveldb/leveldb_store.go
-
61weed/filer2/leveldb/leveldb_store_test.go
-
25weed/filer2/memdb/memdb_store.go
-
8weed/filer2/memdb/memdb_store_test.go
-
21weed/server/filer_grpc_server.go
-
62weed/server/filer_server.go
-
2weed/server/filer_server_handlers_admin.go
-
8weed/server/filer_server_handlers_read.go
-
11weed/server/filer_server_handlers_write.go
@ -0,0 +1,87 @@ |
|||||
|
package filer2 |
||||
|
|
||||
|
import ( |
||||
|
"os" |
||||
|
|
||||
|
"github.com/spf13/viper" |
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
) |
||||
|
|
||||
|
const ( |
||||
|
FILER_TOML_EXAMPLE = ` |
||||
|
# A sample TOML config file for SeaweedFS filer store |
||||
|
|
||||
|
# local in memory, mostly for testing purpose |
||||
|
[memory] |
||||
|
enabled = false |
||||
|
|
||||
|
[leveldb] |
||||
|
enabled = false |
||||
|
dir = "." # directory to store level db files |
||||
|
|
||||
|
[mysql] |
||||
|
enabled = true |
||||
|
server = "192.168.1.1" |
||||
|
port = 8080 |
||||
|
username = "" |
||||
|
password = "" |
||||
|
database = "" |
||||
|
connection_max_idle = 100 |
||||
|
connection_max_open = 100 |
||||
|
|
||||
|
[postgres] |
||||
|
enabled = false |
||||
|
server = "192.168.1.1" |
||||
|
port = 8080 |
||||
|
username = "" |
||||
|
password = "" |
||||
|
database = "" |
||||
|
connection_max_idle = 100 |
||||
|
connection_max_open = 100 |
||||
|
|
||||
|
` |
||||
|
) |
||||
|
|
||||
|
var ( |
||||
|
Stores []FilerStore |
||||
|
) |
||||
|
|
||||
|
func (f *Filer) LoadConfiguration() { |
||||
|
|
||||
|
// find a filer store
|
||||
|
viper.SetConfigName("filer") // name of config file (without extension)
|
||||
|
viper.AddConfigPath(".") // optionally look for config in the working directory
|
||||
|
viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths
|
||||
|
viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in
|
||||
|
if err := viper.ReadInConfig(); err != nil { // Handle errors reading the config file
|
||||
|
glog.Fatalf("Failed to load filer.toml file from current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/" + |
||||
|
"\n\nPlease follow this example and add a filer.toml file to " + |
||||
|
"current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/:\n" + FILER_TOML_EXAMPLE) |
||||
|
} |
||||
|
|
||||
|
glog.V(0).Infof("Reading filer configuration from %s", viper.ConfigFileUsed()) |
||||
|
for _, store := range Stores { |
||||
|
if viper.GetBool(store.GetName() + ".enabled") { |
||||
|
viperSub := viper.Sub(store.GetName()) |
||||
|
if err := store.Initialize(viperSub); err != nil { |
||||
|
glog.Fatalf("Failed to initialize store for %s: %+v", |
||||
|
store.GetName(), err) |
||||
|
} |
||||
|
f.SetStore(store) |
||||
|
glog.V(0).Infof("Configure filer for %s from %s", store.GetName(), viper.ConfigFileUsed()) |
||||
|
return |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
println() |
||||
|
println("Supported filer stores are:") |
||||
|
for _, store := range Stores { |
||||
|
println(" " + store.GetName()) |
||||
|
} |
||||
|
|
||||
|
println() |
||||
|
println("Please configure a supported filer store in", viper.ConfigFileUsed()) |
||||
|
println() |
||||
|
|
||||
|
os.Exit(-1) |
||||
|
} |
||||
@ -1,38 +0,0 @@ |
|||||
package embedded |
|
||||
|
|
||||
import ( |
|
||||
"github.com/syndtr/goleveldb/leveldb" |
|
||||
"github.com/chrislusf/seaweedfs/weed/filer2" |
|
||||
) |
|
||||
|
|
||||
type EmbeddedStore struct { |
|
||||
db *leveldb.DB |
|
||||
} |
|
||||
|
|
||||
func NewEmbeddedStore(dir string) (filer *EmbeddedStore, err error) { |
|
||||
filer = &EmbeddedStore{} |
|
||||
if filer.db, err = leveldb.OpenFile(dir, nil); err != nil { |
|
||||
return |
|
||||
} |
|
||||
return |
|
||||
} |
|
||||
|
|
||||
func (filer *EmbeddedStore) InsertEntry(entry *filer2.Entry) (err error) { |
|
||||
return nil |
|
||||
} |
|
||||
|
|
||||
func (filer *EmbeddedStore) UpdateEntry(entry *filer2.Entry) (err error) { |
|
||||
return nil |
|
||||
} |
|
||||
|
|
||||
func (filer *EmbeddedStore) FindEntry(fullpath filer2.FullPath) (found bool, entry *filer2.Entry, err error) { |
|
||||
return false, nil, nil |
|
||||
} |
|
||||
|
|
||||
func (filer *EmbeddedStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { |
|
||||
return nil, nil |
|
||||
} |
|
||||
|
|
||||
func (filer *EmbeddedStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { |
|
||||
return nil, nil |
|
||||
} |
|
||||
@ -0,0 +1,171 @@ |
|||||
|
package leveldb |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"bytes" |
||||
|
|
||||
|
"github.com/syndtr/goleveldb/leveldb" |
||||
|
"github.com/chrislusf/seaweedfs/weed/filer2" |
||||
|
leveldb_util "github.com/syndtr/goleveldb/leveldb/util" |
||||
|
"github.com/chrislusf/seaweedfs/weed/glog" |
||||
|
"github.com/spf13/viper" |
||||
|
weed_util "github.com/chrislusf/seaweedfs/weed/util" |
||||
|
) |
||||
|
|
||||
|
const ( |
||||
|
DIR_FILE_SEPARATOR = byte(0x00) |
||||
|
) |
||||
|
|
||||
|
func init() { |
||||
|
filer2.Stores = append(filer2.Stores, &LevelDBStore{}) |
||||
|
} |
||||
|
|
||||
|
type LevelDBStore struct { |
||||
|
db *leveldb.DB |
||||
|
} |
||||
|
|
||||
|
func (filer *LevelDBStore) GetName() string { |
||||
|
return "leveldb" |
||||
|
} |
||||
|
|
||||
|
func (filer *LevelDBStore) Initialize(viper *viper.Viper) (err error) { |
||||
|
dir := viper.GetString("dir") |
||||
|
return filer.initialize(dir) |
||||
|
} |
||||
|
|
||||
|
func (filer *LevelDBStore) initialize(dir string) (err error) { |
||||
|
if err := weed_util.TestFolderWritable(dir); err != nil { |
||||
|
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) |
||||
|
} |
||||
|
|
||||
|
if filer.db, err = leveldb.OpenFile(dir, nil); err != nil { |
||||
|
return |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func (store *LevelDBStore) InsertEntry(entry *filer2.Entry) (err error) { |
||||
|
key := genKey(entry.DirAndName()) |
||||
|
|
||||
|
value, err := entry.EncodeAttributesAndChunks() |
||||
|
if err != nil { |
||||
|
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) |
||||
|
} |
||||
|
|
||||
|
err = store.db.Put(key, value, nil) |
||||
|
|
||||
|
if err != nil { |
||||
|
return fmt.Errorf("persisting %s : %v", entry.FullPath, err) |
||||
|
} |
||||
|
|
||||
|
// println("saved", entry.FullPath, "chunks", len(entry.Chunks))
|
||||
|
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (store *LevelDBStore) UpdateEntry(entry *filer2.Entry) (err error) { |
||||
|
|
||||
|
return store.InsertEntry(entry) |
||||
|
} |
||||
|
|
||||
|
func (store *LevelDBStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { |
||||
|
key := genKey(fullpath.DirAndName()) |
||||
|
|
||||
|
data, err := store.db.Get(key, nil) |
||||
|
|
||||
|
if err == leveldb.ErrNotFound { |
||||
|
return nil, filer2.ErrNotFound |
||||
|
} |
||||
|
if err != nil { |
||||
|
return nil, fmt.Errorf("get %s : %v", entry.FullPath, err) |
||||
|
} |
||||
|
|
||||
|
entry = &filer2.Entry{ |
||||
|
FullPath: fullpath, |
||||
|
} |
||||
|
err = entry.DecodeAttributesAndChunks(data) |
||||
|
if err != nil { |
||||
|
return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) |
||||
|
} |
||||
|
|
||||
|
// println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data))
|
||||
|
|
||||
|
return entry, nil |
||||
|
} |
||||
|
|
||||
|
func (store *LevelDBStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { |
||||
|
key := genKey(fullpath.DirAndName()) |
||||
|
|
||||
|
entry, _ = store.FindEntry(fullpath) |
||||
|
|
||||
|
err = store.db.Delete(key, nil) |
||||
|
if err != nil { |
||||
|
return entry, fmt.Errorf("delete %s : %v", entry.FullPath, err) |
||||
|
} |
||||
|
|
||||
|
return entry, nil |
||||
|
} |
||||
|
|
||||
|
func (store *LevelDBStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, |
||||
|
limit int) (entries []*filer2.Entry, err error) { |
||||
|
|
||||
|
directoryPrefix := genDirectoryKeyPrefix(fullpath, "") |
||||
|
|
||||
|
iter := store.db.NewIterator(&leveldb_util.Range{Start: genDirectoryKeyPrefix(fullpath, startFileName)}, nil) |
||||
|
for iter.Next() { |
||||
|
key := iter.Key() |
||||
|
if !bytes.HasPrefix(key, directoryPrefix) { |
||||
|
break |
||||
|
} |
||||
|
fileName := getNameFromKey(key) |
||||
|
if fileName == "" { |
||||
|
continue |
||||
|
} |
||||
|
if fileName == startFileName && !inclusive { |
||||
|
continue |
||||
|
} |
||||
|
limit-- |
||||
|
if limit < 0 { |
||||
|
break |
||||
|
} |
||||
|
entry := &filer2.Entry{ |
||||
|
FullPath: filer2.NewFullPath(string(fullpath), fileName), |
||||
|
} |
||||
|
if decodeErr := entry.DecodeAttributesAndChunks(iter.Value()); decodeErr != nil { |
||||
|
err = decodeErr |
||||
|
glog.V(0).Infof("list %s : %v", entry.FullPath, err) |
||||
|
break |
||||
|
} |
||||
|
entries = append(entries, entry) |
||||
|
} |
||||
|
iter.Release() |
||||
|
|
||||
|
return entries, err |
||||
|
} |
||||
|
|
||||
|
func genKey(dirPath, fileName string) (key []byte) { |
||||
|
key = []byte(dirPath) |
||||
|
key = append(key, DIR_FILE_SEPARATOR) |
||||
|
key = append(key, []byte(fileName)...) |
||||
|
return key |
||||
|
} |
||||
|
|
||||
|
func genDirectoryKeyPrefix(fullpath filer2.FullPath, startFileName string) (keyPrefix []byte) { |
||||
|
keyPrefix = []byte(string(fullpath)) |
||||
|
keyPrefix = append(keyPrefix, DIR_FILE_SEPARATOR) |
||||
|
if len(startFileName) > 0 { |
||||
|
keyPrefix = append(keyPrefix, []byte(startFileName)...) |
||||
|
} |
||||
|
return keyPrefix |
||||
|
} |
||||
|
|
||||
|
func getNameFromKey(key []byte) (string) { |
||||
|
|
||||
|
sepIndex := len(key) - 1 |
||||
|
for sepIndex >= 0 && key[sepIndex] != DIR_FILE_SEPARATOR { |
||||
|
sepIndex-- |
||||
|
} |
||||
|
|
||||
|
return string(key[sepIndex+1:]) |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,61 @@ |
|||||
|
package leveldb |
||||
|
|
||||
|
import ( |
||||
|
"testing" |
||||
|
"github.com/chrislusf/seaweedfs/weed/filer2" |
||||
|
"io/ioutil" |
||||
|
"os" |
||||
|
) |
||||
|
|
||||
|
func TestCreateAndFind(t *testing.T) { |
||||
|
filer := filer2.NewFiler("") |
||||
|
dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") |
||||
|
defer os.RemoveAll(dir) |
||||
|
store := &LevelDBStore{} |
||||
|
store.initialize(dir) |
||||
|
filer.SetStore(store) |
||||
|
filer.DisableDirectoryCache() |
||||
|
|
||||
|
fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg") |
||||
|
|
||||
|
entry1 := &filer2.Entry{ |
||||
|
FullPath: fullpath, |
||||
|
Attr: filer2.Attr{ |
||||
|
Mode: 0440, |
||||
|
Uid: 1234, |
||||
|
Gid: 5678, |
||||
|
}, |
||||
|
} |
||||
|
|
||||
|
if err := filer.CreateEntry(entry1); err != nil { |
||||
|
t.Errorf("create entry %v: %v", entry1.FullPath, err) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
entry, err := filer.FindEntry(fullpath) |
||||
|
|
||||
|
if err != nil { |
||||
|
t.Errorf("find entry: %v", err) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
if entry.FullPath != entry1.FullPath { |
||||
|
t.Errorf("find wrong entry: %v", entry.FullPath) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// checking one upper directory
|
||||
|
entries, _ := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one"), "", false, 100) |
||||
|
if len(entries) != 1 { |
||||
|
t.Errorf("list entries count: %v", len(entries)) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
// checking one upper directory
|
||||
|
entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100) |
||||
|
if len(entries) != 1 { |
||||
|
t.Errorf("list entries count: %v", len(entries)) |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue