From 8a1d640dc4fd451e34f8e21a66d1ca1231c03585 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 11 May 2018 02:20:15 -0700 Subject: [PATCH] starting filer2 --- weed/filer2/embedded/embedded_store.go | 25 +++++++ weed/filer2/filer.go | 94 ++++++++++++++++++++++++++ weed/filer2/filer_structure.go | 62 +++++++++++++++++ weed/filer2/filer_test.go | 32 +++++++++ 4 files changed, 213 insertions(+) create mode 100644 weed/filer2/embedded/embedded_store.go create mode 100644 weed/filer2/filer.go create mode 100644 weed/filer2/filer_structure.go create mode 100644 weed/filer2/filer_test.go diff --git a/weed/filer2/embedded/embedded_store.go b/weed/filer2/embedded/embedded_store.go new file mode 100644 index 000000000..a0f7f0385 --- /dev/null +++ b/weed/filer2/embedded/embedded_store.go @@ -0,0 +1,25 @@ +package embedded + +import ( + "github.com/syndtr/goleveldb/leveldb" +) + +type EmbeddedStore struct { + db *leveldb.DB +} + +func NewEmbeddedStore(dir string) (filer *EmbeddedStore, err error) { + filer = &EmbeddedStore{} + if filer.db, err = leveldb.OpenFile(dir, nil); err != nil { + return + } + return +} + +func (filer *EmbeddedStore) CreateFile(filePath string, fid string) (err error) { + return nil +} + +func (filer *EmbeddedStore) Mkdir(filePath string) (err error) { + return nil +} diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go new file mode 100644 index 000000000..0851b4142 --- /dev/null +++ b/weed/filer2/filer.go @@ -0,0 +1,94 @@ +package filer2 + +import ( + "fmt" + + "github.com/chrislusf/seaweedfs/weed/filer2/embedded" + "github.com/karlseguin/ccache" + "strings" +) + +type Filer struct { + master string + store FilerStore + directoryCache *ccache.Cache +} + +func NewFiler(master string) *Filer { + return &Filer{ + master: master, + directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)), + } +} + +func NewEmbeddedFiler(master string, dir string) (*Filer, error) { + _, err := embedded.NewEmbeddedStore(dir) + if err != nil { + return nil, fmt.Errorf("failed to create embedded filer store: %v", err) + } + return &Filer{ + master: master, + // store: store, + }, nil +} + +func (f *Filer) CreateEntry(entry Entry) (error) { + /* + 1. recursively ensure parent directory is created. + 2. get current parent directory, add link, + 3. add the file entry + */ + + f.recursivelyEnsureDirectory(entry.Dir, func(parent, name string) error { + return nil + }) + + return f.store.CreateEntry(entry) +} + +func (f *Filer) AppendFileChunk(p FullPath, c FileChunk) (err error) { + return f.store.AppendFileChunk(p, c) +} + +func (f *Filer) FindEntry(p FullPath) (found bool, fileEntry Entry, err error) { + return f.store.FindEntry(p) +} + +func (f *Filer) DeleteEntry(p FullPath) (fileEntry Entry, err error) { + return f.store.DeleteEntry(p) +} + +func (f *Filer) ListDirectoryEntries(p FullPath) ([]Entry, error) { + return f.store.ListDirectoryEntries(p) +} + +func (f *Filer) UpdateEntry(entry Entry) (error) { + return f.store.UpdateEntry(entry) +} + +func (f *Filer) recursivelyEnsureDirectory(fullPath string, fn func(parent, name string) error) (error) { + if strings.HasSuffix(fullPath, "/") { + fullPath = fullPath[0:len(fullPath)-1] + } + nextPathEnd := strings.LastIndex(fullPath, "/") + if nextPathEnd < 0 { + return nil + } + + dirName := fullPath[nextPathEnd+1:] + parentDirPath := fullPath[0:nextPathEnd] + + if parentDirPath == "" { + parentDirPath = "/" + } + + if err := fn(parentDirPath, dirName); err != nil { + return err + } + + return f.recursivelyEnsureDirectory(parentDirPath, fn) +} + +func (f *Filer) cacheGetDirectory(dirpath string) (error) { + return nil +} diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go new file mode 100644 index 000000000..603645e34 --- /dev/null +++ b/weed/filer2/filer_structure.go @@ -0,0 +1,62 @@ +package filer2 + +import ( + "errors" + "os" + "time" +) + +type FileId string //file id in SeaweedFS +type FullPath struct { + Dir string //full path of the parent dir + Name string //file name without path +} + +type Attr struct { + Mtime time.Time // time of last modification + Crtime time.Time // time of creation (OS X only) + Mode os.FileMode // file mode + Uid uint32 // owner uid + Gid uint32 // group gid + IsDirectory bool + Size uint64 // total size in bytes + Nlink uint32 // number of links (usually 1) +} + +type Entry struct { + Dir string `json:"dir,omitempty"` //full path of the parent dir + Name string `json:"name,omitempty"` //file name without path + + Attr + + // the following is for files + Chunks []FileChunk `json:"chunks,omitempty"` +} + +type FileChunk struct { + Fid FileId `json:"fid,omitempty"` + Offset int64 `json:"offset,omitempty"` + Size uint64 `json:"size,omitempty"` // size in bytes +} + +type AbstractFiler interface { + CreateEntry(Entry) (error) + AppendFileChunk(FullPath, FileChunk) (err error) + FindEntry(FullPath) (found bool, fileEntry Entry, err error) + DeleteEntry(FullPath) (fileEntry Entry, err error) + + ListDirectoryEntries(dirPath FullPath) ([]Entry, error) + UpdateEntry(Entry) (error) +} + +var ErrNotFound = errors.New("filer: no entry is found in filer store") + +type FilerStore interface { + CreateEntry(Entry) (error) + AppendFileChunk(FullPath, FileChunk) (err error) + FindEntry(FullPath) (found bool, fileEntry Entry, err error) + DeleteEntry(FullPath) (fileEntry Entry, err error) + + ListDirectoryEntries(dirPath FullPath) ([]Entry, error) + UpdateEntry(Entry) (error) +} diff --git a/weed/filer2/filer_test.go b/weed/filer2/filer_test.go new file mode 100644 index 000000000..0cefb679a --- /dev/null +++ b/weed/filer2/filer_test.go @@ -0,0 +1,32 @@ +package filer2 + +import ( + "testing" + "path/filepath" + "fmt" +) + +func TestRecursion(t *testing.T) { + filer := NewFiler("") + + fullPath := "/home/chris/some/file/abc.jpg" + expected := []string{ + "/home/chris/some", "file", + "/home/chris", "some", + "/home", "chris", + "/", "home", + } + + dir, _ := filepath.Split(fullPath) + + i := 0 + + filer.recursivelyEnsureDirectory(dir, func(parent, name string) error { + if parent != expected[i] || name != expected[i+1] { + t.Errorf("recursive directory is wrong! parent=%s dirName=%s", parent, name) + } + fmt.Printf("processing folder %s \t parent=%s\n", name, parent) + i += 2 + return nil + }) +}