Refactoring for supporing cassandra as filer meta data store

go/filer/directory.go

@@ -1,18 +0,0 @@
-package filer
-
-type DirectoryId int32
-
-type DirectoryEntry struct {
-	Name string //dir name without path
-	Id   DirectoryId
-}
-
-type DirectoryManager interface {
-	FindDirectory(dirPath string) (DirectoryId, error)
-	ListDirectories(dirPath string) (dirs []DirectoryEntry, err error)
-	MakeDirectory(currentDirPath string, dirName string) (DirectoryId, error)
-	MoveUnderDirectory(oldDirPath string, newParentDirPath string) error
-	DeleteDirectory(dirPath string) error
-	//functions used by FUSE
-	FindDirectoryById(DirectoryId, error)
-}

go/filer/embedded_filer/directory.go

@@ -0,0 +1,15 @@
+package embedded_filer
+
+import (
+	"github.com/chrislusf/weed-fs/go/filer"
+)
+
+type DirectoryManager interface {
+	FindDirectory(dirPath string) (filer.DirectoryId, error)
+	ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error)
+	MakeDirectory(currentDirPath string, dirName string) (filer.DirectoryId, error)
+	MoveUnderDirectory(oldDirPath string, newParentDirPath string) error
+	DeleteDirectory(dirPath string) error
+	//functions used by FUSE
+	FindDirectoryById(filer.DirectoryId, error)
+}

go/filer/directory_in_map.go → go/filer/embedded_filer/directory_in_map.go

@@ -1,4 +1,4 @@
-package filer
+package embedded_filer
 
 import (
 	"bufio"
@@ -10,6 +10,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/chrislusf/weed-fs/go/filer"
 	"github.com/chrislusf/weed-fs/go/util"
 )
 
@@ -19,12 +20,12 @@ type DirectoryEntryInMap struct {
 	Name           string
 	Parent         *DirectoryEntryInMap
 	SubDirectories map[string]*DirectoryEntryInMap
-	Id             DirectoryId
+	Id             filer.DirectoryId
 }
 
 type DirectoryManagerInMap struct {
 	Root      *DirectoryEntryInMap
-	max       DirectoryId
+	max       filer.DirectoryId
 	logFile   *os.File
 	isLoading bool
 }
@@ -83,7 +84,7 @@ func (dm *DirectoryManagerInMap) processEachLine(line string) error {
 		if pe != nil {
 			return pe
 		}
-		if e := dm.loadDirectory(parts[1], DirectoryId(v)); e != nil {
+		if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil {
 			return e
 		}
 	case "mov":
@@ -142,7 +143,7 @@ func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryI
 	}
 	return dir, nil
 }
-func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (DirectoryId, error) {
+func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) {
 	d, e := dm.findDirectory(dirPath)
 	if e == nil {
 		return d.Id, nil
@@ -150,7 +151,7 @@ func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (DirectoryId, err
 	return dm.Root.Id, e
 }
 
-func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId DirectoryId) error {
+func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error {
 	dirPath = filepath.Clean(dirPath)
 	if dirPath == "/" {
 		return nil
@@ -201,7 +202,7 @@ func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEn
 	return dir, created
 }
 
-func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (DirectoryId, error) {
+func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) {
 	dir, _ := dm.makeDirectory(dirPath)
 	return dir.Id, nil
 }
@@ -228,13 +229,13 @@ func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParent
 	return nil
 }
 
-func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []DirectoryEntry, err error) {
+func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) {
 	d, e := dm.findDirectory(dirPath)
 	if e != nil {
 		return dirNames, e
 	}
 	for k, v := range d.SubDirectories {
-		dirNames = append(dirNames, DirectoryEntry{Name: k, Id: v.Id})
+		dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
 	}
 	return dirNames, nil
 }

go/filer/directory_test.go → go/filer/embedded_filer/directory_test.go

@@ -1,4 +1,4 @@
-package filer
+package embedded_filer
 
 import (
 	"os"

go/filer/filer_embedded.go → go/filer/embedded_filer/filer_embedded.go

@@ -1,4 +1,4 @@
-package filer
+package embedded_filer
 
 import (
 	"errors"
@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/chrislusf/weed-fs/go/filer"
 	"github.com/chrislusf/weed-fs/go/operation"
 )
 
@@ -48,13 +49,13 @@ func (filer *FilerEmbedded) FindFile(filePath string) (fid string, err error) {
 	}
 	return filer.files.FindFile(dirId, file)
 }
-func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId DirectoryId, err error) {
+func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
 	return filer.directories.FindDirectory(dirPath)
 }
-func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []DirectoryEntry, err error) {
+func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
 	return filer.directories.ListDirectories(dirPath)
 }
-func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error) {
+func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
 	dirId, e := filer.directories.FindDirectory(dirPath)
 	if e != nil {
 		return nil, e

go/filer/files_in_leveldb.go → go/filer/embedded_filer/files_in_leveldb.go

@@ -1,8 +1,9 @@
-package filer
+package embedded_filer
 
 import (
 	"bytes"
 
+	"github.com/chrislusf/weed-fs/go/filer"
 	"github.com/chrislusf/weed-fs/go/glog"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/util"
@@ -27,7 +28,7 @@ func NewFileListInLevelDb(dir string) (fl *FileListInLevelDb, err error) {
 	return
 }
 
-func genKey(dirId DirectoryId, fileName string) []byte {
+func genKey(dirId filer.DirectoryId, fileName string) []byte {
 	ret := make([]byte, 0, 4+len(fileName))
 	for i := 3; i >= 0; i-- {
 		ret = append(ret, byte(dirId>>(uint(i)*8)))
@@ -36,25 +37,25 @@ func genKey(dirId DirectoryId, fileName string) []byte {
 	return ret
 }
 
-func (fl *FileListInLevelDb) CreateFile(dirId DirectoryId, fileName string, fid string) (err error) {
+func (fl *FileListInLevelDb) CreateFile(dirId filer.DirectoryId, fileName string, fid string) (err error) {
 	glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid)
 	return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil)
 }
-func (fl *FileListInLevelDb) DeleteFile(dirId DirectoryId, fileName string) (fid string, err error) {
+func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
 	if fid, err = fl.FindFile(dirId, fileName); err != nil {
 		return
 	}
 	err = fl.db.Delete(genKey(dirId, fileName), nil)
 	return fid, err
 }
-func (fl *FileListInLevelDb) FindFile(dirId DirectoryId, fileName string) (fid string, err error) {
+func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
 	data, e := fl.db.Get(genKey(dirId, fileName), nil)
 	if e != nil {
 		return "", e
 	}
 	return string(data), nil
 }
-func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, limit int) (files []FileEntry) {
+func (fl *FileListInLevelDb) ListFiles(dirId filer.DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) {
 	glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit)
 	dirKey := genKey(dirId, "")
 	iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil)
@@ -74,7 +75,7 @@ func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, l
 				break
 			}
 		}
-		files = append(files, FileEntry{Name: fileName, Id: FileId(string(iter.Value()))})
+		files = append(files, filer.FileEntry{Name: fileName, Id: filer.FileId(string(iter.Value()))})
 	}
 	iter.Release()
 	return

go/filer/filer.go

@@ -7,13 +7,22 @@ type FileEntry struct {
 	Id   FileId `json:"fid,omitempty"`
 }
 
+type DirectoryId int32
+
+type DirectoryEntry struct {
+	Name string //dir name without path
+	Id   DirectoryId
+}
+
 type Filer interface {
-	CreateFile(filePath string, fid string) (err error)
-	FindFile(filePath string) (fid string, err error)
+	CreateFile(fullFileName string, fid string) (err error)
+	FindFile(fullFileName string) (fid string, err error)
+	DeleteFile(fullFileName string) (fid string, err error)
+
+	//Optional functions. embedded filer support these
 	FindDirectory(dirPath string) (dirId DirectoryId, err error)
 	ListDirectories(dirPath string) (dirs []DirectoryEntry, err error)
 	ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error)
 	DeleteDirectory(dirPath string, recursive bool) (err error)
-	DeleteFile(filePath string) (fid string, err error)
 	Move(fromPath string, toPath string) (err error)
 }

go/weed/weed_server/filer_server.go

@@ -5,6 +5,7 @@ import (
 	"strconv"
 
 	"github.com/chrislusf/weed-fs/go/filer"
+	"github.com/chrislusf/weed-fs/go/filer/embedded_filer"
 	"github.com/chrislusf/weed-fs/go/glog"
 )
 
@@ -28,7 +29,7 @@ func NewFilerServer(r *http.ServeMux, port int, master string, dir string, colle
 		port:               ":" + strconv.Itoa(port),
 	}
 
-	if fs.filer, err = filer.NewFilerEmbedded(master, dir); err != nil {
+	if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil {
 		glog.Fatal("Can not start filer in dir", dir, ": ", err.Error())
 		return
 	}