Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

implement files api, finish related unit tests

pull/254/head
zhaozhi 10 years ago
parent
6ca1475654
commit
f01b2f852b
2 changed files with 165 additions and 11 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 133
      go/filer/redis_store/directory_manager.go
  2. 43
      go/filer/redis_store/directory_manager_test.go

133
go/filer/redis_store/directory_manager.go
View File

@ -58,10 +58,10 @@ func (dm *DirectoryManager) MakeDirectory(dirPath string) (filer.DirectoryId, er
script := redis.NewScript(`
local root=KEYS[1]
local dirMaxIdKey=KEYS[2]
local did, exists
local did
for i, v in ipairs(ARGV) do
exists=redis.call('hexists', root, v)
if exists == 0 then
did=redis.call('hget', root, v)
if did == false then
did=redis.call('incr', dirMaxIdKey)
redis.call('hset', root, v, did)
end
@ -71,7 +71,6 @@ func (dm *DirectoryManager) MakeDirectory(dirPath string) (filer.DirectoryId, er
root=root..'/'..v
end
end
redis.call('set', 'last-dir-id', did)
return did
`)
result, err := script.Run(dm.Client, []string{root, dm.dirMaxIdKey}, parts[1:]).Result()
@ -86,7 +85,7 @@ func (dm *DirectoryManager) MakeDirectory(dirPath string) (filer.DirectoryId, er
return filer.DirectoryId(did), err
}
//delete directory dirPath and its sub-directories recursively;
//delete directory dirPath and its sub-directories or files recursively ;
//it's not this function's responsibility to check whether dirPath is en empty directory.
func (dm *DirectoryManager) DeleteDirectory(dirPath string) error {
dirPath = embedded_filer.CleanFilePath(dirPath)
@ -97,28 +96,33 @@ func (dm *DirectoryManager) DeleteDirectory(dirPath string) error {
dirPathName := filepath.Base(dirPath)
/*
lua comments:
1. delete dirPath's sub-directories recursively
1. delete dirPath's sub-directories and files recursively
2. delete dirPath itself from its parent dir
*/
script := redis.NewScript(`
local delSubs
local dirKeyPrefix=KEYS[1]
local dirFileKeyPrefix=KEYS[2]
local dirPathParent=ARGV[1]
local dirPathName=ARGV[2]
delSubs = function(dir)
local subs=redis.call('hgetall', dir);
local subs=redis.call('hgetall', dirKeyPrefix..dir);
for i, v in ipairs(subs) do
if i%2 ~= 0 then
redis.call('hdel', dir, v)
redis.call('hdel', dirKeyPrefix..dir, v)
local subd=dir..'/'..v;
delSubs(subd);
end
end
redis.call('del', dirFileKeyPrefix..dir)
end
delSubs(KEYS[1])
redis.call('hdel', KEYS[2], KEYS[3])
delSubs(dirPathParent..'/'..dirPathName)
redis.call('hdel', dirKeyPrefix..dirPathParent, dirPathName)
return 0
`)
//we do not use lua to get dirPath's parent dir and its basename, so do it in golang
err := script.Run(dm.Client, []string{dm.dirKeyPrefix + dirPath, dm.dirKeyPrefix + dirPathParent, dirPathName}, nil).Err()
err := script.Run(dm.Client, []string{dm.dirKeyPrefix, dm.dirFileKeyPrefix}, []string{dirPathParent, dirPathName}).Err()
return err
}
@ -245,3 +249,110 @@ func (dm *DirectoryManager) ListDirectories(dirPath string) (dirNames []filer.Di
}
return dirNames, nil
}
//use lua script to make directories and then put file for atomic
func (dm *DirectoryManager) PutFile(fullFileName string, fid string) error {
fullFileName = embedded_filer.CleanFilePath(fullFileName)
dirPath := filepath.Dir(fullFileName)
fname := filepath.Base(fullFileName)
parts := strings.Split(dirPath, "/")
//'d' stands for directory. root must end with a slash
root := dm.dirKeyPrefix + "/"
script := redis.NewScript(`
local root=KEYS[1]
local dirMaxIdKey=KEYS[2]
local dirFileKeyPrefix=KEYS[3]
local fname=KEYS[4]
local fid=KEYS[5]
local dirPath=table.concat(ARGV, '/')
local did
for i, v in ipairs(ARGV) do
did=redis.call('hget', root, v)
if did == false then
did=redis.call('incr', dirMaxIdKey)
redis.call('hset', root, v, did)
end
if i==1 then
root=root .. v
else
root=root..'/'..v
end
end
redis.call('hset', dirFileKeyPrefix..'/'..dirPath, fname, fid)
return did
`)
err := script.Run(dm.Client, []string{root, dm.dirMaxIdKey, dm.dirFileKeyPrefix, fname, fid}, parts[1:]).Err()
if err != nil {
glog.Errorln("redis eval put file script error:", err)
}
return err
}
func (dm *DirectoryManager) FindFile(fullFileName string) (fid string, err error) {
fullFileName = embedded_filer.CleanFilePath(fullFileName)
dirPath := filepath.Dir(fullFileName)
fname := filepath.Base(fullFileName)
result, err := dm.Client.HGet(dm.dirFileKeyPrefix+dirPath, fname).Result()
if err == redis.Nil {
err = nil
}
if err != nil {
glog.Errorf("get file %s error:%v", fullFileName, err)
}
return result, err
}
func (dm *DirectoryManager) DeleteFile(fullFileName string) (fid string, err error) {
fullFileName = embedded_filer.CleanFilePath(fullFileName)
dirPath := filepath.Dir(fullFileName)
fname := filepath.Base(fullFileName)
script := redis.NewScript(`
local dirPathKey=KEYS[1]
local fname = ARGV[1]
local fid=redis.call('hget', dirPathKey, fname)
redis.call('hdel', dirPathKey, fname)
return fid
`)
result, err := script.Run(dm.Client, []string{dm.dirFileKeyPrefix + dirPath}, []string{fname}).Result()
if err != nil {
glog.Errorf("delete file %s error:%v\n", fullFileName, err)
}
fid, ok := result.(string)
if !ok {
glog.Errorf("convert result %v to string failed", result)
}
return fid, err
}
//list files under dirPath, use lastFileName and limit for pagination
//in fact, this implemention has a bug, you can not get first file of the first page;
//the api needs to be modified
func (dm *DirectoryManager) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
dirPath = embedded_filer.CleanFilePath(dirPath)
result, le := dm.Client.HGetAllMap(dm.dirFileKeyPrefix + dirPath).Result()
if le != nil {
glog.Errorf("get files of %s error:%v", dirPath, err)
return files, le
}
//if the lastFileName argument is ok
if _, ok := result[lastFileName]; ok {
//sort entries by file names
//when files amount is large, here may be slow
nResult := len(result)
keys := make(sort.StringSlice, nResult)
i := 0
for k, _ := range result {
keys[i] = k
i++
}
keys.Sort()
index := keys.Search(lastFileName)
for _, k := range keys[index+1:] {
fid := result[k]
entry := filer.FileEntry{Name: k, Id: filer.FileId(fid)}
files = append(files, entry)
}
}
return files, le
}

43
go/filer/redis_store/directory_manager_test.go
View File

@ -182,6 +182,49 @@ func TestDirectory(t *testing.T) {
}
}
func TestFiles(t *testing.T) {
// put one file
fname := "/a/b/c/test.txt"
fid := "1,23"
err := dm.PutFile(fname, fid)
if err != nil {
t.Errorf("put file %s error:%v", fname, err)
}
// put another file
fname = "/a/b/c/abc.txt"
fid = "1,234"
err = dm.PutFile(fname, fid)
if err != nil {
t.Errorf("put file %s error:%v", fname, err)
}
//get file
id, err := dm.FindFile(fname)
if err != nil {
t.Errorf("get file %s error:%v", fname, err)
}
if id != fid {
t.Errorf("get wrong fid %s, expect %s", id, fid)
}
//list files
lastFileName := "abc.txt"
dir := filepath.Dir(fname)
files, err := dm.ListFiles(dir, lastFileName, 10)
if err != nil {
t.Errorf("list files for %s error:%v", dir, err)
}
if files[0].Name != "test.txt" {
t.Errorf("files list order wrong, first file is %s, expect %s", files[0].Name, "test.txt")
}
//delete file
id, err = dm.DeleteFile(fname)
if err != nil {
t.Errorf("delete file get error:%v", err)
}
if id != fid {
t.Errorf("delete file return wrong fid %s, expect %s", id, fid)
}
}
func clearRedisKeys(client *redis.Client, dirKeyPrefix string, dirMaxIdKey string) error {
result, err := client.Keys(dirKeyPrefix + "*").Result()
if err != nil {

Write Preview
Loading…
Cancel
Save