Chris Lu
5 years ago
22 changed files with 310 additions and 401 deletions
-
9weed/command/scaffold.go
-
30weed/filer2/abstract_sql/abstract_sql_store.go
-
11weed/filer2/cassandra/cassandra_store.go
-
10weed/filer2/etcd/etcd_store.go
-
62weed/filer2/filer.go
-
49weed/filer2/filer_client_util.go
-
102weed/filer2/filer_delete_entry.go
-
7weed/filer2/filer_deletion.go
-
11weed/filer2/filerstore.go
-
35weed/filer2/leveldb/leveldb_store.go
-
35weed/filer2/leveldb2/leveldb2_store.go
-
132weed/filer2/memdb/memdb_store.go
-
149weed/filer2/memdb/memdb_store_test.go
-
1weed/filer2/mysql/mysql_store.go
-
1weed/filer2/postgres/postgres_store.go
-
18weed/filer2/redis/universal_redis_store.go
-
32weed/filer2/tikv/tikv_store.go
-
6weed/s3api/s3api_server.go
-
6weed/server/filer_grpc_server.go
-
1weed/server/filer_server.go
-
2weed/server/filer_server_handlers_write.go
-
2weed/server/filer_server_handlers_write_autochunk.go
@ -0,0 +1,102 @@ |
|||
package filer2 |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
) |
|||
|
|||
func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecursive bool, ignoreRecursiveError, shouldDeleteChunks bool) (err error) { |
|||
if p == "/" { |
|||
return nil |
|||
} |
|||
|
|||
entry, findErr := f.FindEntry(ctx, p) |
|||
if findErr != nil { |
|||
return findErr |
|||
} |
|||
|
|||
var chunks []*filer_pb.FileChunk |
|||
chunks = append(chunks, entry.Chunks...) |
|||
if entry.IsDirectory() { |
|||
// delete the folder children, not including the folder itself
|
|||
var dirChunks []*filer_pb.FileChunk |
|||
dirChunks, err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks) |
|||
if err != nil { |
|||
return fmt.Errorf("delete directory %s: %v", p, err) |
|||
} |
|||
chunks = append(chunks, dirChunks...) |
|||
f.cacheDelDirectory(string(p)) |
|||
} |
|||
// delete the file or folder
|
|||
err = f.doDeleteEntryMetaAndData(ctx, entry, shouldDeleteChunks) |
|||
if err != nil { |
|||
return fmt.Errorf("delete file %s: %v", p, err) |
|||
} |
|||
|
|||
if shouldDeleteChunks { |
|||
go f.DeleteChunks(chunks) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive bool, ignoreRecursiveError, shouldDeleteChunks bool) (chunks []*filer_pb.FileChunk, err error) { |
|||
|
|||
lastFileName := "" |
|||
includeLastFile := false |
|||
for { |
|||
entries, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize) |
|||
if err != nil { |
|||
glog.Errorf("list folder %s: %v", entry.FullPath, err) |
|||
return nil, fmt.Errorf("list folder %s: %v", entry.FullPath, err) |
|||
} |
|||
if lastFileName == "" && !isRecursive && len(entries) > 0 { |
|||
// only for first iteration in the loop
|
|||
return nil, fmt.Errorf("fail to delete non-empty folder: %s", entry.FullPath) |
|||
} |
|||
|
|||
for _, sub := range entries { |
|||
lastFileName = sub.Name() |
|||
var dirChunks []*filer_pb.FileChunk |
|||
if sub.IsDirectory() { |
|||
dirChunks, err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks) |
|||
} |
|||
if err != nil && !ignoreRecursiveError { |
|||
return nil, err |
|||
} |
|||
if shouldDeleteChunks { |
|||
chunks = append(chunks, dirChunks...) |
|||
} |
|||
} |
|||
|
|||
if len(entries) < PaginationSize { |
|||
break |
|||
} |
|||
} |
|||
|
|||
f.cacheDelDirectory(string(entry.FullPath)) |
|||
|
|||
glog.V(3).Infof("deleting directory %v", entry.FullPath) |
|||
|
|||
if storeDeletionErr := f.store.DeleteFolderChildren(ctx, entry.FullPath); storeDeletionErr != nil { |
|||
return nil, fmt.Errorf("filer store delete: %v", storeDeletionErr) |
|||
} |
|||
f.NotifyUpdateEvent(entry, nil, shouldDeleteChunks) |
|||
|
|||
return chunks, nil |
|||
} |
|||
|
|||
func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shouldDeleteChunks bool) (err error) { |
|||
|
|||
glog.V(3).Infof("deleting entry %v", entry.FullPath) |
|||
|
|||
if storeDeletionErr := f.store.DeleteEntry(ctx, entry.FullPath); storeDeletionErr != nil { |
|||
return fmt.Errorf("filer store delete: %v", storeDeletionErr) |
|||
} |
|||
f.NotifyUpdateEvent(entry, nil, shouldDeleteChunks) |
|||
|
|||
return nil |
|||
} |
|||
@ -1,132 +0,0 @@ |
|||
package memdb |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"github.com/chrislusf/seaweedfs/weed/filer2" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"github.com/google/btree" |
|||
"strings" |
|||
"sync" |
|||
) |
|||
|
|||
func init() { |
|||
filer2.Stores = append(filer2.Stores, &MemDbStore{}) |
|||
} |
|||
|
|||
type MemDbStore struct { |
|||
tree *btree.BTree |
|||
treeLock sync.Mutex |
|||
} |
|||
|
|||
type entryItem struct { |
|||
*filer2.Entry |
|||
} |
|||
|
|||
func (a entryItem) Less(b btree.Item) bool { |
|||
return strings.Compare(string(a.FullPath), string(b.(entryItem).FullPath)) < 0 |
|||
} |
|||
|
|||
func (store *MemDbStore) GetName() string { |
|||
return "memory" |
|||
} |
|||
|
|||
func (store *MemDbStore) Initialize(configuration util.Configuration) (err error) { |
|||
store.tree = btree.New(8) |
|||
return nil |
|||
} |
|||
|
|||
func (store *MemDbStore) BeginTransaction(ctx context.Context) (context.Context, error) { |
|||
return ctx, nil |
|||
} |
|||
func (store *MemDbStore) CommitTransaction(ctx context.Context) error { |
|||
return nil |
|||
} |
|||
func (store *MemDbStore) RollbackTransaction(ctx context.Context) error { |
|||
return nil |
|||
} |
|||
|
|||
func (store *MemDbStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) { |
|||
// println("inserting", entry.FullPath)
|
|||
store.treeLock.Lock() |
|||
store.tree.ReplaceOrInsert(entryItem{entry}) |
|||
store.treeLock.Unlock() |
|||
return nil |
|||
} |
|||
|
|||
func (store *MemDbStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) { |
|||
if _, err = store.FindEntry(ctx, entry.FullPath); err != nil { |
|||
return fmt.Errorf("no such file %s : %v", entry.FullPath, err) |
|||
} |
|||
store.treeLock.Lock() |
|||
store.tree.ReplaceOrInsert(entryItem{entry}) |
|||
store.treeLock.Unlock() |
|||
return nil |
|||
} |
|||
|
|||
func (store *MemDbStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) { |
|||
item := store.tree.Get(entryItem{&filer2.Entry{FullPath: fullpath}}) |
|||
if item == nil { |
|||
return nil, filer2.ErrNotFound |
|||
} |
|||
entry = item.(entryItem).Entry |
|||
return entry, nil |
|||
} |
|||
|
|||
func (store *MemDbStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) { |
|||
store.treeLock.Lock() |
|||
store.tree.Delete(entryItem{&filer2.Entry{FullPath: fullpath}}) |
|||
store.treeLock.Unlock() |
|||
return nil |
|||
} |
|||
|
|||
func (store *MemDbStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { |
|||
|
|||
startFrom := string(fullpath) |
|||
if startFileName != "" { |
|||
startFrom = startFrom + "/" + startFileName |
|||
} |
|||
|
|||
store.tree.AscendGreaterOrEqual(entryItem{&filer2.Entry{FullPath: filer2.FullPath(startFrom)}}, |
|||
func(item btree.Item) bool { |
|||
if limit <= 0 { |
|||
return false |
|||
} |
|||
entry := item.(entryItem).Entry |
|||
// println("checking", entry.FullPath)
|
|||
|
|||
if entry.FullPath == fullpath { |
|||
// skipping the current directory
|
|||
// println("skipping the folder", entry.FullPath)
|
|||
return true |
|||
} |
|||
|
|||
dir, name := entry.FullPath.DirAndName() |
|||
if name == startFileName { |
|||
if inclusive { |
|||
limit-- |
|||
entries = append(entries, entry) |
|||
} |
|||
return true |
|||
} |
|||
|
|||
// only iterate the same prefix
|
|||
if !strings.HasPrefix(string(entry.FullPath), string(fullpath)) { |
|||
// println("breaking from", entry.FullPath)
|
|||
return false |
|||
} |
|||
|
|||
if dir != string(fullpath) { |
|||
// this could be items in deeper directories
|
|||
// println("skipping deeper folder", entry.FullPath)
|
|||
return true |
|||
} |
|||
// now process the directory items
|
|||
// println("adding entry", entry.FullPath)
|
|||
limit-- |
|||
entries = append(entries, entry) |
|||
return true |
|||
}, |
|||
) |
|||
return entries, nil |
|||
} |
|||
@ -1,149 +0,0 @@ |
|||
package memdb |
|||
|
|||
import ( |
|||
"context" |
|||
"github.com/chrislusf/seaweedfs/weed/filer2" |
|||
"testing" |
|||
) |
|||
|
|||
func TestCreateAndFind(t *testing.T) { |
|||
filer := filer2.NewFiler(nil, nil) |
|||
store := &MemDbStore{} |
|||
store.Initialize(nil) |
|||
filer.SetStore(store) |
|||
filer.DisableDirectoryCache() |
|||
|
|||
ctx := context.Background() |
|||
|
|||
fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg") |
|||
|
|||
entry1 := &filer2.Entry{ |
|||
FullPath: fullpath, |
|||
Attr: filer2.Attr{ |
|||
Mode: 0440, |
|||
Uid: 1234, |
|||
Gid: 5678, |
|||
}, |
|||
} |
|||
|
|||
if err := filer.CreateEntry(ctx, entry1); err != nil { |
|||
t.Errorf("create entry %v: %v", entry1.FullPath, err) |
|||
return |
|||
} |
|||
|
|||
entry, err := filer.FindEntry(ctx, fullpath) |
|||
|
|||
if err != nil { |
|||
t.Errorf("find entry: %v", err) |
|||
return |
|||
} |
|||
|
|||
if entry.FullPath != entry1.FullPath { |
|||
t.Errorf("find wrong entry: %v", entry.FullPath) |
|||
return |
|||
} |
|||
|
|||
} |
|||
|
|||
func TestCreateFileAndList(t *testing.T) { |
|||
filer := filer2.NewFiler(nil, nil) |
|||
store := &MemDbStore{} |
|||
store.Initialize(nil) |
|||
filer.SetStore(store) |
|||
filer.DisableDirectoryCache() |
|||
|
|||
ctx := context.Background() |
|||
|
|||
entry1 := &filer2.Entry{ |
|||
FullPath: filer2.FullPath("/home/chris/this/is/one/file1.jpg"), |
|||
Attr: filer2.Attr{ |
|||
Mode: 0440, |
|||
Uid: 1234, |
|||
Gid: 5678, |
|||
}, |
|||
} |
|||
|
|||
entry2 := &filer2.Entry{ |
|||
FullPath: filer2.FullPath("/home/chris/this/is/one/file2.jpg"), |
|||
Attr: filer2.Attr{ |
|||
Mode: 0440, |
|||
Uid: 1234, |
|||
Gid: 5678, |
|||
}, |
|||
} |
|||
|
|||
filer.CreateEntry(ctx, entry1) |
|||
filer.CreateEntry(ctx, entry2) |
|||
|
|||
// checking the 2 files
|
|||
entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one/"), "", false, 100) |
|||
|
|||
if err != nil { |
|||
t.Errorf("list entries: %v", err) |
|||
return |
|||
} |
|||
|
|||
if len(entries) != 2 { |
|||
t.Errorf("list entries count: %v", len(entries)) |
|||
return |
|||
} |
|||
|
|||
if entries[0].FullPath != entry1.FullPath { |
|||
t.Errorf("find wrong entry 1: %v", entries[0].FullPath) |
|||
return |
|||
} |
|||
|
|||
if entries[1].FullPath != entry2.FullPath { |
|||
t.Errorf("find wrong entry 2: %v", entries[1].FullPath) |
|||
return |
|||
} |
|||
|
|||
// checking the offset
|
|||
entries, err = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one/"), "file1.jpg", false, 100) |
|||
if len(entries) != 1 { |
|||
t.Errorf("list entries count: %v", len(entries)) |
|||
return |
|||
} |
|||
|
|||
// checking one upper directory
|
|||
entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is"), "", false, 100) |
|||
if len(entries) != 1 { |
|||
t.Errorf("list entries count: %v", len(entries)) |
|||
return |
|||
} |
|||
|
|||
// checking root directory
|
|||
entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100) |
|||
if len(entries) != 1 { |
|||
t.Errorf("list entries count: %v", len(entries)) |
|||
return |
|||
} |
|||
|
|||
// add file3
|
|||
file3Path := filer2.FullPath("/home/chris/this/is/file3.jpg") |
|||
entry3 := &filer2.Entry{ |
|||
FullPath: file3Path, |
|||
Attr: filer2.Attr{ |
|||
Mode: 0440, |
|||
Uid: 1234, |
|||
Gid: 5678, |
|||
}, |
|||
} |
|||
filer.CreateEntry(ctx, entry3) |
|||
|
|||
// checking one upper directory
|
|||
entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is"), "", false, 100) |
|||
if len(entries) != 2 { |
|||
t.Errorf("list entries count: %v", len(entries)) |
|||
return |
|||
} |
|||
|
|||
// delete file and count
|
|||
filer.DeleteEntryMetaAndData(ctx, file3Path, false, false, false) |
|||
entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is"), "", false, 100) |
|||
if len(entries) != 1 { |
|||
t.Errorf("list entries count: %v", len(entries)) |
|||
return |
|||
} |
|||
|
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue