From c359a5117f9400a08b18f01daaaea13b09e3f235 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Wed, 30 Mar 2016 12:34:11 -0700 Subject: [PATCH 1/4] add lock to dir children map fix https://github.com/chrislusf/seaweedfs/issues/273 --- go/filer/embedded_filer/directory_in_map.go | 60 +++++++++++++++------ 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/go/filer/embedded_filer/directory_in_map.go b/go/filer/embedded_filer/directory_in_map.go index a1679f93e..b142639ab 100644 --- a/go/filer/embedded_filer/directory_in_map.go +++ b/go/filer/embedded_filer/directory_in_map.go @@ -17,12 +17,43 @@ import ( var writeLock sync.Mutex //serialize changes to dir.log type DirectoryEntryInMap struct { + sync.Mutex Name string Parent *DirectoryEntryInMap - SubDirectories map[string]*DirectoryEntryInMap + subDirectories map[string]*DirectoryEntryInMap Id filer.DirectoryId } +func (de *DirectoryEntryInMap) getChild(dirName string) (*DirectoryEntryInMap, bool) { + de.Lock() + defer de.Unlock() + child, ok := de.subDirectories[dirName] + return child, ok +} +func (de *DirectoryEntryInMap) addChild(dirName string, child *DirectoryEntryInMap) { + de.Lock() + defer de.Unlock() + de.subDirectories[dirName] = child +} +func (de *DirectoryEntryInMap) removeChild(dirName string) { + de.Lock() + defer de.Unlock() + delete(de.subDirectories, dirName) +} +func (de *DirectoryEntryInMap) hasChildren() bool { + de.Lock() + defer de.Unlock() + return len(de.subDirectories) > 0 +} +func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryEntry) { + de.Lock() + defer de.Unlock() + for k, v := range de.subDirectories { + dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id}) + } + return dirNames +} + type DirectoryManagerInMap struct { Root *DirectoryEntryInMap max filer.DirectoryId @@ -33,7 +64,7 @@ type DirectoryManagerInMap struct { func (dm *DirectoryManagerInMap) NewDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap, err error) { writeLock.Lock() defer writeLock.Unlock() - d = &DirectoryEntryInMap{Name: name, Parent: parent, SubDirectories: make(map[string]*DirectoryEntryInMap)} + d = &DirectoryEntryInMap{Name: name, Parent: parent, subDirectories: make(map[string]*DirectoryEntryInMap)} var parts []string for p := d; p != nil && p.Name != ""; p = p.Parent { parts = append(parts, p.Name) @@ -60,7 +91,7 @@ func (dm *DirectoryManagerInMap) log(words ...string) { func NewDirectoryManagerInMap(dirLogFile string) (dm *DirectoryManagerInMap, err error) { dm = &DirectoryManagerInMap{} //dm.Root do not use NewDirectoryEntryInMap, since dm.max will be changed - dm.Root = &DirectoryEntryInMap{SubDirectories: make(map[string]*DirectoryEntryInMap)} + dm.Root = &DirectoryEntryInMap{subDirectories: make(map[string]*DirectoryEntryInMap)} if dm.logFile, err = os.OpenFile(dirLogFile, os.O_RDWR|os.O_CREATE, 0644); err != nil { return nil, fmt.Errorf("cannot write directory log file %s: %v", dirLogFile, err) } @@ -135,7 +166,7 @@ func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryI parts := strings.Split(dirPath, "/") dir := dm.Root for i := 1; i < len(parts); i++ { - if sub, ok := dir.SubDirectories[parts[i]]; ok { + if sub, ok := dir.getChild(parts[i]); ok { dir = sub } else { return dm.Root, fmt.Errorf("Directory %s Not Found", dirPath) @@ -159,7 +190,7 @@ func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.Direc parts := strings.Split(dirPath, "/") dir := dm.Root for i := 1; i < len(parts); i++ { - sub, ok := dir.SubDirectories[parts[i]] + sub, ok := dir.getChild(parts[i]) if !ok { if i != len(parts)-1 { return fmt.Errorf("%s should be created after parent %s", dirPath, parts[i]) @@ -172,7 +203,7 @@ func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.Direc if sub.Id != dirId { return fmt.Errorf("%s should be have id %v instead of %v", dirPath, sub.Id, dirId) } - dir.SubDirectories[parts[i]] = sub + dir.addChild(parts[i], sub) } dir = sub } @@ -187,14 +218,14 @@ func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEn parts := strings.Split(dirPath, "/") dir = dm.Root for i := 1; i < len(parts); i++ { - sub, ok := dir.SubDirectories[parts[i]] + sub, ok := dir.getChild(parts[i]) if !ok { var err error sub, err = dm.NewDirectoryEntryInMap(dir, parts[i]) if err != nil { return nil, false } - dir.SubDirectories[parts[i]] = sub + dir.addChild(parts[i], sub) created = true } dir = sub @@ -219,11 +250,11 @@ func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParent return pe } dm.log("mov", oldDirPath, newParentDirPath, newName) - delete(oldDir.Parent.SubDirectories, oldDir.Name) + oldDir.Parent.removeChild(oldDir.Name) if newName == "" { newName = oldDir.Name } - parentDir.SubDirectories[newName] = oldDir + parentDir.addChild(newName, oldDir) oldDir.Name = newName oldDir.Parent = parentDir return nil @@ -234,10 +265,7 @@ func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []fil if e != nil { return dirNames, e } - for k, v := range d.SubDirectories { - dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id}) - } - return dirNames, nil + return d.children(), nil } func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error { writeLock.Lock() @@ -249,10 +277,10 @@ func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error { if e != nil { return e } - if len(d.SubDirectories) != 0 { + if d.hasChildren() { return fmt.Errorf("dir %s still has sub directories", dirPath) } - delete(d.Parent.SubDirectories, d.Name) + d.Parent.removeChild(d.Name) d.Parent = nil dm.log("del", dirPath) return nil From 67cd8959bcc81cbe32a6c1998e8facb36436bb88 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Wed, 30 Mar 2016 12:53:37 -0700 Subject: [PATCH 2/4] fix tests --- go/filer/embedded_filer/directory_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/filer/embedded_filer/directory_test.go b/go/filer/embedded_filer/directory_test.go index cb0090bf2..c8b3f1f30 100644 --- a/go/filer/embedded_filer/directory_test.go +++ b/go/filer/embedded_filer/directory_test.go @@ -57,13 +57,13 @@ func TestDirectory(t *testing.T) { func printTree(node *DirectoryEntryInMap, path []string) { println(strings.Join(path, "/") + "/" + node.Name) path = append(path, node.Name) - for _, v := range node.SubDirectories { + for _, v := range node.subDirectories { printTree(v, path) } } func compare(root1 *DirectoryEntryInMap, root2 *DirectoryEntryInMap) bool { - if len(root1.SubDirectories) != len(root2.SubDirectories) { + if len(root1.subDirectories) != len(root2.subDirectories) { return false } if root1.Name != root2.Name { @@ -77,8 +77,8 @@ func compare(root1 *DirectoryEntryInMap, root2 *DirectoryEntryInMap) bool { return false } } - for k, v := range root1.SubDirectories { - if !compare(v, root2.SubDirectories[k]) { + for k, v := range root1.subDirectories { + if !compare(v, root2.subDirectories[k]) { return false } } From ca2a6cc36bf91e1072751c4d8567829f47f3855b Mon Sep 17 00:00:00 2001 From: Alexandr Razumov Date: Fri, 1 Apr 2016 22:18:58 +0300 Subject: [PATCH 3/4] use klauspost/crc32 instead of hash/crc32 --- go/storage/crc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/storage/crc.go b/go/storage/crc.go index b10354d64..21e384854 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -2,7 +2,7 @@ package storage import ( "fmt" - "hash/crc32" + "github.com/klauspost/crc32" "github.com/chrislusf/seaweedfs/go/util" ) From a2bb5a34fda6241c242a9d37f7f9d1d0e155b756 Mon Sep 17 00:00:00 2001 From: Alexandr Razumov Date: Fri, 1 Apr 2016 22:23:37 +0300 Subject: [PATCH 4/4] update .travis.yml --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 27d6b58bf..059699b8b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,8 @@ sudo: false language: go -go: +go: - 1.5 + - 1.6 - tip before_install: @@ -9,6 +10,6 @@ before_install: install: - go get ./go/... - + script: - go test ./go/...