Browse Source

WIP

no memory issue

if some directory is removed, it may have this error

$ rm -Rf ~/tmp/m2/s1
rm: fts_read: Device not configured
pull/2013/head
Chris Lu 4 years ago
parent
commit
d9a2a7f1c4
  1. 2
      go.mod
  2. 26
      weed/filesys/dir.go
  3. 8
      weed/filesys/dir_rename.go
  4. 6
      weed/filesys/file.go
  5. 25
      weed/filesys/wfs.go

2
go.mod

@ -102,7 +102,7 @@ require (
gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect
) )
// replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse
replace github.com/seaweedfs/fuse => /Users/chris/go/src/github.com/seaweedfs/fuse
// replace github.com/chrislusf/raft => /Users/chris/go/src/github.com/chrislusf/raft // replace github.com/chrislusf/raft => /Users/chris/go/src/github.com/chrislusf/raft
replace go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547 replace go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547

26
weed/filesys/dir.go

@ -27,6 +27,7 @@ type Dir struct {
} }
var _ = fs.Node(&Dir{}) var _ = fs.Node(&Dir{})
var _ = fs.NodeIdentifier(&Dir{})
var _ = fs.NodeCreater(&Dir{}) var _ = fs.NodeCreater(&Dir{})
var _ = fs.NodeMknoder(&Dir{}) var _ = fs.NodeMknoder(&Dir{})
var _ = fs.NodeMkdirer(&Dir{}) var _ = fs.NodeMkdirer(&Dir{})
@ -42,6 +43,10 @@ var _ = fs.NodeRemovexattrer(&Dir{})
var _ = fs.NodeListxattrer(&Dir{}) var _ = fs.NodeListxattrer(&Dir{})
var _ = fs.NodeForgetter(&Dir{}) var _ = fs.NodeForgetter(&Dir{})
func (dir *Dir) Id() uint64 {
return util.FullPath(dir.FullPath()).AsInode()
}
func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error { func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
// https://github.com/bazil/fuse/issues/196 // https://github.com/bazil/fuse/issues/196
@ -105,24 +110,17 @@ func (dir *Dir) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
} }
func (dir *Dir) newFile(name string) fs.Node { func (dir *Dir) newFile(name string) fs.Node {
f := dir.wfs.fsNodeCache.EnsureFsNode(util.NewFullPath(dir.FullPath(), name), func() fs.Node {
return &File{ return &File{
Name: name, Name: name,
dir: dir, dir: dir,
wfs: dir.wfs, wfs: dir.wfs,
} }
})
f.(*File).dir = dir // in case dir node was created later
return f
} }
func (dir *Dir) newDirectory(fullpath util.FullPath) fs.Node { func (dir *Dir) newDirectory(fullpath util.FullPath) fs.Node {
d := dir.wfs.fsNodeCache.EnsureFsNode(fullpath, func() fs.Node {
return &Dir{name: fullpath.Name(), wfs: dir.wfs, parent: dir} return &Dir{name: fullpath.Name(), wfs: dir.wfs, parent: dir}
})
d.(*Dir).parent = dir // in case dir node was created later
return d
} }
func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
@ -396,15 +394,6 @@ func (dir *Dir) removeOneFile(req *fuse.RemoveRequest) error {
return fuse.ESTALE return fuse.ESTALE
} }
// clear entry inside the file
fsNode := dir.wfs.fsNodeCache.GetFsNode(filePath)
dir.wfs.fsNodeCache.DeleteFsNode(filePath)
if fsNode != nil {
if file, ok := fsNode.(*File); ok {
file.entry = nil
}
}
// remove current file handle if any // remove current file handle if any
dir.wfs.handlesLock.Lock() dir.wfs.handlesLock.Lock()
defer dir.wfs.handlesLock.Unlock() defer dir.wfs.handlesLock.Unlock()
@ -431,7 +420,6 @@ func (dir *Dir) removeFolder(req *fuse.RemoveRequest) error {
t := util.NewFullPath(dirFullPath, req.Name) t := util.NewFullPath(dirFullPath, req.Name)
dir.wfs.metaCache.DeleteEntry(context.Background(), t) dir.wfs.metaCache.DeleteEntry(context.Background(), t)
dir.wfs.fsNodeCache.DeleteFsNode(t)
return nil return nil
@ -519,8 +507,6 @@ func (dir *Dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp
func (dir *Dir) Forget() { func (dir *Dir) Forget() {
glog.V(4).Infof("Forget dir %s", dir.FullPath()) glog.V(4).Infof("Forget dir %s", dir.FullPath())
dir.wfs.fsNodeCache.DeleteFsNode(util.FullPath(dir.FullPath()))
} }
func (dir *Dir) maybeLoadEntry() (*filer_pb.Entry, error) { func (dir *Dir) maybeLoadEntry() (*filer_pb.Entry, error) {

8
weed/filesys/dir_rename.go

@ -64,19 +64,17 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
return fuse.EIO return fuse.EIO
} }
// fmt.Printf("rename path: %v => %v\n", oldPath, newPath)
dir.wfs.fsNodeCache.Move(oldPath, newPath)
// change file handle // change file handle
dir.wfs.handlesLock.Lock() dir.wfs.handlesLock.Lock()
defer dir.wfs.handlesLock.Unlock() defer dir.wfs.handlesLock.Unlock()
inodeId := oldPath.AsInode() inodeId := oldPath.AsInode()
existingHandle, found := dir.wfs.handles[inodeId] existingHandle, found := dir.wfs.handles[inodeId]
if !found || existingHandle == nil { if !found || existingHandle == nil {
return err
return nil
} }
glog.V(4).Infof("opened filehandle %s => %s", oldPath, newPath)
delete(dir.wfs.handles, inodeId) delete(dir.wfs.handles, inodeId)
dir.wfs.handles[newPath.AsInode()] = existingHandle dir.wfs.handles[newPath.AsInode()] = existingHandle
return err
return nil
} }

6
weed/filesys/file.go

@ -18,6 +18,7 @@ import (
const blockSize = 512 const blockSize = 512
var _ = fs.Node(&File{}) var _ = fs.Node(&File{})
var _ = fs.NodeIdentifier(&File{})
var _ = fs.NodeOpener(&File{}) var _ = fs.NodeOpener(&File{})
var _ = fs.NodeFsyncer(&File{}) var _ = fs.NodeFsyncer(&File{})
var _ = fs.NodeSetattrer(&File{}) var _ = fs.NodeSetattrer(&File{})
@ -40,6 +41,10 @@ func (file *File) fullpath() util.FullPath {
return util.NewFullPath(file.dir.FullPath(), file.Name) return util.NewFullPath(file.dir.FullPath(), file.Name)
} }
func (file *File) Id() uint64 {
return file.fullpath().AsInode()
}
func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) { func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) {
glog.V(4).Infof("file Attr %s, open:%v existing:%v", file.fullpath(), file.isOpen, attr) glog.V(4).Infof("file Attr %s, open:%v existing:%v", file.fullpath(), file.isOpen, attr)
@ -253,7 +258,6 @@ func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
func (file *File) Forget() { func (file *File) Forget() {
t := util.NewFullPath(file.dir.FullPath(), file.Name) t := util.NewFullPath(file.dir.FullPath(), file.Name)
glog.V(4).Infof("Forget file %s", t) glog.V(4).Infof("Forget file %s", t)
file.wfs.fsNodeCache.DeleteFsNode(t)
file.wfs.ReleaseHandle(t, 0) file.wfs.ReleaseHandle(t, 0)
} }

25
weed/filesys/wfs.go

@ -103,25 +103,16 @@ func NewSeaweedFileSystem(option *Option) *WFS {
} }
wfs.metaCache = meta_cache.NewMetaCache(path.Join(cacheDir, "meta"), util.FullPath(option.FilerMountRootPath), option.UidGidMapper, func(filePath util.FullPath) { wfs.metaCache = meta_cache.NewMetaCache(path.Join(cacheDir, "meta"), util.FullPath(option.FilerMountRootPath), option.UidGidMapper, func(filePath util.FullPath) {
fsNode := wfs.fsNodeCache.GetFsNode(filePath)
if fsNode != nil {
if file, ok := fsNode.(*File); ok {
if err := wfs.Server.InvalidateNodeData(file); err != nil {
fsNode := NodeWithId(filePath.AsInode())
if err := wfs.Server.InvalidateNodeData(fsNode); err != nil {
glog.V(4).Infof("InvalidateNodeData %s : %v", filePath, err) glog.V(4).Infof("InvalidateNodeData %s : %v", filePath, err)
} }
file.entry = nil
}
}
dir, name := filePath.DirAndName() dir, name := filePath.DirAndName()
parent := wfs.root
if dir != "/" {
parent = wfs.fsNodeCache.GetFsNode(util.FullPath(dir))
}
if parent != nil {
parent := NodeWithId(util.FullPath(dir).AsInode())
if err := wfs.Server.InvalidateEntry(parent, name); err != nil { if err := wfs.Server.InvalidateEntry(parent, name); err != nil {
glog.V(4).Infof("InvalidateEntry %s : %v", filePath, err) glog.V(4).Infof("InvalidateEntry %s : %v", filePath, err)
} }
}
}) })
startTime := time.Now() startTime := time.Now()
go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano()) go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano())
@ -267,3 +258,11 @@ func (wfs *WFS) LookupFn() wdclient.LookupFileIdFunctionType {
return filer.LookupFn(wfs) return filer.LookupFn(wfs)
} }
type NodeWithId uint64
func (n NodeWithId) Id() uint64 {
return uint64(n)
}
func (n NodeWithId) Attr(ctx context.Context, attr *fuse.Attr) error {
return nil
}
Loading…
Cancel
Save