You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

377 lines
9.0 KiB

7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
6 years ago
  1. package filesys
  2. import (
  3. "context"
  4. "os"
  5. "path"
  6. "path/filepath"
  7. "time"
  8. "bazil.org/fuse"
  9. "bazil.org/fuse/fs"
  10. "github.com/chrislusf/seaweedfs/weed/filer2"
  11. "github.com/chrislusf/seaweedfs/weed/glog"
  12. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  13. )
  14. type Dir struct {
  15. Path string
  16. wfs *WFS
  17. attributes *filer_pb.FuseAttributes
  18. }
  19. var _ = fs.Node(&Dir{})
  20. var _ = fs.NodeCreater(&Dir{})
  21. var _ = fs.NodeMkdirer(&Dir{})
  22. var _ = fs.NodeRequestLookuper(&Dir{})
  23. var _ = fs.HandleReadDirAller(&Dir{})
  24. var _ = fs.NodeRemover(&Dir{})
  25. var _ = fs.NodeRenamer(&Dir{})
  26. var _ = fs.NodeSetattrer(&Dir{})
  27. func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
  28. // https://github.com/bazil/fuse/issues/196
  29. attr.Valid = time.Second
  30. if dir.Path == "/" {
  31. attr.Mode = os.ModeDir | 0777
  32. return nil
  33. }
  34. item := dir.wfs.listDirectoryEntriesCache.Get(dir.Path)
  35. if item != nil && !item.Expired() {
  36. entry := item.Value().(*filer_pb.Entry)
  37. attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
  38. attr.Ctime = time.Unix(entry.Attributes.Crtime, 0)
  39. attr.Mode = os.FileMode(entry.Attributes.FileMode)
  40. attr.Gid = entry.Attributes.Gid
  41. attr.Uid = entry.Attributes.Uid
  42. return nil
  43. }
  44. parent, name := filepath.Split(dir.Path)
  45. err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  46. request := &filer_pb.LookupDirectoryEntryRequest{
  47. Directory: parent,
  48. Name: name,
  49. }
  50. glog.V(1).Infof("read dir %s attr: %v", dir.Path, request)
  51. resp, err := client.LookupDirectoryEntry(context, request)
  52. if err != nil {
  53. if err == filer2.ErrNotFound {
  54. return nil
  55. }
  56. glog.V(0).Infof("read dir %s attr %v: %v", dir.Path, request, err)
  57. return err
  58. }
  59. if resp.Entry != nil {
  60. dir.attributes = resp.Entry.Attributes
  61. }
  62. dir.wfs.listDirectoryEntriesCache.Set(dir.Path, resp.Entry, dir.wfs.option.EntryCacheTtl)
  63. return nil
  64. })
  65. if err != nil {
  66. return err
  67. }
  68. // glog.V(1).Infof("dir %s: %v", dir.Path, attributes)
  69. // glog.V(1).Infof("dir %s permission: %v", dir.Path, os.FileMode(attributes.FileMode))
  70. attr.Mode = os.FileMode(dir.attributes.FileMode) | os.ModeDir
  71. attr.Mtime = time.Unix(dir.attributes.Mtime, 0)
  72. attr.Ctime = time.Unix(dir.attributes.Crtime, 0)
  73. attr.Gid = dir.attributes.Gid
  74. attr.Uid = dir.attributes.Uid
  75. return nil
  76. }
  77. func (dir *Dir) newFile(name string, entry *filer_pb.Entry) *File {
  78. return &File{
  79. Name: name,
  80. dir: dir,
  81. wfs: dir.wfs,
  82. entry: entry,
  83. }
  84. }
  85. func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
  86. resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
  87. request := &filer_pb.CreateEntryRequest{
  88. Directory: dir.Path,
  89. Entry: &filer_pb.Entry{
  90. Name: req.Name,
  91. IsDirectory: req.Mode&os.ModeDir > 0,
  92. Attributes: &filer_pb.FuseAttributes{
  93. Mtime: time.Now().Unix(),
  94. Crtime: time.Now().Unix(),
  95. FileMode: uint32(req.Mode),
  96. Uid: req.Uid,
  97. Gid: req.Gid,
  98. Collection: dir.wfs.option.Collection,
  99. Replication: dir.wfs.option.Replication,
  100. TtlSec: dir.wfs.option.TtlSec,
  101. },
  102. },
  103. }
  104. glog.V(1).Infof("create: %v", request)
  105. if request.Entry.IsDirectory {
  106. if err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  107. if _, err := client.CreateEntry(ctx, request); err != nil {
  108. glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
  109. return fuse.EIO
  110. }
  111. return nil
  112. }); err != nil {
  113. return nil, nil, err
  114. }
  115. }
  116. file := dir.newFile(req.Name, request.Entry)
  117. if !request.Entry.IsDirectory {
  118. file.isOpen = true
  119. }
  120. fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid)
  121. fh.dirtyMetadata = true
  122. return file, fh, nil
  123. }
  124. func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
  125. err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  126. request := &filer_pb.CreateEntryRequest{
  127. Directory: dir.Path,
  128. Entry: &filer_pb.Entry{
  129. Name: req.Name,
  130. IsDirectory: true,
  131. Attributes: &filer_pb.FuseAttributes{
  132. Mtime: time.Now().Unix(),
  133. Crtime: time.Now().Unix(),
  134. FileMode: uint32(req.Mode),
  135. Uid: req.Uid,
  136. Gid: req.Gid,
  137. },
  138. },
  139. }
  140. glog.V(1).Infof("mkdir: %v", request)
  141. if _, err := client.CreateEntry(ctx, request); err != nil {
  142. glog.V(0).Infof("mkdir %s/%s: %v", dir.Path, req.Name, err)
  143. return fuse.EIO
  144. }
  145. return nil
  146. })
  147. if err == nil {
  148. node := &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs}
  149. return node, nil
  150. }
  151. return nil, err
  152. }
  153. func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) {
  154. var entry *filer_pb.Entry
  155. item := dir.wfs.listDirectoryEntriesCache.Get(path.Join(dir.Path, req.Name))
  156. if item != nil && !item.Expired() {
  157. entry = item.Value().(*filer_pb.Entry)
  158. }
  159. if entry == nil {
  160. err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  161. request := &filer_pb.LookupDirectoryEntryRequest{
  162. Directory: dir.Path,
  163. Name: req.Name,
  164. }
  165. glog.V(4).Infof("lookup directory entry: %v", request)
  166. resp, err := client.LookupDirectoryEntry(ctx, request)
  167. if err != nil {
  168. // glog.V(0).Infof("lookup %s/%s: %v", dir.Path, name, err)
  169. return fuse.ENOENT
  170. }
  171. entry = resp.Entry
  172. dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, dir.wfs.option.EntryCacheTtl)
  173. return nil
  174. })
  175. }
  176. if entry != nil {
  177. if entry.IsDirectory {
  178. node = &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs, attributes: entry.Attributes}
  179. } else {
  180. node = dir.newFile(req.Name, entry)
  181. }
  182. resp.EntryValid = time.Duration(0)
  183. resp.Attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
  184. resp.Attr.Ctime = time.Unix(entry.Attributes.Crtime, 0)
  185. resp.Attr.Mode = os.FileMode(entry.Attributes.FileMode)
  186. resp.Attr.Gid = entry.Attributes.Gid
  187. resp.Attr.Uid = entry.Attributes.Uid
  188. return node, nil
  189. }
  190. return nil, fuse.ENOENT
  191. }
  192. func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
  193. err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  194. paginationLimit := 1024
  195. remaining := dir.wfs.option.DirListingLimit
  196. lastEntryName := ""
  197. for remaining >= 0 {
  198. request := &filer_pb.ListEntriesRequest{
  199. Directory: dir.Path,
  200. StartFromFileName: lastEntryName,
  201. Limit: uint32(paginationLimit),
  202. }
  203. glog.V(4).Infof("read directory: %v", request)
  204. resp, err := client.ListEntries(ctx, request)
  205. if err != nil {
  206. glog.V(0).Infof("list %s: %v", dir.Path, err)
  207. return fuse.EIO
  208. }
  209. cacheTtl := estimatedCacheTtl(len(resp.Entries))
  210. for _, entry := range resp.Entries {
  211. if entry.IsDirectory {
  212. dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
  213. ret = append(ret, dirent)
  214. } else {
  215. dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
  216. ret = append(ret, dirent)
  217. }
  218. dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, cacheTtl)
  219. lastEntryName = entry.Name
  220. }
  221. remaining -= len(resp.Entries)
  222. if len(resp.Entries) < paginationLimit {
  223. break
  224. }
  225. }
  226. return nil
  227. })
  228. return ret, err
  229. }
  230. func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
  231. return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  232. request := &filer_pb.DeleteEntryRequest{
  233. Directory: dir.Path,
  234. Name: req.Name,
  235. IsDeleteData: true,
  236. }
  237. glog.V(1).Infof("remove directory entry: %v", request)
  238. _, err := client.DeleteEntry(ctx, request)
  239. if err != nil {
  240. glog.V(0).Infof("remove %s/%s: %v", dir.Path, req.Name, err)
  241. return fuse.EIO
  242. }
  243. dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))
  244. return nil
  245. })
  246. }
  247. func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
  248. glog.V(3).Infof("%v dir setattr %+v, fh=%d", dir.Path, req, req.Handle)
  249. if req.Valid.Mode() {
  250. dir.attributes.FileMode = uint32(req.Mode)
  251. }
  252. if req.Valid.Uid() {
  253. dir.attributes.Uid = req.Uid
  254. }
  255. if req.Valid.Gid() {
  256. dir.attributes.Gid = req.Gid
  257. }
  258. if req.Valid.Mtime() {
  259. dir.attributes.Mtime = req.Mtime.Unix()
  260. }
  261. parentDir, name := filer2.FullPath(dir.Path).DirAndName()
  262. return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  263. request := &filer_pb.UpdateEntryRequest{
  264. Directory: parentDir,
  265. Entry: &filer_pb.Entry{
  266. Name: name,
  267. Attributes: dir.attributes,
  268. },
  269. }
  270. glog.V(1).Infof("set attr directory entry: %v", request)
  271. _, err := client.UpdateEntry(ctx, request)
  272. if err != nil {
  273. glog.V(0).Infof("UpdateEntry %s: %v", dir.Path, err)
  274. return fuse.EIO
  275. }
  276. dir.wfs.listDirectoryEntriesCache.Delete(dir.Path)
  277. return nil
  278. })
  279. }
  280. func estimatedCacheTtl(numEntries int) time.Duration {
  281. if numEntries < 100 {
  282. // 30 ms per entry
  283. return 3 * time.Second
  284. }
  285. if numEntries < 1000 {
  286. // 10 ms per entry
  287. return 10 * time.Second
  288. }
  289. if numEntries < 10000 {
  290. // 10 ms per entry
  291. return 100 * time.Second
  292. }
  293. // 2 ms per entry
  294. return time.Duration(numEntries*2) * time.Millisecond
  295. }