You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

193 lines
5.0 KiB

7 years ago
7 years ago
6 years ago
6 years ago
6 years ago
6 years ago
  1. package filesys
  2. import (
  3. "context"
  4. "fmt"
  5. "math"
  6. "os"
  7. "sync"
  8. "time"
  9. "github.com/chrislusf/seaweedfs/weed/glog"
  10. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  11. "github.com/chrislusf/seaweedfs/weed/util"
  12. "github.com/karlseguin/ccache"
  13. "github.com/seaweedfs/fuse"
  14. "github.com/seaweedfs/fuse/fs"
  15. "google.golang.org/grpc"
  16. )
  17. type Option struct {
  18. FilerGrpcAddress string
  19. FilerMountRootPath string
  20. Collection string
  21. Replication string
  22. TtlSec int32
  23. ChunkSizeLimit int64
  24. DataCenter string
  25. DirListingLimit int
  26. EntryCacheTtl time.Duration
  27. MountUid uint32
  28. MountGid uint32
  29. MountMode os.FileMode
  30. }
  31. var _ = fs.FS(&WFS{})
  32. var _ = fs.FSStatfser(&WFS{})
  33. type WFS struct {
  34. option *Option
  35. listDirectoryEntriesCache *ccache.Cache
  36. // contains all open handles
  37. handles []*FileHandle
  38. pathToHandleIndex map[string]int
  39. pathToHandleLock sync.Mutex
  40. bufPool sync.Pool
  41. fileIdsDeletionChan chan []string
  42. stats statsCache
  43. }
  44. type statsCache struct {
  45. filer_pb.StatisticsResponse
  46. lastChecked int64 // unix time in seconds
  47. }
  48. func NewSeaweedFileSystem(option *Option) *WFS {
  49. wfs := &WFS{
  50. option: option,
  51. listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(1024 * 8).ItemsToPrune(100)),
  52. pathToHandleIndex: make(map[string]int),
  53. bufPool: sync.Pool{
  54. New: func() interface{} {
  55. return make([]byte, option.ChunkSizeLimit)
  56. },
  57. },
  58. fileIdsDeletionChan: make(chan []string, 32),
  59. }
  60. go wfs.loopProcessingDeletion()
  61. return wfs
  62. }
  63. func (wfs *WFS) Root() (fs.Node, error) {
  64. return &Dir{Path: wfs.option.FilerMountRootPath, wfs: wfs}, nil
  65. }
  66. func (wfs *WFS) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
  67. return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
  68. client := filer_pb.NewSeaweedFilerClient(grpcConnection)
  69. return fn(client)
  70. }, wfs.option.FilerGrpcAddress)
  71. }
  72. func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHandle) {
  73. wfs.pathToHandleLock.Lock()
  74. defer wfs.pathToHandleLock.Unlock()
  75. fullpath := file.fullpath()
  76. index, found := wfs.pathToHandleIndex[fullpath]
  77. if found && wfs.handles[index] != nil {
  78. glog.V(2).Infoln(fullpath, "found fileHandle id", index)
  79. return wfs.handles[index]
  80. }
  81. fileHandle = newFileHandle(file, uid, gid)
  82. for i, h := range wfs.handles {
  83. if h == nil {
  84. wfs.handles[i] = fileHandle
  85. fileHandle.handle = uint64(i)
  86. wfs.pathToHandleIndex[fullpath] = i
  87. glog.V(4).Infoln(fullpath, "reuse fileHandle id", fileHandle.handle)
  88. return
  89. }
  90. }
  91. wfs.handles = append(wfs.handles, fileHandle)
  92. fileHandle.handle = uint64(len(wfs.handles) - 1)
  93. glog.V(2).Infoln(fullpath, "new fileHandle id", fileHandle.handle)
  94. wfs.pathToHandleIndex[fullpath] = int(fileHandle.handle)
  95. return
  96. }
  97. func (wfs *WFS) ReleaseHandle(fullpath string, handleId fuse.HandleID) {
  98. wfs.pathToHandleLock.Lock()
  99. defer wfs.pathToHandleLock.Unlock()
  100. glog.V(4).Infof("%s releasing handle id %d current handles length %d", fullpath, handleId, len(wfs.handles))
  101. delete(wfs.pathToHandleIndex, fullpath)
  102. if int(handleId) < len(wfs.handles) {
  103. wfs.handles[int(handleId)] = nil
  104. }
  105. return
  106. }
  107. // Statfs is called to obtain file system metadata. Implements fuse.FSStatfser
  108. func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) error {
  109. glog.V(4).Infof("reading fs stats: %+v", req)
  110. if wfs.stats.lastChecked < time.Now().Unix()-20 {
  111. err := wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  112. request := &filer_pb.StatisticsRequest{
  113. Collection: wfs.option.Collection,
  114. Replication: wfs.option.Replication,
  115. Ttl: fmt.Sprintf("%ds", wfs.option.TtlSec),
  116. }
  117. glog.V(4).Infof("reading filer stats: %+v", request)
  118. resp, err := client.Statistics(ctx, request)
  119. if err != nil {
  120. glog.V(0).Infof("reading filer stats %v: %v", request, err)
  121. return err
  122. }
  123. glog.V(4).Infof("read filer stats: %+v", resp)
  124. wfs.stats.TotalSize = resp.TotalSize
  125. wfs.stats.UsedSize = resp.UsedSize
  126. wfs.stats.FileCount = resp.FileCount
  127. wfs.stats.lastChecked = time.Now().Unix()
  128. return nil
  129. })
  130. if err != nil {
  131. glog.V(0).Infof("filer Statistics: %v", err)
  132. return err
  133. }
  134. }
  135. totalDiskSize := wfs.stats.TotalSize
  136. usedDiskSize := wfs.stats.UsedSize
  137. actualFileCount := wfs.stats.FileCount
  138. // Compute the total number of available blocks
  139. resp.Blocks = totalDiskSize / blockSize
  140. // Compute the number of used blocks
  141. numBlocks := uint64(usedDiskSize / blockSize)
  142. // Report the number of free and available blocks for the block size
  143. resp.Bfree = resp.Blocks - numBlocks
  144. resp.Bavail = resp.Blocks - numBlocks
  145. resp.Bsize = uint32(blockSize)
  146. // Report the total number of possible files in the file system (and those free)
  147. resp.Files = math.MaxInt64
  148. resp.Ffree = math.MaxInt64 - actualFileCount
  149. // Report the maximum length of a name and the minimum fragment size
  150. resp.Namelen = 1024
  151. resp.Frsize = uint32(blockSize)
  152. return nil
  153. }