You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

288 lines
7.6 KiB

7 years ago
7 years ago
7 years ago
7 years ago
4 years ago
4 years ago
5 years ago
4 years ago
4 years ago
6 years ago
4 years ago
6 years ago
5 years ago
4 years ago
4 years ago
5 years ago
6 years ago
6 years ago
4 years ago
6 years ago
  1. package filesys
  2. import (
  3. "context"
  4. "fmt"
  5. "io"
  6. "math"
  7. "net/http"
  8. "os"
  9. "sync"
  10. "time"
  11. "github.com/seaweedfs/fuse"
  12. "github.com/seaweedfs/fuse/fs"
  13. "github.com/chrislusf/seaweedfs/weed/filer2"
  14. "github.com/chrislusf/seaweedfs/weed/glog"
  15. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  16. )
  17. type FileHandle struct {
  18. // cache file has been written to
  19. dirtyPages *ContinuousDirtyPages
  20. contentType string
  21. handle uint64
  22. sync.RWMutex
  23. f *File
  24. RequestId fuse.RequestID // unique ID for request
  25. NodeId fuse.NodeID // file or directory the request is about
  26. Uid uint32 // user ID of process making request
  27. Gid uint32 // group ID of process making request
  28. }
  29. func newFileHandle(file *File, uid, gid uint32) *FileHandle {
  30. fh := &FileHandle{
  31. f: file,
  32. dirtyPages: newDirtyPages(file),
  33. Uid: uid,
  34. Gid: gid,
  35. }
  36. if fh.f.entry != nil {
  37. fh.f.entry.Attributes.FileSize = filer2.FileSize(fh.f.entry)
  38. }
  39. return fh
  40. }
  41. var _ = fs.Handle(&FileHandle{})
  42. // var _ = fs.HandleReadAller(&FileHandle{})
  43. var _ = fs.HandleReader(&FileHandle{})
  44. var _ = fs.HandleFlusher(&FileHandle{})
  45. var _ = fs.HandleWriter(&FileHandle{})
  46. var _ = fs.HandleReleaser(&FileHandle{})
  47. func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
  48. glog.V(4).Infof("%s read fh %d: [%d,%d) size %d resp.Data cap=%d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, cap(resp.Data))
  49. fh.RLock()
  50. defer fh.RUnlock()
  51. if req.Size <= 0 {
  52. return nil
  53. }
  54. buff := resp.Data[:cap(resp.Data)]
  55. if req.Size > cap(resp.Data) {
  56. // should not happen
  57. buff = make([]byte, req.Size)
  58. }
  59. totalRead, err := fh.readFromChunks(buff, req.Offset)
  60. if err == nil {
  61. maxStop := fh.readFromDirtyPages(buff, req.Offset)
  62. totalRead = max(maxStop-req.Offset, totalRead)
  63. }
  64. if err != nil {
  65. glog.Warningf("file handle read %s %d: %v", fh.f.fullpath(), totalRead, err)
  66. return nil
  67. }
  68. if totalRead > int64(len(buff)) {
  69. glog.Warningf("%s FileHandle Read %d: [%d,%d) size %d totalRead %d", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size), req.Size, totalRead)
  70. totalRead = min(int64(len(buff)), totalRead)
  71. }
  72. // resp.Data = buff[:totalRead]
  73. resp.Data = buff
  74. return err
  75. }
  76. func (fh *FileHandle) readFromDirtyPages(buff []byte, startOffset int64) (maxStop int64) {
  77. maxStop = fh.dirtyPages.ReadDirtyDataAt(buff, startOffset)
  78. return
  79. }
  80. func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
  81. fileSize := int64(filer2.FileSize(fh.f.entry))
  82. if fileSize == 0 {
  83. glog.V(1).Infof("empty fh %v", fh.f.fullpath())
  84. return 0, io.EOF
  85. }
  86. var chunkResolveErr error
  87. if fh.f.entryViewCache == nil {
  88. fh.f.entryViewCache, chunkResolveErr = filer2.NonOverlappingVisibleIntervals(filer2.LookupFn(fh.f.wfs), fh.f.entry.Chunks)
  89. if chunkResolveErr != nil {
  90. return 0, fmt.Errorf("fail to resolve chunk manifest: %v", chunkResolveErr)
  91. }
  92. fh.f.reader = nil
  93. }
  94. if fh.f.reader == nil {
  95. chunkViews := filer2.ViewFromVisibleIntervals(fh.f.entryViewCache, 0, math.MaxInt64)
  96. fh.f.reader = filer2.NewChunkReaderAtFromClient(fh.f.wfs, chunkViews, fh.f.wfs.chunkCache, fileSize)
  97. }
  98. totalRead, err := fh.f.reader.ReadAt(buff, offset)
  99. if err == io.EOF {
  100. err = nil
  101. }
  102. if err != nil {
  103. glog.Errorf("file handle read %s: %v", fh.f.fullpath(), err)
  104. }
  105. glog.V(4).Infof("file handle read %s [%d,%d] %d : %v", fh.f.fullpath(), offset, offset+int64(totalRead), totalRead, err)
  106. return int64(totalRead), err
  107. }
  108. // Write to the file handle
  109. func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {
  110. fh.Lock()
  111. defer fh.Unlock()
  112. // write the request to volume servers
  113. data := make([]byte, len(req.Data))
  114. copy(data, req.Data)
  115. fh.f.entry.Attributes.FileSize = uint64(max(req.Offset+int64(len(data)), int64(fh.f.entry.Attributes.FileSize)))
  116. glog.V(4).Infof("%v write [%d,%d) %d", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)), len(req.Data))
  117. chunks, err := fh.dirtyPages.AddPage(req.Offset, data)
  118. if err != nil {
  119. glog.Errorf("%v write fh %d: [%d,%d): %v", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(len(data)), err)
  120. return fuse.EIO
  121. }
  122. resp.Size = len(data)
  123. if req.Offset == 0 {
  124. // detect mime type
  125. fh.contentType = http.DetectContentType(data)
  126. fh.f.dirtyMetadata = true
  127. }
  128. if len(chunks) > 0 {
  129. fh.f.addChunks(chunks)
  130. fh.f.dirtyMetadata = true
  131. }
  132. return nil
  133. }
  134. func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
  135. glog.V(4).Infof("Release %v fh %d", fh.f.fullpath(), fh.handle)
  136. fh.Lock()
  137. defer fh.Unlock()
  138. fh.f.isOpen--
  139. if fh.f.isOpen < 0 {
  140. glog.V(0).Infof("Release reset %s open count %d => %d", fh.f.Name, fh.f.isOpen, 0)
  141. fh.f.isOpen = 0
  142. return nil
  143. }
  144. if fh.f.isOpen == 0 {
  145. fh.doFlush(ctx, req.Header)
  146. fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
  147. }
  148. return nil
  149. }
  150. func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
  151. fh.Lock()
  152. defer fh.Unlock()
  153. return fh.doFlush(ctx, req.Header)
  154. }
  155. func (fh *FileHandle) doFlush(ctx context.Context, header fuse.Header) error {
  156. // fflush works at fh level
  157. // send the data to the OS
  158. glog.V(4).Infof("doFlush %s fh %d", fh.f.fullpath(), fh.handle)
  159. chunks, err := fh.dirtyPages.saveExistingPagesToStorage()
  160. if err != nil {
  161. glog.Errorf("flush %s: %v", fh.f.fullpath(), err)
  162. return fuse.EIO
  163. }
  164. if len(chunks) > 0 {
  165. fh.f.addChunks(chunks)
  166. fh.f.dirtyMetadata = true
  167. }
  168. if !fh.f.dirtyMetadata {
  169. return nil
  170. }
  171. err = fh.f.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
  172. if fh.f.entry.Attributes != nil {
  173. fh.f.entry.Attributes.Mime = fh.contentType
  174. if fh.f.entry.Attributes.Uid == 0 {
  175. fh.f.entry.Attributes.Uid = header.Uid
  176. }
  177. if fh.f.entry.Attributes.Gid == 0 {
  178. fh.f.entry.Attributes.Gid = header.Gid
  179. }
  180. if fh.f.entry.Attributes.Crtime == 0 {
  181. fh.f.entry.Attributes.Crtime = time.Now().Unix()
  182. }
  183. fh.f.entry.Attributes.Mtime = time.Now().Unix()
  184. fh.f.entry.Attributes.FileMode = uint32(os.FileMode(fh.f.entry.Attributes.FileMode) &^ fh.f.wfs.option.Umask)
  185. fh.f.entry.Attributes.Collection = fh.dirtyPages.collection
  186. fh.f.entry.Attributes.Replication = fh.dirtyPages.replication
  187. }
  188. request := &filer_pb.CreateEntryRequest{
  189. Directory: fh.f.dir.FullPath(),
  190. Entry: fh.f.entry,
  191. Signatures: []int32{fh.f.wfs.signature},
  192. }
  193. glog.V(4).Infof("%s set chunks: %v", fh.f.fullpath(), len(fh.f.entry.Chunks))
  194. for i, chunk := range fh.f.entry.Chunks {
  195. glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fh.f.fullpath(), i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
  196. }
  197. manifestChunks, nonManifestChunks := filer2.SeparateManifestChunks(fh.f.entry.Chunks)
  198. chunks, _ := filer2.CompactFileChunks(filer2.LookupFn(fh.f.wfs), nonManifestChunks)
  199. chunks, manifestErr := filer2.MaybeManifestize(fh.f.wfs.saveDataAsChunk(fh.f.dir.FullPath()), chunks)
  200. if manifestErr != nil {
  201. // not good, but should be ok
  202. glog.V(0).Infof("MaybeManifestize: %v", manifestErr)
  203. }
  204. fh.f.entry.Chunks = append(chunks, manifestChunks...)
  205. fh.f.entryViewCache = nil
  206. if err := filer_pb.CreateEntry(client, request); err != nil {
  207. glog.Errorf("fh flush create %s: %v", fh.f.fullpath(), err)
  208. return fmt.Errorf("fh flush create %s: %v", fh.f.fullpath(), err)
  209. }
  210. fh.f.wfs.metaCache.InsertEntry(context.Background(), filer2.FromPbEntry(request.Directory, request.Entry))
  211. return nil
  212. })
  213. if err == nil {
  214. fh.f.dirtyMetadata = false
  215. }
  216. if err != nil {
  217. glog.Errorf("%v fh %d flush: %v", fh.f.fullpath(), fh.handle, err)
  218. return fuse.EIO
  219. }
  220. return nil
  221. }