You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

189 lines
5.1 KiB

3 years ago
3 years ago
3 years ago
3 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
3 years ago
3 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
3 years ago
  1. package mount
  2. import (
  3. "context"
  4. "fmt"
  5. "github.com/hanwen/go-fuse/v2/fuse"
  6. "github.com/seaweedfs/seaweedfs/weed/filer"
  7. "github.com/seaweedfs/seaweedfs/weed/glog"
  8. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  9. "syscall"
  10. "time"
  11. )
  12. /**
  13. * Flush method
  14. *
  15. * This is called on each close() of the opened file.
  16. *
  17. * Since file descriptors can be duplicated (dup, dup2, fork), for
  18. * one open call there may be many flush calls.
  19. *
  20. * Filesystems shouldn't assume that flush will always be called
  21. * after some writes, or that if will be called at all.
  22. *
  23. * fi->fh will contain the value set by the open method, or will
  24. * be undefined if the open method didn't set any value.
  25. *
  26. * NOTE: the name of the method is misleading, since (unlike
  27. * fsync) the filesystem is not forced to flush pending writes.
  28. * One reason to flush data is if the filesystem wants to return
  29. * write errors during close. However, such use is non-portable
  30. * because POSIX does not require [close] to wait for delayed I/O to
  31. * complete.
  32. *
  33. * If the filesystem supports file locking operations (setlk,
  34. * getlk) it should remove all locks belonging to 'fi->owner'.
  35. *
  36. * If this request is answered with an error code of ENOSYS,
  37. * this is treated as success and future calls to flush() will
  38. * succeed automatically without being send to the filesystem
  39. * process.
  40. *
  41. * Valid replies:
  42. * fuse_reply_err
  43. *
  44. * @param req request handle
  45. * @param ino the inode number
  46. * @param fi file information
  47. *
  48. * [close]: http://pubs.opengroup.org/onlinepubs/9699919799/functions/close.html
  49. */
  50. func (wfs *WFS) Flush(cancel <-chan struct{}, in *fuse.FlushIn) fuse.Status {
  51. fh := wfs.GetHandle(FileHandleId(in.Fh))
  52. if fh == nil {
  53. return fuse.ENOENT
  54. }
  55. return wfs.doFlush(fh, in.Uid, in.Gid)
  56. }
  57. /**
  58. * Synchronize file contents
  59. *
  60. * If the datasync parameter is non-zero, then only the user data
  61. * should be flushed, not the meta data.
  62. *
  63. * If this request is answered with an error code of ENOSYS,
  64. * this is treated as success and future calls to fsync() will
  65. * succeed automatically without being send to the filesystem
  66. * process.
  67. *
  68. * Valid replies:
  69. * fuse_reply_err
  70. *
  71. * @param req request handle
  72. * @param ino the inode number
  73. * @param datasync flag indicating if only data should be flushed
  74. * @param fi file information
  75. */
  76. func (wfs *WFS) Fsync(cancel <-chan struct{}, in *fuse.FsyncIn) (code fuse.Status) {
  77. fh := wfs.GetHandle(FileHandleId(in.Fh))
  78. if fh == nil {
  79. return fuse.ENOENT
  80. }
  81. return wfs.doFlush(fh, in.Uid, in.Gid)
  82. }
  83. func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
  84. fh.Lock()
  85. defer fh.Unlock()
  86. // flush works at fh level
  87. fileFullPath := fh.FullPath()
  88. dir, name := fileFullPath.DirAndName()
  89. // send the data to the OS
  90. glog.V(4).Infof("doFlush %s fh %d", fileFullPath, fh.handle)
  91. if !wfs.IsOverQuota {
  92. if err := fh.dirtyPages.FlushData(); err != nil {
  93. glog.Errorf("%v doFlush: %v", fileFullPath, err)
  94. return fuse.EIO
  95. }
  96. }
  97. if !fh.dirtyMetadata {
  98. return fuse.OK
  99. }
  100. if wfs.IsOverQuota {
  101. return fuse.Status(syscall.ENOSPC)
  102. }
  103. err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
  104. fh.entryLock.Lock()
  105. defer fh.entryLock.Unlock()
  106. entry := fh.GetEntry()
  107. if entry == nil {
  108. return nil
  109. }
  110. entry.Name = name // this flush may be just after a rename operation
  111. if entry.Attributes != nil {
  112. entry.Attributes.Mime = fh.contentType
  113. if entry.Attributes.Uid == 0 {
  114. entry.Attributes.Uid = uid
  115. }
  116. if entry.Attributes.Gid == 0 {
  117. entry.Attributes.Gid = gid
  118. }
  119. if entry.Attributes.Crtime == 0 {
  120. entry.Attributes.Crtime = time.Now().Unix()
  121. }
  122. entry.Attributes.Mtime = time.Now().Unix()
  123. }
  124. request := &filer_pb.CreateEntryRequest{
  125. Directory: string(dir),
  126. Entry: entry,
  127. Signatures: []int32{wfs.signature},
  128. SkipCheckParentDirectory: true,
  129. }
  130. glog.V(4).Infof("%s set chunks: %v", fileFullPath, len(entry.GetChunks()))
  131. //for i, chunk := range entry.GetChunks() {
  132. // glog.V(4).Infof("%s chunks %d: %v [%d,%d)", fileFullPath, i, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
  133. //}
  134. manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(entry.GetChunks())
  135. chunks, _ := filer.CompactFileChunks(wfs.LookupFn(), nonManifestChunks)
  136. chunks, manifestErr := filer.MaybeManifestize(wfs.saveDataAsChunk(fileFullPath), chunks)
  137. if manifestErr != nil {
  138. // not good, but should be ok
  139. glog.V(0).Infof("MaybeManifestize: %v", manifestErr)
  140. }
  141. entry.Chunks = append(chunks, manifestChunks...)
  142. wfs.mapPbIdFromLocalToFiler(request.Entry)
  143. defer wfs.mapPbIdFromFilerToLocal(request.Entry)
  144. if err := filer_pb.CreateEntry(client, request); err != nil {
  145. glog.Errorf("fh flush create %s: %v", fileFullPath, err)
  146. return fmt.Errorf("fh flush create %s: %v", fileFullPath, err)
  147. }
  148. wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry))
  149. return nil
  150. })
  151. if err == nil {
  152. fh.dirtyMetadata = false
  153. }
  154. if err != nil {
  155. glog.Errorf("%v fh %d flush: %v", fileFullPath, fh.handle, err)
  156. return fuse.EIO
  157. }
  158. if IsDebugFileReadWrite {
  159. fh.mirrorFile.Sync()
  160. }
  161. return fuse.OK
  162. }