You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

263 lines
6.6 KiB

2 months ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
3 months ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
3 years ago
3 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
3 years ago
3 years ago
3 years ago
3 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
3 years ago
3 years ago
3 years ago
3 years ago
  1. package mount
  2. import (
  3. "github.com/hanwen/go-fuse/v2/fuse"
  4. "github.com/seaweedfs/seaweedfs/weed/filer"
  5. "github.com/seaweedfs/seaweedfs/weed/glog"
  6. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  7. "os"
  8. "syscall"
  9. "time"
  10. )
  11. func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse.AttrOut) (code fuse.Status) {
  12. glog.V(4).Infof("GetAttr %v", input.NodeId)
  13. if input.NodeId == 1 {
  14. wfs.setRootAttr(out)
  15. return fuse.OK
  16. }
  17. inode := input.NodeId
  18. _, _, entry, status := wfs.maybeReadEntry(inode)
  19. if status == fuse.OK {
  20. out.AttrValid = 1
  21. wfs.setAttrByPbEntry(&out.Attr, inode, entry, true)
  22. return status
  23. } else {
  24. if fh, found := wfs.fhMap.FindFileHandle(inode); found {
  25. out.AttrValid = 1
  26. wfs.setAttrByPbEntry(&out.Attr, inode, fh.entry.GetEntry(), true)
  27. out.Nlink = 0
  28. return fuse.OK
  29. }
  30. }
  31. return status
  32. }
  33. func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse.AttrOut) (code fuse.Status) {
  34. if wfs.IsOverQuota {
  35. return fuse.Status(syscall.ENOSPC)
  36. }
  37. path, fh, entry, status := wfs.maybeReadEntry(input.NodeId)
  38. if status != fuse.OK {
  39. return status
  40. }
  41. if fh != nil {
  42. fh.entryLock.Lock()
  43. defer fh.entryLock.Unlock()
  44. }
  45. if size, ok := input.GetSize(); ok && entry != nil {
  46. glog.V(4).Infof("%v setattr set size=%v chunks=%d", path, size, len(entry.GetChunks()))
  47. if size < filer.FileSize(entry) {
  48. // fmt.Printf("truncate %v \n", fullPath)
  49. var chunks []*filer_pb.FileChunk
  50. var truncatedChunks []*filer_pb.FileChunk
  51. for _, chunk := range entry.GetChunks() {
  52. int64Size := int64(chunk.Size)
  53. if chunk.Offset+int64Size > int64(size) {
  54. // this chunk is truncated
  55. int64Size = int64(size) - chunk.Offset
  56. if int64Size > 0 {
  57. chunks = append(chunks, chunk)
  58. glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
  59. chunk.Size = uint64(int64Size)
  60. } else {
  61. glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
  62. truncatedChunks = append(truncatedChunks, chunk)
  63. }
  64. } else {
  65. chunks = append(chunks, chunk)
  66. }
  67. }
  68. // set the new chunks and reset entry cache
  69. entry.Chunks = chunks
  70. if fh != nil {
  71. fh.entryChunkGroup.SetChunks(chunks)
  72. }
  73. }
  74. entry.Attributes.Mtime = time.Now().Unix()
  75. entry.Attributes.FileSize = size
  76. }
  77. if mode, ok := input.GetMode(); ok {
  78. // glog.V(4).Infof("setAttr mode %o", mode)
  79. entry.Attributes.FileMode = chmod(entry.Attributes.FileMode, mode)
  80. if input.NodeId == 1 {
  81. wfs.option.MountMode = os.FileMode(chmod(uint32(wfs.option.MountMode), mode))
  82. }
  83. }
  84. if uid, ok := input.GetUID(); ok {
  85. entry.Attributes.Uid = uid
  86. if input.NodeId == 1 {
  87. wfs.option.MountUid = uid
  88. }
  89. }
  90. if gid, ok := input.GetGID(); ok {
  91. entry.Attributes.Gid = gid
  92. if input.NodeId == 1 {
  93. wfs.option.MountGid = gid
  94. }
  95. }
  96. if atime, ok := input.GetATime(); ok {
  97. entry.Attributes.Mtime = atime.Unix()
  98. }
  99. if mtime, ok := input.GetMTime(); ok {
  100. entry.Attributes.Mtime = mtime.Unix()
  101. }
  102. out.AttrValid = 1
  103. size, includeSize := input.GetSize()
  104. if includeSize {
  105. out.Attr.Size = size
  106. }
  107. wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry, !includeSize)
  108. if fh != nil {
  109. fh.dirtyMetadata = true
  110. return fuse.OK
  111. }
  112. return wfs.saveEntry(path, entry)
  113. }
  114. func (wfs *WFS) setRootAttr(out *fuse.AttrOut) {
  115. now := uint64(time.Now().Unix())
  116. out.AttrValid = 119
  117. out.Ino = 1
  118. setBlksize(&out.Attr, blockSize)
  119. out.Uid = wfs.option.MountUid
  120. out.Gid = wfs.option.MountGid
  121. out.Mtime = now
  122. out.Ctime = now
  123. out.Atime = now
  124. out.Mode = toSyscallType(os.ModeDir) | uint32(wfs.option.MountMode)
  125. out.Nlink = 1
  126. }
  127. func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry, calculateSize bool) {
  128. out.Ino = inode
  129. setBlksize(out, blockSize)
  130. if entry == nil {
  131. return
  132. }
  133. if entry.Attributes != nil && entry.Attributes.Inode != 0 {
  134. out.Ino = entry.Attributes.Inode
  135. }
  136. if calculateSize {
  137. out.Size = filer.FileSize(entry)
  138. }
  139. if entry.FileMode()&os.ModeSymlink != 0 {
  140. out.Size = uint64(len(entry.Attributes.SymlinkTarget))
  141. }
  142. out.Blocks = (out.Size + blockSize - 1) / blockSize
  143. out.Mtime = uint64(entry.Attributes.Mtime)
  144. out.Ctime = uint64(entry.Attributes.Mtime)
  145. out.Atime = uint64(entry.Attributes.Mtime)
  146. out.Mode = toSyscallMode(os.FileMode(entry.Attributes.FileMode))
  147. if entry.HardLinkCounter > 0 {
  148. out.Nlink = uint32(entry.HardLinkCounter)
  149. } else {
  150. out.Nlink = 1
  151. }
  152. out.Uid = entry.Attributes.Uid
  153. out.Gid = entry.Attributes.Gid
  154. out.Rdev = entry.Attributes.Rdev
  155. }
  156. func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.Entry) {
  157. out.Ino = inode
  158. out.Size = entry.FileSize
  159. if entry.Mode&os.ModeSymlink != 0 {
  160. out.Size = uint64(len(entry.SymlinkTarget))
  161. }
  162. out.Blocks = (out.Size + blockSize - 1) / blockSize
  163. setBlksize(out, blockSize)
  164. out.Atime = uint64(entry.Attr.Mtime.Unix())
  165. out.Mtime = uint64(entry.Attr.Mtime.Unix())
  166. out.Ctime = uint64(entry.Attr.Mtime.Unix())
  167. out.Mode = toSyscallMode(entry.Attr.Mode)
  168. if entry.HardLinkCounter > 0 {
  169. out.Nlink = uint32(entry.HardLinkCounter)
  170. } else {
  171. out.Nlink = 1
  172. }
  173. out.Uid = entry.Attr.Uid
  174. out.Gid = entry.Attr.Gid
  175. out.Rdev = entry.Attr.Rdev
  176. }
  177. func (wfs *WFS) outputPbEntry(out *fuse.EntryOut, inode uint64, entry *filer_pb.Entry) {
  178. out.NodeId = inode
  179. out.Generation = 1
  180. out.EntryValid = 1
  181. out.AttrValid = 1
  182. wfs.setAttrByPbEntry(&out.Attr, inode, entry, true)
  183. }
  184. func (wfs *WFS) outputFilerEntry(out *fuse.EntryOut, inode uint64, entry *filer.Entry) {
  185. out.NodeId = inode
  186. out.Generation = 1
  187. out.EntryValid = 1
  188. out.AttrValid = 1
  189. wfs.setAttrByFilerEntry(&out.Attr, inode, entry)
  190. }
  191. func chmod(existing uint32, mode uint32) uint32 {
  192. return existing&^07777 | mode&07777
  193. }
  194. func toSyscallMode(mode os.FileMode) uint32 {
  195. return toSyscallType(mode) | uint32(mode)
  196. }
  197. func toSyscallType(mode os.FileMode) uint32 {
  198. switch mode & os.ModeType {
  199. case os.ModeDir:
  200. return syscall.S_IFDIR
  201. case os.ModeSymlink:
  202. return syscall.S_IFLNK
  203. case os.ModeNamedPipe:
  204. return syscall.S_IFIFO
  205. case os.ModeSocket:
  206. return syscall.S_IFSOCK
  207. case os.ModeDevice:
  208. return syscall.S_IFBLK
  209. case os.ModeCharDevice:
  210. return syscall.S_IFCHR
  211. default:
  212. return syscall.S_IFREG
  213. }
  214. }
  215. func toOsFileType(mode uint32) os.FileMode {
  216. switch mode & (syscall.S_IFMT & 0xffff) {
  217. case syscall.S_IFDIR:
  218. return os.ModeDir
  219. case syscall.S_IFLNK:
  220. return os.ModeSymlink
  221. case syscall.S_IFIFO:
  222. return os.ModeNamedPipe
  223. case syscall.S_IFSOCK:
  224. return os.ModeSocket
  225. case syscall.S_IFBLK:
  226. return os.ModeDevice
  227. case syscall.S_IFCHR:
  228. return os.ModeCharDevice
  229. default:
  230. return 0
  231. }
  232. }
  233. func toOsFileMode(mode uint32) os.FileMode {
  234. return toOsFileType(mode) | os.FileMode(mode&07777)
  235. }