You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

262 lines
6.6 KiB

more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
3 months ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
3 years ago
3 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
3 years ago
3 years ago
3 years ago
3 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
3 years ago
3 years ago
3 years ago
3 years ago
  1. package mount
  2. import (
  3. "github.com/hanwen/go-fuse/v2/fuse"
  4. "github.com/seaweedfs/seaweedfs/weed/filer"
  5. "github.com/seaweedfs/seaweedfs/weed/glog"
  6. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  7. "os"
  8. "syscall"
  9. "time"
  10. )
  11. func (wfs *WFS) GetAttr(cancel <-chan struct{}, input *fuse.GetAttrIn, out *fuse.AttrOut) (code fuse.Status) {
  12. if input.NodeId == 1 {
  13. wfs.setRootAttr(out)
  14. return fuse.OK
  15. }
  16. inode := input.NodeId
  17. _, _, entry, status := wfs.maybeReadEntry(inode)
  18. if status == fuse.OK {
  19. out.AttrValid = 1
  20. wfs.setAttrByPbEntry(&out.Attr, inode, entry, true)
  21. return status
  22. } else {
  23. if fh, found := wfs.fhMap.FindFileHandle(inode); found {
  24. out.AttrValid = 1
  25. wfs.setAttrByPbEntry(&out.Attr, inode, fh.entry.GetEntry(), true)
  26. out.Nlink = 0
  27. return fuse.OK
  28. }
  29. }
  30. return status
  31. }
  32. func (wfs *WFS) SetAttr(cancel <-chan struct{}, input *fuse.SetAttrIn, out *fuse.AttrOut) (code fuse.Status) {
  33. if wfs.IsOverQuota {
  34. return fuse.Status(syscall.ENOSPC)
  35. }
  36. path, fh, entry, status := wfs.maybeReadEntry(input.NodeId)
  37. if status != fuse.OK {
  38. return status
  39. }
  40. if fh != nil {
  41. fh.entryLock.Lock()
  42. defer fh.entryLock.Unlock()
  43. }
  44. if size, ok := input.GetSize(); ok && entry != nil {
  45. glog.V(4).Infof("%v setattr set size=%v chunks=%d", path, size, len(entry.GetChunks()))
  46. if size < filer.FileSize(entry) {
  47. // fmt.Printf("truncate %v \n", fullPath)
  48. var chunks []*filer_pb.FileChunk
  49. var truncatedChunks []*filer_pb.FileChunk
  50. for _, chunk := range entry.GetChunks() {
  51. int64Size := int64(chunk.Size)
  52. if chunk.Offset+int64Size > int64(size) {
  53. // this chunk is truncated
  54. int64Size = int64(size) - chunk.Offset
  55. if int64Size > 0 {
  56. chunks = append(chunks, chunk)
  57. glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
  58. chunk.Size = uint64(int64Size)
  59. } else {
  60. glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
  61. truncatedChunks = append(truncatedChunks, chunk)
  62. }
  63. } else {
  64. chunks = append(chunks, chunk)
  65. }
  66. }
  67. // set the new chunks and reset entry cache
  68. entry.Chunks = chunks
  69. if fh != nil {
  70. fh.entryChunkGroup.SetChunks(chunks)
  71. }
  72. }
  73. entry.Attributes.Mtime = time.Now().Unix()
  74. entry.Attributes.FileSize = size
  75. }
  76. if mode, ok := input.GetMode(); ok {
  77. // glog.V(4).Infof("setAttr mode %o", mode)
  78. entry.Attributes.FileMode = chmod(entry.Attributes.FileMode, mode)
  79. if input.NodeId == 1 {
  80. wfs.option.MountMode = os.FileMode(chmod(uint32(wfs.option.MountMode), mode))
  81. }
  82. }
  83. if uid, ok := input.GetUID(); ok {
  84. entry.Attributes.Uid = uid
  85. if input.NodeId == 1 {
  86. wfs.option.MountUid = uid
  87. }
  88. }
  89. if gid, ok := input.GetGID(); ok {
  90. entry.Attributes.Gid = gid
  91. if input.NodeId == 1 {
  92. wfs.option.MountGid = gid
  93. }
  94. }
  95. if atime, ok := input.GetATime(); ok {
  96. entry.Attributes.Mtime = atime.Unix()
  97. }
  98. if mtime, ok := input.GetMTime(); ok {
  99. entry.Attributes.Mtime = mtime.Unix()
  100. }
  101. out.AttrValid = 1
  102. size, includeSize := input.GetSize()
  103. if includeSize {
  104. out.Attr.Size = size
  105. }
  106. wfs.setAttrByPbEntry(&out.Attr, input.NodeId, entry, !includeSize)
  107. if fh != nil {
  108. fh.dirtyMetadata = true
  109. return fuse.OK
  110. }
  111. return wfs.saveEntry(path, entry)
  112. }
  113. func (wfs *WFS) setRootAttr(out *fuse.AttrOut) {
  114. now := uint64(time.Now().Unix())
  115. out.AttrValid = 119
  116. out.Ino = 1
  117. setBlksize(&out.Attr, blockSize)
  118. out.Uid = wfs.option.MountUid
  119. out.Gid = wfs.option.MountGid
  120. out.Mtime = now
  121. out.Ctime = now
  122. out.Atime = now
  123. out.Mode = toSyscallType(os.ModeDir) | uint32(wfs.option.MountMode)
  124. out.Nlink = 1
  125. }
  126. func (wfs *WFS) setAttrByPbEntry(out *fuse.Attr, inode uint64, entry *filer_pb.Entry, calculateSize bool) {
  127. out.Ino = inode
  128. setBlksize(out, blockSize)
  129. if entry == nil {
  130. return
  131. }
  132. if entry.Attributes != nil && entry.Attributes.Inode != 0 {
  133. out.Ino = entry.Attributes.Inode
  134. }
  135. if calculateSize {
  136. out.Size = filer.FileSize(entry)
  137. }
  138. if entry.FileMode()&os.ModeSymlink != 0 {
  139. out.Size = uint64(len(entry.Attributes.SymlinkTarget))
  140. }
  141. out.Blocks = (out.Size + blockSize - 1) / blockSize
  142. out.Mtime = uint64(entry.Attributes.Mtime)
  143. out.Ctime = uint64(entry.Attributes.Mtime)
  144. out.Atime = uint64(entry.Attributes.Mtime)
  145. out.Mode = toSyscallMode(os.FileMode(entry.Attributes.FileMode))
  146. if entry.HardLinkCounter > 0 {
  147. out.Nlink = uint32(entry.HardLinkCounter)
  148. } else {
  149. out.Nlink = 1
  150. }
  151. out.Uid = entry.Attributes.Uid
  152. out.Gid = entry.Attributes.Gid
  153. out.Rdev = entry.Attributes.Rdev
  154. }
  155. func (wfs *WFS) setAttrByFilerEntry(out *fuse.Attr, inode uint64, entry *filer.Entry) {
  156. out.Ino = inode
  157. out.Size = entry.FileSize
  158. if entry.Mode&os.ModeSymlink != 0 {
  159. out.Size = uint64(len(entry.SymlinkTarget))
  160. }
  161. out.Blocks = (out.Size + blockSize - 1) / blockSize
  162. setBlksize(out, blockSize)
  163. out.Atime = uint64(entry.Attr.Mtime.Unix())
  164. out.Mtime = uint64(entry.Attr.Mtime.Unix())
  165. out.Ctime = uint64(entry.Attr.Mtime.Unix())
  166. out.Mode = toSyscallMode(entry.Attr.Mode)
  167. if entry.HardLinkCounter > 0 {
  168. out.Nlink = uint32(entry.HardLinkCounter)
  169. } else {
  170. out.Nlink = 1
  171. }
  172. out.Uid = entry.Attr.Uid
  173. out.Gid = entry.Attr.Gid
  174. out.Rdev = entry.Attr.Rdev
  175. }
  176. func (wfs *WFS) outputPbEntry(out *fuse.EntryOut, inode uint64, entry *filer_pb.Entry) {
  177. out.NodeId = inode
  178. out.Generation = 1
  179. out.EntryValid = 1
  180. out.AttrValid = 1
  181. wfs.setAttrByPbEntry(&out.Attr, inode, entry, true)
  182. }
  183. func (wfs *WFS) outputFilerEntry(out *fuse.EntryOut, inode uint64, entry *filer.Entry) {
  184. out.NodeId = inode
  185. out.Generation = 1
  186. out.EntryValid = 1
  187. out.AttrValid = 1
  188. wfs.setAttrByFilerEntry(&out.Attr, inode, entry)
  189. }
  190. func chmod(existing uint32, mode uint32) uint32 {
  191. return existing&^07777 | mode&07777
  192. }
  193. func toSyscallMode(mode os.FileMode) uint32 {
  194. return toSyscallType(mode) | uint32(mode)
  195. }
  196. func toSyscallType(mode os.FileMode) uint32 {
  197. switch mode & os.ModeType {
  198. case os.ModeDir:
  199. return syscall.S_IFDIR
  200. case os.ModeSymlink:
  201. return syscall.S_IFLNK
  202. case os.ModeNamedPipe:
  203. return syscall.S_IFIFO
  204. case os.ModeSocket:
  205. return syscall.S_IFSOCK
  206. case os.ModeDevice:
  207. return syscall.S_IFBLK
  208. case os.ModeCharDevice:
  209. return syscall.S_IFCHR
  210. default:
  211. return syscall.S_IFREG
  212. }
  213. }
  214. func toOsFileType(mode uint32) os.FileMode {
  215. switch mode & (syscall.S_IFMT & 0xffff) {
  216. case syscall.S_IFDIR:
  217. return os.ModeDir
  218. case syscall.S_IFLNK:
  219. return os.ModeSymlink
  220. case syscall.S_IFIFO:
  221. return os.ModeNamedPipe
  222. case syscall.S_IFSOCK:
  223. return os.ModeSocket
  224. case syscall.S_IFBLK:
  225. return os.ModeDevice
  226. case syscall.S_IFCHR:
  227. return os.ModeCharDevice
  228. default:
  229. return 0
  230. }
  231. }
  232. func toOsFileMode(mode uint32) os.FileMode {
  233. return toOsFileType(mode) | os.FileMode(mode&07777)
  234. }