You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

203 lines
5.2 KiB

4 years ago
4 years ago
4 years ago
2 months ago
4 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
4 years ago
3 years ago
3 years ago
4 years ago
2 months ago
4 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
4 years ago
4 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
4 years ago
3 years ago
3 years ago
4 years ago
4 years ago
2 months ago
4 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
3 years ago
3 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
more solid weed mount (#4089) * compare chunks by timestamp * fix slab clearing error * fix test compilation * move oldest chunk to sealed, instead of by fullness * lock on fh.entryViewCache * remove verbose logs * revert slat clearing * less logs * less logs * track write and read by timestamp * remove useless logic * add entry lock on file handle release * use mem chunk only, swap file chunk has problems * comment out code that maybe used later * add debug mode to compare data read and write * more efficient readResolvedChunks with linked list * small optimization * fix test compilation * minor fix on writer * add SeparateGarbageChunks * group chunks into sections * turn off debug mode * fix tests * fix tests * tmp enable swap file chunk * Revert "tmp enable swap file chunk" This reverts commit 985137ec472924e4815f258189f6ca9f2168a0a7. * simple refactoring * simple refactoring * do not re-use swap file chunk. Sealed chunks should not be re-used. * comment out debugging facilities * either mem chunk or swap file chunk is fine now * remove orderedMutex as *semaphore.Weighted not found impactful * optimize size calculation for changing large files * optimize performance to avoid going through the long list of chunks * still problems with swap file chunk * rename * tiny optimization * swap file chunk save only successfully read data * fix * enable both mem and swap file chunk * resolve chunks with range * rename * fix chunk interval list * also change file handle chunk group when adding chunks * pick in-active chunk with time-decayed counter * fix compilation * avoid nil with empty fh.entry * refactoring * rename * rename * refactor visible intervals to *list.List * refactor chunkViews to *list.List * add IntervalList for generic interval list * change visible interval to use IntervalList in generics * cahnge chunkViews to *IntervalList[*ChunkView] * use NewFileChunkSection to create * rename variables * refactor * fix renaming leftover * renaming * renaming * add insert interval * interval list adds lock * incrementally add chunks to readers Fixes: 1. set start and stop offset for the value object 2. clone the value object 3. use pointer instead of copy-by-value when passing to interval.Value 4. use insert interval since adding chunk could be out of order * fix tests compilation * fix tests compilation
2 years ago
  1. package filer
  2. import (
  3. "bytes"
  4. "io"
  5. "math"
  6. "strconv"
  7. "testing"
  8. )
  9. type mockChunkCache struct {
  10. }
  11. func (m *mockChunkCache) GetChunk(fileId string, minSize uint64) (data []byte) {
  12. x, _ := strconv.Atoi(fileId)
  13. data = make([]byte, minSize)
  14. for i := 0; i < int(minSize); i++ {
  15. data[i] = byte(x)
  16. }
  17. return data
  18. }
  19. func (m *mockChunkCache) ReadChunkAt(data []byte, fileId string, offset uint64) (n int, err error) {
  20. x, _ := strconv.Atoi(fileId)
  21. for i := 0; i < len(data); i++ {
  22. data[i] = byte(x)
  23. }
  24. return len(data), nil
  25. }
  26. func (m *mockChunkCache) SetChunk(fileId string, data []byte) {
  27. }
  28. func (m *mockChunkCache) GetMaxFilePartSizeInCache() uint64 {
  29. return 0
  30. }
  31. func (m *mockChunkCache) IsInCache(fileId string, lockNeeded bool) (answer bool) {
  32. return false
  33. }
  34. func TestReaderAt(t *testing.T) {
  35. visibles := NewIntervalList[*VisibleInterval]()
  36. addVisibleInterval(visibles, &VisibleInterval{
  37. start: 1,
  38. stop: 2,
  39. fileId: "1",
  40. chunkSize: 9,
  41. })
  42. addVisibleInterval(visibles, &VisibleInterval{
  43. start: 3,
  44. stop: 4,
  45. fileId: "3",
  46. chunkSize: 1,
  47. })
  48. addVisibleInterval(visibles, &VisibleInterval{
  49. start: 5,
  50. stop: 6,
  51. fileId: "5",
  52. chunkSize: 2,
  53. })
  54. addVisibleInterval(visibles, &VisibleInterval{
  55. start: 7,
  56. stop: 9,
  57. fileId: "7",
  58. chunkSize: 2,
  59. })
  60. addVisibleInterval(visibles, &VisibleInterval{
  61. start: 9,
  62. stop: 10,
  63. fileId: "9",
  64. chunkSize: 2,
  65. })
  66. readerAt := &ChunkReadAt{
  67. chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
  68. fileSize: 10,
  69. readerCache: NewReaderCache(3, &mockChunkCache{}, nil),
  70. readerPattern: NewReaderPattern(),
  71. }
  72. testReadAt(t, readerAt, 0, 10, 10, io.EOF, nil, nil)
  73. testReadAt(t, readerAt, 0, 12, 10, io.EOF, nil, nil)
  74. testReadAt(t, readerAt, 2, 8, 8, io.EOF, nil, nil)
  75. testReadAt(t, readerAt, 3, 6, 6, nil, nil, nil)
  76. }
  77. func testReadAt(t *testing.T, readerAt *ChunkReadAt, offset int64, size int, expectedN int, expectedErr error, data, expectedData []byte) {
  78. if data == nil {
  79. data = make([]byte, size)
  80. }
  81. n, _, err := readerAt.doReadAt(data, offset)
  82. if expectedN != n {
  83. t.Errorf("unexpected read size: %d, expect: %d", n, expectedN)
  84. }
  85. if err != expectedErr {
  86. t.Errorf("unexpected read error: %v, expect: %v", err, expectedErr)
  87. }
  88. if expectedData != nil && !bytes.Equal(data, expectedData) {
  89. t.Errorf("unexpected read data: %v, expect: %v", data, expectedData)
  90. }
  91. }
  92. func TestReaderAt0(t *testing.T) {
  93. visibles := NewIntervalList[*VisibleInterval]()
  94. addVisibleInterval(visibles, &VisibleInterval{
  95. start: 2,
  96. stop: 5,
  97. fileId: "1",
  98. chunkSize: 9,
  99. })
  100. addVisibleInterval(visibles, &VisibleInterval{
  101. start: 7,
  102. stop: 9,
  103. fileId: "2",
  104. chunkSize: 9,
  105. })
  106. readerAt := &ChunkReadAt{
  107. chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
  108. fileSize: 10,
  109. readerCache: NewReaderCache(3, &mockChunkCache{}, nil),
  110. readerPattern: NewReaderPattern(),
  111. }
  112. testReadAt(t, readerAt, 0, 10, 10, io.EOF, nil, nil)
  113. testReadAt(t, readerAt, 3, 16, 7, io.EOF, nil, nil)
  114. testReadAt(t, readerAt, 3, 5, 5, nil, nil, nil)
  115. testReadAt(t, readerAt, 11, 5, 0, io.EOF, nil, nil)
  116. testReadAt(t, readerAt, 10, 5, 0, io.EOF, nil, nil)
  117. }
  118. func TestReaderAt1(t *testing.T) {
  119. visibles := NewIntervalList[*VisibleInterval]()
  120. addVisibleInterval(visibles, &VisibleInterval{
  121. start: 2,
  122. stop: 5,
  123. fileId: "1",
  124. chunkSize: 9,
  125. })
  126. readerAt := &ChunkReadAt{
  127. chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
  128. fileSize: 20,
  129. readerCache: NewReaderCache(3, &mockChunkCache{}, nil),
  130. readerPattern: NewReaderPattern(),
  131. }
  132. testReadAt(t, readerAt, 0, 20, 20, io.EOF, nil, nil)
  133. testReadAt(t, readerAt, 1, 7, 7, nil, nil, nil)
  134. testReadAt(t, readerAt, 0, 1, 1, nil, nil, nil)
  135. testReadAt(t, readerAt, 18, 4, 2, io.EOF, nil, nil)
  136. testReadAt(t, readerAt, 12, 4, 4, nil, nil, nil)
  137. testReadAt(t, readerAt, 4, 20, 16, io.EOF, nil, nil)
  138. testReadAt(t, readerAt, 4, 10, 10, nil, nil, nil)
  139. testReadAt(t, readerAt, 1, 10, 10, nil, nil, nil)
  140. }
  141. func TestReaderAtGappedChunksDoNotLeak(t *testing.T) {
  142. visibles := NewIntervalList[*VisibleInterval]()
  143. addVisibleInterval(visibles, &VisibleInterval{
  144. start: 2,
  145. stop: 3,
  146. fileId: "1",
  147. chunkSize: 5,
  148. })
  149. addVisibleInterval(visibles, &VisibleInterval{
  150. start: 7,
  151. stop: 9,
  152. fileId: "1",
  153. chunkSize: 4,
  154. })
  155. readerAt := &ChunkReadAt{
  156. chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
  157. fileSize: 9,
  158. readerCache: NewReaderCache(3, &mockChunkCache{}, nil),
  159. readerPattern: NewReaderPattern(),
  160. }
  161. testReadAt(t, readerAt, 0, 9, 9, io.EOF, []byte{2, 2, 2, 2, 2, 2, 2, 2, 2}, []byte{0, 0, 1, 0, 0, 0, 0, 1, 1})
  162. testReadAt(t, readerAt, 1, 8, 8, io.EOF, []byte{2, 2, 2, 2, 2, 2, 2, 2}, []byte{0, 1, 0, 0, 0, 0, 1, 1})
  163. }
  164. func TestReaderAtSparseFileDoesNotLeak(t *testing.T) {
  165. readerAt := &ChunkReadAt{
  166. chunkViews: ViewFromVisibleIntervals(NewIntervalList[*VisibleInterval](), 0, math.MaxInt64),
  167. fileSize: 3,
  168. readerCache: NewReaderCache(3, &mockChunkCache{}, nil),
  169. readerPattern: NewReaderPattern(),
  170. }
  171. testReadAt(t, readerAt, 0, 3, 3, io.EOF, []byte{2, 2, 2}, []byte{0, 0, 0})
  172. testReadAt(t, readerAt, 1, 2, 2, io.EOF, []byte{2, 2}, []byte{0, 0})
  173. }