Browse Source

fix: skip exhausted blocks before creating an interval (#8180)

* fix: skip exhausted blocks before creating an interval

* refactor: optimize interval creation and fix logic duplication

* docs: add docstring for LocateData

* refactor: extract moveToNextBlock helper to deduplicate logic

* fix: use int64 for block index comparison to prevent overflow

* test: add unit test for LocateData boundary crossing (issue #8179)

* fix: skip exhausted blocks to prevent negative interval size and panics (issue #8179)

* refactor: apply review suggestions for test maintainability and code style
pull/5637/merge
Chris Lu 1 day ago
committed by GitHub
parent
commit
f23e09f58b
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 34
      weed/storage/erasure_coding/ec_locate.go
  2. 17
      weed/storage/erasure_coding/ec_test.go

34
weed/storage/erasure_coding/ec_locate.go

@ -12,10 +12,23 @@ type Interval struct {
LargeBlockRowsCount int LargeBlockRowsCount int
} }
// LocateData finds the intervals of data within erasure coding blocks for a given offset and size.
func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64, size types.Size) (intervals []Interval) { func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64, size types.Size) (intervals []Interval) {
blockIndex, isLargeBlock, nLargeBlockRows, innerBlockOffset := locateOffset(largeBlockLength, smallBlockLength, shardDatSize, offset) blockIndex, isLargeBlock, nLargeBlockRows, innerBlockOffset := locateOffset(largeBlockLength, smallBlockLength, shardDatSize, offset)
for size > 0 { for size > 0 {
blockRemaining := largeBlockLength - innerBlockOffset
if !isLargeBlock {
blockRemaining = smallBlockLength - innerBlockOffset
}
if blockRemaining <= 0 {
// move to next block
blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows)
innerBlockOffset = 0
continue
}
interval := Interval{ interval := Interval{
BlockIndex: blockIndex, BlockIndex: blockIndex,
InnerBlockOffset: innerBlockOffset, InnerBlockOffset: innerBlockOffset,
@ -23,11 +36,6 @@ func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, of
LargeBlockRowsCount: int(nLargeBlockRows), LargeBlockRowsCount: int(nLargeBlockRows),
} }
blockRemaining := largeBlockLength - innerBlockOffset
if !isLargeBlock {
blockRemaining = smallBlockLength - innerBlockOffset
}
if int64(size) <= blockRemaining { if int64(size) <= blockRemaining {
interval.Size = size interval.Size = size
intervals = append(intervals, interval) intervals = append(intervals, interval)
@ -37,17 +45,23 @@ func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, of
intervals = append(intervals, interval) intervals = append(intervals, interval)
size -= interval.Size size -= interval.Size
blockIndex += 1
if isLargeBlock && blockIndex == interval.LargeBlockRowsCount*DataShardsCount {
isLargeBlock = false
blockIndex = 0
}
blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows)
innerBlockOffset = 0 innerBlockOffset = 0
} }
return return
} }
func moveToNextBlock(blockIndex int, isLargeBlock bool, nLargeBlockRows int64) (int, bool) {
nextBlockIndex := blockIndex + 1
nextIsLargeBlock := isLargeBlock
if isLargeBlock && int64(nextBlockIndex) == nLargeBlockRows*DataShardsCount {
nextIsLargeBlock = false
nextBlockIndex = 0
}
return nextBlockIndex, nextIsLargeBlock
}
func locateOffset(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64) (blockIndex int, isLargeBlock bool, nLargeBlockRows int64, innerBlockOffset int64) { func locateOffset(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64) (blockIndex int, isLargeBlock bool, nLargeBlockRows int64, innerBlockOffset int64) {
largeRowSize := largeBlockLength * DataShardsCount largeRowSize := largeBlockLength * DataShardsCount
nLargeBlockRows = (shardDatSize - 1) / largeBlockLength nLargeBlockRows = (shardDatSize - 1) / largeBlockLength

17
weed/storage/erasure_coding/ec_test.go

@ -236,3 +236,20 @@ func TestLocateData3(t *testing.T) {
{BlockIndex: 8876, InnerBlockOffset: 912752, Size: 112568, IsLargeBlock: false, LargeBlockRowsCount: 2}, {BlockIndex: 8876, InnerBlockOffset: 912752, Size: 112568, IsLargeBlock: false, LargeBlockRowsCount: 2},
}) })
} }
func TestLocateData_Issue8179(t *testing.T) {
large := int64(10000)
small := int64(100)
shardSize := int64(259092) // Resulting in nLargeBlockRows = 25 as seen in panic log
// Testing range through the large-to-small transition boundary
nLargeBlockRows := (shardSize - 1) / large
largeAreaSize := nLargeBlockRows * int64(DataShardsCount) * large
for offset := largeAreaSize - 500; offset < largeAreaSize+500; offset++ {
intervals := LocateData(large, small, shardSize, offset, 200)
for _, interval := range intervals {
assert.True(t, interval.Size > 0, "Interval size must be positive at offset %d, got %+v", offset, interval)
}
}
}
Loading…
Cancel
Save