diff --git a/weed/storage/erasure_coding/ec_locate.go b/weed/storage/erasure_coding/ec_locate.go index 22839a809..84ba8e04f 100644 --- a/weed/storage/erasure_coding/ec_locate.go +++ b/weed/storage/erasure_coding/ec_locate.go @@ -12,10 +12,23 @@ type Interval struct { LargeBlockRowsCount int } +// LocateData finds the intervals of data within erasure coding blocks for a given offset and size. func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64, size types.Size) (intervals []Interval) { blockIndex, isLargeBlock, nLargeBlockRows, innerBlockOffset := locateOffset(largeBlockLength, smallBlockLength, shardDatSize, offset) for size > 0 { + blockRemaining := largeBlockLength - innerBlockOffset + if !isLargeBlock { + blockRemaining = smallBlockLength - innerBlockOffset + } + + if blockRemaining <= 0 { + // move to next block + blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows) + innerBlockOffset = 0 + continue + } + interval := Interval{ BlockIndex: blockIndex, InnerBlockOffset: innerBlockOffset, @@ -23,11 +36,6 @@ func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, of LargeBlockRowsCount: int(nLargeBlockRows), } - blockRemaining := largeBlockLength - innerBlockOffset - if !isLargeBlock { - blockRemaining = smallBlockLength - innerBlockOffset - } - if int64(size) <= blockRemaining { interval.Size = size intervals = append(intervals, interval) @@ -37,17 +45,23 @@ func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, of intervals = append(intervals, interval) size -= interval.Size - blockIndex += 1 - if isLargeBlock && blockIndex == interval.LargeBlockRowsCount*DataShardsCount { - isLargeBlock = false - blockIndex = 0 - } + blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows) innerBlockOffset = 0 } return } +func moveToNextBlock(blockIndex int, isLargeBlock bool, nLargeBlockRows int64) (int, bool) { + nextBlockIndex := blockIndex + 1 + nextIsLargeBlock := isLargeBlock + if isLargeBlock && int64(nextBlockIndex) == nLargeBlockRows*DataShardsCount { + nextIsLargeBlock = false + nextBlockIndex = 0 + } + return nextBlockIndex, nextIsLargeBlock +} + func locateOffset(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64) (blockIndex int, isLargeBlock bool, nLargeBlockRows int64, innerBlockOffset int64) { largeRowSize := largeBlockLength * DataShardsCount nLargeBlockRows = (shardDatSize - 1) / largeBlockLength diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go index 0930d1b01..0db9f30fa 100644 --- a/weed/storage/erasure_coding/ec_test.go +++ b/weed/storage/erasure_coding/ec_test.go @@ -236,3 +236,20 @@ func TestLocateData3(t *testing.T) { {BlockIndex: 8876, InnerBlockOffset: 912752, Size: 112568, IsLargeBlock: false, LargeBlockRowsCount: 2}, }) } + +func TestLocateData_Issue8179(t *testing.T) { + large := int64(10000) + small := int64(100) + shardSize := int64(259092) // Resulting in nLargeBlockRows = 25 as seen in panic log + + // Testing range through the large-to-small transition boundary + nLargeBlockRows := (shardSize - 1) / large + largeAreaSize := nLargeBlockRows * int64(DataShardsCount) * large + + for offset := largeAreaSize - 500; offset < largeAreaSize+500; offset++ { + intervals := LocateData(large, small, shardSize, offset, 200) + for _, interval := range intervals { + assert.True(t, interval.Size > 0, "Interval size must be positive at offset %d, got %+v", offset, interval) + } + } +}