diff --git a/weed/filer/filechunk_section.go b/weed/filer/filechunk_section.go index 5804c7160..3fbfb396d 100644 --- a/weed/filer/filechunk_section.go +++ b/weed/filer/filechunk_section.go @@ -33,16 +33,7 @@ func (section *FileChunkSection) addChunk(chunk *filer_pb.FileChunk) error { if section.visibleIntervals != nil { MergeIntoVisibles(section.visibleIntervals, start, stop, chunk) garbageFileIds := FindGarbageChunks(section.visibleIntervals, start, stop) - for _, garbageFileId := range garbageFileIds { - length := len(section.chunks) - for i, t := range section.chunks { - if t.FileId == garbageFileId { - section.chunks[i] = section.chunks[length-1] - section.chunks = section.chunks[:length-1] - break - } - } - } + removeGarbageChunks(section, garbageFileIds) } if section.chunkViews != nil { @@ -52,6 +43,22 @@ func (section *FileChunkSection) addChunk(chunk *filer_pb.FileChunk) error { return nil } +func removeGarbageChunks(section *FileChunkSection, garbageFileIds map[string]struct{}) { + now := time.Now() + for i := 0; i < len(section.chunks); { + t := section.chunks[i] + length := len(section.chunks) + if _, found := garbageFileIds[t.FileId]; found { + if i < length-1 { + section.chunks[i] = section.chunks[length-1] + } + section.chunks = section.chunks[:length-1] + } else { + i++ + } + } +} + func (section *FileChunkSection) setupForRead(group *ChunkGroup, fileSize int64) { if section.visibleIntervals == nil { section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize) diff --git a/weed/filer/filechunk_section_test.go b/weed/filer/filechunk_section_test.go new file mode 100644 index 000000000..e4536540b --- /dev/null +++ b/weed/filer/filechunk_section_test.go @@ -0,0 +1,48 @@ +package filer + +import ( + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "testing" +) + +func Test_removeGarbageChunks(t *testing.T) { + section := NewFileChunkSection(0) + section.addChunk(&filer_pb.FileChunk{ + FileId: "0", + Offset: 0, + Size: 1, + ModifiedTsNs: 0, + }) + section.addChunk(&filer_pb.FileChunk{ + FileId: "1", + Offset: 1, + Size: 1, + ModifiedTsNs: 1, + }) + section.addChunk(&filer_pb.FileChunk{ + FileId: "2", + Offset: 2, + Size: 1, + ModifiedTsNs: 2, + }) + section.addChunk(&filer_pb.FileChunk{ + FileId: "3", + Offset: 3, + Size: 1, + ModifiedTsNs: 3, + }) + section.addChunk(&filer_pb.FileChunk{ + FileId: "4", + Offset: 4, + Size: 1, + ModifiedTsNs: 4, + }) + garbageFileIds := make(map[string]struct{}) + garbageFileIds["0"] = struct{}{} + garbageFileIds["2"] = struct{}{} + garbageFileIds["4"] = struct{}{} + removeGarbageChunks(section, garbageFileIds) + if len(section.chunks) != 2 { + t.Errorf("remove chunk 2 failed") + } +} diff --git a/weed/filer/filechunks.go b/weed/filer/filechunks.go index 480478fd7..7c8bb2fe1 100644 --- a/weed/filer/filechunks.go +++ b/weed/filer/filechunks.go @@ -86,12 +86,13 @@ func SeparateGarbageChunks(visibles *IntervalList[*VisibleInterval], chunks []*f return compacted, garbage } -func FindGarbageChunks(visibles *IntervalList[*VisibleInterval], start int64, stop int64) (garbageFileId []string) { +func FindGarbageChunks(visibles *IntervalList[*VisibleInterval], start int64, stop int64) (garbageFileIds map[string]struct{}) { + garbageFileIds = make(map[string]struct{}) for x := visibles.Front(); x != nil; x = x.Next { interval := x.Value offset := interval.start - interval.offsetInChunk if start <= offset && offset+int64(interval.chunkSize) <= stop { - garbageFileId = append(garbageFileId, interval.fileId) + garbageFileIds[interval.fileId] = struct{}{} } } return