diff --git a/weed/storage/needle_map/compact_map.go b/weed/storage/needle_map/compact_map.go index 3146c7502..c3365c788 100644 --- a/weed/storage/needle_map/compact_map.go +++ b/weed/storage/needle_map/compact_map.go @@ -175,9 +175,14 @@ func (cs *CompactMapSegment) get(key types.NeedleId) (*CompactNeedleValue, bool) // delete deletes a map entry by key. Returns the entries' previous Size, if available. func (cs *CompactMapSegment) delete(key types.NeedleId) types.Size { if i, found := cs.bsearchKey(key); found { - if cs.list[i].size > 0 && cs.list[i].size.IsValid() { + if !cs.list[i].size.IsDeleted() { ret := cs.list[i].size - cs.list[i].size = -cs.list[i].size + if cs.list[i].size == 0 { + // size=0 needles can't be marked deleted by negating, use tombstone + cs.list[i].size = types.TombstoneFileSize + } else { + cs.list[i].size = -cs.list[i].size + } return ret } } diff --git a/weed/storage/needle_map_leveldb.go b/weed/storage/needle_map_leveldb.go index 3439e0361..3a3935244 100644 --- a/weed/storage/needle_map_leveldb.go +++ b/weed/storage/needle_map_leveldb.go @@ -122,7 +122,7 @@ func generateLevelDbFile(dbFileName string, indexFile *os.File) error { glog.V(0).Infof("generateLevelDbFile %s, watermark %d, num of entries:%d", dbFileName, watermark, (uint64(stat.Size())-watermark*NeedleMapEntrySize)/NeedleMapEntrySize) } return idx.WalkIndexFile(indexFile, watermark, func(key NeedleId, offset Offset, size Size) error { - if !offset.IsZero() && size.IsValid() { + if !offset.IsZero() && !size.IsDeleted() { levelDbWrite(db, key, offset, size, false, 0) } else { levelDbDelete(db, key) @@ -380,10 +380,10 @@ func (m *LevelDbNeedleMap) DoOffsetLoading(v *Volume, indexFile *os.File, startF // needle is found oldSize := BytesToSize(data[OffsetSize : OffsetSize+SizeSize]) oldOffset := BytesToOffset(data[0:OffsetSize]) - if !offset.IsZero() && size.IsValid() { + if !offset.IsZero() && !size.IsDeleted() { // updated needle m.mapMetric.FileByteCounter += uint64(size) - if !oldOffset.IsZero() && oldSize.IsValid() { + if !oldOffset.IsZero() && !oldSize.IsDeleted() { m.mapMetric.DeletionCounter++ m.mapMetric.DeletionByteCounter += uint64(oldSize) } diff --git a/weed/storage/needle_map_memory.go b/weed/storage/needle_map_memory.go index c00c75010..37f757b9d 100644 --- a/weed/storage/needle_map_memory.go +++ b/weed/storage/needle_map_memory.go @@ -40,7 +40,7 @@ func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) { nm.FileCounter++ nm.FileByteCounter = nm.FileByteCounter + uint64(size) oldOffset, oldSize := nm.m.Set(NeedleId(key), offset, size) - if !oldOffset.IsZero() && oldSize.IsValid() { + if !oldOffset.IsZero() && !oldSize.IsDeleted() { nm.DeletionCounter++ nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize) } @@ -112,10 +112,10 @@ func (nm *NeedleMap) DoOffsetLoading(v *Volume, indexFile *os.File, startFrom ui e := idx.WalkIndexFile(indexFile, startFrom, func(key NeedleId, offset Offset, size Size) error { nm.MaybeSetMaxFileKey(key) nm.FileCounter++ - if !offset.IsZero() && size.IsValid() { + if !offset.IsZero() && !size.IsDeleted() { nm.FileByteCounter = nm.FileByteCounter + uint64(size) oldOffset, oldSize := nm.m.Set(NeedleId(key), offset, size) - if !oldOffset.IsZero() && oldSize.IsValid() { + if !oldOffset.IsZero() && !oldSize.IsDeleted() { nm.DeletionCounter++ nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize) } diff --git a/weed/storage/types/needle_types.go b/weed/storage/types/needle_types.go index f769fab0d..7df890938 100644 --- a/weed/storage/types/needle_types.go +++ b/weed/storage/types/needle_types.go @@ -14,9 +14,18 @@ type Offset struct { type Size int32 +// IsDeleted checks if the needle entry has been marked as deleted (tombstoned). +// Use this when checking if an entry should exist in the needle map. +// Returns true for negative sizes or TombstoneFileSize. +// Note: size=0 is NOT considered deleted - it's an anomalous but active entry. func (s Size) IsDeleted() bool { return s < 0 || s == TombstoneFileSize } + +// IsValid checks if the needle has actual readable data. +// Use this when checking if needle data can be read or when counting data bytes. +// Returns true only for positive sizes (size > 0). +// Note: size=0 returns false - such needles exist in the map but have no readable data. func (s Size) IsValid() bool { return s > 0 && s != TombstoneFileSize }