From 6ccd7f0a4d8ff5167054147cc66774fec84d80b6 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Tue, 18 Aug 2020 18:01:37 -0700
Subject: [PATCH] refactoring
---
weed/storage/needle_map/compact_map.go | 2 +-
weed/storage/needle_map/compact_map_test.go | 4 ++--
weed/storage/needle_map_leveldb.go | 2 +-
weed/storage/needle_map_memory.go | 4 ++--
weed/storage/needle_map_metric.go | 6 +++---
weed/storage/types/needle_types.go | 3 +++
weed/storage/volume_backup.go | 2 +-
weed/storage/volume_read_write.go | 6 +++---
weed/storage/volume_vacuum.go | 4 ++--
9 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/weed/storage/needle_map/compact_map.go b/weed/storage/needle_map/compact_map.go
index b8d242e2d..c1fb00268 100644
--- a/weed/storage/needle_map/compact_map.go
+++ b/weed/storage/needle_map/compact_map.go
@@ -130,7 +130,7 @@ func (cs *CompactSection) Delete(key NeedleId) Size {
cs.Lock()
ret := Size(0)
if i := cs.binarySearchValues(skey); i >= 0 {
- if cs.values[i].Size > 0 && cs.values[i].Size != TombstoneFileSize {
+ if cs.values[i].Size > 0 && cs.values[i].Size.IsValid() {
ret = cs.values[i].Size
cs.values[i].Size = TombstoneFileSize
}
diff --git a/weed/storage/needle_map/compact_map_test.go b/weed/storage/needle_map/compact_map_test.go
index 33ed4f1ce..c6bfb97b4 100644
--- a/weed/storage/needle_map/compact_map_test.go
+++ b/weed/storage/needle_map/compact_map_test.go
@@ -76,7 +76,7 @@ func TestCompactMap(t *testing.T) {
t.Fatal("key", i, "size", v.Size)
}
} else if i%37 == 0 {
- if ok && v.Size != TombstoneFileSize {
+ if ok && v.Size.IsValid() {
t.Fatal("key", i, "should have been deleted needle value", v)
}
} else if i%2 == 0 {
@@ -89,7 +89,7 @@ func TestCompactMap(t *testing.T) {
for i := uint32(10 * batch); i < 100*batch; i++ {
v, ok := m.Get(NeedleId(i))
if i%37 == 0 {
- if ok && v.Size != TombstoneFileSize {
+ if ok && v.Size.IsValid() {
t.Fatal("key", i, "should have been deleted needle value", v)
}
} else if i%2 == 0 {
diff --git a/weed/storage/needle_map_leveldb.go b/weed/storage/needle_map_leveldb.go
index b4b04c07f..4a17e6d0e 100644
--- a/weed/storage/needle_map_leveldb.go
+++ b/weed/storage/needle_map_leveldb.go
@@ -74,7 +74,7 @@ func generateLevelDbFile(dbFileName string, indexFile *os.File) error {
}
defer db.Close()
return idx.WalkIndexFile(indexFile, func(key NeedleId, offset Offset, size Size) error {
- if !offset.IsZero() && size != TombstoneFileSize {
+ if !offset.IsZero() && size.IsValid() {
levelDbWrite(db, key, offset, size)
} else {
levelDbDelete(db, key)
diff --git a/weed/storage/needle_map_memory.go b/weed/storage/needle_map_memory.go
index 8e7e51973..d0891dc98 100644
--- a/weed/storage/needle_map_memory.go
+++ b/weed/storage/needle_map_memory.go
@@ -30,11 +30,11 @@ func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) {
func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
e := idx.WalkIndexFile(file, func(key NeedleId, offset Offset, size Size) error {
nm.MaybeSetMaxFileKey(key)
- if !offset.IsZero() && size != TombstoneFileSize {
+ if !offset.IsZero() && size.IsValid() {
nm.FileCounter++
nm.FileByteCounter = nm.FileByteCounter + uint64(size)
oldOffset, oldSize := nm.m.Set(NeedleId(key), offset, size)
- if !oldOffset.IsZero() && oldSize != TombstoneFileSize {
+ if !oldOffset.IsZero() && oldSize.IsValid() {
nm.DeletionCounter++
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
}
diff --git a/weed/storage/needle_map_metric.go b/weed/storage/needle_map_metric.go
index 85addc27d..3618dada9 100644
--- a/weed/storage/needle_map_metric.go
+++ b/weed/storage/needle_map_metric.go
@@ -31,7 +31,7 @@ func (mm *mapMetric) logPut(key NeedleId, oldSize Size, newSize Size) {
}
mm.MaybeSetMaxFileKey(key)
mm.LogFileCounter(newSize)
- if oldSize > 0 && oldSize != TombstoneFileSize {
+ if oldSize > 0 && oldSize.IsValid() {
mm.LogDeletionCounter(oldSize)
}
}
@@ -101,7 +101,7 @@ func newNeedleMapMetricFromIndexFile(r *os.File) (mm *mapMetric, err error) {
mm.MaybeSetMaxFileKey(key)
NeedleIdToBytes(buf, key)
- if size != TombstoneFileSize {
+ if size.IsValid() {
mm.FileByteCounter += uint64(size)
}
@@ -111,7 +111,7 @@ func newNeedleMapMetricFromIndexFile(r *os.File) (mm *mapMetric, err error) {
} else {
// deleted file
mm.DeletionCounter++
- if size != TombstoneFileSize {
+ if size.IsValid() {
// previously already deleted file
mm.DeletionByteCounter += uint64(size)
}
diff --git a/weed/storage/types/needle_types.go b/weed/storage/types/needle_types.go
index 138643f7f..0e9115c0d 100644
--- a/weed/storage/types/needle_types.go
+++ b/weed/storage/types/needle_types.go
@@ -18,6 +18,9 @@ type Size uint32
func (s Size) IsDeleted() bool {
return s == TombstoneFileSize
}
+func (s Size) IsValid() bool {
+ return s != TombstoneFileSize
+}
type OffsetLower struct {
b3 byte
diff --git a/weed/storage/volume_backup.go b/weed/storage/volume_backup.go
index f7075fe2b..595bd8a35 100644
--- a/weed/storage/volume_backup.go
+++ b/weed/storage/volume_backup.go
@@ -253,7 +253,7 @@ func (scanner *VolumeFileScanner4GenIdx) ReadNeedleBody() bool {
}
func (scanner *VolumeFileScanner4GenIdx) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
- if n.Size > 0 && n.Size != TombstoneFileSize {
+ if n.Size > 0 && n.Size.IsValid() {
return scanner.v.nm.Put(n.Id, ToOffset(offset), n.Size)
}
return scanner.v.nm.Delete(n.Id, ToOffset(offset))
diff --git a/weed/storage/volume_read_write.go b/weed/storage/volume_read_write.go
index 31be0640a..b2487dde0 100644
--- a/weed/storage/volume_read_write.go
+++ b/weed/storage/volume_read_write.go
@@ -25,7 +25,7 @@ func (v *Volume) isFileUnchanged(n *needle.Needle) bool {
}
nv, ok := v.nm.Get(n.Id)
- if ok && !nv.Offset.IsZero() && nv.Size != TombstoneFileSize {
+ if ok && !nv.Offset.IsZero() && nv.Size.IsValid() {
oldNeedle := new(needle.Needle)
err := oldNeedle.ReadData(v.DataBackend, nv.Offset.ToAcutalOffset(), nv.Size, v.Version())
if err != nil {
@@ -196,7 +196,7 @@ func (v *Volume) syncDelete(n *needle.Needle) (Size, error) {
nv, ok := v.nm.Get(n.Id)
//fmt.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
- if ok && nv.Size != TombstoneFileSize {
+ if ok && nv.Size.IsValid() {
size := nv.Size
n.Data = nil
n.AppendAtNs = uint64(time.Now().UnixNano())
@@ -234,7 +234,7 @@ func (v *Volume) doDeleteRequest(n *needle.Needle) (Size, error) {
glog.V(4).Infof("delete needle %s", needle.NewFileIdFromNeedle(v.Id, n).String())
nv, ok := v.nm.Get(n.Id)
//fmt.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
- if ok && nv.Size != TombstoneFileSize {
+ if ok && nv.Size.IsValid() {
size := nv.Size
n.Data = nil
n.AppendAtNs = uint64(time.Now().UnixNano())
diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go
index 9d366a27d..a3e5800df 100644
--- a/weed/storage/volume_vacuum.go
+++ b/weed/storage/volume_vacuum.go
@@ -274,7 +274,7 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
}
//updated needle
- if !increIdxEntry.offset.IsZero() && increIdxEntry.size != 0 && increIdxEntry.size != TombstoneFileSize {
+ if !increIdxEntry.offset.IsZero() && increIdxEntry.size != 0 && increIdxEntry.size.IsValid() {
//even the needle cache in memory is hit, the need_bytes is correct
glog.V(4).Infof("file %d offset %d size %d", key, increIdxEntry.offset.ToAcutalOffset(), increIdxEntry.size)
var needleBytes []byte
@@ -335,7 +335,7 @@ func (scanner *VolumeFileScanner4Vacuum) VisitNeedle(n *needle.Needle, offset in
}
nv, ok := scanner.v.nm.Get(n.Id)
glog.V(4).Infoln("needle expected offset ", offset, "ok", ok, "nv", nv)
- if ok && nv.Offset.ToAcutalOffset() == offset && nv.Size > 0 && nv.Size != TombstoneFileSize {
+ if ok && nv.Offset.ToAcutalOffset() == offset && nv.Size > 0 && nv.Size.IsValid() {
if err := scanner.nm.Set(n.Id, ToOffset(scanner.newOffset), n.Size); err != nil {
return fmt.Errorf("cannot put needle: %s", err)
}