optimize a little bit

weed/storage/needle/compact_map.go

@@ -117,14 +117,16 @@ func (cm *CompactMap) Set(key NeedleId, offset Offset, size uint32) (oldOffset O
 	x := cm.binarySearchCompactSection(key)
 	if x < 0 {
 		//println(x, "creating", len(cm.list), "section, starting", key)
-		cm.list = append(cm.list, NewCompactSection(key))
+		cs := NewCompactSection(key)
+		cm.list = append(cm.list, cs)
 		x = len(cm.list) - 1
 		//keep compact section sorted by start
 		for x > 0 {
-			if cm.list[x-1].start > cm.list[x].start {
-				cm.list[x-1], cm.list[x] = cm.list[x], cm.list[x-1]
+			if cm.list[x-1].start > key {
+				cm.list[x] = cm.list[x-1]
 				x = x - 1
 			} else {
+				cm.list[x] = cs
 				break
 			}
 		}

weed/storage/needle/compact_map_perf_test.go

@@ -1,15 +1,23 @@
 package needle
 
 import (
-	"log"
-	"os"
 	"testing"
+	"os"
+	"log"
 
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	. "github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	. "github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
+/*
+
+To see the memory usage:
+
+go test -run TestMemoryUsage -memprofile=mem.out
+go tool pprof needle.test mem.out
+
+ */
+
 func TestMemoryUsage(t *testing.T) {
 
 	indexFile, ie := os.OpenFile("../../../test/sample.idx", os.O_RDWR|os.O_RDONLY, 0644)
@@ -18,18 +26,16 @@ func TestMemoryUsage(t *testing.T) {
 	}
 	loadNewNeedleMap(indexFile)
 
+	indexFile.Close()
+
 }
 
 func loadNewNeedleMap(file *os.File) {
 	m := NewCompactMap()
-	bytes := make([]byte, 16*1024)
+	bytes := make([]byte, NeedleEntrySize*1024)
 	count, e := file.Read(bytes)
-	if count > 0 {
-		fstat, _ := file.Stat()
-		glog.V(0).Infoln("Loading index file", fstat.Name(), "size", fstat.Size())
-	}
 	for count > 0 && e == nil {
-		for i := 0; i < count; i += 16 {
+		for i := 0; i < count; i += NeedleEntrySize {
 			key := BytesToNeedleId(bytes[i : i+NeedleIdSize])
 			offset := BytesToOffset(bytes[i+NeedleIdSize : i+NeedleIdSize+OffsetSize])
 			size := util.BytesToUint32(bytes[i+NeedleIdSize+OffsetSize : i+NeedleIdSize+OffsetSize+SizeSize])
@@ -43,4 +49,38 @@ func loadNewNeedleMap(file *os.File) {
 
 		count, e = file.Read(bytes)
 	}
+
+	m.report()
+
+}
+
+// report memory usage
+func (cm *CompactMap) report() {
+	overFlowCount := 0;
+	overwrittenByOverflow := 0;
+	entryCount := 0
+	compactSectionCount := 0
+	compactSectionEntryCount := 0
+	for _, cs := range cm.list {
+		compactSectionCount++
+		cs.RLock()
+		for range cs.overflow {
+			overFlowCount++
+			entryCount++
+		}
+		for _, v := range cs.values {
+			compactSectionEntryCount++
+			if _, found := cs.overflow[v.Key]; !found {
+				entryCount++
+			} else {
+				overwrittenByOverflow++
+			}
+		}
+		cs.RUnlock()
+	}
+	println("overFlowCount", overFlowCount)
+	println("overwrittenByOverflow", overwrittenByOverflow)
+	println("entryCount", entryCount)
+	println("compactSectionCount", compactSectionCount)
+	println("compactSectionEntryCount", compactSectionEntryCount)
 }