From 0871d2cff091e43190065bd17dba624cf6da1d6d Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Mon, 9 Mar 2020 22:29:02 -0700
Subject: [PATCH] volume: fix memory leak during compaction

fix https://github.com/chrislusf/seaweedfs/issues/1222
---
 weed/command/export.go                      |  2 ++
 weed/command/fix.go                         |  1 +
 weed/server/master_server_handlers_admin.go |  2 +-
 weed/storage/erasure_coding/ec_encoder.go   |  7 +++++--
 weed/storage/erasure_coding/ec_test.go      |  4 +++-
 weed/storage/needle_map/memdb.go            |  4 ++++
 weed/storage/needle_map/memdb_test.go       | 23 +++++++++++++++++++++
 7 files changed, 39 insertions(+), 4 deletions(-)
 create mode 100644 weed/storage/needle_map/memdb_test.go

diff --git a/weed/command/export.go b/weed/command/export.go
index 8d664ad3b..8c32b3f4d 100644
--- a/weed/command/export.go
+++ b/weed/command/export.go
@@ -195,6 +195,8 @@ func runExport(cmd *Command, args []string) bool {
 	vid := needle.VolumeId(*export.volumeId)
 
 	needleMap := needle_map.NewMemDb()
+	defer needleMap.Close()
+
 	if err := needleMap.LoadFromIdx(path.Join(*export.dir, fileName+".idx")); err != nil {
 		glog.Fatalf("cannot load needle map from %s.idx: %s", fileName, err)
 	}
diff --git a/weed/command/fix.go b/weed/command/fix.go
index 8903595fa..90d1c4893 100644
--- a/weed/command/fix.go
+++ b/weed/command/fix.go
@@ -70,6 +70,7 @@ func runFix(cmd *Command, args []string) bool {
 	indexFileName := path.Join(*fixVolumePath, baseFileName+".idx")
 
 	nm := needle_map.NewMemDb()
+	defer nm.Close()
 
 	vid := needle.VolumeId(*fixVolumeId)
 	scanner := &VolumeFileScanner4Fix{
diff --git a/weed/server/master_server_handlers_admin.go b/weed/server/master_server_handlers_admin.go
index 2965a4863..5d0986f97 100644
--- a/weed/server/master_server_handlers_admin.go
+++ b/weed/server/master_server_handlers_admin.go
@@ -61,7 +61,7 @@ func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Reque
 			return
 		}
 	}
-	glog.Infoln("garbageThreshold =", gcThreshold)
+	// glog.Infoln("garbageThreshold =", gcThreshold)
 	ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.preallocateSize)
 	ms.dirStatusHandler(w, r)
 }
diff --git a/weed/storage/erasure_coding/ec_encoder.go b/weed/storage/erasure_coding/ec_encoder.go
index 9e2edf57d..97c3ccbd9 100644
--- a/weed/storage/erasure_coding/ec_encoder.go
+++ b/weed/storage/erasure_coding/ec_encoder.go
@@ -27,6 +27,9 @@ const (
 func WriteSortedFileFromIdx(baseFileName string, ext string) (e error) {
 
 	nm, err := readNeedleMap(baseFileName)
+	if nm != nil {
+		defer nm.Close()
+	}
 	if err != nil {
 		return fmt.Errorf("readNeedleMap: %v", err)
 	}
@@ -196,7 +199,7 @@ func encodeDatFile(remainingSize int64, err error, baseFileName string, bufferSi
 	}
 
 	buffers := make([][]byte, TotalShardsCount)
-	for i, _ := range buffers {
+	for i := range buffers {
 		buffers[i] = make([]byte, bufferSize)
 	}
 
@@ -233,7 +236,7 @@ func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*o
 	}
 
 	buffers := make([][]byte, TotalShardsCount)
-	for i, _ := range buffers {
+	for i := range buffers {
 		if shardHasData[i] {
 			buffers[i] = make([]byte, ErasureCodingSmallBlockSize)
 		}
diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go
index b2c94cfd7..92b83cdc8 100644
--- a/weed/storage/erasure_coding/ec_test.go
+++ b/weed/storage/erasure_coding/ec_test.go
@@ -7,9 +7,10 @@ import (
 	"os"
 	"testing"
 
+	"github.com/klauspost/reedsolomon"
+
 	"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
-	"github.com/klauspost/reedsolomon"
 )
 
 const (
@@ -42,6 +43,7 @@ func TestEncodingDecoding(t *testing.T) {
 
 func validateFiles(baseFileName string) error {
 	nm, err := readNeedleMap(baseFileName)
+	defer nm.Close()
 	if err != nil {
 		return fmt.Errorf("readNeedleMap: %v", err)
 	}
diff --git a/weed/storage/needle_map/memdb.go b/weed/storage/needle_map/memdb.go
index 9eb4d9f56..a52d52a10 100644
--- a/weed/storage/needle_map/memdb.go
+++ b/weed/storage/needle_map/memdb.go
@@ -113,3 +113,7 @@ func (cm *MemDb) LoadFromIdx(idxName string) (ret error) {
 	})
 
 }
+
+func (cm *MemDb) Close() {
+	cm.db.Close()
+}
diff --git a/weed/storage/needle_map/memdb_test.go b/weed/storage/needle_map/memdb_test.go
new file mode 100644
index 000000000..7b45d23f8
--- /dev/null
+++ b/weed/storage/needle_map/memdb_test.go
@@ -0,0 +1,23 @@
+package needle_map
+
+import (
+	"testing"
+
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
+)
+
+func BenchmarkMemDb(b *testing.B) {
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		nm := NewMemDb()
+
+		nid := types.NeedleId(345)
+		offset := types.Offset{
+			OffsetHigher: types.OffsetHigher{},
+			OffsetLower:  types.OffsetLower{},
+		}
+		nm.Set(nid, offset, 324)
+		nm.Close()
+	}
+
+}