Browse Source

volume: fix memory leak during compaction

fix https://github.com/chrislusf/seaweedfs/issues/1222
tikv
Chris Lu 5 years ago
parent
commit
0871d2cff0
  1. 2
      weed/command/export.go
  2. 1
      weed/command/fix.go
  3. 2
      weed/server/master_server_handlers_admin.go
  4. 7
      weed/storage/erasure_coding/ec_encoder.go
  5. 4
      weed/storage/erasure_coding/ec_test.go
  6. 4
      weed/storage/needle_map/memdb.go
  7. 23
      weed/storage/needle_map/memdb_test.go

2
weed/command/export.go

@ -195,6 +195,8 @@ func runExport(cmd *Command, args []string) bool {
vid := needle.VolumeId(*export.volumeId) vid := needle.VolumeId(*export.volumeId)
needleMap := needle_map.NewMemDb() needleMap := needle_map.NewMemDb()
defer needleMap.Close()
if err := needleMap.LoadFromIdx(path.Join(*export.dir, fileName+".idx")); err != nil { if err := needleMap.LoadFromIdx(path.Join(*export.dir, fileName+".idx")); err != nil {
glog.Fatalf("cannot load needle map from %s.idx: %s", fileName, err) glog.Fatalf("cannot load needle map from %s.idx: %s", fileName, err)
} }

1
weed/command/fix.go

@ -70,6 +70,7 @@ func runFix(cmd *Command, args []string) bool {
indexFileName := path.Join(*fixVolumePath, baseFileName+".idx") indexFileName := path.Join(*fixVolumePath, baseFileName+".idx")
nm := needle_map.NewMemDb() nm := needle_map.NewMemDb()
defer nm.Close()
vid := needle.VolumeId(*fixVolumeId) vid := needle.VolumeId(*fixVolumeId)
scanner := &VolumeFileScanner4Fix{ scanner := &VolumeFileScanner4Fix{

2
weed/server/master_server_handlers_admin.go

@ -61,7 +61,7 @@ func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Reque
return return
} }
} }
glog.Infoln("garbageThreshold =", gcThreshold)
// glog.Infoln("garbageThreshold =", gcThreshold)
ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.preallocateSize) ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.preallocateSize)
ms.dirStatusHandler(w, r) ms.dirStatusHandler(w, r)
} }

7
weed/storage/erasure_coding/ec_encoder.go

@ -27,6 +27,9 @@ const (
func WriteSortedFileFromIdx(baseFileName string, ext string) (e error) { func WriteSortedFileFromIdx(baseFileName string, ext string) (e error) {
nm, err := readNeedleMap(baseFileName) nm, err := readNeedleMap(baseFileName)
if nm != nil {
defer nm.Close()
}
if err != nil { if err != nil {
return fmt.Errorf("readNeedleMap: %v", err) return fmt.Errorf("readNeedleMap: %v", err)
} }
@ -196,7 +199,7 @@ func encodeDatFile(remainingSize int64, err error, baseFileName string, bufferSi
} }
buffers := make([][]byte, TotalShardsCount) buffers := make([][]byte, TotalShardsCount)
for i, _ := range buffers {
for i := range buffers {
buffers[i] = make([]byte, bufferSize) buffers[i] = make([]byte, bufferSize)
} }
@ -233,7 +236,7 @@ func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*o
} }
buffers := make([][]byte, TotalShardsCount) buffers := make([][]byte, TotalShardsCount)
for i, _ := range buffers {
for i := range buffers {
if shardHasData[i] { if shardHasData[i] {
buffers[i] = make([]byte, ErasureCodingSmallBlockSize) buffers[i] = make([]byte, ErasureCodingSmallBlockSize)
} }

4
weed/storage/erasure_coding/ec_test.go

@ -7,9 +7,10 @@ import (
"os" "os"
"testing" "testing"
"github.com/klauspost/reedsolomon"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map" "github.com/chrislusf/seaweedfs/weed/storage/needle_map"
"github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/klauspost/reedsolomon"
) )
const ( const (
@ -42,6 +43,7 @@ func TestEncodingDecoding(t *testing.T) {
func validateFiles(baseFileName string) error { func validateFiles(baseFileName string) error {
nm, err := readNeedleMap(baseFileName) nm, err := readNeedleMap(baseFileName)
defer nm.Close()
if err != nil { if err != nil {
return fmt.Errorf("readNeedleMap: %v", err) return fmt.Errorf("readNeedleMap: %v", err)
} }

4
weed/storage/needle_map/memdb.go

@ -113,3 +113,7 @@ func (cm *MemDb) LoadFromIdx(idxName string) (ret error) {
}) })
} }
func (cm *MemDb) Close() {
cm.db.Close()
}

23
weed/storage/needle_map/memdb_test.go

@ -0,0 +1,23 @@
package needle_map
import (
"testing"
"github.com/chrislusf/seaweedfs/weed/storage/types"
)
func BenchmarkMemDb(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
nm := NewMemDb()
nid := types.NeedleId(345)
offset := types.Offset{
OffsetHigher: types.OffsetHigher{},
OffsetLower: types.OffsetLower{},
}
nm.Set(nid, offset, 324)
nm.Close()
}
}
Loading…
Cancel
Save