Browse Source

Optimiz leveldb metric (#3830)

* optimiz updating mapmetric for leveldb

* import loading leveldb

* add comments
pull/3831/head
Guo Lei 2 years ago
committed by GitHub
parent
commit
84c401e693
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      weed/storage/needle_map.go
  2. 46
      weed/storage/needle_map_leveldb.go
  3. 4
      weed/storage/needle_map_memory.go
  4. 8
      weed/storage/volume_vacuum.go

1
weed/storage/needle_map.go

@ -48,7 +48,6 @@ type TempNeedleMapper interface {
NeedleMapper NeedleMapper
DoOffsetLoading(v *Volume, indexFile *os.File, startFrom uint64) error DoOffsetLoading(v *Volume, indexFile *os.File, startFrom uint64) error
UpdateNeedleMap(v *Volume, indexFile *os.File, opts *opt.Options) error UpdateNeedleMap(v *Volume, indexFile *os.File, opts *opt.Options) error
UpdateNeedleMapMetric(indexFile *os.File) error
} }
func (nm *baseNeedleMapper) IndexFileSize() uint64 { func (nm *baseNeedleMapper) IndexFileSize() uint64 {

46
weed/storage/needle_map_leveldb.go

@ -4,6 +4,7 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/opt"
@ -179,6 +180,7 @@ func levelDbWrite(db *leveldb.DB, key NeedleId, offset Offset, size Size, update
} }
return nil return nil
} }
func levelDbDelete(db *leveldb.DB, key NeedleId) error { func levelDbDelete(db *leveldb.DB, key NeedleId) error {
bytes := make([]byte, NeedleIdSize) bytes := make([]byte, NeedleIdSize)
NeedleIdToBytes(bytes, key) NeedleIdToBytes(bytes, key)
@ -305,23 +307,45 @@ func (m *LevelDbNeedleMap) DoOffsetLoading(v *Volume, indexFile *os.File, startF
} }
err = idx.WalkIndexFile(indexFile, startFrom, func(key NeedleId, offset Offset, size Size) (e error) { err = idx.WalkIndexFile(indexFile, startFrom, func(key NeedleId, offset Offset, size Size) (e error) {
m.mapMetric.FileCounter++
bytes := make([]byte, NeedleIdSize)
NeedleIdToBytes(bytes[0:NeedleIdSize], key)
// fresh loading
if startFrom == 0 {
m.mapMetric.FileByteCounter += uint64(size)
e = levelDbWrite(db, key, offset, size, false, 0)
return e
}
// increment loading
data, err := db.Get(bytes, nil)
if err != nil {
if !strings.Contains(strings.ToLower(err.Error()), "not found") {
// unexpected error
return err
}
// new needle, unlikely happen
m.mapMetric.FileByteCounter += uint64(size)
e = levelDbWrite(db, key, offset, size, false, 0)
} else {
// needle is found
oldSize := BytesToSize(data[OffsetSize : OffsetSize+SizeSize])
oldOffset := BytesToOffset(data[0:OffsetSize])
if !offset.IsZero() && size.IsValid() { if !offset.IsZero() && size.IsValid() {
// updated needle
m.mapMetric.FileByteCounter += uint64(size)
if !oldOffset.IsZero() && oldSize.IsValid() {
m.mapMetric.DeletionCounter++
m.mapMetric.DeletionByteCounter += uint64(oldSize)
}
e = levelDbWrite(db, key, offset, size, false, 0) e = levelDbWrite(db, key, offset, size, false, 0)
} else { } else {
// deleted needle
m.mapMetric.DeletionCounter++
m.mapMetric.DeletionByteCounter += uint64(oldSize)
e = levelDbDelete(db, key) e = levelDbDelete(db, key)
} }
}
return e return e
}) })
if err != nil {
return err return err
}
if startFrom != 0 {
return needleMapMetricFromIndexFile(indexFile, &m.mapMetric)
}
return nil
}
func (m *LevelDbNeedleMap) UpdateNeedleMapMetric(indexFile *os.File) error {
return needleMapMetricFromIndexFile(indexFile, &m.mapMetric)
} }

4
weed/storage/needle_map_memory.go

@ -129,7 +129,3 @@ func (nm *NeedleMap) DoOffsetLoading(v *Volume, indexFile *os.File, startFrom ui
return e return e
} }
func (m *NeedleMap) UpdateNeedleMapMetric(indexFile *os.File) error {
return nil
}

8
weed/storage/volume_vacuum.go

@ -219,16 +219,8 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
return fmt.Errorf("verifyIndexFileIntegrity %s failed: %v", oldIdxFileName, err) return fmt.Errorf("verifyIndexFileIntegrity %s failed: %v", oldIdxFileName, err)
} }
if indexSize == 0 || uint64(indexSize) <= v.lastCompactIndexOffset { if indexSize == 0 || uint64(indexSize) <= v.lastCompactIndexOffset {
if v.needleMapKind == NeedleMapInMemory {
return nil return nil
} }
newIdx, err := os.OpenFile(newIdxFileName, os.O_RDWR, 0644)
if err != nil {
return fmt.Errorf("open idx file %s failed: %v", newIdxFileName, err)
}
defer newIdx.Close()
return v.tmpNm.UpdateNeedleMapMetric(newIdx)
}
// fail if the old .dat file has changed to a new revision // fail if the old .dat file has changed to a new revision
oldDatCompactRevision, err := fetchCompactRevisionFromDatFile(oldDatBackend) oldDatCompactRevision, err := fetchCompactRevisionFromDatFile(oldDatBackend)

Loading…
Cancel
Save