Browse Source

add bloom filter for filer leveldb/rocksdb

pull/2211/head
qieqieplus 3 years ago
parent
commit
2b46df13f8
  1. 7
      weed/filer/leveldb2/leveldb2_store.go
  2. 14
      weed/filer/leveldb3/leveldb3_store.go
  3. 8
      weed/filer/rocksdb/rocksdb_store.go

7
weed/filer/leveldb2/leveldb2_store.go

@ -5,12 +5,14 @@ import (
"context" "context"
"crypto/md5" "crypto/md5"
"fmt" "fmt"
"io"
"os"
"github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb"
leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt" "github.com/syndtr/goleveldb/leveldb/opt"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util" leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
"io"
"os"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
@ -47,6 +49,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
CompactionTableSizeMultiplier: 4, CompactionTableSizeMultiplier: 4,
Filter: filter.NewBloomFilter(8), // false positive rate 0.02
} }
for d := 0; d < dbCount; d++ { for d := 0; d < dbCount; d++ {

14
weed/filer/leveldb3/leveldb3_store.go

@ -5,15 +5,17 @@ import (
"context" "context"
"crypto/md5" "crypto/md5"
"fmt" "fmt"
"github.com/syndtr/goleveldb/leveldb"
leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/opt"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
"io" "io"
"os" "os"
"strings" "strings"
"sync" "sync"
"github.com/syndtr/goleveldb/leveldb"
leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
"github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@ -62,17 +64,19 @@ func (store *LevelDB3Store) initialize(dir string) (err error) {
} }
func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) { func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) {
bloom := filter.NewBloomFilter(8) // false positive rate 0.02
opts := &opt.Options{ opts := &opt.Options{
BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB
CompactionTableSizeMultiplier: 4, CompactionTableSizeMultiplier: 4,
Filter: bloom,
} }
if name != DEFAULT { if name != DEFAULT {
opts = &opt.Options{ opts = &opt.Options{
BlockCacheCapacity: 4 * 1024 * 1024, // default value is 8MiB BlockCacheCapacity: 4 * 1024 * 1024, // default value is 8MiB
WriteBuffer: 2 * 1024 * 1024, // default value is 4MiB WriteBuffer: 2 * 1024 * 1024, // default value is 4MiB
CompactionTableSizeMultiplier: 4, CompactionTableSizeMultiplier: 4,
Filter: bloom,
} }
} }

8
weed/filer/rocksdb/rocksdb_store.go

@ -24,18 +24,21 @@ func init() {
type options struct { type options struct {
opt *gorocksdb.Options opt *gorocksdb.Options
bto *gorocksdb.BlockBasedTableOptions
ro *gorocksdb.ReadOptions ro *gorocksdb.ReadOptions
wo *gorocksdb.WriteOptions wo *gorocksdb.WriteOptions
} }
func (opt *options) init() { func (opt *options) init() {
opt.opt = gorocksdb.NewDefaultOptions() opt.opt = gorocksdb.NewDefaultOptions()
opt.bto = gorocksdb.NewDefaultBlockBasedTableOptions()
opt.ro = gorocksdb.NewDefaultReadOptions() opt.ro = gorocksdb.NewDefaultReadOptions()
opt.wo = gorocksdb.NewDefaultWriteOptions() opt.wo = gorocksdb.NewDefaultWriteOptions()
} }
func (opt *options) close() { func (opt *options) close() {
opt.opt.Destroy() opt.opt.Destroy()
opt.bto.Destroy()
opt.ro.Destroy() opt.ro.Destroy()
opt.wo.Destroy() opt.wo.Destroy()
} }
@ -69,6 +72,11 @@ func (store *RocksDBStore) initialize(dir string) (err error) {
store.opt.SetCompactionFilter(NewTTLFilter()) store.opt.SetCompactionFilter(NewTTLFilter())
// store.opt.SetMaxBackgroundCompactions(2) // store.opt.SetMaxBackgroundCompactions(2)
// https://github.com/tecbot/gorocksdb/issues/132
store.bto.SetFilterPolicy(gorocksdb.NewBloomFilterFull(8))
store.opt.SetBlockBasedTableFactory(store.bto)
// store.opt.EnableStatistics()
store.db, err = gorocksdb.OpenDb(store.opt, dir) store.db, err = gorocksdb.OpenDb(store.opt, dir)
return return

Loading…
Cancel
Save