diff --git a/weed/filer/leveldb2/leveldb2_store.go b/weed/filer/leveldb2/leveldb2_store.go index 124d61c1c..4c4409c4d 100644 --- a/weed/filer/leveldb2/leveldb2_store.go +++ b/weed/filer/leveldb2/leveldb2_store.go @@ -5,13 +5,15 @@ import ( "context" "crypto/md5" "fmt" - "github.com/syndtr/goleveldb/leveldb" - leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/opt" - leveldb_util "github.com/syndtr/goleveldb/leveldb/util" "io" "os" + "github.com/syndtr/goleveldb/leveldb" + leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/filter" + "github.com/syndtr/goleveldb/leveldb/opt" + leveldb_util "github.com/syndtr/goleveldb/leveldb/util" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -47,6 +49,7 @@ func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) { BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB CompactionTableSizeMultiplier: 4, + Filter: filter.NewBloomFilter(8), // false positive rate 0.02 } for d := 0; d < dbCount; d++ { diff --git a/weed/filer/leveldb3/leveldb3_store.go b/weed/filer/leveldb3/leveldb3_store.go index d1cdfbbf6..bc57a6605 100644 --- a/weed/filer/leveldb3/leveldb3_store.go +++ b/weed/filer/leveldb3/leveldb3_store.go @@ -5,15 +5,17 @@ import ( "context" "crypto/md5" "fmt" - "github.com/syndtr/goleveldb/leveldb" - leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/opt" - leveldb_util "github.com/syndtr/goleveldb/leveldb/util" "io" "os" "strings" "sync" + "github.com/syndtr/goleveldb/leveldb" + leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/filter" + "github.com/syndtr/goleveldb/leveldb/opt" + leveldb_util "github.com/syndtr/goleveldb/leveldb/util" + "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -62,17 +64,19 @@ func (store *LevelDB3Store) initialize(dir string) (err error) { } func (store *LevelDB3Store) loadDB(name string) (*leveldb.DB, error) { - + bloom := filter.NewBloomFilter(8) // false positive rate 0.02 opts := &opt.Options{ BlockCacheCapacity: 32 * 1024 * 1024, // default value is 8MiB WriteBuffer: 16 * 1024 * 1024, // default value is 4MiB CompactionTableSizeMultiplier: 4, + Filter: bloom, } if name != DEFAULT { opts = &opt.Options{ BlockCacheCapacity: 4 * 1024 * 1024, // default value is 8MiB WriteBuffer: 2 * 1024 * 1024, // default value is 4MiB CompactionTableSizeMultiplier: 4, + Filter: bloom, } } diff --git a/weed/filer/rocksdb/rocksdb_store.go b/weed/filer/rocksdb/rocksdb_store.go index 379a18c62..729da7d9b 100644 --- a/weed/filer/rocksdb/rocksdb_store.go +++ b/weed/filer/rocksdb/rocksdb_store.go @@ -24,18 +24,21 @@ func init() { type options struct { opt *gorocksdb.Options + bto *gorocksdb.BlockBasedTableOptions ro *gorocksdb.ReadOptions wo *gorocksdb.WriteOptions } func (opt *options) init() { opt.opt = gorocksdb.NewDefaultOptions() + opt.bto = gorocksdb.NewDefaultBlockBasedTableOptions() opt.ro = gorocksdb.NewDefaultReadOptions() opt.wo = gorocksdb.NewDefaultWriteOptions() } func (opt *options) close() { opt.opt.Destroy() + opt.bto.Destroy() opt.ro.Destroy() opt.wo.Destroy() } @@ -69,6 +72,11 @@ func (store *RocksDBStore) initialize(dir string) (err error) { store.opt.SetCompactionFilter(NewTTLFilter()) // store.opt.SetMaxBackgroundCompactions(2) + // https://github.com/tecbot/gorocksdb/issues/132 + store.bto.SetFilterPolicy(gorocksdb.NewBloomFilterFull(8)) + store.opt.SetBlockBasedTableFactory(store.bto) + // store.opt.EnableStatistics() + store.db, err = gorocksdb.OpenDb(store.opt, dir) return