mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
weed volume: remove boltdb, btree options, add options for leveldb medium, large
This commit is contained in:
parent
9924fa3b1a
commit
8a5ce16e96
|
@ -90,7 +90,7 @@ func init() {
|
|||
|
||||
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
|
||||
serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
|
||||
serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
|
||||
serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
|
||||
serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
|
||||
serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
|
||||
serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
|
||||
|
|
|
@ -58,7 +58,7 @@ func init() {
|
|||
v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
|
||||
v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
|
||||
v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
|
||||
v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
|
||||
v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
|
||||
v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
|
||||
v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
|
||||
v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
|
||||
|
@ -142,10 +142,10 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
|
|||
switch *v.indexType {
|
||||
case "leveldb":
|
||||
volumeNeedleMapKind = storage.NeedleMapLevelDb
|
||||
case "boltdb":
|
||||
case "leveldbMedium":
|
||||
volumeNeedleMapKind = storage.NeedleMapBoltDb
|
||||
case "leveldbLarge":
|
||||
volumeNeedleMapKind = storage.NeedleMapBoltDb
|
||||
case "btree":
|
||||
volumeNeedleMapKind = storage.NeedleMapBtree
|
||||
}
|
||||
|
||||
masters := *v.masters
|
||||
|
|
|
@ -15,7 +15,9 @@ type NeedleMapType int
|
|||
|
||||
const (
|
||||
NeedleMapInMemory NeedleMapType = iota
|
||||
NeedleMapLevelDb
|
||||
NeedleMapLevelDb // small memory footprint, 4MB total, 1 write buffer, 3 block buffer
|
||||
NeedleMapLevelDbMedium // medium memory footprint, 8MB total, 3 write buffer, 5 block buffer
|
||||
NeedleMapLevelDbLarge // large memory footprint, 12MB total, 4write buffer, 8 block buffer
|
||||
NeedleMapBoltDb
|
||||
NeedleMapBtree
|
||||
)
|
||||
|
|
|
@ -1,185 +0,0 @@
|
|||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/boltdb/bolt"
|
||||
|
||||
"errors"
|
||||
"github.com/chrislusf/seaweedfs/weed/glog"
|
||||
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
||||
. "github.com/chrislusf/seaweedfs/weed/storage/types"
|
||||
"github.com/chrislusf/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
type BoltDbNeedleMap struct {
|
||||
dbFileName string
|
||||
db *bolt.DB
|
||||
baseNeedleMapper
|
||||
}
|
||||
|
||||
var boltdbBucket = []byte("weed")
|
||||
|
||||
var NotFound = errors.New("not found")
|
||||
|
||||
func NewBoltDbNeedleMap(dbFileName string, indexFile *os.File) (m *BoltDbNeedleMap, err error) {
|
||||
m = &BoltDbNeedleMap{dbFileName: dbFileName}
|
||||
m.indexFile = indexFile
|
||||
if !isBoltDbFresh(dbFileName, indexFile) {
|
||||
glog.V(0).Infof("Start to Generate %s from %s", dbFileName, indexFile.Name())
|
||||
generateBoltDbFile(dbFileName, indexFile)
|
||||
glog.V(0).Infof("Finished Generating %s from %s", dbFileName, indexFile.Name())
|
||||
}
|
||||
glog.V(1).Infof("Opening %s...", dbFileName)
|
||||
if m.db, err = bolt.Open(dbFileName, 0644, nil); err != nil {
|
||||
return
|
||||
}
|
||||
glog.V(1).Infof("Loading %s...", indexFile.Name())
|
||||
mm, indexLoadError := newNeedleMapMetricFromIndexFile(indexFile)
|
||||
if indexLoadError != nil {
|
||||
return nil, indexLoadError
|
||||
}
|
||||
m.mapMetric = *mm
|
||||
return
|
||||
}
|
||||
|
||||
func isBoltDbFresh(dbFileName string, indexFile *os.File) bool {
|
||||
// normally we always write to index file first
|
||||
dbLogFile, err := os.Open(dbFileName)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer dbLogFile.Close()
|
||||
dbStat, dbStatErr := dbLogFile.Stat()
|
||||
indexStat, indexStatErr := indexFile.Stat()
|
||||
if dbStatErr != nil || indexStatErr != nil {
|
||||
glog.V(0).Infof("Can not stat file: %v and %v", dbStatErr, indexStatErr)
|
||||
return false
|
||||
}
|
||||
|
||||
return dbStat.ModTime().After(indexStat.ModTime())
|
||||
}
|
||||
|
||||
func generateBoltDbFile(dbFileName string, indexFile *os.File) error {
|
||||
db, err := bolt.Open(dbFileName, 0644, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
return WalkIndexFile(indexFile, func(key NeedleId, offset Offset, size uint32) error {
|
||||
if !offset.IsZero() && size != TombstoneFileSize {
|
||||
boltDbWrite(db, key, offset, size)
|
||||
} else {
|
||||
boltDbDelete(db, key)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (m *BoltDbNeedleMap) Get(key NeedleId) (element *needle.NeedleValue, ok bool) {
|
||||
var offset Offset
|
||||
var size uint32
|
||||
bytes := make([]byte, NeedleIdSize)
|
||||
NeedleIdToBytes(bytes, key)
|
||||
err := m.db.View(func(tx *bolt.Tx) error {
|
||||
bucket := tx.Bucket(boltdbBucket)
|
||||
if bucket == nil {
|
||||
return fmt.Errorf("Bucket %q not found!", boltdbBucket)
|
||||
}
|
||||
|
||||
data := bucket.Get(bytes)
|
||||
|
||||
if len(data) == 0 {
|
||||
return NotFound
|
||||
}
|
||||
|
||||
if len(data) != OffsetSize+SizeSize {
|
||||
glog.V(0).Infof("key:%v has wrong data length: %d", key, len(data))
|
||||
return fmt.Errorf("key:%v has wrong data length: %d", key, len(data))
|
||||
}
|
||||
|
||||
offset = BytesToOffset(data[0:OffsetSize])
|
||||
size = util.BytesToUint32(data[OffsetSize : OffsetSize+SizeSize])
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return &needle.NeedleValue{Key: key, Offset: offset, Size: size}, true
|
||||
}
|
||||
|
||||
func (m *BoltDbNeedleMap) Put(key NeedleId, offset Offset, size uint32) error {
|
||||
var oldSize uint32
|
||||
if oldNeedle, ok := m.Get(key); ok {
|
||||
oldSize = oldNeedle.Size
|
||||
}
|
||||
m.logPut(key, oldSize, size)
|
||||
// write to index file first
|
||||
if err := m.appendToIndexFile(key, offset, size); err != nil {
|
||||
return fmt.Errorf("cannot write to indexfile %s: %v", m.indexFile.Name(), err)
|
||||
}
|
||||
return boltDbWrite(m.db, key, offset, size)
|
||||
}
|
||||
|
||||
func boltDbWrite(db *bolt.DB,
|
||||
key NeedleId, offset Offset, size uint32) error {
|
||||
|
||||
bytes := make([]byte, NeedleIdSize+OffsetSize+SizeSize)
|
||||
NeedleIdToBytes(bytes[0:NeedleIdSize], key)
|
||||
OffsetToBytes(bytes[NeedleIdSize:NeedleIdSize+OffsetSize], offset)
|
||||
util.Uint32toBytes(bytes[NeedleIdSize+OffsetSize:NeedleIdSize+OffsetSize+SizeSize], size)
|
||||
|
||||
return db.Update(func(tx *bolt.Tx) error {
|
||||
bucket, err := tx.CreateBucketIfNotExists(boltdbBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = bucket.Put(bytes[0:NeedleIdSize], bytes[NeedleIdSize:NeedleIdSize+OffsetSize+SizeSize])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
func boltDbDelete(db *bolt.DB, key NeedleId) error {
|
||||
bytes := make([]byte, NeedleIdSize)
|
||||
NeedleIdToBytes(bytes, key)
|
||||
return db.Update(func(tx *bolt.Tx) error {
|
||||
bucket, err := tx.CreateBucketIfNotExists(boltdbBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = bucket.Delete(bytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (m *BoltDbNeedleMap) Delete(key NeedleId, offset Offset) error {
|
||||
if oldNeedle, ok := m.Get(key); ok {
|
||||
m.logDelete(oldNeedle.Size)
|
||||
}
|
||||
// write to index file first
|
||||
if err := m.appendToIndexFile(key, offset, TombstoneFileSize); err != nil {
|
||||
return err
|
||||
}
|
||||
return boltDbDelete(m.db, key)
|
||||
}
|
||||
|
||||
func (m *BoltDbNeedleMap) Close() {
|
||||
m.indexFile.Close()
|
||||
m.db.Close()
|
||||
}
|
||||
|
||||
func (m *BoltDbNeedleMap) Destroy() error {
|
||||
m.Close()
|
||||
os.Remove(m.indexFile.Name())
|
||||
return os.Remove(m.dbFileName)
|
||||
}
|
|
@ -19,7 +19,7 @@ type LevelDbNeedleMap struct {
|
|||
baseNeedleMapper
|
||||
}
|
||||
|
||||
func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File) (m *LevelDbNeedleMap, err error) {
|
||||
func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File, opts *opt.Options) (m *LevelDbNeedleMap, err error) {
|
||||
m = &LevelDbNeedleMap{dbFileName: dbFileName}
|
||||
m.indexFile = indexFile
|
||||
if !isLevelDbFresh(dbFileName, indexFile) {
|
||||
|
@ -28,9 +28,8 @@ func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File) (m *LevelDbNeedl
|
|||
glog.V(0).Infof("Finished Generating %s from %s", dbFileName, indexFile.Name())
|
||||
}
|
||||
glog.V(1).Infof("Opening %s...", dbFileName)
|
||||
if m.db, err = leveldb.OpenFile(dbFileName, &opt.Options{
|
||||
BlockCacheCapacity: -1, // default value is 8MiB
|
||||
}); err != nil {
|
||||
|
||||
if m.db, err = leveldb.OpenFile(dbFileName, opts); err != nil {
|
||||
return
|
||||
}
|
||||
glog.V(1).Infof("Loading %s...", indexFile.Name())
|
||||
|
|
|
@ -2,6 +2,7 @@ package storage
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/syndtr/goleveldb/leveldb/opt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
|
@ -82,18 +83,30 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
|
|||
}
|
||||
case NeedleMapLevelDb:
|
||||
glog.V(0).Infoln("loading leveldb", fileName+".ldb")
|
||||
if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile); e != nil {
|
||||
opts := &opt.Options{
|
||||
BlockCacheCapacity: 2 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 1 * 1024 * 1024, // default value is 4MiB
|
||||
}
|
||||
if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil {
|
||||
glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
|
||||
}
|
||||
case NeedleMapBoltDb:
|
||||
glog.V(0).Infoln("loading boltdb", fileName+".bdb")
|
||||
if v.nm, e = NewBoltDbNeedleMap(fileName+".bdb", indexFile); e != nil {
|
||||
glog.V(0).Infof("loading boltdb %s error: %v", fileName+".bdb", e)
|
||||
case NeedleMapLevelDbMedium:
|
||||
glog.V(0).Infoln("loading leveldb medium", fileName+".ldb")
|
||||
opts := &opt.Options{
|
||||
BlockCacheCapacity: 4 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 2 * 1024 * 1024, // default value is 4MiB
|
||||
}
|
||||
case NeedleMapBtree:
|
||||
glog.V(0).Infoln("loading index", fileName+".idx", "to btree readonly", v.readOnly)
|
||||
if v.nm, e = LoadBtreeNeedleMap(indexFile); e != nil {
|
||||
glog.V(0).Infof("loading index %s to btree error: %v", fileName+".idx", e)
|
||||
if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil {
|
||||
glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
|
||||
}
|
||||
case NeedleMapLevelDbLarge:
|
||||
glog.V(0).Infoln("loading leveldb large", fileName+".ldb")
|
||||
opts := &opt.Options{
|
||||
BlockCacheCapacity: 8 * 1024 * 1024, // default value is 8MiB
|
||||
WriteBuffer: 4 * 1024 * 1024, // default value is 4MiB
|
||||
}
|
||||
if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil {
|
||||
glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue