weed volume: remove boltdb, btree options, add options for leveldb medium, large

weed/command/server.go

@@ -90,7 +90,7 @@ func init() {
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
-	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
+	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
 	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")

weed/command/volume.go

@@ -58,7 +58,7 @@ func init() {
 	v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
 	v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
 	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
-	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
+	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
 	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
@@ -142,10 +142,10 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	switch *v.indexType {
 	case "leveldb":
 		volumeNeedleMapKind = storage.NeedleMapLevelDb
-	case "boltdb":
+	case "leveldbMedium":
+		volumeNeedleMapKind = storage.NeedleMapBoltDb
+	case "leveldbLarge":
 		volumeNeedleMapKind = storage.NeedleMapBoltDb
-	case "btree":
-		volumeNeedleMapKind = storage.NeedleMapBtree
 	}
 
 	masters := *v.masters

weed/storage/needle_map.go

@@ -15,7 +15,9 @@ type NeedleMapType int
 
 const (
 	NeedleMapInMemory      NeedleMapType = iota
-	NeedleMapLevelDb
+	NeedleMapLevelDb        // small memory footprint, 4MB total, 1 write buffer, 3 block buffer
+	NeedleMapLevelDbMedium  // medium memory footprint, 8MB total, 3 write buffer, 5 block buffer
+	NeedleMapLevelDbLarge   // large memory footprint, 12MB total, 4write buffer, 8 block buffer
 	NeedleMapBoltDb
 	NeedleMapBtree
 )

weed/storage/needle_map_boltdb.go

@@ -1,185 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/boltdb/bolt"
-
-	"errors"
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/storage/needle"
-	. "github.com/chrislusf/seaweedfs/weed/storage/types"
-	"github.com/chrislusf/seaweedfs/weed/util"
-)
-
-type BoltDbNeedleMap struct {
-	dbFileName string
-	db         *bolt.DB
-	baseNeedleMapper
-}
-
-var boltdbBucket = []byte("weed")
-
-var NotFound = errors.New("not found")
-
-func NewBoltDbNeedleMap(dbFileName string, indexFile *os.File) (m *BoltDbNeedleMap, err error) {
-	m = &BoltDbNeedleMap{dbFileName: dbFileName}
-	m.indexFile = indexFile
-	if !isBoltDbFresh(dbFileName, indexFile) {
-		glog.V(0).Infof("Start to Generate %s from %s", dbFileName, indexFile.Name())
-		generateBoltDbFile(dbFileName, indexFile)
-		glog.V(0).Infof("Finished Generating %s from %s", dbFileName, indexFile.Name())
-	}
-	glog.V(1).Infof("Opening %s...", dbFileName)
-	if m.db, err = bolt.Open(dbFileName, 0644, nil); err != nil {
-		return
-	}
-	glog.V(1).Infof("Loading %s...", indexFile.Name())
-	mm, indexLoadError := newNeedleMapMetricFromIndexFile(indexFile)
-	if indexLoadError != nil {
-		return nil, indexLoadError
-	}
-	m.mapMetric = *mm
-	return
-}
-
-func isBoltDbFresh(dbFileName string, indexFile *os.File) bool {
-	// normally we always write to index file first
-	dbLogFile, err := os.Open(dbFileName)
-	if err != nil {
-		return false
-	}
-	defer dbLogFile.Close()
-	dbStat, dbStatErr := dbLogFile.Stat()
-	indexStat, indexStatErr := indexFile.Stat()
-	if dbStatErr != nil || indexStatErr != nil {
-		glog.V(0).Infof("Can not stat file: %v and %v", dbStatErr, indexStatErr)
-		return false
-	}
-
-	return dbStat.ModTime().After(indexStat.ModTime())
-}
-
-func generateBoltDbFile(dbFileName string, indexFile *os.File) error {
-	db, err := bolt.Open(dbFileName, 0644, nil)
-	if err != nil {
-		return err
-	}
-	defer db.Close()
-	return WalkIndexFile(indexFile, func(key NeedleId, offset Offset, size uint32) error {
-		if !offset.IsZero() && size != TombstoneFileSize {
-			boltDbWrite(db, key, offset, size)
-		} else {
-			boltDbDelete(db, key)
-		}
-		return nil
-	})
-}
-
-func (m *BoltDbNeedleMap) Get(key NeedleId) (element *needle.NeedleValue, ok bool) {
-	var offset Offset
-	var size uint32
-	bytes := make([]byte, NeedleIdSize)
-	NeedleIdToBytes(bytes, key)
-	err := m.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket(boltdbBucket)
-		if bucket == nil {
-			return fmt.Errorf("Bucket %q not found!", boltdbBucket)
-		}
-
-		data := bucket.Get(bytes)
-
-		if len(data) == 0 {
-			return NotFound
-		}
-
-		if len(data) != OffsetSize+SizeSize {
-			glog.V(0).Infof("key:%v has wrong data length: %d", key, len(data))
-			return fmt.Errorf("key:%v has wrong data length: %d", key, len(data))
-		}
-
-		offset = BytesToOffset(data[0:OffsetSize])
-		size = util.BytesToUint32(data[OffsetSize : OffsetSize+SizeSize])
-
-		return nil
-	})
-
-	if err != nil {
-		return nil, false
-	}
-	return &needle.NeedleValue{Key: key, Offset: offset, Size: size}, true
-}
-
-func (m *BoltDbNeedleMap) Put(key NeedleId, offset Offset, size uint32) error {
-	var oldSize uint32
-	if oldNeedle, ok := m.Get(key); ok {
-		oldSize = oldNeedle.Size
-	}
-	m.logPut(key, oldSize, size)
-	// write to index file first
-	if err := m.appendToIndexFile(key, offset, size); err != nil {
-		return fmt.Errorf("cannot write to indexfile %s: %v", m.indexFile.Name(), err)
-	}
-	return boltDbWrite(m.db, key, offset, size)
-}
-
-func boltDbWrite(db *bolt.DB,
-	key NeedleId, offset Offset, size uint32) error {
-
-	bytes := make([]byte, NeedleIdSize+OffsetSize+SizeSize)
-	NeedleIdToBytes(bytes[0:NeedleIdSize], key)
-	OffsetToBytes(bytes[NeedleIdSize:NeedleIdSize+OffsetSize], offset)
-	util.Uint32toBytes(bytes[NeedleIdSize+OffsetSize:NeedleIdSize+OffsetSize+SizeSize], size)
-
-	return db.Update(func(tx *bolt.Tx) error {
-		bucket, err := tx.CreateBucketIfNotExists(boltdbBucket)
-		if err != nil {
-			return err
-		}
-
-		err = bucket.Put(bytes[0:NeedleIdSize], bytes[NeedleIdSize:NeedleIdSize+OffsetSize+SizeSize])
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-func boltDbDelete(db *bolt.DB, key NeedleId) error {
-	bytes := make([]byte, NeedleIdSize)
-	NeedleIdToBytes(bytes, key)
-	return db.Update(func(tx *bolt.Tx) error {
-		bucket, err := tx.CreateBucketIfNotExists(boltdbBucket)
-		if err != nil {
-			return err
-		}
-
-		err = bucket.Delete(bytes)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-}
-
-func (m *BoltDbNeedleMap) Delete(key NeedleId, offset Offset) error {
-	if oldNeedle, ok := m.Get(key); ok {
-		m.logDelete(oldNeedle.Size)
-	}
-	// write to index file first
-	if err := m.appendToIndexFile(key, offset, TombstoneFileSize); err != nil {
-		return err
-	}
-	return boltDbDelete(m.db, key)
-}
-
-func (m *BoltDbNeedleMap) Close() {
-	m.indexFile.Close()
-	m.db.Close()
-}
-
-func (m *BoltDbNeedleMap) Destroy() error {
-	m.Close()
-	os.Remove(m.indexFile.Name())
-	return os.Remove(m.dbFileName)
-}

weed/storage/needle_map_leveldb.go

@@ -19,7 +19,7 @@ type LevelDbNeedleMap struct {
 	baseNeedleMapper
 }
 
-func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File) (m *LevelDbNeedleMap, err error) {
+func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File, opts *opt.Options) (m *LevelDbNeedleMap, err error) {
 	m = &LevelDbNeedleMap{dbFileName: dbFileName}
 	m.indexFile = indexFile
 	if !isLevelDbFresh(dbFileName, indexFile) {
@@ -28,9 +28,8 @@ func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File) (m *LevelDbNeedl
 		glog.V(0).Infof("Finished Generating %s from %s", dbFileName, indexFile.Name())
 	}
 	glog.V(1).Infof("Opening %s...", dbFileName)
-	if m.db, err = leveldb.OpenFile(dbFileName, &opt.Options{
+
-		BlockCacheCapacity: -1, // default value is 8MiB
+	if m.db, err = leveldb.OpenFile(dbFileName, opts); err != nil {
-	}); err != nil {
 		return
 	}
 	glog.V(1).Infof("Loading %s...", indexFile.Name())

weed/storage/volume_loading.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"fmt"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 	"os"
 	"time"
 
@@ -82,18 +83,30 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
 			}
 		case NeedleMapLevelDb:
 			glog.V(0).Infoln("loading leveldb", fileName+".ldb")
-			if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile); e != nil {
+			opts := &opt.Options{
+				BlockCacheCapacity: 2 * 1024 * 1024, // default value is 8MiB
+				WriteBuffer:        1 * 1024 * 1024, // default value is 4MiB
+			}
+			if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil {
 				glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
 			}
-		case NeedleMapBoltDb:
+		case NeedleMapLevelDbMedium:
-			glog.V(0).Infoln("loading boltdb", fileName+".bdb")
+			glog.V(0).Infoln("loading leveldb medium", fileName+".ldb")
-			if v.nm, e = NewBoltDbNeedleMap(fileName+".bdb", indexFile); e != nil {
+			opts := &opt.Options{
-				glog.V(0).Infof("loading boltdb %s error: %v", fileName+".bdb", e)
+				BlockCacheCapacity: 4 * 1024 * 1024, // default value is 8MiB
+				WriteBuffer:        2 * 1024 * 1024, // default value is 4MiB
 			}
-		case NeedleMapBtree:
+			if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil {
-			glog.V(0).Infoln("loading index", fileName+".idx", "to btree readonly", v.readOnly)
+				glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
-			if v.nm, e = LoadBtreeNeedleMap(indexFile); e != nil {
+			}
-				glog.V(0).Infof("loading index %s to btree error: %v", fileName+".idx", e)
+		case NeedleMapLevelDbLarge:
+			glog.V(0).Infoln("loading leveldb large", fileName+".ldb")
+			opts := &opt.Options{
+				BlockCacheCapacity: 8 * 1024 * 1024, // default value is 8MiB
+				WriteBuffer:        4 * 1024 * 1024, // default value is 4MiB
+			}
+			if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil {
+				glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
 			}
 		}
 	}