2011-12-16 14:51:26 +00:00
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
2013-02-10 22:00:06 +00:00
|
|
|
"fmt"
|
2015-05-26 07:58:41 +00:00
|
|
|
"io/ioutil"
|
2013-02-10 22:00:06 +00:00
|
|
|
"os"
|
2015-05-26 07:58:41 +00:00
|
|
|
"sync"
|
2016-04-10 07:24:22 +00:00
|
|
|
|
2016-06-03 01:09:14 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/util"
|
2011-12-16 14:51:26 +00:00
|
|
|
)
|
|
|
|
|
Add boltdb for volume needle map
boltdb is fairly slow to write, about 6 minutes for recreating index
for 1553934 files. Boltdb loads 1,553,934 x 16 = 24,862,944bytes from
disk, and generate the boltdb as large as 134,217,728 bytes in 6
minutes.
To compare, for leveldb, it recreates index in leveldb as large as
27,188,148 bytes in 8 seconds.
For in memory version, it loads the index in
To test the memory consumption, the leveldb or boltdb index are
created. And the server is restarted. Using the benchmark tool to read
lots of files. There are 7 volumes in benchmark collection, each with
about 1553K files.
For leveldb, the memory starts at 142,884KB, and stays at 179,340KB.
For boltdb, the memory starts at 73,756KB, and stays at 144,564KB.
For in-memory, the memory starts at 368,152KB, and stays at 448,032KB.
2015-03-29 18:04:32 +00:00
|
|
|
type NeedleMapType int
|
|
|
|
|
|
|
|
const (
|
|
|
|
NeedleMapInMemory NeedleMapType = iota
|
|
|
|
NeedleMapLevelDb
|
|
|
|
NeedleMapBoltDb
|
|
|
|
)
|
|
|
|
|
2013-04-18 07:23:14 +00:00
|
|
|
type NeedleMapper interface {
|
2015-03-27 23:34:58 +00:00
|
|
|
Put(key uint64, offset uint32, size uint32) error
|
2013-04-18 07:23:14 +00:00
|
|
|
Get(key uint64) (element *NeedleValue, ok bool)
|
|
|
|
Delete(key uint64) error
|
|
|
|
Close()
|
2014-03-10 18:43:54 +00:00
|
|
|
Destroy() error
|
2013-04-18 07:23:14 +00:00
|
|
|
ContentSize() uint64
|
|
|
|
DeletedSize() uint64
|
|
|
|
FileCount() int
|
|
|
|
DeletedCount() int
|
2014-04-17 06:43:27 +00:00
|
|
|
MaxFileKey() uint64
|
2015-05-26 07:58:41 +00:00
|
|
|
IndexFileSize() uint64
|
|
|
|
IndexFileContent() ([]byte, error)
|
|
|
|
IndexFileName() string
|
2013-04-18 07:23:14 +00:00
|
|
|
}
|
|
|
|
|
2015-05-26 07:58:41 +00:00
|
|
|
type baseNeedleMapper struct {
|
|
|
|
indexFile *os.File
|
|
|
|
indexFileAccessLock sync.Mutex
|
|
|
|
|
|
|
|
mapMetric
|
2013-04-18 07:23:14 +00:00
|
|
|
}
|
|
|
|
|
2016-04-10 05:47:03 +00:00
|
|
|
func (nm *baseNeedleMapper) IndexFileSize() uint64 {
|
2015-05-26 07:58:41 +00:00
|
|
|
stat, err := nm.indexFile.Stat()
|
|
|
|
if err == nil {
|
|
|
|
return uint64(stat.Size())
|
|
|
|
}
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
2016-04-10 05:47:03 +00:00
|
|
|
func (nm *baseNeedleMapper) IndexFileName() string {
|
2015-05-26 07:58:41 +00:00
|
|
|
return nm.indexFile.Name()
|
|
|
|
}
|
|
|
|
|
|
|
|
func idxFileEntry(bytes []byte) (key uint64, offset uint32, size uint32) {
|
2016-04-10 07:24:22 +00:00
|
|
|
key = util.BytesToUint64(bytes[:8])
|
|
|
|
offset = util.BytesToUint32(bytes[8:12])
|
|
|
|
size = util.BytesToUint32(bytes[12:16])
|
2015-05-26 07:58:41 +00:00
|
|
|
return
|
|
|
|
}
|
2016-04-10 05:47:03 +00:00
|
|
|
func (nm *baseNeedleMapper) appendToIndexFile(key uint64, offset uint32, size uint32) error {
|
2014-03-19 11:38:10 +00:00
|
|
|
bytes := make([]byte, 16)
|
2016-04-10 07:24:22 +00:00
|
|
|
util.Uint64toBytes(bytes[0:8], key)
|
|
|
|
util.Uint32toBytes(bytes[8:12], offset)
|
|
|
|
util.Uint32toBytes(bytes[12:16], size)
|
2015-05-26 07:58:41 +00:00
|
|
|
|
|
|
|
nm.indexFileAccessLock.Lock()
|
|
|
|
defer nm.indexFileAccessLock.Unlock()
|
|
|
|
if _, err := nm.indexFile.Seek(0, 2); err != nil {
|
2015-03-27 23:34:58 +00:00
|
|
|
return fmt.Errorf("cannot seek end of indexfile %s: %v",
|
2015-05-26 07:58:41 +00:00
|
|
|
nm.indexFile.Name(), err)
|
2014-05-30 02:19:03 +00:00
|
|
|
}
|
2015-05-26 07:58:41 +00:00
|
|
|
_, err := nm.indexFile.Write(bytes)
|
2015-03-27 23:34:58 +00:00
|
|
|
return err
|
2011-12-19 05:59:37 +00:00
|
|
|
}
|
2016-04-10 05:47:03 +00:00
|
|
|
func (nm *baseNeedleMapper) IndexFileContent() ([]byte, error) {
|
2015-05-26 07:58:41 +00:00
|
|
|
nm.indexFileAccessLock.Lock()
|
|
|
|
defer nm.indexFileAccessLock.Unlock()
|
|
|
|
return ioutil.ReadFile(nm.indexFile.Name())
|
|
|
|
}
|
|
|
|
|
|
|
|
type mapMetric struct {
|
|
|
|
indexFile *os.File
|
|
|
|
|
|
|
|
DeletionCounter int `json:"DeletionCounter"`
|
|
|
|
FileCounter int `json:"FileCounter"`
|
|
|
|
DeletionByteCounter uint64 `json:"DeletionByteCounter"`
|
|
|
|
FileByteCounter uint64 `json:"FileByteCounter"`
|
|
|
|
MaximumFileKey uint64 `json:"MaxFileKey"`
|
|
|
|
}
|
2015-03-27 23:34:58 +00:00
|
|
|
|
|
|
|
func (mm *mapMetric) logDelete(deletedByteCount uint32) {
|
|
|
|
mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(deletedByteCount)
|
|
|
|
mm.DeletionCounter++
|
2011-12-16 14:51:26 +00:00
|
|
|
}
|
2015-03-27 23:34:58 +00:00
|
|
|
|
|
|
|
func (mm *mapMetric) logPut(key uint64, oldSize uint32, newSize uint32) {
|
|
|
|
if key > mm.MaximumFileKey {
|
|
|
|
mm.MaximumFileKey = key
|
2014-05-30 02:19:03 +00:00
|
|
|
}
|
2015-03-27 23:34:58 +00:00
|
|
|
mm.FileCounter++
|
|
|
|
mm.FileByteCounter = mm.FileByteCounter + uint64(newSize)
|
|
|
|
if oldSize > 0 {
|
|
|
|
mm.DeletionCounter++
|
|
|
|
mm.DeletionByteCounter = mm.DeletionByteCounter + uint64(oldSize)
|
2013-02-10 22:00:06 +00:00
|
|
|
}
|
2013-09-02 06:58:21 +00:00
|
|
|
}
|
2015-05-26 07:58:41 +00:00
|
|
|
|
|
|
|
func (mm mapMetric) ContentSize() uint64 {
|
|
|
|
return mm.FileByteCounter
|
|
|
|
}
|
|
|
|
func (mm mapMetric) DeletedSize() uint64 {
|
|
|
|
return mm.DeletionByteCounter
|
|
|
|
}
|
|
|
|
func (mm mapMetric) FileCount() int {
|
|
|
|
return mm.FileCounter
|
|
|
|
}
|
|
|
|
func (mm mapMetric) DeletedCount() int {
|
|
|
|
return mm.DeletionCounter
|
|
|
|
}
|
|
|
|
func (mm mapMetric) MaxFileKey() uint64 {
|
|
|
|
return mm.MaximumFileKey
|
|
|
|
}
|