gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

reduce needle map memory usage by 25%

Browse source
This commit is contained in:
Chris Lu 2018-12-15 05:55:56 -08:00
parent 21315f709d
commit 42cb9b76c4
3 changed files with 63 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
weed/storage/needle/compact_map.go
View file

@ -10,21 +10,31 @@ const (
batch = 100000 batch = 100000
) )
type SectionalNeedleId uint32
const SectionalNeedleIdLimit = 1<<32 - 1
type SectionalNeedleValue struct {
Key SectionalNeedleId
Offset Offset `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
Size uint32 `comment:"Size of the data portion"`
}
type CompactSection struct { type CompactSection struct {
sync.RWMutex sync.RWMutex
values []NeedleValue values []SectionalNeedleValue
overflow Overflow overflow Overflow
start NeedleId start NeedleId
end NeedleId end NeedleId
counter int counter int
} }
type Overflow []NeedleValue type Overflow []SectionalNeedleValue
func NewCompactSection(start NeedleId) *CompactSection { func NewCompactSection(start NeedleId) *CompactSection {
return &CompactSection{ return &CompactSection{
values: make([]NeedleValue, batch), values: make([]SectionalNeedleValue, batch),
overflow: Overflow(make([]NeedleValue, 0)), overflow: Overflow(make([]SectionalNeedleValue, 0)),
start: start, start: start,
} }
} }
@ -35,22 +45,23 @@ func (cs *CompactSection) Set(key NeedleId, offset Offset, size uint32) (oldOffs
if key > cs.end { if key > cs.end {
cs.end = key cs.end = key
} }
if i := cs.binarySearchValues(key); i >= 0 { skey := SectionalNeedleId(key - cs.start)
if i := cs.binarySearchValues(skey); i >= 0 {
oldOffset, oldSize = cs.values[i].Offset, cs.values[i].Size oldOffset, oldSize = cs.values[i].Offset, cs.values[i].Size
//println("key", key, "old size", ret) //println("key", key, "old size", ret)
cs.values[i].Offset, cs.values[i].Size = offset, size cs.values[i].Offset, cs.values[i].Size = offset, size
} else { } else {
needOverflow := cs.counter >= batch needOverflow := cs.counter >= batch
needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > key needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > skey
if needOverflow { if needOverflow {
//println("start", cs.start, "counter", cs.counter, "key", key) //println("start", cs.start, "counter", cs.counter, "key", key)
if oldValue, found := cs.overflow.findOverflowEntry(key); found { if oldValue, found := cs.overflow.findOverflowEntry(skey); found {
oldOffset, oldSize = oldValue.Offset, oldValue.Size oldOffset, oldSize = oldValue.Offset, oldValue.Size
} }
cs.overflow = cs.overflow.setOverflowEntry(NeedleValue{Key: key, Offset: offset, Size: size}) cs.overflow = cs.overflow.setOverflowEntry(SectionalNeedleValue{Key: skey, Offset: offset, Size: size})
} else { } else {
p := &cs.values[cs.counter] p := &cs.values[cs.counter]
p.Key, p.Offset, p.Size = key, offset, size p.Key, p.Offset, p.Size = skey, offset, size
//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key) //println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
cs.counter++ cs.counter++
} }
@ -61,16 +72,17 @@ func (cs *CompactSection) Set(key NeedleId, offset Offset, size uint32) (oldOffs
//return old entry size //return old entry size
func (cs *CompactSection) Delete(key NeedleId) uint32 { func (cs *CompactSection) Delete(key NeedleId) uint32 {
skey := SectionalNeedleId(key - cs.start)
cs.Lock() cs.Lock()
ret := uint32(0) ret := uint32(0)
if i := cs.binarySearchValues(key); i >= 0 { if i := cs.binarySearchValues(skey); i >= 0 {
if cs.values[i].Size > 0 { if cs.values[i].Size > 0 {
ret = cs.values[i].Size ret = cs.values[i].Size
cs.values[i].Size = 0 cs.values[i].Size = 0
} }
} }
if v, found := cs.overflow.findOverflowEntry(key); found { if v, found := cs.overflow.findOverflowEntry(skey); found {
cs.overflow = cs.overflow.deleteOverflowEntry(key) cs.overflow = cs.overflow.deleteOverflowEntry(skey)
ret = v.Size ret = v.Size
} }
cs.Unlock() cs.Unlock()
@ -78,18 +90,21 @@ func (cs *CompactSection) Delete(key NeedleId) uint32 {
} }
func (cs *CompactSection) Get(key NeedleId) (*NeedleValue, bool) { func (cs *CompactSection) Get(key NeedleId) (*NeedleValue, bool) {
cs.RLock() cs.RLock()
if v, ok := cs.overflow.findOverflowEntry(key); ok { skey := SectionalNeedleId(key - cs.start)
if v, ok := cs.overflow.findOverflowEntry(skey); ok {
cs.RUnlock() cs.RUnlock()
return &v, true nv := v.toNeedleValue(cs)
return &nv, true
} }
if i := cs.binarySearchValues(key); i >= 0 { if i := cs.binarySearchValues(skey); i >= 0 {
cs.RUnlock() cs.RUnlock()
return &cs.values[i], true nv := cs.values[i].toNeedleValue(cs)
return &nv, true
} }
cs.RUnlock() cs.RUnlock()
return nil, false return nil, false
} }
func (cs *CompactSection) binarySearchValues(key NeedleId) int { func (cs *CompactSection) binarySearchValues(key SectionalNeedleId) int {
l, h := 0, cs.counter-1 l, h := 0, cs.counter-1
if h >= 0 && cs.values[h].Key < key { if h >= 0 && cs.values[h].Key < key {
return -2 return -2
@ -122,7 +137,7 @@ func NewCompactMap() *CompactMap {
func (cm *CompactMap) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) { func (cm *CompactMap) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
x := cm.binarySearchCompactSection(key) x := cm.binarySearchCompactSection(key)
if x < 0 { if x < 0 || (key-cm.list[x].start) > SectionalNeedleIdLimit {
//println(x, "creating", len(cm.list), "section, starting", key) //println(x, "creating", len(cm.list), "section, starting", key)
cs := NewCompactSection(key) cs := NewCompactSection(key)
cm.list = append(cm.list, cs) cm.list = append(cm.list, cs)
@ -185,14 +200,14 @@ func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
for _, cs := range cm.list { for _, cs := range cm.list {
cs.RLock() cs.RLock()
for _, v := range cs.overflow { for _, v := range cs.overflow {
if err := visit(v); err != nil { if err := visit(v.toNeedleValue(cs)); err != nil {
cs.RUnlock() cs.RUnlock()
return err return err
} }
} }
for _, v := range cs.values { for _, v := range cs.values {
if _, found := cs.overflow.findOverflowEntry(v.Key); !found { if _, found := cs.overflow.findOverflowEntry(v.Key); !found {
if err := visit(v); err != nil { if err := visit(v.toNeedleValue(cs)); err != nil {
cs.RUnlock() cs.RUnlock()
return err return err
} }
@ -203,7 +218,7 @@ func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
return nil return nil
} }
func (o Overflow) deleteOverflowEntry(key NeedleId) Overflow { func (o Overflow) deleteOverflowEntry(key SectionalNeedleId) Overflow {
length := len(o) length := len(o)
deleteCandidate := sort.Search(length, func(i int) bool { deleteCandidate := sort.Search(length, func(i int) bool {
return o[i].Key >= key return o[i].Key >= key
@ -217,7 +232,7 @@ func (o Overflow) deleteOverflowEntry(key NeedleId) Overflow {
return o return o
} }
func (o Overflow) setOverflowEntry(needleValue NeedleValue) Overflow { func (o Overflow) setOverflowEntry(needleValue SectionalNeedleValue) Overflow {
insertCandidate := sort.Search(len(o), func(i int) bool { insertCandidate := sort.Search(len(o), func(i int) bool {
return o[i].Key >= needleValue.Key return o[i].Key >= needleValue.Key
}) })
@ -233,7 +248,7 @@ func (o Overflow) setOverflowEntry(needleValue NeedleValue) Overflow {
return o return o
} }
func (o Overflow) findOverflowEntry(key NeedleId) (nv NeedleValue, found bool) { func (o Overflow) findOverflowEntry(key SectionalNeedleId) (nv SectionalNeedleValue, found bool) {
foundCandidate := sort.Search(len(o), func(i int) bool { foundCandidate := sort.Search(len(o), func(i int) bool {
return o[i].Key >= key return o[i].Key >= key
}) })
@ -242,3 +257,11 @@ func (o Overflow) findOverflowEntry(key NeedleId) (nv NeedleValue, found bool) {
} }
return nv, false return nv, false
} }
func (snv SectionalNeedleValue) toNeedleValue(cs *CompactSection) NeedleValue {
return NeedleValue{NeedleId(snv.Key) + cs.start, snv.Offset, snv.Size}
}
func (nv NeedleValue) toSectionalNeedleValue(cs *CompactSection) SectionalNeedleValue {
return SectionalNeedleValue{SectionalNeedleId(nv.Key - cs.start), nv.Offset, nv.Size}
}

9
weed/storage/needle/compact_map_perf_test.go
View file

@ -6,6 +6,7 @@ import (
"os" "os"
"runtime" "runtime"
"testing" "testing"
"time"
. "github.com/chrislusf/seaweedfs/weed/storage/types" . "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
@ -28,6 +29,7 @@ func TestMemoryUsage(t *testing.T) {
var maps []*CompactMap var maps []*CompactMap
startTime := time.Now()
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
indexFile, ie := os.OpenFile("../../../test/sample.idx", os.O_RDWR|os.O_RDONLY, 0644) indexFile, ie := os.OpenFile("../../../test/sample.idx", os.O_RDWR|os.O_RDONLY, 0644)
if ie != nil { if ie != nil {
@ -38,6 +40,9 @@ func TestMemoryUsage(t *testing.T) {
indexFile.Close() indexFile.Close()
PrintMemUsage() PrintMemUsage()
now := time.Now()
fmt.Printf("\tTaken = %v\n", now.Sub(startTime))
startTime = now
} }
} }
@ -67,13 +72,15 @@ func loadNewNeedleMap(file *os.File) *CompactMap {
} }
func PrintMemUsage() { func PrintMemUsage() {
runtime.GC()
var m runtime.MemStats var m runtime.MemStats
runtime.ReadMemStats(&m) runtime.ReadMemStats(&m)
// For info on each, see: https://golang.org/pkg/runtime/#MemStats // For info on each, see: https://golang.org/pkg/runtime/#MemStats
fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc)) fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))
fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc)) fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
fmt.Printf("\tSys = %v MiB", bToMb(m.Sys)) fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
fmt.Printf("\tNumGC = %v\n", m.NumGC) fmt.Printf("\tNumGC = %v", m.NumGC)
} }
func bToMb(b uint64) uint64 { func bToMb(b uint64) uint64 {
return b / 1024 / 1024 return b / 1024 / 1024

18
weed/storage/needle/compact_map_test.go
View file

@ -78,19 +78,19 @@ func TestCompactMap(t *testing.T) {
} }
func TestOverflow(t *testing.T) { func TestOverflow(t *testing.T) {
o := Overflow(make([]NeedleValue, 0)) o := Overflow(make([]SectionalNeedleValue, 0))
o = o.setOverflowEntry(NeedleValue{Key: 1, Offset: 12, Size: 12}) o = o.setOverflowEntry(SectionalNeedleValue{Key: 1, Offset: 12, Size: 12})
o = o.setOverflowEntry(NeedleValue{Key: 2, Offset: 12, Size: 12}) o = o.setOverflowEntry(SectionalNeedleValue{Key: 2, Offset: 12, Size: 12})
o = o.setOverflowEntry(NeedleValue{Key: 3, Offset: 12, Size: 12}) o = o.setOverflowEntry(SectionalNeedleValue{Key: 3, Offset: 12, Size: 12})
o = o.setOverflowEntry(NeedleValue{Key: 4, Offset: 12, Size: 12}) o = o.setOverflowEntry(SectionalNeedleValue{Key: 4, Offset: 12, Size: 12})
o = o.setOverflowEntry(NeedleValue{Key: 5, Offset: 12, Size: 12}) o = o.setOverflowEntry(SectionalNeedleValue{Key: 5, Offset: 12, Size: 12})
if o[2].Key != 3 { if o[2].Key != 3 {
t.Fatalf("expecting o[2] has key 3: %+v", o[2].Key) t.Fatalf("expecting o[2] has key 3: %+v", o[2].Key)
} }
o = o.setOverflowEntry(NeedleValue{Key: 3, Offset: 24, Size: 24}) o = o.setOverflowEntry(SectionalNeedleValue{Key: 3, Offset: 24, Size: 24})
if o[2].Key != 3 { if o[2].Key != 3 {
t.Fatalf("expecting o[2] has key 3: %+v", o[2].Key) t.Fatalf("expecting o[2] has key 3: %+v", o[2].Key)
@ -123,13 +123,13 @@ func TestOverflow(t *testing.T) {
} }
println() println()
o = o.setOverflowEntry(NeedleValue{Key: 4, Offset: 44, Size: 44}) o = o.setOverflowEntry(SectionalNeedleValue{Key: 4, Offset: 44, Size: 44})
for i, x := range o { for i, x := range o {
println("overflow[", i, "]:", x.Key) println("overflow[", i, "]:", x.Key)
} }
println() println()
o = o.setOverflowEntry(NeedleValue{Key: 1, Offset: 11, Size: 11}) o = o.setOverflowEntry(SectionalNeedleValue{Key: 1, Offset: 11, Size: 11})
for i, x := range o { for i, x := range o {
println("overflow[", i, "]:", x.Key) println("overflow[", i, "]:", x.Key)