seaweedfs/weed/storage/needle/compact_map.go

266 lines
6.6 KiB
Go
Raw Normal View History

2017-05-27 05:51:25 +00:00
package needle
import (
. "github.com/chrislusf/seaweedfs/weed/storage/types"
2018-12-09 09:27:11 +00:00
"sort"
2018-07-22 00:39:10 +00:00
"sync"
2017-05-27 05:51:25 +00:00
)
2018-12-09 08:12:37 +00:00
const (
batch = 100000
)
2018-12-15 13:55:56 +00:00
type SectionalNeedleId uint32
const SectionalNeedleIdLimit = 1<<32 - 1
type SectionalNeedleValue struct {
Key SectionalNeedleId
Offset Offset `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
Size uint32 `comment:"Size of the data portion"`
}
2017-05-27 05:51:25 +00:00
type CompactSection struct {
sync.RWMutex
2018-12-15 13:55:56 +00:00
values []SectionalNeedleValue
2018-12-09 09:27:11 +00:00
overflow Overflow
start NeedleId
end NeedleId
2017-05-27 05:51:25 +00:00
counter int
}
2018-12-15 13:55:56 +00:00
type Overflow []SectionalNeedleValue
2018-12-09 09:27:11 +00:00
func NewCompactSection(start NeedleId) *CompactSection {
2017-05-27 05:51:25 +00:00
return &CompactSection{
2018-12-15 13:55:56 +00:00
values: make([]SectionalNeedleValue, batch),
overflow: Overflow(make([]SectionalNeedleValue, 0)),
2017-05-27 05:51:25 +00:00
start: start,
}
}
//return old entry size
func (cs *CompactSection) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
2017-05-27 05:51:25 +00:00
cs.Lock()
if key > cs.end {
cs.end = key
}
2018-12-15 13:55:56 +00:00
skey := SectionalNeedleId(key - cs.start)
if i := cs.binarySearchValues(skey); i >= 0 {
2017-05-27 05:51:25 +00:00
oldOffset, oldSize = cs.values[i].Offset, cs.values[i].Size
//println("key", key, "old size", ret)
cs.values[i].Offset, cs.values[i].Size = offset, size
} else {
needOverflow := cs.counter >= batch
2018-12-15 13:55:56 +00:00
needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > skey
2017-05-27 05:51:25 +00:00
if needOverflow {
//println("start", cs.start, "counter", cs.counter, "key", key)
2018-12-15 13:55:56 +00:00
if oldValue, found := cs.overflow.findOverflowEntry(skey); found {
2017-05-27 05:51:25 +00:00
oldOffset, oldSize = oldValue.Offset, oldValue.Size
}
2018-12-15 13:55:56 +00:00
cs.overflow = cs.overflow.setOverflowEntry(SectionalNeedleValue{Key: skey, Offset: offset, Size: size})
2017-05-27 05:51:25 +00:00
} else {
p := &cs.values[cs.counter]
2018-12-15 13:55:56 +00:00
p.Key, p.Offset, p.Size = skey, offset, size
2017-05-27 05:51:25 +00:00
//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
cs.counter++
}
}
cs.Unlock()
return
}
//return old entry size
func (cs *CompactSection) Delete(key NeedleId) uint32 {
2018-12-15 13:55:56 +00:00
skey := SectionalNeedleId(key - cs.start)
2017-05-27 05:51:25 +00:00
cs.Lock()
ret := uint32(0)
2018-12-15 13:55:56 +00:00
if i := cs.binarySearchValues(skey); i >= 0 {
2017-05-27 05:51:25 +00:00
if cs.values[i].Size > 0 {
ret = cs.values[i].Size
cs.values[i].Size = 0
}
}
2018-12-15 13:55:56 +00:00
if v, found := cs.overflow.findOverflowEntry(skey); found {
cs.overflow = cs.overflow.deleteOverflowEntry(skey)
2017-05-27 05:51:25 +00:00
ret = v.Size
}
cs.Unlock()
return ret
}
func (cs *CompactSection) Get(key NeedleId) (*NeedleValue, bool) {
2017-05-27 05:51:25 +00:00
cs.RLock()
2018-12-15 13:55:56 +00:00
skey := SectionalNeedleId(key - cs.start)
if v, ok := cs.overflow.findOverflowEntry(skey); ok {
2017-05-27 05:51:25 +00:00
cs.RUnlock()
2018-12-15 13:55:56 +00:00
nv := v.toNeedleValue(cs)
return &nv, true
2017-05-27 05:51:25 +00:00
}
2018-12-15 13:55:56 +00:00
if i := cs.binarySearchValues(skey); i >= 0 {
2017-05-27 05:51:25 +00:00
cs.RUnlock()
2018-12-15 13:55:56 +00:00
nv := cs.values[i].toNeedleValue(cs)
return &nv, true
2017-05-27 05:51:25 +00:00
}
cs.RUnlock()
return nil, false
}
2018-12-15 13:55:56 +00:00
func (cs *CompactSection) binarySearchValues(key SectionalNeedleId) int {
2018-12-22 19:05:29 +00:00
x := sort.Search(cs.counter, func(i int) bool {
return cs.values[i].Key >= key
})
if x == cs.counter {
return -1
2017-05-27 05:51:25 +00:00
}
2018-12-22 19:05:29 +00:00
if cs.values[x].Key > key {
return -2
2017-05-27 05:51:25 +00:00
}
2018-12-22 19:05:29 +00:00
return x
2017-05-27 05:51:25 +00:00
}
//This map assumes mostly inserting increasing keys
//This map assumes mostly inserting increasing keys
type CompactMap struct {
list []*CompactSection
}
func NewCompactMap() *CompactMap {
return &CompactMap{}
}
func (cm *CompactMap) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
2017-05-27 05:51:25 +00:00
x := cm.binarySearchCompactSection(key)
2018-12-15 13:55:56 +00:00
if x < 0 || (key-cm.list[x].start) > SectionalNeedleIdLimit {
2018-12-22 19:05:29 +00:00
// println(x, "adding to existing", len(cm.list), "sections, starting", key)
2018-12-09 05:45:14 +00:00
cs := NewCompactSection(key)
cm.list = append(cm.list, cs)
2017-05-27 05:51:25 +00:00
x = len(cm.list) - 1
//keep compact section sorted by start
2018-12-22 19:05:29 +00:00
for x >= 0 {
if x > 0 && cm.list[x-1].start > key {
2018-12-09 05:45:14 +00:00
cm.list[x] = cm.list[x-1]
2018-12-22 19:05:29 +00:00
// println("shift", x, "start", cs.start, "to", x-1)
2017-05-27 05:51:25 +00:00
x = x - 1
} else {
2018-12-09 05:45:14 +00:00
cm.list[x] = cs
2018-12-22 19:05:29 +00:00
// println("cs", x, "start", cs.start)
2017-05-27 05:51:25 +00:00
break
}
}
}
2018-12-22 19:05:29 +00:00
// println(key, "set to section[", x, "].start", cm.list[x].start)
2017-05-27 05:51:25 +00:00
return cm.list[x].Set(key, offset, size)
}
func (cm *CompactMap) Delete(key NeedleId) uint32 {
2017-05-27 05:51:25 +00:00
x := cm.binarySearchCompactSection(key)
if x < 0 {
return uint32(0)
}
return cm.list[x].Delete(key)
}
func (cm *CompactMap) Get(key NeedleId) (*NeedleValue, bool) {
2017-05-27 05:51:25 +00:00
x := cm.binarySearchCompactSection(key)
if x < 0 {
return nil, false
}
return cm.list[x].Get(key)
}
func (cm *CompactMap) binarySearchCompactSection(key NeedleId) int {
2018-12-22 19:52:58 +00:00
l, h := 0, len(cm.list)-1
if h < 0 {
return -5
2017-05-27 05:51:25 +00:00
}
2018-12-22 19:52:58 +00:00
if cm.list[h].start <= key {
if cm.list[h].counter < batch || key <= cm.list[h].end {
return h
}
return -4
}
for l <= h {
m := (l + h) / 2
if key < cm.list[m].start {
h = m - 1
} else { // cm.list[m].start <= key
if cm.list[m+1].start <= key {
l = m + 1
} else {
return m
}
}
2018-12-22 19:05:29 +00:00
}
2018-12-22 19:52:58 +00:00
return -3
2017-05-27 05:51:25 +00:00
}
// Visit visits all entries or stop if any error when visiting
func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
for _, cs := range cm.list {
cs.RLock()
for _, v := range cs.overflow {
2018-12-15 13:55:56 +00:00
if err := visit(v.toNeedleValue(cs)); err != nil {
2017-05-27 05:51:25 +00:00
cs.RUnlock()
return err
}
}
2018-12-22 19:05:29 +00:00
for i, v := range cs.values {
if i >= cs.counter {
break
}
2018-12-09 09:27:11 +00:00
if _, found := cs.overflow.findOverflowEntry(v.Key); !found {
2018-12-15 13:55:56 +00:00
if err := visit(v.toNeedleValue(cs)); err != nil {
2017-05-27 05:51:25 +00:00
cs.RUnlock()
return err
}
}
}
cs.RUnlock()
}
return nil
}
2018-12-09 09:27:11 +00:00
2018-12-15 13:55:56 +00:00
func (o Overflow) deleteOverflowEntry(key SectionalNeedleId) Overflow {
2018-12-09 09:27:11 +00:00
length := len(o)
deleteCandidate := sort.Search(length, func(i int) bool {
return o[i].Key >= key
})
if deleteCandidate != length && o[deleteCandidate].Key == key {
for i := deleteCandidate; i < length-1; i++ {
o[i] = o[i+1]
}
o = o[0 : length-1]
}
return o
}
2018-12-15 13:55:56 +00:00
func (o Overflow) setOverflowEntry(needleValue SectionalNeedleValue) Overflow {
2018-12-09 09:27:11 +00:00
insertCandidate := sort.Search(len(o), func(i int) bool {
return o[i].Key >= needleValue.Key
})
if insertCandidate != len(o) && o[insertCandidate].Key == needleValue.Key {
o[insertCandidate] = needleValue
} else {
o = append(o, needleValue)
for i := len(o) - 1; i > insertCandidate; i-- {
o[i] = o[i-1]
}
o[insertCandidate] = needleValue
}
return o
}
2018-12-15 13:55:56 +00:00
func (o Overflow) findOverflowEntry(key SectionalNeedleId) (nv SectionalNeedleValue, found bool) {
2018-12-09 09:27:11 +00:00
foundCandidate := sort.Search(len(o), func(i int) bool {
return o[i].Key >= key
})
if foundCandidate != len(o) && o[foundCandidate].Key == key {
return o[foundCandidate], true
}
return nv, false
}
2018-12-15 13:55:56 +00:00
func (snv SectionalNeedleValue) toNeedleValue(cs *CompactSection) NeedleValue {
return NeedleValue{NeedleId(snv.Key) + cs.start, snv.Offset, snv.Size}
}
func (nv NeedleValue) toSectionalNeedleValue(cs *CompactSection) SectionalNeedleValue {
return SectionalNeedleValue{SectionalNeedleId(nv.Key - cs.start), nv.Offset, nv.Size}
}