gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

divide offset into higher and lower sections

Browse source
This commit is contained in:
Chris Lu 2019-04-08 21:44:06 -07:00
parent 000ee725fc
commit 7a4b234ea1
3 changed files with 139 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

159
weed/storage/needle/compact_map.go
View file

@ -15,27 +15,36 @@ type SectionalNeedleId uint32
const SectionalNeedleIdLimit = 1<<32 - 1 const SectionalNeedleIdLimit = 1<<32 - 1
type SectionalNeedleValue struct { type SectionalNeedleValue struct {
Key SectionalNeedleId Key SectionalNeedleId
Offset Offset `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G OffsetLower OffsetLower `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
Size uint32 `comment:"Size of the data portion"` Size uint32 `comment:"Size of the data portion"`
}
type SectionalNeedleValueExtra struct {
OffsetHigher OffsetHigher
} }
type CompactSection struct { type CompactSection struct {
sync.RWMutex sync.RWMutex
values []SectionalNeedleValue values []SectionalNeedleValue
overflow Overflow valuesExtra []SectionalNeedleValueExtra
start NeedleId overflow Overflow
end NeedleId overflowExtra OverflowExtra
counter int start NeedleId
end NeedleId
counter int
} }
type Overflow []SectionalNeedleValue type Overflow []SectionalNeedleValue
type OverflowExtra []SectionalNeedleValueExtra
func NewCompactSection(start NeedleId) *CompactSection { func NewCompactSection(start NeedleId) *CompactSection {
return &CompactSection{ return &CompactSection{
values: make([]SectionalNeedleValue, batch), values: make([]SectionalNeedleValue, batch),
overflow: Overflow(make([]SectionalNeedleValue, 0)), valuesExtra: make([]SectionalNeedleValueExtra, batch),
start: start, overflow: Overflow(make([]SectionalNeedleValue, 0)),
overflowExtra: OverflowExtra(make([]SectionalNeedleValueExtra, 0)),
start: start,
} }
} }
@ -47,21 +56,21 @@ func (cs *CompactSection) Set(key NeedleId, offset Offset, size uint32) (oldOffs
} }
skey := SectionalNeedleId(key - cs.start) skey := SectionalNeedleId(key - cs.start)
if i := cs.binarySearchValues(skey); i >= 0 { if i := cs.binarySearchValues(skey); i >= 0 {
oldOffset, oldSize = cs.values[i].Offset, cs.values[i].Size oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = cs.valuesExtra[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size
//println("key", key, "old size", ret) //println("key", key, "old size", ret)
cs.values[i].Offset, cs.values[i].Size = offset, size cs.valuesExtra[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size = offset.OffsetHigher, offset.OffsetLower, size
} else { } else {
needOverflow := cs.counter >= batch needOverflow := cs.counter >= batch
needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > skey needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > skey
if needOverflow { if needOverflow {
//println("start", cs.start, "counter", cs.counter, "key", key) //println("start", cs.start, "counter", cs.counter, "key", key)
if oldValue, found := cs.overflow.findOverflowEntry(skey); found { if oldValueExtra, oldValue, found := cs.findOverflowEntry(skey); found {
oldOffset, oldSize = oldValue.Offset, oldValue.Size oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = oldValueExtra.OffsetHigher, oldValue.OffsetLower, oldValue.Size
} }
cs.overflow = cs.overflow.setOverflowEntry(SectionalNeedleValue{Key: skey, Offset: offset, Size: size}) cs.setOverflowEntry(skey, offset, size)
} else { } else {
p := &cs.values[cs.counter] p := &cs.values[cs.counter]
p.Key, p.Offset, p.Size = skey, offset, size p.Key, cs.valuesExtra[cs.counter].OffsetHigher, p.OffsetLower, p.Size = skey, offset.OffsetHigher, offset.OffsetLower, size
//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key) //println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
cs.counter++ cs.counter++
} }
@ -70,6 +79,50 @@ func (cs *CompactSection) Set(key NeedleId, offset Offset, size uint32) (oldOffs
return return
} }
func (cs *CompactSection) setOverflowEntry(skey SectionalNeedleId, offset Offset, size uint32) {
needleValue := SectionalNeedleValue{Key: skey, OffsetLower: offset.OffsetLower, Size: size}
needleValueExtra := SectionalNeedleValueExtra{OffsetHigher: OffsetHigher{}}
insertCandidate := sort.Search(len(cs.overflow), func(i int) bool {
return cs.overflow[i].Key >= needleValue.Key
})
if insertCandidate != len(cs.overflow) && cs.overflow[insertCandidate].Key == needleValue.Key {
cs.overflow[insertCandidate] = needleValue
} else {
cs.overflow = append(cs.overflow, needleValue)
cs.overflowExtra = append(cs.overflowExtra, needleValueExtra)
for i := len(cs.overflow) - 1; i > insertCandidate; i-- {
cs.overflow[i] = cs.overflow[i-1]
cs.overflowExtra[i] = cs.overflowExtra[i-1]
}
cs.overflow[insertCandidate] = needleValue
}
}
func (cs *CompactSection) findOverflowEntry(key SectionalNeedleId) (nve SectionalNeedleValueExtra, nv SectionalNeedleValue, found bool) {
foundCandidate := sort.Search(len(cs.overflow), func(i int) bool {
return cs.overflow[i].Key >= key
})
if foundCandidate != len(cs.overflow) && cs.overflow[foundCandidate].Key == key {
return cs.overflowExtra[foundCandidate], cs.overflow[foundCandidate], true
}
return nve, nv, false
}
func (cs *CompactSection) deleteOverflowEntry(key SectionalNeedleId) {
length := len(cs.overflow)
deleteCandidate := sort.Search(length, func(i int) bool {
return cs.overflow[i].Key >= key
})
if deleteCandidate != length && cs.overflow[deleteCandidate].Key == key {
for i := deleteCandidate; i < length-1; i++ {
cs.overflow[i] = cs.overflow[i+1]
cs.overflowExtra[i] = cs.overflowExtra[i+1]
}
cs.overflow = cs.overflow[0 : length-1]
cs.overflowExtra = cs.overflowExtra[0 : length-1]
}
}
//return old entry size //return old entry size
func (cs *CompactSection) Delete(key NeedleId) uint32 { func (cs *CompactSection) Delete(key NeedleId) uint32 {
skey := SectionalNeedleId(key - cs.start) skey := SectionalNeedleId(key - cs.start)
@ -81,8 +134,8 @@ func (cs *CompactSection) Delete(key NeedleId) uint32 {
cs.values[i].Size = TombstoneFileSize cs.values[i].Size = TombstoneFileSize
} }
} }
if v, found := cs.overflow.findOverflowEntry(skey); found { if _, v, found := cs.findOverflowEntry(skey); found {
cs.overflow = cs.overflow.deleteOverflowEntry(skey) cs.deleteOverflowEntry(skey)
ret = v.Size ret = v.Size
} }
cs.Unlock() cs.Unlock()
@ -91,14 +144,14 @@ func (cs *CompactSection) Delete(key NeedleId) uint32 {
func (cs *CompactSection) Get(key NeedleId) (*NeedleValue, bool) { func (cs *CompactSection) Get(key NeedleId) (*NeedleValue, bool) {
cs.RLock() cs.RLock()
skey := SectionalNeedleId(key - cs.start) skey := SectionalNeedleId(key - cs.start)
if v, ok := cs.overflow.findOverflowEntry(skey); ok { if ve, v, ok := cs.findOverflowEntry(skey); ok {
cs.RUnlock() cs.RUnlock()
nv := v.toNeedleValue(cs) nv := toNeedleValue(ve, v, cs)
return &nv, true return &nv, true
} }
if i := cs.binarySearchValues(skey); i >= 0 { if i := cs.binarySearchValues(skey); i >= 0 {
cs.RUnlock() cs.RUnlock()
nv := cs.values[i].toNeedleValue(cs) nv := toNeedleValue(cs.valuesExtra[i], cs.values[i], cs)
return &nv, true return &nv, true
} }
cs.RUnlock() cs.RUnlock()
@ -194,8 +247,8 @@ func (cm *CompactMap) binarySearchCompactSection(key NeedleId) int {
func (cm *CompactMap) Visit(visit func(NeedleValue) error) error { func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
for _, cs := range cm.list { for _, cs := range cm.list {
cs.RLock() cs.RLock()
for _, v := range cs.overflow { for i, v := range cs.overflow {
if err := visit(v.toNeedleValue(cs)); err != nil { if err := visit(toNeedleValue(cs.overflowExtra[i], v, cs)); err != nil {
cs.RUnlock() cs.RUnlock()
return err return err
} }
@ -204,8 +257,8 @@ func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
if i >= cs.counter { if i >= cs.counter {
break break
} }
if _, found := cs.overflow.findOverflowEntry(v.Key); !found { if _, _, found := cs.findOverflowEntry(v.Key); !found {
if err := visit(v.toNeedleValue(cs)); err != nil { if err := visit(toNeedleValue(cs.valuesExtra[i], v, cs)); err != nil {
cs.RUnlock() cs.RUnlock()
return err return err
} }
@ -216,50 +269,20 @@ func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
return nil return nil
} }
func (o Overflow) deleteOverflowEntry(key SectionalNeedleId) Overflow { func toNeedleValue(snve SectionalNeedleValueExtra, snv SectionalNeedleValue, cs *CompactSection) NeedleValue {
length := len(o) offset := Offset{
deleteCandidate := sort.Search(length, func(i int) bool { OffsetHigher: snve.OffsetHigher,
return o[i].Key >= key OffsetLower: snv.OffsetLower,
})
if deleteCandidate != length && o[deleteCandidate].Key == key {
for i := deleteCandidate; i < length-1; i++ {
o[i] = o[i+1]
}
o = o[0 : length-1]
} }
return o return NeedleValue{Key: NeedleId(snv.Key) + cs.start, Offset: offset, Size: snv.Size}
} }
func (o Overflow) setOverflowEntry(needleValue SectionalNeedleValue) Overflow { func (nv NeedleValue) toSectionalNeedleValue(cs *CompactSection) (SectionalNeedleValue, SectionalNeedleValueExtra) {
insertCandidate := sort.Search(len(o), func(i int) bool { return SectionalNeedleValue{
return o[i].Key >= needleValue.Key SectionalNeedleId(nv.Key - cs.start),
}) nv.Offset.OffsetLower,
if insertCandidate != len(o) && o[insertCandidate].Key == needleValue.Key { nv.Size,
o[insertCandidate] = needleValue }, SectionalNeedleValueExtra{
} else { nv.Offset.OffsetHigher,
o = append(o, needleValue)
for i := len(o) - 1; i > insertCandidate; i-- {
o[i] = o[i-1]
}
o[insertCandidate] = needleValue
} }
return o
}
func (o Overflow) findOverflowEntry(key SectionalNeedleId) (nv SectionalNeedleValue, found bool) {
foundCandidate := sort.Search(len(o), func(i int) bool {
return o[i].Key >= key
})
if foundCandidate != len(o) && o[foundCandidate].Key == key {
return o[foundCandidate], true
}
return nv, false
}
func (snv SectionalNeedleValue) toNeedleValue(cs *CompactSection) NeedleValue {
return NeedleValue{NeedleId(snv.Key) + cs.start, snv.Offset, snv.Size}
}
func (nv NeedleValue) toSectionalNeedleValue(cs *CompactSection) SectionalNeedleValue {
return SectionalNeedleValue{SectionalNeedleId(nv.Key - cs.start), nv.Offset, nv.Size}
} }

48
weed/storage/needle/compact_map_test.go
View file

@ -98,60 +98,60 @@ func TestCompactMap(t *testing.T) {
} }
func TestOverflow(t *testing.T) { func TestOverflow(t *testing.T) {
o := Overflow(make([]SectionalNeedleValue, 0)) cs := NewCompactSection(1)
o = o.setOverflowEntry(SectionalNeedleValue{Key: 1, Offset: ToOffset(12), Size: 12}) cs.setOverflowEntry(1, ToOffset(12), 12)
o = o.setOverflowEntry(SectionalNeedleValue{Key: 2, Offset: ToOffset(12), Size: 12}) cs.setOverflowEntry(2, ToOffset(12), 12)
o = o.setOverflowEntry(SectionalNeedleValue{Key: 3, Offset: ToOffset(12), Size: 12}) cs.setOverflowEntry(3, ToOffset(12), 12)
o = o.setOverflowEntry(SectionalNeedleValue{Key: 4, Offset: ToOffset(12), Size: 12}) cs.setOverflowEntry(4, ToOffset(12), 12)
o = o.setOverflowEntry(SectionalNeedleValue{Key: 5, Offset: ToOffset(12), Size: 12}) cs.setOverflowEntry(5, ToOffset(12), 12)
if o[2].Key != 3 { if cs.overflow[2].Key != 3 {
t.Fatalf("expecting o[2] has key 3: %+v", o[2].Key) t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
} }
o = o.setOverflowEntry(SectionalNeedleValue{Key: 3, Offset: ToOffset(24), Size: 24}) cs.setOverflowEntry(3, ToOffset(24), 24)
if o[2].Key != 3 { if cs.overflow[2].Key != 3 {
t.Fatalf("expecting o[2] has key 3: %+v", o[2].Key) t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
} }
if o[2].Size != 24 { if cs.overflow[2].Size != 24 {
t.Fatalf("expecting o[2] has size 24: %+v", o[2].Size) t.Fatalf("expecting o[2] has size 24: %+v", cs.overflow[2].Size)
} }
o = o.deleteOverflowEntry(4) cs.deleteOverflowEntry(4)
if len(o) != 4 { if len(cs.overflow) != 4 {
t.Fatalf("expecting 4 entries now: %+v", o) t.Fatalf("expecting 4 entries now: %+v", cs.overflow)
} }
x, _ := o.findOverflowEntry(5) _, x, _ := cs.findOverflowEntry(5)
if x.Key != 5 { if x.Key != 5 {
t.Fatalf("expecting entry 5 now: %+v", x) t.Fatalf("expecting entry 5 now: %+v", x)
} }
for i, x := range o { for i, x := range cs.overflow {
println("overflow[", i, "]:", x.Key) println("overflow[", i, "]:", x.Key)
} }
println() println()
o = o.deleteOverflowEntry(1) cs.deleteOverflowEntry(1)
for i, x := range o { for i, x := range cs.overflow {
println("overflow[", i, "]:", x.Key) println("overflow[", i, "]:", x.Key)
} }
println() println()
o = o.setOverflowEntry(SectionalNeedleValue{Key: 4, Offset: ToOffset(44), Size: 44}) cs.setOverflowEntry(4, ToOffset(44), 44)
for i, x := range o { for i, x := range cs.overflow {
println("overflow[", i, "]:", x.Key) println("overflow[", i, "]:", x.Key)
} }
println() println()
o = o.setOverflowEntry(SectionalNeedleValue{Key: 1, Offset: ToOffset(11), Size: 11}) cs.setOverflowEntry(1, ToOffset(11), 11)
for i, x := range o { for i, x := range cs.overflow {
println("overflow[", i, "]:", x.Key) println("overflow[", i, "]:", x.Key)
} }
println() println()

36
weed/storage/types/needle_types.go
View file

@ -8,8 +8,15 @@ import (
) )
type Offset struct { type Offset struct {
// b5 byte // unused OffsetHigher
// b4 byte // unused OffsetLower
}
type OffsetHigher struct {
// b4 byte
}
type OffsetLower struct {
b3 byte b3 byte
b2 byte b2 byte
b1 byte b1 byte
@ -19,9 +26,9 @@ type Offset struct {
type Cookie uint32 type Cookie uint32
const ( const (
OffsetSize = 4 OffsetSize = 4 // + 1
SizeSize = 4 // uint32 size SizeSize = 4 // uint32 size
NeedleEntrySize = NeedleIdSize + OffsetSize + SizeSize NeedleEntrySize = CookieSize + NeedleIdSize + SizeSize
TimestampSize = 8 // int64 size TimestampSize = 8 // int64 size
NeedlePaddingSize = 8 NeedlePaddingSize = 8
MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8 MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8
@ -55,21 +62,26 @@ func OffsetToBytes(bytes []byte, offset Offset) {
bytes[0] = offset.b3 bytes[0] = offset.b3
} }
// only for testing, will be removed later.
func Uint32ToOffset(offset uint32) Offset { func Uint32ToOffset(offset uint32) Offset {
return Offset{ return Offset{
b0: byte(offset), OffsetLower: OffsetLower{
b1: byte(offset >> 8), b0: byte(offset),
b2: byte(offset >> 16), b1: byte(offset >> 8),
b3: byte(offset >> 24), b2: byte(offset >> 16),
b3: byte(offset >> 24),
},
} }
} }
func BytesToOffset(bytes []byte) Offset { func BytesToOffset(bytes []byte) Offset {
return Offset{ return Offset{
b0: bytes[3], OffsetLower: OffsetLower{
b1: bytes[2], b0: bytes[3],
b2: bytes[1], b1: bytes[2],
b3: bytes[0], b2: bytes[1],
b3: bytes[0],
},
} }
} }