gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

add NodeStore

Browse source
This commit is contained in:
Chris Lu 2021-09-18 15:32:17 -07:00
parent 198fa58e3c
commit e066e2642c
7 changed files with 147 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
weed/util/bptree/bpmap.go
View file

@ -9,9 +9,9 @@ import (
*/
type BpMap BpTree
func NewBpMap(node_size int) *BpMap {
func NewBpMap(node_size int, nodeStore NodeStore) *BpMap {
return &BpMap{
root: NewLeaf(node_size),
root: NewLeaf(node_size, nodeStore),
}
}
@ -47,7 +47,7 @@ func (self *BpMap) Remove(key ItemKey) (value ItemValue, err error) {
return nil, err
}
if new_root == nil {
new_root = NewLeaf(ns)
new_root = NewLeaf(ns, self.root.nodeStore)
err = new_root.persist()
self.setRoot(new_root)
} else {

6
weed/util/bptree/bptree.go
View file

@ -12,9 +12,9 @@ type BpTree struct {
type loc_iterator func() (i int, leaf *BpNode, li loc_iterator)
func NewBpTree(node_size int) *BpTree {
func NewBpTree(node_size int, nodeStore NodeStore) *BpTree {
return &BpTree{
root: NewLeaf(node_size),
root: NewLeaf(node_size, nodeStore),
}
}
@ -93,7 +93,7 @@ func (self *BpTree) RemoveWhere(key ItemKey, where WhereFunc) (err error) {
return err
}
if new_root == nil {
new_root = NewLeaf(ns)
new_root = NewLeaf(ns, self.root.nodeStore)
err = new_root.persist()
self.setRoot(new_root)
} else {

27
weed/util/bptree/bptree_node.go
View file

@ -2,13 +2,11 @@ package bptree
type ItemKey Hashable
type ItemValue Equatable
type PersistFunc func(node *BpNode) error
type DestroyFunc func(node *BpNode) error
var (
PersistFn PersistFunc
DestroyFn DestroyFunc
)
type NodeStore interface {
PersistFunc(node *BpNode) error
DestroyFunc(node *BpNode) error
}
type BpNode struct {
keys []ItemKey
@ -18,9 +16,10 @@ type BpNode struct {
prev *BpNode
protoNodeId int64
protoNode *ProtoNode
nodeStore NodeStore
}
func NewInternal(size int) *BpNode {
func NewInternal(size int, nodeStore NodeStore) *BpNode {
if size < 0 {
panic(NegativeSize())
}
@ -28,10 +27,11 @@ func NewInternal(size int) *BpNode {
keys: make([]ItemKey, 0, size),
pointers: make([]*BpNode, 0, size),
protoNodeId: GetProtoNodeId(),
nodeStore: nodeStore,
}
}
func NewLeaf(size int) *BpNode {
func NewLeaf(size int, nodeStore NodeStore) *BpNode {
if size < 0 {
panic(NegativeSize())
}
@ -39,6 +39,7 @@ func NewLeaf(size int) *BpNode {
keys: make([]ItemKey, 0, size),
values: make([]ItemValue, 0, size),
protoNodeId: GetProtoNodeId(),
nodeStore: nodeStore,
}
}
@ -187,7 +188,7 @@ func (self *BpNode) put(key ItemKey, value ItemValue) (root *BpNode, err error)
return a, nil
}
// else we have root split
root = NewInternal(self.Capacity())
root = NewInternal(self.Capacity(), self.nodeStore)
root.put_kp(a.keys[0], a)
root.put_kp(b.keys[0], b)
return root, root.persist()
@ -256,7 +257,7 @@ func (self *BpNode) internal_split(key ItemKey, ptr *BpNode) (a, b *BpNode, err
return nil, nil, BpTreeError("Tried to split an internal block on duplicate key")
}
a = self
b = NewInternal(self.Capacity())
b = NewInternal(self.Capacity(), self.nodeStore)
balance_nodes(a, b, key)
if b.Len() > 0 && key.Less(b.keys[0]) {
if err := a.put_kp(key, ptr); err != nil {
@ -310,7 +311,7 @@ func (self *BpNode) leaf_split(key ItemKey, value ItemValue) (a, b *BpNode, err
return self.pure_leaf_split(key, value)
}
a = self
b = NewLeaf(self.Capacity())
b = NewLeaf(self.Capacity(), self.nodeStore)
insert_linked_list_node(b, a, a.getNext())
balance_nodes(a, b, key)
if b.Len() > 0 && key.Less(b.keys[0]) {
@ -342,7 +343,7 @@ func (self *BpNode) pure_leaf_split(key ItemKey, value ItemValue) (a, b *BpNode,
return nil, nil, BpTreeError("Expected a pure leaf node")
}
if key.Less(self.keys[0]) {
a = NewLeaf(self.Capacity())
a = NewLeaf(self.Capacity(), self.nodeStore)
b = self
if err := a.put_kv(key, value); err != nil {
return nil, nil, err
@ -358,7 +359,7 @@ func (self *BpNode) pure_leaf_split(key ItemKey, value ItemValue) (a, b *BpNode,
}
return a, nil, a.persist()
} else {
b = NewLeaf(self.Capacity())
b = NewLeaf(self.Capacity(), self.nodeStore)
if err := b.put_kv(key, value); err != nil {
return nil, nil, err
}

39
weed/util/bptree/bptree_store_test.go
View file

@ -5,29 +5,38 @@ import (
"testing"
)
type nodeStorePrintlnImpl struct {
}
func (n *nodeStorePrintlnImpl) PersistFunc(node *BpNode) error {
println("saving node", node.protoNodeId)
return nil
}
func (n *nodeStorePrintlnImpl) DestroyFunc(node *BpNode) error {
println("delete node", node.protoNodeId)
return nil
}
func TestAddRemove(t *testing.T) {
tree := NewBpTree(5)
PersistFn = func(node *BpNode) error {
println("saving", node.protoNodeId)
return nil
}
DestroyFn = func(node *BpNode) error {
println("delete", node.protoNodeId)
return nil
}
for i:=0;i<32;i++{
tree := NewBpTree(3, &nodeStorePrintlnImpl{})
for i:=0;i<9;i++{
println("++++++++++", i)
tree.Add(String(fmt.Sprintf("%02d", i)), nil)
printTree(tree.root, "")
}
if !tree.Has(String("30")) {
if !tree.Has(String("08")) {
t.Errorf("lookup error")
}
tree.RemoveWhere(String("30"), func(value ItemValue) bool {
return true
})
if tree.Has(String("30")) {
for i:=5;i<9;i++{
println("----------", i)
tree.RemoveWhere(String(fmt.Sprintf("%02d", i)), func(value ItemValue) bool {
return true
})
printTree(tree.root, "")
}
if tree.Has(String("08")) {
t.Errorf("remove error")
}
}

106
weed/util/bptree/bptree_test.go
View file

@ -79,7 +79,7 @@ func BenchmarkBpTree(b *testing.B) {
b.StartTimer()
for i := 0; i < b.N; i++ {
t := NewBpTree(23)
t := NewBpTree(23, nil)
for _, r := range recs {
t.Add(r.key, r.value)
}
@ -207,7 +207,7 @@ func TestAddHasCountFindIterateRemove(t *testing.T) {
}
}
for i := 2; i < 64; i++ {
test(NewBpTree(i))
test(NewBpTree(i, nil))
}
}
@ -271,11 +271,11 @@ func TestBpMap(t *testing.T) {
}
}
test(NewBpMap(23))
test(NewBpMap(23, nil))
}
func Test_get_start(t *testing.T) {
root := NewLeaf(2)
root := NewLeaf(2, nil)
root, err := root.put(Int(1), Int(1))
if err != nil {
t.Error(err)
@ -344,7 +344,7 @@ func Test_get_start(t *testing.T) {
}
func Test_get_end(t *testing.T) {
root := NewLeaf(3)
root := NewLeaf(3, nil)
root, err := root.put(Int(1), Int(1))
if err != nil {
t.Fatal(err)
@ -388,7 +388,7 @@ func Test_get_end(t *testing.T) {
}
func Test_put_no_root_split(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
}
@ -423,7 +423,7 @@ func Test_put_no_root_split(t *testing.T) {
}
func Test_put_root_split(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
p, err := a.put(Int(1), Int(1))
if err != nil {
t.Error(err)
@ -472,8 +472,8 @@ func Test_put_root_split(t *testing.T) {
}
func Test_internal_insert_no_split(t *testing.T) {
a := NewInternal(3)
leaf := NewLeaf(1)
a := NewInternal(3, nil)
leaf := NewLeaf(1, nil)
if err := leaf.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
}
@ -500,8 +500,8 @@ func Test_internal_insert_no_split(t *testing.T) {
}
func Test_internal_insert_split_less(t *testing.T) {
a := NewInternal(3)
leaf := NewLeaf(1)
a := NewInternal(3, nil)
leaf := NewLeaf(1, nil)
if err := leaf.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
}
@ -534,7 +534,7 @@ func Test_internal_insert_split_less(t *testing.T) {
}
func Test_internal_split_less(t *testing.T) {
a := NewInternal(3)
a := NewInternal(3, nil)
if err := a.put_kp(Int(1), nil); err != nil {
t.Error(err)
}
@ -564,7 +564,7 @@ func Test_internal_split_less(t *testing.T) {
}
func Test_internal_split_equal(t *testing.T) {
a := NewInternal(3)
a := NewInternal(3, nil)
if err := a.put_kp(Int(1), nil); err != nil {
t.Error(err)
}
@ -581,7 +581,7 @@ func Test_internal_split_equal(t *testing.T) {
}
func Test_internal_split_greater(t *testing.T) {
a := NewInternal(3)
a := NewInternal(3, nil)
if err := a.put_kp(Int(1), nil); err != nil {
t.Error(err)
}
@ -611,7 +611,7 @@ func Test_internal_split_greater(t *testing.T) {
}
func Test_leaf_insert_no_split(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, nil)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
@ -637,7 +637,7 @@ func Test_leaf_insert_no_split(t *testing.T) {
// tests the defer to split logic
func Test_leaf_insert_split_less(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, nil)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
@ -668,7 +668,7 @@ func Test_leaf_insert_split_less(t *testing.T) {
}
func Test_leaf_split_less(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, nil)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
@ -699,7 +699,7 @@ func Test_leaf_split_less(t *testing.T) {
}
func Test_leaf_split_equal(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, nil)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
@ -730,7 +730,7 @@ func Test_leaf_split_equal(t *testing.T) {
}
func Test_leaf_split_greater(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, nil)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
@ -762,13 +762,13 @@ func Test_leaf_split_greater(t *testing.T) {
// tests the defer logic
func Test_pure_leaf_insert_split_less(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, nil)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), Int(1)); err != nil {
t.Error(err)
@ -835,13 +835,13 @@ func Test_pure_leaf_insert_split_less(t *testing.T) {
}
func Test_pure_leaf_split_less(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, nil)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), Int(1)); err != nil {
t.Error(err)
@ -908,13 +908,13 @@ func Test_pure_leaf_split_less(t *testing.T) {
}
func Test_pure_leaf_split_equal(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, nil)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), Int(1)); err != nil {
t.Error(err)
@ -972,13 +972,13 @@ func Test_pure_leaf_split_equal(t *testing.T) {
}
func Test_pure_leaf_split_greater(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, nil)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), Int(1)); err != nil {
t.Error(err)
@ -1042,13 +1042,13 @@ func Test_pure_leaf_split_greater(t *testing.T) {
}
func Test_find_end_of_pure_run(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, nil)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), Int(1)); err != nil {
t.Error(err)
@ -1078,13 +1078,13 @@ func Test_find_end_of_pure_run(t *testing.T) {
}
func Test_insert_linked_list_node(t *testing.T) {
a := NewLeaf(1)
a := NewLeaf(1, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(3)
c := NewLeaf(3, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(4)
d := NewLeaf(4, nil)
insert_linked_list_node(d, a, b)
if a.getPrev() != nil {
t.Errorf("expected a.prev == nil")
@ -1113,13 +1113,13 @@ func Test_insert_linked_list_node(t *testing.T) {
}
func Test_remove_linked_list_node(t *testing.T) {
a := NewLeaf(1)
a := NewLeaf(1, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(3)
c := NewLeaf(3, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(4)
d := NewLeaf(4, nil)
insert_linked_list_node(d, a, b)
if a.getPrev() != nil {
t.Errorf("expected a.prev == nil")
@ -1188,8 +1188,8 @@ func Test_remove_linked_list_node(t *testing.T) {
}
func Test_balance_leaf_nodes_with_dup(t *testing.T) {
a := NewLeaf(3)
b := NewLeaf(3)
a := NewLeaf(3, nil)
b := NewLeaf(3, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
}
@ -1209,8 +1209,8 @@ func Test_balance_leaf_nodes_with_dup(t *testing.T) {
}
func Test_balance_leaf_nodes(t *testing.T) {
a := NewLeaf(7)
b := NewLeaf(7)
a := NewLeaf(7, nil)
b := NewLeaf(7, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
}
@ -1258,8 +1258,8 @@ func Test_balance_leaf_nodes(t *testing.T) {
}
func Test_balance_internal_nodes(t *testing.T) {
a := NewInternal(6)
b := NewInternal(6)
a := NewInternal(6, nil)
b := NewInternal(6, nil)
if err := a.put_kp(Int(1), nil); err != nil {
t.Error(err)
}

8
weed/util/bptree/getter_setter.go
View file

@ -45,14 +45,14 @@ func (self *BpNode) maybePersist(shouldPersist bool) error {
return self.persist()
}
func (self *BpNode) persist() error {
if PersistFn != nil {
return PersistFn(self)
if self.nodeStore != nil {
return self.nodeStore.PersistFunc(self)
}
return nil
}
func (self *BpNode) destroy() error {
if DestroyFn != nil {
return DestroyFn(self)
if self.nodeStore != nil {
return self.nodeStore.DestroyFunc(self)
}
return nil
}

46
weed/util/bptree/serde_test.go Normal file
View file

@ -0,0 +1,46 @@
package bptree
import (
"fmt"
"testing"
)
type nodeStoreMapImpl struct {
m map[int64]*ProtoNode
}
func (n *nodeStoreMapImpl) PersistFunc(node *BpNode) error {
println("saving node", node.protoNodeId)
n.m[node.protoNodeId] = node.protoNode
return nil
}
func (n *nodeStoreMapImpl) DestroyFunc(node *BpNode) error {
println("delete node", node.protoNodeId)
delete(n.m, node.protoNodeId)
return nil
}
func TestSerDe(t *testing.T) {
nodeStore := &nodeStoreMapImpl{
m: make(map[int64]*ProtoNode),
}
tree := NewBpTree(3, nodeStore)
for i:=0;i<32;i++{
println("add", i)
tree.Add(String(fmt.Sprintf("%02d", i)), nil)
}
for i:=5;i<9;i++{
println("----------", i)
tree.RemoveWhere(String(fmt.Sprintf("%02d", i)), func(value ItemValue) bool {
return true
})
printTree(tree.root, "")
}
}