diff --git a/weed/util/bptree/bpmap.go b/weed/util/bptree/bpmap.go index 399ac7b86..0c13a132f 100644 --- a/weed/util/bptree/bpmap.go +++ b/weed/util/bptree/bpmap.go @@ -9,9 +9,9 @@ import ( */ type BpMap BpTree -func NewBpMap(node_size int) *BpMap { +func NewBpMap(node_size int, nodeStore NodeStore) *BpMap { return &BpMap{ - root: NewLeaf(node_size), + root: NewLeaf(node_size, nodeStore), } } @@ -47,7 +47,7 @@ func (self *BpMap) Remove(key ItemKey) (value ItemValue, err error) { return nil, err } if new_root == nil { - new_root = NewLeaf(ns) + new_root = NewLeaf(ns, self.root.nodeStore) err = new_root.persist() self.setRoot(new_root) } else { diff --git a/weed/util/bptree/bptree.go b/weed/util/bptree/bptree.go index 3ad73ad30..141c595f3 100644 --- a/weed/util/bptree/bptree.go +++ b/weed/util/bptree/bptree.go @@ -12,9 +12,9 @@ type BpTree struct { type loc_iterator func() (i int, leaf *BpNode, li loc_iterator) -func NewBpTree(node_size int) *BpTree { +func NewBpTree(node_size int, nodeStore NodeStore) *BpTree { return &BpTree{ - root: NewLeaf(node_size), + root: NewLeaf(node_size, nodeStore), } } @@ -93,7 +93,7 @@ func (self *BpTree) RemoveWhere(key ItemKey, where WhereFunc) (err error) { return err } if new_root == nil { - new_root = NewLeaf(ns) + new_root = NewLeaf(ns, self.root.nodeStore) err = new_root.persist() self.setRoot(new_root) } else { diff --git a/weed/util/bptree/bptree_node.go b/weed/util/bptree/bptree_node.go index 5c3461cfd..507d9d318 100644 --- a/weed/util/bptree/bptree_node.go +++ b/weed/util/bptree/bptree_node.go @@ -2,13 +2,11 @@ package bptree type ItemKey Hashable type ItemValue Equatable -type PersistFunc func(node *BpNode) error -type DestroyFunc func(node *BpNode) error -var ( - PersistFn PersistFunc - DestroyFn DestroyFunc -) +type NodeStore interface { + PersistFunc(node *BpNode) error + DestroyFunc(node *BpNode) error +} type BpNode struct { keys []ItemKey @@ -18,9 +16,10 @@ type BpNode struct { prev *BpNode protoNodeId int64 protoNode *ProtoNode + nodeStore NodeStore } -func NewInternal(size int) *BpNode { +func NewInternal(size int, nodeStore NodeStore) *BpNode { if size < 0 { panic(NegativeSize()) } @@ -28,10 +27,11 @@ func NewInternal(size int) *BpNode { keys: make([]ItemKey, 0, size), pointers: make([]*BpNode, 0, size), protoNodeId: GetProtoNodeId(), + nodeStore: nodeStore, } } -func NewLeaf(size int) *BpNode { +func NewLeaf(size int, nodeStore NodeStore) *BpNode { if size < 0 { panic(NegativeSize()) } @@ -39,6 +39,7 @@ func NewLeaf(size int) *BpNode { keys: make([]ItemKey, 0, size), values: make([]ItemValue, 0, size), protoNodeId: GetProtoNodeId(), + nodeStore: nodeStore, } } @@ -187,7 +188,7 @@ func (self *BpNode) put(key ItemKey, value ItemValue) (root *BpNode, err error) return a, nil } // else we have root split - root = NewInternal(self.Capacity()) + root = NewInternal(self.Capacity(), self.nodeStore) root.put_kp(a.keys[0], a) root.put_kp(b.keys[0], b) return root, root.persist() @@ -256,7 +257,7 @@ func (self *BpNode) internal_split(key ItemKey, ptr *BpNode) (a, b *BpNode, err return nil, nil, BpTreeError("Tried to split an internal block on duplicate key") } a = self - b = NewInternal(self.Capacity()) + b = NewInternal(self.Capacity(), self.nodeStore) balance_nodes(a, b, key) if b.Len() > 0 && key.Less(b.keys[0]) { if err := a.put_kp(key, ptr); err != nil { @@ -310,7 +311,7 @@ func (self *BpNode) leaf_split(key ItemKey, value ItemValue) (a, b *BpNode, err return self.pure_leaf_split(key, value) } a = self - b = NewLeaf(self.Capacity()) + b = NewLeaf(self.Capacity(), self.nodeStore) insert_linked_list_node(b, a, a.getNext()) balance_nodes(a, b, key) if b.Len() > 0 && key.Less(b.keys[0]) { @@ -342,7 +343,7 @@ func (self *BpNode) pure_leaf_split(key ItemKey, value ItemValue) (a, b *BpNode, return nil, nil, BpTreeError("Expected a pure leaf node") } if key.Less(self.keys[0]) { - a = NewLeaf(self.Capacity()) + a = NewLeaf(self.Capacity(), self.nodeStore) b = self if err := a.put_kv(key, value); err != nil { return nil, nil, err @@ -358,7 +359,7 @@ func (self *BpNode) pure_leaf_split(key ItemKey, value ItemValue) (a, b *BpNode, } return a, nil, a.persist() } else { - b = NewLeaf(self.Capacity()) + b = NewLeaf(self.Capacity(), self.nodeStore) if err := b.put_kv(key, value); err != nil { return nil, nil, err } diff --git a/weed/util/bptree/bptree_store_test.go b/weed/util/bptree/bptree_store_test.go index 82dcbbf55..2e034171c 100644 --- a/weed/util/bptree/bptree_store_test.go +++ b/weed/util/bptree/bptree_store_test.go @@ -5,29 +5,38 @@ import ( "testing" ) +type nodeStorePrintlnImpl struct { +} + +func (n *nodeStorePrintlnImpl) PersistFunc(node *BpNode) error { + println("saving node", node.protoNodeId) + return nil +} +func (n *nodeStorePrintlnImpl) DestroyFunc(node *BpNode) error { + println("delete node", node.protoNodeId) + return nil +} + func TestAddRemove(t *testing.T) { - tree := NewBpTree(5) - PersistFn = func(node *BpNode) error { - println("saving", node.protoNodeId) - return nil - } - DestroyFn = func(node *BpNode) error { - println("delete", node.protoNodeId) - return nil - } - for i:=0;i<32;i++{ + + tree := NewBpTree(3, &nodeStorePrintlnImpl{}) + for i:=0;i<9;i++{ println("++++++++++", i) tree.Add(String(fmt.Sprintf("%02d", i)), nil) printTree(tree.root, "") } - if !tree.Has(String("30")) { + if !tree.Has(String("08")) { t.Errorf("lookup error") } - tree.RemoveWhere(String("30"), func(value ItemValue) bool { - return true - }) - if tree.Has(String("30")) { + for i:=5;i<9;i++{ + println("----------", i) + tree.RemoveWhere(String(fmt.Sprintf("%02d", i)), func(value ItemValue) bool { + return true + }) + printTree(tree.root, "") + } + if tree.Has(String("08")) { t.Errorf("remove error") } } diff --git a/weed/util/bptree/bptree_test.go b/weed/util/bptree/bptree_test.go index fc5b6f900..1fd6d1122 100644 --- a/weed/util/bptree/bptree_test.go +++ b/weed/util/bptree/bptree_test.go @@ -79,7 +79,7 @@ func BenchmarkBpTree(b *testing.B) { b.StartTimer() for i := 0; i < b.N; i++ { - t := NewBpTree(23) + t := NewBpTree(23, nil) for _, r := range recs { t.Add(r.key, r.value) } @@ -207,7 +207,7 @@ func TestAddHasCountFindIterateRemove(t *testing.T) { } } for i := 2; i < 64; i++ { - test(NewBpTree(i)) + test(NewBpTree(i, nil)) } } @@ -271,11 +271,11 @@ func TestBpMap(t *testing.T) { } } - test(NewBpMap(23)) + test(NewBpMap(23, nil)) } func Test_get_start(t *testing.T) { - root := NewLeaf(2) + root := NewLeaf(2, nil) root, err := root.put(Int(1), Int(1)) if err != nil { t.Error(err) @@ -344,7 +344,7 @@ func Test_get_start(t *testing.T) { } func Test_get_end(t *testing.T) { - root := NewLeaf(3) + root := NewLeaf(3, nil) root, err := root.put(Int(1), Int(1)) if err != nil { t.Fatal(err) @@ -388,7 +388,7 @@ func Test_get_end(t *testing.T) { } func Test_put_no_root_split(t *testing.T) { - a := NewLeaf(2) + a := NewLeaf(2, nil) if err := a.put_kv(Int(1), Int(1)); err != nil { t.Error(err) } @@ -423,7 +423,7 @@ func Test_put_no_root_split(t *testing.T) { } func Test_put_root_split(t *testing.T) { - a := NewLeaf(2) + a := NewLeaf(2, nil) p, err := a.put(Int(1), Int(1)) if err != nil { t.Error(err) @@ -472,8 +472,8 @@ func Test_put_root_split(t *testing.T) { } func Test_internal_insert_no_split(t *testing.T) { - a := NewInternal(3) - leaf := NewLeaf(1) + a := NewInternal(3, nil) + leaf := NewLeaf(1, nil) if err := leaf.put_kv(Int(1), Int(1)); err != nil { t.Error(err) } @@ -500,8 +500,8 @@ func Test_internal_insert_no_split(t *testing.T) { } func Test_internal_insert_split_less(t *testing.T) { - a := NewInternal(3) - leaf := NewLeaf(1) + a := NewInternal(3, nil) + leaf := NewLeaf(1, nil) if err := leaf.put_kv(Int(1), Int(1)); err != nil { t.Error(err) } @@ -534,7 +534,7 @@ func Test_internal_insert_split_less(t *testing.T) { } func Test_internal_split_less(t *testing.T) { - a := NewInternal(3) + a := NewInternal(3, nil) if err := a.put_kp(Int(1), nil); err != nil { t.Error(err) } @@ -564,7 +564,7 @@ func Test_internal_split_less(t *testing.T) { } func Test_internal_split_equal(t *testing.T) { - a := NewInternal(3) + a := NewInternal(3, nil) if err := a.put_kp(Int(1), nil); err != nil { t.Error(err) } @@ -581,7 +581,7 @@ func Test_internal_split_equal(t *testing.T) { } func Test_internal_split_greater(t *testing.T) { - a := NewInternal(3) + a := NewInternal(3, nil) if err := a.put_kp(Int(1), nil); err != nil { t.Error(err) } @@ -611,7 +611,7 @@ func Test_internal_split_greater(t *testing.T) { } func Test_leaf_insert_no_split(t *testing.T) { - a := NewLeaf(3) + a := NewLeaf(3, nil) insert_linked_list_node(a, nil, nil) if err := a.put_kv(Int(1), Int(1)); err != nil { t.Error(err) @@ -637,7 +637,7 @@ func Test_leaf_insert_no_split(t *testing.T) { // tests the defer to split logic func Test_leaf_insert_split_less(t *testing.T) { - a := NewLeaf(3) + a := NewLeaf(3, nil) insert_linked_list_node(a, nil, nil) if err := a.put_kv(Int(1), Int(1)); err != nil { t.Error(err) @@ -668,7 +668,7 @@ func Test_leaf_insert_split_less(t *testing.T) { } func Test_leaf_split_less(t *testing.T) { - a := NewLeaf(3) + a := NewLeaf(3, nil) insert_linked_list_node(a, nil, nil) if err := a.put_kv(Int(1), Int(1)); err != nil { t.Error(err) @@ -699,7 +699,7 @@ func Test_leaf_split_less(t *testing.T) { } func Test_leaf_split_equal(t *testing.T) { - a := NewLeaf(3) + a := NewLeaf(3, nil) insert_linked_list_node(a, nil, nil) if err := a.put_kv(Int(1), Int(1)); err != nil { t.Error(err) @@ -730,7 +730,7 @@ func Test_leaf_split_equal(t *testing.T) { } func Test_leaf_split_greater(t *testing.T) { - a := NewLeaf(3) + a := NewLeaf(3, nil) insert_linked_list_node(a, nil, nil) if err := a.put_kv(Int(1), Int(1)); err != nil { t.Error(err) @@ -762,13 +762,13 @@ func Test_leaf_split_greater(t *testing.T) { // tests the defer logic func Test_pure_leaf_insert_split_less(t *testing.T) { - a := NewLeaf(2) + a := NewLeaf(2, nil) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2) + b := NewLeaf(2, nil) insert_linked_list_node(b, a, nil) - c := NewLeaf(2) + c := NewLeaf(2, nil) insert_linked_list_node(c, b, nil) - d := NewLeaf(2) + d := NewLeaf(2, nil) insert_linked_list_node(d, c, nil) if err := a.put_kv(Int(3), Int(1)); err != nil { t.Error(err) @@ -835,13 +835,13 @@ func Test_pure_leaf_insert_split_less(t *testing.T) { } func Test_pure_leaf_split_less(t *testing.T) { - a := NewLeaf(2) + a := NewLeaf(2, nil) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2) + b := NewLeaf(2, nil) insert_linked_list_node(b, a, nil) - c := NewLeaf(2) + c := NewLeaf(2, nil) insert_linked_list_node(c, b, nil) - d := NewLeaf(2) + d := NewLeaf(2, nil) insert_linked_list_node(d, c, nil) if err := a.put_kv(Int(3), Int(1)); err != nil { t.Error(err) @@ -908,13 +908,13 @@ func Test_pure_leaf_split_less(t *testing.T) { } func Test_pure_leaf_split_equal(t *testing.T) { - a := NewLeaf(2) + a := NewLeaf(2, nil) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2) + b := NewLeaf(2, nil) insert_linked_list_node(b, a, nil) - c := NewLeaf(2) + c := NewLeaf(2, nil) insert_linked_list_node(c, b, nil) - d := NewLeaf(2) + d := NewLeaf(2, nil) insert_linked_list_node(d, c, nil) if err := a.put_kv(Int(3), Int(1)); err != nil { t.Error(err) @@ -972,13 +972,13 @@ func Test_pure_leaf_split_equal(t *testing.T) { } func Test_pure_leaf_split_greater(t *testing.T) { - a := NewLeaf(2) + a := NewLeaf(2, nil) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2) + b := NewLeaf(2, nil) insert_linked_list_node(b, a, nil) - c := NewLeaf(2) + c := NewLeaf(2, nil) insert_linked_list_node(c, b, nil) - d := NewLeaf(2) + d := NewLeaf(2, nil) insert_linked_list_node(d, c, nil) if err := a.put_kv(Int(3), Int(1)); err != nil { t.Error(err) @@ -1042,13 +1042,13 @@ func Test_pure_leaf_split_greater(t *testing.T) { } func Test_find_end_of_pure_run(t *testing.T) { - a := NewLeaf(2) + a := NewLeaf(2, nil) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2) + b := NewLeaf(2, nil) insert_linked_list_node(b, a, nil) - c := NewLeaf(2) + c := NewLeaf(2, nil) insert_linked_list_node(c, b, nil) - d := NewLeaf(2) + d := NewLeaf(2, nil) insert_linked_list_node(d, c, nil) if err := a.put_kv(Int(3), Int(1)); err != nil { t.Error(err) @@ -1078,13 +1078,13 @@ func Test_find_end_of_pure_run(t *testing.T) { } func Test_insert_linked_list_node(t *testing.T) { - a := NewLeaf(1) + a := NewLeaf(1, nil) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2) + b := NewLeaf(2, nil) insert_linked_list_node(b, a, nil) - c := NewLeaf(3) + c := NewLeaf(3, nil) insert_linked_list_node(c, b, nil) - d := NewLeaf(4) + d := NewLeaf(4, nil) insert_linked_list_node(d, a, b) if a.getPrev() != nil { t.Errorf("expected a.prev == nil") @@ -1113,13 +1113,13 @@ func Test_insert_linked_list_node(t *testing.T) { } func Test_remove_linked_list_node(t *testing.T) { - a := NewLeaf(1) + a := NewLeaf(1, nil) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2) + b := NewLeaf(2, nil) insert_linked_list_node(b, a, nil) - c := NewLeaf(3) + c := NewLeaf(3, nil) insert_linked_list_node(c, b, nil) - d := NewLeaf(4) + d := NewLeaf(4, nil) insert_linked_list_node(d, a, b) if a.getPrev() != nil { t.Errorf("expected a.prev == nil") @@ -1188,8 +1188,8 @@ func Test_remove_linked_list_node(t *testing.T) { } func Test_balance_leaf_nodes_with_dup(t *testing.T) { - a := NewLeaf(3) - b := NewLeaf(3) + a := NewLeaf(3, nil) + b := NewLeaf(3, nil) if err := a.put_kv(Int(1), Int(1)); err != nil { t.Error(err) } @@ -1209,8 +1209,8 @@ func Test_balance_leaf_nodes_with_dup(t *testing.T) { } func Test_balance_leaf_nodes(t *testing.T) { - a := NewLeaf(7) - b := NewLeaf(7) + a := NewLeaf(7, nil) + b := NewLeaf(7, nil) if err := a.put_kv(Int(1), Int(1)); err != nil { t.Error(err) } @@ -1258,8 +1258,8 @@ func Test_balance_leaf_nodes(t *testing.T) { } func Test_balance_internal_nodes(t *testing.T) { - a := NewInternal(6) - b := NewInternal(6) + a := NewInternal(6, nil) + b := NewInternal(6, nil) if err := a.put_kp(Int(1), nil); err != nil { t.Error(err) } diff --git a/weed/util/bptree/getter_setter.go b/weed/util/bptree/getter_setter.go index caafc1bbd..dcaa7a0b6 100644 --- a/weed/util/bptree/getter_setter.go +++ b/weed/util/bptree/getter_setter.go @@ -45,14 +45,14 @@ func (self *BpNode) maybePersist(shouldPersist bool) error { return self.persist() } func (self *BpNode) persist() error { - if PersistFn != nil { - return PersistFn(self) + if self.nodeStore != nil { + return self.nodeStore.PersistFunc(self) } return nil } func (self *BpNode) destroy() error { - if DestroyFn != nil { - return DestroyFn(self) + if self.nodeStore != nil { + return self.nodeStore.DestroyFunc(self) } return nil } diff --git a/weed/util/bptree/serde_test.go b/weed/util/bptree/serde_test.go new file mode 100644 index 000000000..27ccccb78 --- /dev/null +++ b/weed/util/bptree/serde_test.go @@ -0,0 +1,46 @@ +package bptree + +import ( + "fmt" + "testing" +) + +type nodeStoreMapImpl struct { + m map[int64]*ProtoNode +} + +func (n *nodeStoreMapImpl) PersistFunc(node *BpNode) error { + println("saving node", node.protoNodeId) + n.m[node.protoNodeId] = node.protoNode + return nil +} +func (n *nodeStoreMapImpl) DestroyFunc(node *BpNode) error { + println("delete node", node.protoNodeId) + delete(n.m, node.protoNodeId) + return nil +} + +func TestSerDe(t *testing.T) { + + nodeStore := &nodeStoreMapImpl{ + m: make(map[int64]*ProtoNode), + } + + tree := NewBpTree(3, nodeStore) + + for i:=0;i<32;i++{ + println("add", i) + tree.Add(String(fmt.Sprintf("%02d", i)), nil) + } + + for i:=5;i<9;i++{ + println("----------", i) + tree.RemoveWhere(String(fmt.Sprintf("%02d", i)), func(value ItemValue) bool { + return true + }) + printTree(tree.root, "") + } + + + +} \ No newline at end of file