gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

add back non_dedup

Browse source
This commit is contained in:
Chris Lu 2021-08-21 15:13:13 -07:00
parent 849f185a20
commit 38c8470d1d
4 changed files with 83 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
weed/util/bptree/bpmap.go
View file

@ -11,7 +11,7 @@ type BpMap BpTree
func NewBpMap(node_size int) *BpMap { func NewBpMap(node_size int) *BpMap {
return &BpMap{ return &BpMap{
root: NewLeaf(node_size), root: NewLeaf(node_size, true),
} }
} }
@ -47,7 +47,7 @@ func (self *BpMap) Remove(key Hashable) (value interface{}, err error) {
return nil, err return nil, err
} }
if new_root == nil { if new_root == nil {
self.setRoot(NewLeaf(ns)) self.setRoot(NewLeaf(ns, true))
} else { } else {
self.setRoot(new_root) self.setRoot(new_root)
} }

18
weed/util/bptree/bptree.go
View file

@ -14,7 +14,7 @@ type loc_iterator func() (i int, leaf *BpNode, li loc_iterator)
func NewBpTree(node_size int) *BpTree { func NewBpTree(node_size int) *BpTree {
return &BpTree{ return &BpTree{
root: NewLeaf(node_size), root: NewLeaf(node_size, false),
} }
} }
@ -26,6 +26,20 @@ func (self *BpTree) Has(key Hashable) bool {
return l.keys[j].Equals(key) return l.keys[j].Equals(key)
} }
func (self *BpTree) Count(key Hashable) int {
if len(self.root.keys) == 0 {
return 0
}
j, l := self.root.get_start(key)
count := 0
end := false
for !end && l.keys[j].Equals(key) {
count++
j, l, end = next_location(j, l)
}
return count
}
func (self *BpTree) Add(key Hashable, value interface{}) (err error) { func (self *BpTree) Add(key Hashable, value interface{}) (err error) {
new_root, err := self.getRoot().put(key, value) new_root, err := self.getRoot().put(key, value)
if err != nil { if err != nil {
@ -75,7 +89,7 @@ func (self *BpTree) RemoveWhere(key Hashable, where WhereFunc) (err error) {
return err return err
} }
if new_root == nil { if new_root == nil {
self.setRoot(NewLeaf(ns)) self.setRoot(NewLeaf(ns, false))
} else { } else {
self.setRoot(new_root) self.setRoot(new_root)
} }

17
weed/util/bptree/bptree_node.go
View file

@ -6,6 +6,7 @@ type BpNode struct {
pointers []*BpNode pointers []*BpNode
next *BpNode next *BpNode
prev *BpNode prev *BpNode
no_dup bool
} }
func NewInternal(size int) *BpNode { func NewInternal(size int) *BpNode {
@ -18,13 +19,14 @@ func NewInternal(size int) *BpNode {
} }
} }
func NewLeaf(size int) *BpNode { func NewLeaf(size int, no_dup bool) *BpNode {
if size < 0 { if size < 0 {
panic(NegativeSize()) panic(NegativeSize())
} }
return &BpNode{ return &BpNode{
keys: make([]Hashable, 0, size), keys: make([]Hashable, 0, size),
values: make([]interface{}, 0, size), values: make([]interface{}, 0, size),
no_dup: no_dup,
} }
} }
@ -274,6 +276,13 @@ func (self *BpNode) leaf_insert(key Hashable, value interface{}) (a, b *BpNode,
if self.Internal() { if self.Internal() {
return nil, nil, BpTreeError("Expected a leaf node") return nil, nil, BpTreeError("Expected a leaf node")
} }
if self.no_dup {
i, has := self.find(key)
if has {
self.values[i] = value
return self, nil, nil
}
}
if self.Full() { if self.Full() {
return self.leaf_split(key, value) return self.leaf_split(key, value)
} else { } else {
@ -298,7 +307,7 @@ func (self *BpNode) leaf_split(key Hashable, value interface{}) (a, b *BpNode, e
return self.pure_leaf_split(key, value) return self.pure_leaf_split(key, value)
} }
a = self a = self
b = NewLeaf(self.NodeSize()) b = NewLeaf(self.NodeSize(), self.no_dup)
insert_linked_list_node(b, a, a.getNext()) insert_linked_list_node(b, a, a.getNext())
balance_nodes(a, b) balance_nodes(a, b)
if key.Less(b.keys[0]) { if key.Less(b.keys[0]) {
@ -330,7 +339,7 @@ func (self *BpNode) pure_leaf_split(key Hashable, value interface{}) (a, b *BpNo
return nil, nil, BpTreeError("Expected a pure leaf node") return nil, nil, BpTreeError("Expected a pure leaf node")
} }
if key.Less(self.keys[0]) { if key.Less(self.keys[0]) {
a = NewLeaf(self.NodeSize()) a = NewLeaf(self.NodeSize(), self.no_dup)
b = self b = self
if err := a.put_kv(key, value); err != nil { if err := a.put_kv(key, value); err != nil {
return nil, nil, err return nil, nil, err
@ -346,7 +355,7 @@ func (self *BpNode) pure_leaf_split(key Hashable, value interface{}) (a, b *BpNo
} }
return a, nil, nil return a, nil, nil
} else { } else {
b = NewLeaf(self.NodeSize()) b = NewLeaf(self.NodeSize(), self.no_dup)
if err := b.put_kv(key, value); err != nil { if err := b.put_kv(key, value); err != nil {
return nil, nil, err return nil, nil, err
} }

95
weed/util/bptree/bptree_test.go
View file

@ -119,6 +119,12 @@ func TestAddHasCountFindIterateRemove(t *testing.T) {
if has := bpt.Has(randstr(10)); has { if has := bpt.Has(randstr(10)); has {
t.Error("Table has extra key") t.Error("Table has extra key")
} }
if count := bpt.Count(r.key); count != 1 {
t.Error(bpt, "Missing key")
}
if count := bpt.Count(randstr(10)); count != 0 {
t.Error("Table has extra key")
}
for k, v, next := bpt.Find(r.key)(); next != nil; k, v, next = next() { for k, v, next := bpt.Find(r.key)(); next != nil; k, v, next = next() {
if !k.Equals(r.key) { if !k.Equals(r.key) {
t.Error(bpt, "Find Failed Key Error") t.Error(bpt, "Find Failed Key Error")
@ -184,6 +190,9 @@ func TestAddHasCountFindIterateRemove(t *testing.T) {
if has := bpt.Has(r.key); !has { if has := bpt.Has(r.key); !has {
t.Error(bpt, "Missing key") t.Error(bpt, "Missing key")
} }
if count := bpt.Count(r.key); count != 1 {
t.Error(bpt, "Missing key")
}
if err := bpt.RemoveWhere(r.key, func(value interface{}) bool { return true }); err != nil { if err := bpt.RemoveWhere(r.key, func(value interface{}) bool { return true }); err != nil {
t.Fatal(bpt, err) t.Fatal(bpt, err)
} }
@ -266,7 +275,7 @@ func TestBpMap(t *testing.T) {
} }
func Test_get_start(t *testing.T) { func Test_get_start(t *testing.T) {
root := NewLeaf(2) root := NewLeaf(2, false)
root, err := root.put(Int(1), 1) root, err := root.put(Int(1), 1)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
@ -335,7 +344,7 @@ func Test_get_start(t *testing.T) {
} }
func Test_get_end(t *testing.T) { func Test_get_end(t *testing.T) {
root := NewLeaf(3) root := NewLeaf(3, false)
root, err := root.put(Int(1), -1) root, err := root.put(Int(1), -1)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -427,7 +436,7 @@ func Test_get_end(t *testing.T) {
} }
func Test_put_no_root_split(t *testing.T) { func Test_put_no_root_split(t *testing.T) {
a := NewLeaf(2) a := NewLeaf(2, false)
if err := a.put_kv(Int(1), 1); err != nil { if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err) t.Error(err)
} }
@ -461,7 +470,7 @@ func Test_put_no_root_split(t *testing.T) {
} }
func Test_put_root_split(t *testing.T) { func Test_put_root_split(t *testing.T) {
a := NewLeaf(2) a := NewLeaf(2, false)
p, err := a.put(Int(1), 1) p, err := a.put(Int(1), 1)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
@ -511,7 +520,7 @@ func Test_put_root_split(t *testing.T) {
func Test_internal_insert_no_split(t *testing.T) { func Test_internal_insert_no_split(t *testing.T) {
a := NewInternal(3) a := NewInternal(3)
leaf := NewLeaf(1) leaf := NewLeaf(1, false)
if err := leaf.put_kv(Int(1), 1); err != nil { if err := leaf.put_kv(Int(1), 1); err != nil {
t.Error(err) t.Error(err)
} }
@ -539,7 +548,7 @@ func Test_internal_insert_no_split(t *testing.T) {
func Test_internal_insert_split_less(t *testing.T) { func Test_internal_insert_split_less(t *testing.T) {
a := NewInternal(3) a := NewInternal(3)
leaf := NewLeaf(1) leaf := NewLeaf(1, false)
if err := leaf.put_kv(Int(1), 1); err != nil { if err := leaf.put_kv(Int(1), 1); err != nil {
t.Error(err) t.Error(err)
} }
@ -649,7 +658,7 @@ func Test_internal_split_greater(t *testing.T) {
} }
func Test_leaf_insert_no_split(t *testing.T) { func Test_leaf_insert_no_split(t *testing.T) {
a := NewLeaf(3) a := NewLeaf(3, false)
insert_linked_list_node(a, nil, nil) insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), 1); err != nil { if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err) t.Error(err)
@ -675,7 +684,7 @@ func Test_leaf_insert_no_split(t *testing.T) {
// tests the defer to split logic // tests the defer to split logic
func Test_leaf_insert_split_less(t *testing.T) { func Test_leaf_insert_split_less(t *testing.T) {
a := NewLeaf(3) a := NewLeaf(3, false)
insert_linked_list_node(a, nil, nil) insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), 1); err != nil { if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err) t.Error(err)
@ -706,7 +715,7 @@ func Test_leaf_insert_split_less(t *testing.T) {
} }
func Test_leaf_split_less(t *testing.T) { func Test_leaf_split_less(t *testing.T) {
a := NewLeaf(3) a := NewLeaf(3, false)
insert_linked_list_node(a, nil, nil) insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), 1); err != nil { if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err) t.Error(err)
@ -737,7 +746,7 @@ func Test_leaf_split_less(t *testing.T) {
} }
func Test_leaf_split_equal(t *testing.T) { func Test_leaf_split_equal(t *testing.T) {
a := NewLeaf(3) a := NewLeaf(3, false)
insert_linked_list_node(a, nil, nil) insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), 1); err != nil { if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err) t.Error(err)
@ -768,7 +777,7 @@ func Test_leaf_split_equal(t *testing.T) {
} }
func Test_leaf_split_greater(t *testing.T) { func Test_leaf_split_greater(t *testing.T) {
a := NewLeaf(3) a := NewLeaf(3, false)
insert_linked_list_node(a, nil, nil) insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), 1); err != nil { if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err) t.Error(err)
@ -800,13 +809,13 @@ func Test_leaf_split_greater(t *testing.T) {
// tests the defer logic // tests the defer logic
func Test_pure_leaf_insert_split_less(t *testing.T) { func Test_pure_leaf_insert_split_less(t *testing.T) {
a := NewLeaf(2) a := NewLeaf(2, false)
insert_linked_list_node(a, nil, nil) insert_linked_list_node(a, nil, nil)
b := NewLeaf(2) b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil) insert_linked_list_node(b, a, nil)
c := NewLeaf(2) c := NewLeaf(2, false)
insert_linked_list_node(c, b, nil) insert_linked_list_node(c, b, nil)
d := NewLeaf(2) d := NewLeaf(2, false)
insert_linked_list_node(d, c, nil) insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), 1); err != nil { if err := a.put_kv(Int(3), 1); err != nil {
t.Error(err) t.Error(err)
@ -873,13 +882,13 @@ func Test_pure_leaf_insert_split_less(t *testing.T) {
} }
func Test_pure_leaf_split_less(t *testing.T) { func Test_pure_leaf_split_less(t *testing.T) {
a := NewLeaf(2) a := NewLeaf(2, false)
insert_linked_list_node(a, nil, nil) insert_linked_list_node(a, nil, nil)
b := NewLeaf(2) b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil) insert_linked_list_node(b, a, nil)
c := NewLeaf(2) c := NewLeaf(2, false)
insert_linked_list_node(c, b, nil) insert_linked_list_node(c, b, nil)
d := NewLeaf(2) d := NewLeaf(2, false)
insert_linked_list_node(d, c, nil) insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), 1); err != nil { if err := a.put_kv(Int(3), 1); err != nil {
t.Error(err) t.Error(err)
@ -946,13 +955,13 @@ func Test_pure_leaf_split_less(t *testing.T) {
} }
func Test_pure_leaf_split_equal(t *testing.T) { func Test_pure_leaf_split_equal(t *testing.T) {
a := NewLeaf(2) a := NewLeaf(2, false)
insert_linked_list_node(a, nil, nil) insert_linked_list_node(a, nil, nil)
b := NewLeaf(2) b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil) insert_linked_list_node(b, a, nil)
c := NewLeaf(2) c := NewLeaf(2, false)
insert_linked_list_node(c, b, nil) insert_linked_list_node(c, b, nil)
d := NewLeaf(2) d := NewLeaf(2, false)
insert_linked_list_node(d, c, nil) insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), 1); err != nil { if err := a.put_kv(Int(3), 1); err != nil {
t.Error(err) t.Error(err)
@ -1010,13 +1019,13 @@ func Test_pure_leaf_split_equal(t *testing.T) {
} }
func Test_pure_leaf_split_greater(t *testing.T) { func Test_pure_leaf_split_greater(t *testing.T) {
a := NewLeaf(2) a := NewLeaf(2, false)
insert_linked_list_node(a, nil, nil) insert_linked_list_node(a, nil, nil)
b := NewLeaf(2) b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil) insert_linked_list_node(b, a, nil)
c := NewLeaf(2) c := NewLeaf(2, false)
insert_linked_list_node(c, b, nil) insert_linked_list_node(c, b, nil)
d := NewLeaf(2) d := NewLeaf(2, false)
insert_linked_list_node(d, c, nil) insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), 1); err != nil { if err := a.put_kv(Int(3), 1); err != nil {
t.Error(err) t.Error(err)
@ -1080,13 +1089,13 @@ func Test_pure_leaf_split_greater(t *testing.T) {
} }
func Test_find_end_of_pure_run(t *testing.T) { func Test_find_end_of_pure_run(t *testing.T) {
a := NewLeaf(2) a := NewLeaf(2, false)
insert_linked_list_node(a, nil, nil) insert_linked_list_node(a, nil, nil)
b := NewLeaf(2) b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil) insert_linked_list_node(b, a, nil)
c := NewLeaf(2) c := NewLeaf(2, false)
insert_linked_list_node(c, b, nil) insert_linked_list_node(c, b, nil)
d := NewLeaf(2) d := NewLeaf(2, false)
insert_linked_list_node(d, c, nil) insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), 1); err != nil { if err := a.put_kv(Int(3), 1); err != nil {
t.Error(err) t.Error(err)
@ -1116,13 +1125,13 @@ func Test_find_end_of_pure_run(t *testing.T) {
} }
func Test_insert_linked_list_node(t *testing.T) { func Test_insert_linked_list_node(t *testing.T) {
a := NewLeaf(1) a := NewLeaf(1, false)
insert_linked_list_node(a, nil, nil) insert_linked_list_node(a, nil, nil)
b := NewLeaf(2) b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil) insert_linked_list_node(b, a, nil)
c := NewLeaf(3) c := NewLeaf(3, false)
insert_linked_list_node(c, b, nil) insert_linked_list_node(c, b, nil)
d := NewLeaf(4) d := NewLeaf(4, false)
insert_linked_list_node(d, a, b) insert_linked_list_node(d, a, b)
if a.getPrev() != nil { if a.getPrev() != nil {
t.Errorf("expected a.prev == nil") t.Errorf("expected a.prev == nil")
@ -1151,13 +1160,13 @@ func Test_insert_linked_list_node(t *testing.T) {
} }
func Test_remove_linked_list_node(t *testing.T) { func Test_remove_linked_list_node(t *testing.T) {
a := NewLeaf(1) a := NewLeaf(1, false)
insert_linked_list_node(a, nil, nil) insert_linked_list_node(a, nil, nil)
b := NewLeaf(2) b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil) insert_linked_list_node(b, a, nil)
c := NewLeaf(3) c := NewLeaf(3, false)
insert_linked_list_node(c, b, nil) insert_linked_list_node(c, b, nil)
d := NewLeaf(4) d := NewLeaf(4, false)
insert_linked_list_node(d, a, b) insert_linked_list_node(d, a, b)
if a.getPrev() != nil { if a.getPrev() != nil {
t.Errorf("expected a.prev == nil") t.Errorf("expected a.prev == nil")
@ -1226,8 +1235,8 @@ func Test_remove_linked_list_node(t *testing.T) {
} }
func Test_balance_leaf_nodes_with_dup(t *testing.T) { func Test_balance_leaf_nodes_with_dup(t *testing.T) {
a := NewLeaf(3) a := NewLeaf(3, false)
b := NewLeaf(3) b := NewLeaf(3, false)
if err := a.put_kv(Int(1), 1); err != nil { if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err) t.Error(err)
} }
@ -1247,8 +1256,8 @@ func Test_balance_leaf_nodes_with_dup(t *testing.T) {
} }
func Test_balance_leaf_nodes(t *testing.T) { func Test_balance_leaf_nodes(t *testing.T) {
a := NewLeaf(7) a := NewLeaf(7, false)
b := NewLeaf(7) b := NewLeaf(7, false)
if err := a.put_kv(Int(1), 1); err != nil { if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err) t.Error(err)
} }