From ec72547c8d25929155e6c797f965872fb8448b47 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 20 Aug 2021 01:12:52 -0700 Subject: [PATCH] started by copying from https://sourcegraph.com/github.com/timtadh/data-structures@master/-/tree/tree/bptree --- weed/util/bptree/bpmap.go | 77 ++ weed/util/bptree/bptree.go | 160 ++++ weed/util/bptree/bptree_node.go | 752 ++++++++++++++++ weed/util/bptree/bptree_test.go | 1460 +++++++++++++++++++++++++++++++ weed/util/bptree/int.go | 357 ++++++++ weed/util/bptree/rand.go | 2 + weed/util/bptree/string.go | 71 ++ weed/util/bptree/types.go | 103 +++ 8 files changed, 2982 insertions(+) create mode 100644 weed/util/bptree/bpmap.go create mode 100644 weed/util/bptree/bptree.go create mode 100644 weed/util/bptree/bptree_node.go create mode 100644 weed/util/bptree/bptree_test.go create mode 100644 weed/util/bptree/int.go create mode 100644 weed/util/bptree/rand.go create mode 100644 weed/util/bptree/string.go create mode 100644 weed/util/bptree/types.go diff --git a/weed/util/bptree/bpmap.go b/weed/util/bptree/bpmap.go new file mode 100644 index 000000000..37b2b25bb --- /dev/null +++ b/weed/util/bptree/bpmap.go @@ -0,0 +1,77 @@ +package bptree + +import ( + "fmt" +) + +/* A BpMap is a B+Tree with support for duplicate keys disabled. This makes it + * behave like a regular Map rather than a MultiMap. + */ +type BpMap BpTree + +func NewBpMap(node_size int) *BpMap { + return &BpMap{ + root: NewLeaf(node_size, true), + size: 0, + } +} + +func (self *BpMap) Size() int { + return (*BpTree)(self).Size() +} + +func (self *BpMap) Has(key Hashable) bool { + return (*BpTree)(self).Has(key) +} + +func (self *BpMap) Put(key Hashable, value interface{}) (err error) { + had := self.Has(key) + new_root, err := self.root.put(key, value) + if err != nil { + return err + } + self.root = new_root + if !had { + self.size += 1 + } + return nil +} + +func (self *BpMap) Get(key Hashable) (value interface{}, err error) { + j, l := self.root.get_start(key) + if l.keys[j].Equals(key) { + return l.values[j], nil + } + return nil, fmt.Errorf("key not found: %s", key) +} + +func (self *BpMap) Remove(key Hashable) (value interface{}, err error) { + value, err = self.Get(key) + if err != nil { + return nil, err + } + ns := self.root.NodeSize() + new_root, err := self.root.remove(key, func(value interface{}) bool { return true }) + if err != nil { + return nil, err + } + if new_root == nil { + self.root = NewLeaf(ns, true) + } else { + self.root = new_root + } + self.size-- + return value, nil +} + +func (self *BpMap) Keys() (ki KIterator) { + return (*BpTree)(self).Keys() +} + +func (self *BpMap) Values() (vi Iterator) { + return (*BpTree)(self).Values() +} + +func (self *BpMap) Iterate() (kvi KVIterator) { + return (*BpTree)(self).Iterate() +} diff --git a/weed/util/bptree/bptree.go b/weed/util/bptree/bptree.go new file mode 100644 index 000000000..4b68adb20 --- /dev/null +++ b/weed/util/bptree/bptree.go @@ -0,0 +1,160 @@ +package bptree + +// started by copying from https://sourcegraph.com/github.com/timtadh/data-structures@master/-/tree/tree/bptree + +/* A BpTree is a B+Tree with support for duplicate keys. This makes it behave as + * a MultiMap. Additionally you can use the Range operator to select k/v in a + * range. If from > to it will iterate backwards. + */ +type BpTree struct { + root *BpNode + size int +} + +type loc_iterator func() (i int, leaf *BpNode, li loc_iterator) + +func NewBpTree(node_size int) *BpTree { + return &BpTree{ + root: NewLeaf(node_size, false), + size: 0, + } +} + +func (self *BpTree) Size() int { + return self.size +} + +func (self *BpTree) Has(key Hashable) bool { + if len(self.root.keys) == 0 { + return false + } + j, l := self.root.get_start(key) + return l.keys[j].Equals(key) +} + +func (self *BpTree) Count(key Hashable) int { + if len(self.root.keys) == 0 { + return 0 + } + j, l := self.root.get_start(key) + count := 0 + end := false + for !end && l.keys[j].Equals(key) { + count++ + j, l, end = next_location(j, l) + } + return count +} + +func (self *BpTree) Add(key Hashable, value interface{}) (err error) { + new_root, err := self.root.put(key, value) + if err != nil { + return err + } + self.root = new_root + self.size += 1 + return nil +} + +func (self *BpTree) Replace(key Hashable, where WhereFunc, value interface{}) (err error) { + li := self.root.forward(key, key) + for i, leaf, next := li(); next != nil; i, leaf, next = next() { + if where(leaf.values[i]) { + leaf.values[i] = value + } + } + return nil +} + +func (self *BpTree) Find(key Hashable) (kvi KVIterator) { + return self.Range(key, key) +} + +func (self *BpTree) Range(from, to Hashable) (kvi KVIterator) { + var li loc_iterator + if !to.Less(from) { + li = self.root.forward(from, to) + } else { + li = self.root.backward(from, to) + } + kvi = func() (key Hashable, value interface{}, next KVIterator) { + var i int + var leaf *BpNode + i, leaf, li = li() + if li == nil { + return nil, nil, nil + } + return leaf.keys[i], leaf.values[i], kvi + } + return kvi +} + +func (self *BpTree) RemoveWhere(key Hashable, where WhereFunc) (err error) { + ns := self.root.NodeSize() + new_root, err := self.root.remove(key, where) + if err != nil { + return err + } + if new_root == nil { + self.root = NewLeaf(ns, false) + } else { + self.root = new_root + } + self.size -= 1 + return nil +} + +func (self *BpTree) Keys() (ki KIterator) { + li := self.root.all() + var prev Equatable + ki = func() (key Hashable, next KIterator) { + var i int + var leaf *BpNode + i, leaf, li = li() + if li == nil { + return nil, nil + } + if leaf.keys[i].Equals(prev) { + return ki() + } + prev = leaf.keys[i] + return leaf.keys[i], ki + } + return ki +} + +func (self *BpTree) Values() (vi Iterator) { + return MakeValuesIterator(self) +} + +func (self *BpTree) Items() (vi KIterator) { + return MakeItemsIterator(self) +} + +func (self *BpTree) Iterate() (kvi KVIterator) { + li := self.root.all() + kvi = func() (key Hashable, value interface{}, next KVIterator) { + var i int + var leaf *BpNode + i, leaf, li = li() + if li == nil { + return nil, nil, nil + } + return leaf.keys[i], leaf.values[i], kvi + } + return kvi +} + +func (self *BpTree) Backward() (kvi KVIterator) { + li := self.root.all_backward() + kvi = func() (key Hashable, value interface{}, next KVIterator) { + var i int + var leaf *BpNode + i, leaf, li = li() + if li == nil { + return nil, nil, nil + } + return leaf.keys[i], leaf.values[i], kvi + } + return kvi +} \ No newline at end of file diff --git a/weed/util/bptree/bptree_node.go b/weed/util/bptree/bptree_node.go new file mode 100644 index 000000000..3574371f5 --- /dev/null +++ b/weed/util/bptree/bptree_node.go @@ -0,0 +1,752 @@ +package bptree + +type BpNode struct { + keys []Hashable + values []interface{} + pointers []*BpNode + next *BpNode + prev *BpNode + no_dup bool +} + +func NewInternal(size int) *BpNode { + if size < 0 { + panic(NegativeSize()) + } + return &BpNode{ + keys: make([]Hashable, 0, size), + pointers: make([]*BpNode, 0, size), + } +} + +func NewLeaf(size int, no_dup bool) *BpNode { + if size < 0 { + panic(NegativeSize()) + } + return &BpNode{ + keys: make([]Hashable, 0, size), + values: make([]interface{}, 0, size), + no_dup: no_dup, + } +} + +func (self *BpNode) Full() bool { + return len(self.keys) == cap(self.keys) +} + +func (self *BpNode) Pure() bool { + if len(self.keys) == 0 { + return true + } + k0 := self.keys[0] + for _, k := range self.keys { + if !k0.Equals(k) { + return false + } + } + return true +} + +func (self *BpNode) Internal() bool { + return cap(self.pointers) > 0 +} + +func (self *BpNode) NodeSize() int { + return cap(self.keys) +} + +func (self *BpNode) Height() int { + if !self.Internal() { + return 1 + } else if len(self.pointers) == 0 { + panic(BpTreeError("Internal node has no pointers but asked for height")) + } + return self.pointers[0].Height() + 1 +} + +func (self *BpNode) count(key Hashable) int { + i, _ := self.find(key) + count := 0 + for ; i < len(self.keys); i++ { + if self.keys[i].Equals(key) { + count++ + } else { + break + } + } + return count +} + +func (self *BpNode) has(key Hashable) bool { + _, has := self.find(key) + return has +} + +func (self *BpNode) left_most_leaf() *BpNode { + if self.Internal() { + return self.pointers[0].left_most_leaf() + } + return self +} + +func (self *BpNode) right_most_leaf() *BpNode { + if self.Internal() { + return self.pointers[len(self.pointers)-1].right_most_leaf() + } + return self +} + +/* returns the index and leaf-block of the first key greater than or equal to + * the search key. (unless the search key is greater than all the keys in the + * tree, in that case it will be the last key in the tree) + */ +func (self *BpNode) get_start(key Hashable) (i int, leaf *BpNode) { + if self.Internal() { + return self.internal_get_start(key) + } else { + return self.leaf_get_start(key) + } +} + +func next_location(i int, leaf *BpNode) (int, *BpNode, bool) { + j := i + 1 + for j >= len(leaf.keys) && leaf.next != nil { + j = 0 + leaf = leaf.next + } + if j >= len(leaf.keys) { + return -1, nil, true + } + return j, leaf, false +} + +func prev_location(i int, leaf *BpNode) (int, *BpNode, bool) { + j := i - 1 + for j < 0 && leaf.prev != nil { + leaf = leaf.prev + j = len(leaf.keys) - 1 + } + if j < 0 { + return -1, nil, true + } + return j, leaf, false +} + +/* returns the index and leaf-block of the last key equal to the search key or + * the first key greater than the search key. (unless the search key is greater + * than all the keys in the tree, in that case it will be the last key in the + * tree) + */ +func (self *BpNode) get_end(key Hashable) (i int, leaf *BpNode) { + end := false + i, leaf = self.get_start(key) + pi, pleaf := i, leaf + for !end && leaf.keys[i].Equals(key) { + pi, pleaf = i, leaf + i, leaf, end = next_location(i, leaf) + } + return pi, pleaf +} + +func (self *BpNode) internal_get_start(key Hashable) (i int, leaf *BpNode) { + if !self.Internal() { + panic(BpTreeError("Expected a internal node")) + } + i, has := self.find(key) + if !has && i > 0 { + // if it doesn't have it and the index > 0 then we have the next block + // so we have to subtract one from the index. + i-- + } + child := self.pointers[i] + return child.get_start(key) +} + +func (self *BpNode) leaf_get_start(key Hashable) (i int, leaf *BpNode) { + i, has := self.find(key) + if i >= len(self.keys) && i > 0 { + i = len(self.keys) - 1 + } + if !has && (len(self.keys) == 0 || self.keys[i].Less(key)) && self.next != nil { + return self.next.leaf_get_start(key) + } + return i, self +} + +/* This puts the k/v pair into the B+Tree rooted at this node and returns the + * (possibly) new root of the tree. + */ +func (self *BpNode) put(key Hashable, value interface{}) (root *BpNode, err error) { + a, b, err := self.insert(key, value) + if err != nil { + return nil, err + } else if b == nil { + return a, nil + } + // else we have root split + root = NewInternal(self.NodeSize()) + root.put_kp(a.keys[0], a) + root.put_kp(b.keys[0], b) + return root, nil +} + +// right is only set on split +// left is always set. When split is false left is the pointer to block +// When split is true left is the pointer to the new left +// block +func (self *BpNode) insert(key Hashable, value interface{}) (a, b *BpNode, err error) { + if self.Internal() { + return self.internal_insert(key, value) + } else { // leaf node + return self.leaf_insert(key, value) + } +} + +/* - first find the child to insert into + * - do the child insert + * - if there was a split: + * - if the block is full, split this block + * - else insert the new key/pointer into this block + */ +func (self *BpNode) internal_insert(key Hashable, value interface{}) (a, b *BpNode, err error) { + if !self.Internal() { + return nil, nil, BpTreeError("Expected a internal node") + } + i, has := self.find(key) + if !has && i > 0 { + // if it doesn't have it and the index > 0 then we have the next block + // so we have to subtract one from the index. + i-- + } + child := self.pointers[i] + p, q, err := child.insert(key, value) + if err != nil { + return nil, nil, err + } + self.keys[i] = p.keys[0] + self.pointers[i] = p + if q != nil { + // we had a split + if self.Full() { + return self.internal_split(q.keys[0], q) + } else { + if err := self.put_kp(q.keys[0], q); err != nil { + return nil, nil, err + } + return self, nil, nil + } + } + return self, nil, nil +} + +/* On split + * - first assert that the key to be inserted is not already in the block. + * - Make a new block + * - balance the two blocks. + * - insert the new key/pointer combo into the correct block + */ +func (self *BpNode) internal_split(key Hashable, ptr *BpNode) (a, b *BpNode, err error) { + if !self.Internal() { + return nil, nil, BpTreeError("Expected a internal node") + } + if self.has(key) { + return nil, nil, BpTreeError("Tried to split an internal block on duplicate key") + } + a = self + b = NewInternal(self.NodeSize()) + balance_nodes(a, b) + if key.Less(b.keys[0]) { + if err := a.put_kp(key, ptr); err != nil { + return nil, nil, err + } + } else { + if err := b.put_kp(key, ptr); err != nil { + return nil, nil, err + } + } + return a, b, nil +} + +/* if the leaf is full then it will defer to a leaf_split + * (but in one case that will not actually split in the case of a insert into + * a pure block with a matching key) + * else this leaf will get a new entry. + */ +func (self *BpNode) leaf_insert(key Hashable, value interface{}) (a, b *BpNode, err error) { + if self.Internal() { + return nil, nil, BpTreeError("Expected a leaf node") + } + if self.no_dup { + i, has := self.find(key) + if has { + self.values[i] = value + return self, nil, nil + } + } + if self.Full() { + return self.leaf_split(key, value) + } else { + if err := self.put_kv(key, value); err != nil { + return nil, nil, err + } + return self, nil, nil + } +} + +/* on leaf split if the block is pure then it will defer to pure_leaf_split + * else + * - a new block will be made and inserted after this one + * - the two blocks will be balanced with balanced_nodes + * - if the key is less than b.keys[0] it will go in a else b + */ +func (self *BpNode) leaf_split(key Hashable, value interface{}) (a, b *BpNode, err error) { + if self.Internal() { + return nil, nil, BpTreeError("Expected a leaf node") + } + if self.Pure() { + return self.pure_leaf_split(key, value) + } + a = self + b = NewLeaf(self.NodeSize(), self.no_dup) + insert_linked_list_node(b, a, a.next) + balance_nodes(a, b) + if key.Less(b.keys[0]) { + if err := a.put_kv(key, value); err != nil { + return nil, nil, err + } + } else { + if err := b.put_kv(key, value); err != nil { + return nil, nil, err + } + } + return a, b, nil +} + +/* a pure leaf split has two cases: + * 1) the inserted key is less than the current pure block. + * - a new block should be created before the current block + * - the key should be put in it + * 2) the inserted key is greater than or equal to the pure block. + * - the end of run of pure blocks should be found + * - if the key is equal to pure block and the last block is not full insert + * the new kv + * - else split by making a new block after the last block in the run + * and putting the new key there. + * - always return the current block as "a" and the new block as "b" + */ +func (self *BpNode) pure_leaf_split(key Hashable, value interface{}) (a, b *BpNode, err error) { + if self.Internal() || !self.Pure() { + return nil, nil, BpTreeError("Expected a pure leaf node") + } + if key.Less(self.keys[0]) { + a = NewLeaf(self.NodeSize(), self.no_dup) + b = self + if err := a.put_kv(key, value); err != nil { + return nil, nil, err + } + insert_linked_list_node(a, b.prev, b) + return a, b, nil + } else { + a = self + e := self.find_end_of_pure_run() + if e.keys[0].Equals(key) && !e.Full() { + if err := e.put_kv(key, value); err != nil { + return nil, nil, err + } + return a, nil, nil + } else { + b = NewLeaf(self.NodeSize(), self.no_dup) + if err := b.put_kv(key, value); err != nil { + return nil, nil, err + } + insert_linked_list_node(b, e, e.next) + if e.keys[0].Equals(key) { + return a, nil, nil + } + return a, b, nil + } + } +} + +func (self *BpNode) put_kp(key Hashable, ptr *BpNode) error { + if self.Full() { + return BpTreeError("Block is full.") + } + if !self.Internal() { + return BpTreeError("Expected a internal node") + } + i, has := self.find(key) + if has { + return BpTreeError("Tried to insert a duplicate key into an internal node") + } else if i < 0 { + panic(BpTreeError("find returned a negative int")) + } else if i >= cap(self.keys) { + panic(BpTreeError("find returned a int > than cap(keys)")) + } + if err := self.put_key_at(i, key); err != nil { + return err + } + if err := self.put_pointer_at(i, ptr); err != nil { + return err + } + return nil +} + +func (self *BpNode) put_kv(key Hashable, value interface{}) error { + if self.Full() { + return BpTreeError("Block is full.") + } + if self.Internal() { + return BpTreeError("Expected a leaf node") + } + i, _ := self.find(key) + if i < 0 { + panic(BpTreeError("find returned a negative int")) + } else if i >= cap(self.keys) { + panic(BpTreeError("find returned a int > than cap(keys)")) + } + if err := self.put_key_at(i, key); err != nil { + return err + } + if err := self.put_value_at(i, value); err != nil { + return err + } + return nil +} + +func (self *BpNode) put_key_at(i int, key Hashable) error { + if self.Full() { + return BpTreeError("Block is full.") + } + self.keys = self.keys[:len(self.keys)+1] + for j := len(self.keys) - 1; j > i; j-- { + self.keys[j] = self.keys[j-1] + } + self.keys[i] = key + return nil +} + +func (self *BpNode) put_value_at(i int, value interface{}) error { + if len(self.values) == cap(self.values) { + return BpTreeError("Block is full.") + } + if self.Internal() { + return BpTreeError("Expected a leaf node") + } + self.values = self.values[:len(self.values)+1] + for j := len(self.values) - 1; j > i; j-- { + self.values[j] = self.values[j-1] + } + self.values[i] = value + return nil +} + +func (self *BpNode) put_pointer_at(i int, pointer *BpNode) error { + if len(self.pointers) == cap(self.pointers) { + return BpTreeError("Block is full.") + } + if !self.Internal() { + return BpTreeError("Expected a internal node") + } + self.pointers = self.pointers[:len(self.pointers)+1] + for j := len(self.pointers) - 1; j > i; j-- { + self.pointers[j] = self.pointers[j-1] + } + self.pointers[i] = pointer + return nil +} + +func (self *BpNode) remove(key Hashable, where WhereFunc) (a *BpNode, err error) { + if self.Internal() { + return self.internal_remove(key, nil, where) + } else { + return self.leaf_remove(key, self.keys[len(self.keys)-1], where) + } +} + +func (self *BpNode) internal_remove(key Hashable, sibling *BpNode, where WhereFunc) (a *BpNode, err error) { + if !self.Internal() { + panic(BpTreeError("Expected a internal node")) + } + i, has := self.find(key) + if !has && i > 0 { + // if it doesn't have it and the index > 0 then we have the next block + // so we have to subtract one from the index. + i-- + } + if i+1 < len(self.keys) { + sibling = self.pointers[i+1] + } else if sibling != nil { + sibling = sibling.left_most_leaf() + } + child := self.pointers[i] + if child.Internal() { + child, err = child.internal_remove(key, sibling, where) + } else { + if sibling == nil { + child, err = child.leaf_remove(key, nil, where) + } else { + child, err = child.leaf_remove(key, sibling.keys[0], where) + } + } + if err != nil { + return nil, err + } + if child == nil { + if err := self.remove_key_at(i); err != nil { + return nil, err + } + if err := self.remove_ptr_at(i); err != nil { + return nil, err + } + } else { + self.keys[i] = child.keys[0] + self.pointers[i] = child + } + if len(self.keys) == 0 { + return nil, nil + } + return self, nil +} + +func (self *BpNode) leaf_remove(key, stop Hashable, where WhereFunc) (a *BpNode, err error) { + if self.Internal() { + return nil, BpTreeError("Expected a leaf node") + } + a = self + for j, l, next := self.forward(key, key)(); next != nil; j, l, next = next() { + if where(l.values[j]) { + if err := l.remove_key_at(j); err != nil { + return nil, err + } + if err := l.remove_value_at(j); err != nil { + return nil, err + } + } + if len(l.keys) == 0 { + remove_linked_list_node(l) + if l.next == nil { + a = nil + } else if stop == nil { + a = nil + } else if !l.next.keys[0].Equals(stop) { + a = l.next + } else { + a = nil + } + } + } + return a, nil +} + +func (self *BpNode) remove_key_at(i int) error { + if i >= len(self.keys) || i < 0 { + return BpTreeError("i, %v, is out of bounds, %v, %v %v.", i, len(self.keys), len(self.values), self) + } + for j := i; j < len(self.keys)-1; j++ { + self.keys[j] = self.keys[j+1] + } + self.keys = self.keys[:len(self.keys)-1] + return nil +} + +func (self *BpNode) remove_value_at(i int) error { + if i >= len(self.values) || i < 0 { + return BpTreeError("i, %v, is out of bounds, %v.", i, len(self.values)) + } + for j := i; j < len(self.values)-1; j++ { + self.values[j] = self.values[j+1] + } + self.values = self.values[:len(self.values)-1] + return nil +} + +func (self *BpNode) remove_ptr_at(i int) error { + if i >= len(self.pointers) || i < 0 { + return BpTreeError("i, %v, is out of bounds, %v.", i, len(self.pointers)) + } + for j := i; j < len(self.pointers)-1; j++ { + self.pointers[j] = self.pointers[j+1] + } + self.pointers = self.pointers[:len(self.pointers)-1] + return nil +} + +func (self *BpNode) find(key Hashable) (int, bool) { + var l int = 0 + var r int = len(self.keys) - 1 + var m int + for l <= r { + m = ((r - l) >> 1) + l + if key.Less(self.keys[m]) { + r = m - 1 + } else if key.Equals(self.keys[m]) { + for j := m; j >= 0; j-- { + if j == 0 || !key.Equals(self.keys[j-1]) { + return j, true + } + } + } else { + l = m + 1 + } + } + return l, false +} + +func (self *BpNode) find_end_of_pure_run() *BpNode { + k := self.keys[0] + p := self + n := self.next + for n != nil && n.Pure() && k.Equals(n.keys[0]) { + p = n + n = n.next + } + return p +} + +func (self *BpNode) all() (li loc_iterator) { + j := -1 + l := self.left_most_leaf() + end := false + j, l, end = next_location(j, l) + li = func() (i int, leaf *BpNode, next loc_iterator) { + if end { + return -1, nil, nil + } + i = j + leaf = l + j, l, end = next_location(j, l) + return i, leaf, li + } + return li +} + +func (self *BpNode) all_backward() (li loc_iterator) { + l := self.right_most_leaf() + j := len(l.keys) + end := false + j, l, end = prev_location(j, l) + li = func() (i int, leaf *BpNode, next loc_iterator) { + if end { + return -1, nil, nil + } + i = j + leaf = l + j, l, end = prev_location(j, l) + return i, leaf, li + } + return li +} + +func (self *BpNode) forward(from, to Hashable) (li loc_iterator) { + j, l := self.get_start(from) + end := false + j-- + li = func() (i int, leaf *BpNode, next loc_iterator) { + j, l, end = next_location(j, l) + if end || to.Less(l.keys[j]) { + return -1, nil, nil + } + return j, l, li + } + return li +} + +func (self *BpNode) backward(from, to Hashable) (li loc_iterator) { + j, l := self.get_end(from) + end := false + li = func() (i int, leaf *BpNode, next loc_iterator) { + if end || l.keys[j].Less(to) { + return -1, nil, nil + } + i = j + leaf = l + j, l, end = prev_location(i, l) + return i, leaf, li + } + return li +} + +func insert_linked_list_node(n, prev, next *BpNode) { + if (prev != nil && prev.next != next) || (next != nil && next.prev != prev) { + panic(BpTreeError("prev and next not hooked up")) + } + n.prev = prev + n.next = next + if prev != nil { + prev.next = n + } + if next != nil { + next.prev = n + } +} + +func remove_linked_list_node(n *BpNode) { + if n.prev != nil { + n.prev.next = n.next + } + if n.next != nil { + n.next.prev = n.prev + } +} + +/* a must be full and b must be empty else there will be a panic + */ +func balance_nodes(a, b *BpNode) { + if len(b.keys) != 0 { + panic(BpTreeError("b was not empty")) + } + if !a.Full() { + panic(BpTreeError("a was not full", a)) + } + if cap(a.keys) != cap(b.keys) { + panic(BpTreeError("cap(a.keys) != cap(b.keys)")) + } + if cap(a.values) != cap(b.values) { + panic(BpTreeError("cap(a.values) != cap(b.values)")) + } + if cap(a.pointers) != cap(b.pointers) { + panic(BpTreeError("cap(a.pointers) != cap(b.pointers)")) + } + m := len(a.keys) / 2 + for m < len(a.keys) && a.keys[m-1].Equals(a.keys[m]) { + m++ + } + if m == len(a.keys) { + m-- + for m > 0 && a.keys[m-1].Equals(a.keys[m]) { + m-- + } + } + var lim int = len(a.keys) - m + b.keys = b.keys[:lim] + if cap(a.values) > 0 { + if cap(a.values) != cap(a.keys) { + panic(BpTreeError("cap(a.values) != cap(a.keys)")) + } + b.values = b.values[:lim] + } + if cap(a.pointers) > 0 { + if cap(a.pointers) != cap(a.keys) { + panic(BpTreeError("cap(a.pointers) != cap(a.keys)")) + } + b.pointers = b.pointers[:lim] + } + for i := 0; i < lim; i++ { + j := m + i + b.keys[i] = a.keys[j] + if cap(a.values) > 0 { + b.values[i] = a.values[j] + } + if cap(a.pointers) > 0 { + b.pointers[i] = a.pointers[j] + } + } + a.keys = a.keys[:m] + if cap(a.values) > 0 { + a.values = a.values[:m] + } + if cap(a.pointers) > 0 { + a.pointers = a.pointers[:m] + } +} diff --git a/weed/util/bptree/bptree_test.go b/weed/util/bptree/bptree_test.go new file mode 100644 index 000000000..cf978ede7 --- /dev/null +++ b/weed/util/bptree/bptree_test.go @@ -0,0 +1,1460 @@ +package bptree + +import ( + "encoding/hex" + "runtime/debug" + "sort" + "sync" + "testing" + + crand "crypto/rand" + "encoding/binary" + mrand "math/rand" + +) + +var rand *mrand.Rand + +func init() { + seed := make([]byte, 8) + if _, err := crand.Read(seed); err == nil { + rand = ThreadSafeRand(int64(binary.BigEndian.Uint64(seed))) + } else { + panic(err) + } +} + +func randslice(length int) []byte { + return RandSlice(length) +} + +func randstr(length int) String { + return String(RandStr(length)) +} + +type Strings []String + +func (self Strings) Len() int { + return len(self) +} + +func (self Strings) Less(i, j int) bool { + return self[i].Less(self[j]) +} + +func (self Strings) Swap(i, j int) { + self[i], self[j] = self[j], self[i] +} + +type record struct { + key String + value String +} + +type records []*record + +func (self records) Len() int { + return len(self) +} + +func (self records) Less(i, j int) bool { + return self[i].key.Less(self[j].key) +} + +func (self records) Swap(i, j int) { + self[i], self[j] = self[j], self[i] +} + +func BenchmarkBpTree(b *testing.B) { + b.StopTimer() + + recs := make(records, 100) + ranrec := func() *record { + return &record{randstr(20), randstr(20)} + } + + for i := range recs { + recs[i] = ranrec() + } + + b.StartTimer() + for i := 0; i < b.N; i++ { + t := NewBpTree(23) + for _, r := range recs { + t.Add(r.key, r.value) + } + for _, r := range recs { + t.RemoveWhere(r.key, func(value interface{}) bool { return true }) + } + } +} + +func TestAddHasCountFindIterateRemove(t *testing.T) { + + ranrec := func() *record { + return &record{ + randstr(12), + randstr(12), + } + } + + test := func(bpt *BpTree) { + var err error + recs := make(records, 128) + new_recs := make(records, 128) + for i := range recs { + r := ranrec() + recs[i] = r + new_recs[i] = &record{r.key, randstr(12)} + err = bpt.Add(r.key, r.value) + if err != nil { + t.Error(err) + } + if bpt.Size() != (i + 1) { + t.Error("size was wrong", bpt.Size(), i+1) + } + } + + for i, r := range recs { + if has := bpt.Has(r.key); !has { + t.Error(bpt, "Missing key") + } + if has := bpt.Has(randstr(10)); has { + t.Error("Table has extra key") + } + if count := bpt.Count(r.key); count != 1 { + t.Error(bpt, "Missing key") + } + if count := bpt.Count(randstr(10)); count != 0 { + t.Error("Table has extra key") + } + for k, v, next := bpt.Find(r.key)(); next != nil; k, v, next = next() { + if !k.Equals(r.key) { + t.Error(bpt, "Find Failed Key Error") + } + if !v.(String).Equals(r.value) { + t.Error(bpt, "Find Failed Value Error") + } + } + err = bpt.Replace(r.key, func(value interface{}) bool { return true }, new_recs[i].value) + if err != nil { + t.Error(err) + } + } + sort.Sort(recs) + sort.Sort(new_recs) + i := 0 + for k, v, next := bpt.Iterate()(); next != nil; k, v, next = next() { + if !recs[i].key.Equals(k) { + t.Error("iterate error wrong key") + } + if !new_recs[i].value.Equals(v.(String)) { + t.Error("iterate error wrong value") + } + i++ + } + i = len(recs) - 1 + for k, v, next := bpt.Backward()(); next != nil; k, v, next = next() { + if !recs[i].key.Equals(k) { + t.Error("iterate error wrong key") + } + if !new_recs[i].value.Equals(v.(String)) { + t.Error("iterate error wrong value") + } + i-- + } + i = 0 + for k, next := bpt.Keys()(); next != nil; k, next = next() { + if !recs[i].key.Equals(k) { + t.Error("iterate error wrong key") + } + i++ + } + i = 7 + for k, v, next := bpt.Range(recs[i].key, recs[i+(len(recs)/2)].key)(); next != nil; k, v, next = next() { + if !recs[i].key.Equals(k) { + t.Error("iterate error wrong key") + } + if !new_recs[i].value.Equals(v.(String)) { + t.Error("iterate error wrong value") + } + i++ + } + for k, v, next := bpt.Range(recs[i].key, recs[7].key)(); next != nil; k, v, next = next() { + if !recs[i].key.Equals(k) { + t.Error("iterate error wrong key") + } + if !new_recs[i].value.Equals(v.(String)) { + t.Error("iterate error wrong value", k, v, recs[i].value, new_recs[i].value) + } + i-- + } + for i, r := range recs { + if has := bpt.Has(r.key); !has { + t.Error(bpt, "Missing key") + } + if count := bpt.Count(r.key); count != 1 { + t.Error(bpt, "Missing key") + } + if err := bpt.RemoveWhere(r.key, func(value interface{}) bool { return true }); err != nil { + t.Fatal(bpt, err) + } + if has := bpt.Has(r.key); has { + t.Error("Table has extra key") + } + for _, x := range recs[i+1:] { + if has := bpt.Has(x.key); !has { + t.Error(bpt, "Missing key", x.key) + } + } + } + } + for i := 2; i < 64; i++ { + test(NewBpTree(i)) + } +} + +func TestBpMap(t *testing.T) { + + ranrec := func() *record { + return &record{ + randstr(12), + randstr(12), + } + } + + test := func(table MapOperable) { + recs := make(records, 400) + for i := range recs { + r := ranrec() + recs[i] = r + err := table.Put(r.key, String("")) + if err != nil { + t.Error(err) + } + err = table.Put(r.key, r.value) + if err != nil { + t.Error(err) + } + if table.Size() != (i + 1) { + t.Error("size was wrong", table.Size(), i+1) + } + } + + for _, r := range recs { + if has := table.Has(r.key); !has { + t.Error(table, "Missing key") + } + if has := table.Has(randstr(12)); has { + t.Error("Table has extra key") + } + if val, err := table.Get(r.key); err != nil { + t.Error(err) + } else if !(val.(String)).Equals(r.value) { + t.Error("wrong value") + } + } + + for i, x := range recs { + if val, err := table.Remove(x.key); err != nil { + t.Error(err) + } else if !(val.(String)).Equals(x.value) { + t.Error("wrong value") + } + for _, r := range recs[i+1:] { + if has := table.Has(r.key); !has { + t.Error("Missing key") + } + if has := table.Has(randstr(12)); has { + t.Error("Table has extra key") + } + if val, err := table.Get(r.key); err != nil { + t.Error(err) + } else if !(val.(String)).Equals(r.value) { + t.Error("wrong value") + } + } + if table.Size() != (len(recs) - (i + 1)) { + t.Error("size was wrong", table.Size(), (len(recs) - (i + 1))) + } + } + } + + test(NewBpMap(23)) +} + +func Test_get_start(t *testing.T) { + root := NewLeaf(2, false) + root, err := root.put(Int(1), 1) + if err != nil { + t.Error(err) + } + root, err = root.put(Int(5), 3) + if err != nil { + t.Error(err) + } + root, err = root.put(Int(3), 2) + if err != nil { + t.Error(err) + } + t.Log(root) + t.Log(root.pointers[0]) + t.Log(root.pointers[1]) + i, n := root.get_start(Int(1)) + if n != root.pointers[0] { + t.Error("wrong node from get_start") + } + if i != 0 { + t.Error("wrong index from get_start") + } + i, n = root.get_start(Int(3)) + if n != root.pointers[0] { + t.Error("wrong node from get_start") + } + if i != 1 { + t.Error("wrong index from get_start") + } + i, n = root.get_start(Int(5)) + if n != root.pointers[1] { + t.Error("wrong node from get_start") + } + if i != 0 { + t.Error("wrong index from get_start") + } + i, n = root.get_start(Int(2)) + if n != root.pointers[0] { + t.Error("wrong node from get_start") + } + if i != 1 { + t.Error("wrong index from get_start") + } + i, n = root.get_start(Int(4)) + t.Log(n) + if n != root.pointers[1] { + t.Error("wrong node from get_start") + } + if i != 0 { + t.Error("wrong index from get_start") + } + i, n = root.get_start(Int(0)) + if n != root.pointers[0] { + t.Error("wrong node from get_start") + } + if i != 0 { + t.Error("wrong index from get_start") + } + i, n = root.get_start(Int(5)) + if n != root.pointers[1] { + t.Error("wrong node from get_start") + } + if i != 0 { + t.Error("wrong index from get_start") + } +} + +func Test_get_end(t *testing.T) { + root := NewLeaf(3, false) + root, err := root.put(Int(1), -1) + if err != nil { + t.Fatal(err) + } + root, err = root.put(Int(4), -1) + if err != nil { + t.Fatal(err) + } + root, err = root.put(Int(3), 1) + if err != nil { + t.Fatal(err) + } + root, err = root.put(Int(3), 2) + if err != nil { + t.Fatal(err) + } + root, err = root.put(Int(3), 3) + if err != nil { + t.Fatal(err) + } + root, err = root.put(Int(3), 4) + if err != nil { + t.Fatal(err) + } + root, err = root.put(Int(3), 5) + if err != nil { + t.Fatal(err) + } + t.Log(root) + t.Log(root.pointers[0]) + t.Log(root.pointers[1]) + t.Log(root.pointers[2]) + i, n := root.get_start(Int(3)) + t.Log(n) + if n != root.pointers[1] { + t.Error("wrong node from get_start") + } + if i != 0 { + t.Error("wrong index from get_start") + } + i, n = root.get_end(Int(3)) + t.Log(n) + if n != root.pointers[1].next { + t.Error("wrong node from get_end") + } + if i != 1 { + t.Error("wrong index from get_end") + } + i, n = root.get_end(Int(1)) + t.Log(n) + if n != root.pointers[0] { + t.Error("wrong node from get_end") + } + if i != 0 { + t.Error("wrong index from get_end") + } + i, n = root.get_end(Int(4)) + t.Log(n) + if n != root.pointers[2] { + t.Error("wrong node from get_end") + } + if i != 0 { + t.Error("wrong index from get_end") + } + i, n = root.get_end(Int(0)) + t.Log(n) + if n != root.pointers[0] { + t.Error("wrong node from get_end") + } + if i != 0 { + t.Error("wrong index from get_end") + } + i, n = root.get_end(Int(5)) + t.Log(n) + if n != root.pointers[2] { + t.Error("wrong node from get_end") + } + if i != 0 { + t.Error("wrong index from get_end") + } + i, n = root.get_end(Int(2)) + t.Log(n) + if n != root.pointers[1] { + t.Error("wrong node from get_end") + } + if i != 0 { + t.Error("wrong index from get_end") + } +} + +func Test_put_no_root_split(t *testing.T) { + a := NewLeaf(2, false) + if err := a.put_kv(Int(1), 1); err != nil { + t.Error(err) + } + p, err := a.put(Int(1), 2) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if !p.has(Int(1)) { + t.Error("p didn't have the right keys", p) + } + } + p, err = a.put(Int(1), 3) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if !p.has(Int(1)) { + t.Error("p didn't have the right keys", p) + } + if p.next == nil { + t.Error("p.next should not be nil") + } + t.Log(p) + t.Log(p.next) + } +} + +func Test_put_root_split(t *testing.T) { + a := NewLeaf(2, false) + p, err := a.put(Int(1), 1) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if !p.has(Int(1)) { + t.Error("p didn't have the right keys", p) + } + } + p, err = a.put(Int(3), 3) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if !p.has(Int(1)) || !p.has(Int(3)) { + t.Error("p didn't have the right keys", p) + } + } + p, err = a.put(Int(2), 2) + if err != nil { + t.Error(err) + } else { + if p == a { + t.Errorf("p == a") + } + if !p.has(Int(1)) || !p.has(Int(3)) { + t.Error("p didn't have the right keys", p) + } + if len(p.pointers) != 2 { + t.Error("p didn't have right number of pointers", p) + } + if !p.pointers[0].has(Int(1)) || !p.pointers[0].has(Int(2)) { + t.Error("p.pointers[0] didn't have the right keys", p.pointers[0]) + } + if !p.pointers[1].has(Int(3)) { + t.Error("p.pointers[1] didn't have the right keys", p.pointers[1]) + } + t.Log(p) + t.Log(p.pointers[0]) + t.Log(p.pointers[1]) + } +} + +func Test_internal_insert_no_split(t *testing.T) { + a := NewInternal(3) + leaf := NewLeaf(1, false) + if err := leaf.put_kv(Int(1), 1); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(1), leaf); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(5), nil); err != nil { + t.Error(err) + } + p, q, err := a.internal_insert(Int(2), nil) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if q != nil { + t.Errorf("q != nil") + } + if !p.has(Int(1)) || !p.has(Int(2)) || !p.has(Int(5)) { + t.Error("p didn't have the right keys", p) + } + } +} + +func Test_internal_insert_split_less(t *testing.T) { + a := NewInternal(3) + leaf := NewLeaf(1, false) + if err := leaf.put_kv(Int(1), 1); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(1), leaf); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(3), nil); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(5), nil); err != nil { + t.Error(err) + } + p, q, err := a.internal_insert(Int(2), nil) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if q == nil { + t.Errorf("q == nil") + } + if !p.has(Int(1)) || !p.has(Int(2)) { + t.Error("p didn't have the right keys", p) + } + if !q.has(Int(3)) || !q.has(Int(5)) { + t.Error("q didn't have the right keys", q) + } + } +} + +func Test_internal_split_less(t *testing.T) { + a := NewInternal(3) + if err := a.put_kp(Int(1), nil); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(3), nil); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(5), nil); err != nil { + t.Error(err) + } + p, q, err := a.internal_split(Int(2), nil) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if q == nil { + t.Errorf("q == nil") + } + if !p.has(Int(1)) || !p.has(Int(2)) { + t.Error("p didn't have the right keys", p) + } + if !q.has(Int(3)) || !q.has(Int(5)) { + t.Error("q didn't have the right keys", q) + } + } +} + +func Test_internal_split_equal(t *testing.T) { + a := NewInternal(3) + if err := a.put_kp(Int(1), nil); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(3), nil); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(5), nil); err != nil { + t.Error(err) + } + p, q, err := a.internal_split(Int(3), nil) + if err == nil { + t.Error("split succeeded should have failed", p, q) + } +} + +func Test_internal_split_greater(t *testing.T) { + a := NewInternal(3) + if err := a.put_kp(Int(1), nil); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(3), nil); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(5), nil); err != nil { + t.Error(err) + } + p, q, err := a.internal_split(Int(4), nil) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if q == nil { + t.Errorf("q == nil") + } + if !p.has(Int(1)) { + t.Error("p didn't have the right keys", p) + } + if !q.has(Int(3)) || !q.has(Int(4)) || !q.has(Int(5)) { + t.Error("q didn't have the right keys", q) + } + } +} + +func Test_leaf_insert_no_split(t *testing.T) { + a := NewLeaf(3, false) + insert_linked_list_node(a, nil, nil) + if err := a.put_kv(Int(1), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(3), 3); err != nil { + t.Error(err) + } + p, q, err := a.leaf_insert(Int(2), 2) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if q != nil { + t.Errorf("q != nil") + } + if !p.has(Int(1)) || !p.has(Int(2)) || !p.has(Int(3)) { + t.Error("p didn't have the right keys", p) + } + } +} + +// tests the defer to split logic +func Test_leaf_insert_split_less(t *testing.T) { + a := NewLeaf(3, false) + insert_linked_list_node(a, nil, nil) + if err := a.put_kv(Int(1), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(3), 3); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(5), 5); err != nil { + t.Error(err) + } + p, q, err := a.leaf_insert(Int(2), 2) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if q == nil { + t.Errorf("q == nil") + } + if !p.has(Int(1)) || !p.has(Int(2)) { + t.Error("p didn't have the right keys", p) + } + if !q.has(Int(3)) || !q.has(Int(5)) { + t.Error("q didn't have the right keys", q) + } + } +} + +func Test_leaf_split_less(t *testing.T) { + a := NewLeaf(3, false) + insert_linked_list_node(a, nil, nil) + if err := a.put_kv(Int(1), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(3), 3); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(5), 5); err != nil { + t.Error(err) + } + p, q, err := a.leaf_split(Int(2), 2) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if q == nil { + t.Errorf("q == nil") + } + if !p.has(Int(1)) || !p.has(Int(2)) { + t.Error("p didn't have the right keys", p) + } + if !q.has(Int(3)) || !q.has(Int(5)) { + t.Error("q didn't have the right keys", q) + } + } +} + +func Test_leaf_split_equal(t *testing.T) { + a := NewLeaf(3, false) + insert_linked_list_node(a, nil, nil) + if err := a.put_kv(Int(1), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(3), 3); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(5), 5); err != nil { + t.Error(err) + } + p, q, err := a.leaf_split(Int(3), 2) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if q == nil { + t.Errorf("q == nil") + } + if !p.has(Int(1)) { + t.Error("p didn't have the right keys", p) + } + if !q.has(Int(3)) || q.count(Int(3)) != 2 || !q.has(Int(5)) { + t.Error("q didn't have the right keys", q, q.count(Int(3))) + } + } +} + +func Test_leaf_split_greater(t *testing.T) { + a := NewLeaf(3, false) + insert_linked_list_node(a, nil, nil) + if err := a.put_kv(Int(1), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(3), 3); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(5), 5); err != nil { + t.Error(err) + } + p, q, err := a.leaf_split(Int(4), 2) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if q == nil { + t.Errorf("q == nil") + } + if !p.has(Int(1)) { + t.Error("p didn't have the right keys", p) + } + if !q.has(Int(3)) || !q.has(Int(4)) || !q.has(Int(5)) { + t.Error("q didn't have the right keys", q) + } + } +} + +// tests the defer logic +func Test_pure_leaf_insert_split_less(t *testing.T) { + a := NewLeaf(2, false) + insert_linked_list_node(a, nil, nil) + b := NewLeaf(2, false) + insert_linked_list_node(b, a, nil) + c := NewLeaf(2, false) + insert_linked_list_node(c, b, nil) + d := NewLeaf(2, false) + insert_linked_list_node(d, c, nil) + if err := a.put_kv(Int(3), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(3), 2); err != nil { + t.Error(err) + } + if err := b.put_kv(Int(3), 3); err != nil { + t.Error(err) + } + if err := b.put_kv(Int(3), 4); err != nil { + t.Error(err) + } + if err := c.put_kv(Int(3), 5); err != nil { + t.Error(err) + } + if err := c.put_kv(Int(3), 6); err != nil { + t.Error(err) + } + if err := d.put_kv(Int(4), 6); err != nil { + t.Error(err) + } + p, q, err := a.leaf_insert(Int(2), 1) + if err != nil { + t.Error(err) + } else { + if q != a { + t.Errorf("q != a") + } + if p == nil || len(p.keys) != 1 || !p.keys[0].Equals(Int(2)) { + t.Errorf("p did not contain the right key") + } + if p.prev != nil { + t.Errorf("expected p.prev == nil") + } + if p.next != a { + t.Errorf("expected p.next == a") + } + if a.prev != p { + t.Errorf("expected a.prev == p") + } + if a.next != b { + t.Errorf("expected a.next == b") + } + if b.prev != a { + t.Errorf("expected b.prev == a") + } + if b.next != c { + t.Errorf("expected b.next == c") + } + if c.prev != b { + t.Errorf("expected c.prev == b") + } + if c.next != d { + t.Errorf("expected c.next == d") + } + if d.prev != c { + t.Errorf("expected d.prev == c") + } + if d.next != nil { + t.Errorf("expected d.next == nil") + } + } +} + +func Test_pure_leaf_split_less(t *testing.T) { + a := NewLeaf(2, false) + insert_linked_list_node(a, nil, nil) + b := NewLeaf(2, false) + insert_linked_list_node(b, a, nil) + c := NewLeaf(2, false) + insert_linked_list_node(c, b, nil) + d := NewLeaf(2, false) + insert_linked_list_node(d, c, nil) + if err := a.put_kv(Int(3), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(3), 2); err != nil { + t.Error(err) + } + if err := b.put_kv(Int(3), 3); err != nil { + t.Error(err) + } + if err := b.put_kv(Int(3), 4); err != nil { + t.Error(err) + } + if err := c.put_kv(Int(3), 5); err != nil { + t.Error(err) + } + if err := c.put_kv(Int(3), 6); err != nil { + t.Error(err) + } + if err := d.put_kv(Int(4), 6); err != nil { + t.Error(err) + } + p, q, err := a.pure_leaf_split(Int(2), 1) + if err != nil { + t.Error(err) + } else { + if q != a { + t.Errorf("q != a") + } + if p == nil || len(p.keys) != 1 || !p.keys[0].Equals(Int(2)) { + t.Errorf("p did not contain the right key") + } + if p.prev != nil { + t.Errorf("expected p.prev == nil") + } + if p.next != a { + t.Errorf("expected p.next == a") + } + if a.prev != p { + t.Errorf("expected a.prev == p") + } + if a.next != b { + t.Errorf("expected a.next == b") + } + if b.prev != a { + t.Errorf("expected b.prev == a") + } + if b.next != c { + t.Errorf("expected b.next == c") + } + if c.prev != b { + t.Errorf("expected c.prev == b") + } + if c.next != d { + t.Errorf("expected c.next == d") + } + if d.prev != c { + t.Errorf("expected d.prev == c") + } + if d.next != nil { + t.Errorf("expected d.next == nil") + } + } +} + +func Test_pure_leaf_split_equal(t *testing.T) { + a := NewLeaf(2, false) + insert_linked_list_node(a, nil, nil) + b := NewLeaf(2, false) + insert_linked_list_node(b, a, nil) + c := NewLeaf(2, false) + insert_linked_list_node(c, b, nil) + d := NewLeaf(2, false) + insert_linked_list_node(d, c, nil) + if err := a.put_kv(Int(3), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(3), 2); err != nil { + t.Error(err) + } + if err := b.put_kv(Int(3), 3); err != nil { + t.Error(err) + } + if err := b.put_kv(Int(3), 4); err != nil { + t.Error(err) + } + if err := c.put_kv(Int(3), 5); err != nil { + t.Error(err) + } + if err := d.put_kv(Int(4), 6); err != nil { + t.Error(err) + } + p, q, err := a.pure_leaf_split(Int(3), 1) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if q != nil { + t.Errorf("q != nil") + } + if a.prev != nil { + t.Errorf("expected a.prev == nil") + } + if a.next != b { + t.Errorf("expected a.next == b") + } + if b.prev != a { + t.Errorf("expected b.prev == a") + } + if b.next != c { + t.Errorf("expected b.next == c") + } + if c.prev != b { + t.Errorf("expected c.prev == b") + } + if c.next != d { + t.Errorf("expected c.next == d") + } + if d.prev != c { + t.Errorf("expected d.prev == c") + } + if d.next != nil { + t.Errorf("expected d.next == nil") + } + } +} + +func Test_pure_leaf_split_greater(t *testing.T) { + a := NewLeaf(2, false) + insert_linked_list_node(a, nil, nil) + b := NewLeaf(2, false) + insert_linked_list_node(b, a, nil) + c := NewLeaf(2, false) + insert_linked_list_node(c, b, nil) + d := NewLeaf(2, false) + insert_linked_list_node(d, c, nil) + if err := a.put_kv(Int(3), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(3), 2); err != nil { + t.Error(err) + } + if err := b.put_kv(Int(3), 3); err != nil { + t.Error(err) + } + if err := b.put_kv(Int(3), 4); err != nil { + t.Error(err) + } + if err := c.put_kv(Int(3), 5); err != nil { + t.Error(err) + } + if err := d.put_kv(Int(5), 6); err != nil { + t.Error(err) + } + p, q, err := a.pure_leaf_split(Int(4), 1) + if err != nil { + t.Error(err) + } else { + if p != a { + t.Errorf("p != a") + } + if q == nil || len(q.keys) != 1 || !q.keys[0].Equals(Int(4)) { + t.Errorf("q != nil") + } + if a.prev != nil { + t.Errorf("expected a.prev == nil") + } + if a.next != b { + t.Errorf("expected a.next == b") + } + if b.prev != a { + t.Errorf("expected b.prev == a") + } + if b.next != c { + t.Errorf("expected b.next == c") + } + if c.prev != b { + t.Errorf("expected c.prev == b") + } + if c.next != q { + t.Errorf("expected c.next == q") + } + if q.prev != c { + t.Errorf("expected q.prev == c") + } + if q.next != d { + t.Errorf("expected q.next == d") + } + if d.prev != q { + t.Errorf("expected d.prev == q") + } + if d.next != nil { + t.Errorf("expected d.next == nil") + } + } +} + +func Test_find_end_of_pure_run(t *testing.T) { + a := NewLeaf(2, false) + insert_linked_list_node(a, nil, nil) + b := NewLeaf(2, false) + insert_linked_list_node(b, a, nil) + c := NewLeaf(2, false) + insert_linked_list_node(c, b, nil) + d := NewLeaf(2, false) + insert_linked_list_node(d, c, nil) + if err := a.put_kv(Int(3), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(3), 2); err != nil { + t.Error(err) + } + if err := b.put_kv(Int(3), 3); err != nil { + t.Error(err) + } + if err := b.put_kv(Int(3), 4); err != nil { + t.Error(err) + } + if err := c.put_kv(Int(3), 5); err != nil { + t.Error(err) + } + if err := c.put_kv(Int(3), 6); err != nil { + t.Error(err) + } + if err := d.put_kv(Int(4), 6); err != nil { + t.Error(err) + } + e := a.find_end_of_pure_run() + if e != c { + t.Errorf("end of run should have been block c %v %v", e, c) + } +} + +func Test_insert_linked_list_node(t *testing.T) { + a := NewLeaf(1, false) + insert_linked_list_node(a, nil, nil) + b := NewLeaf(2, false) + insert_linked_list_node(b, a, nil) + c := NewLeaf(3, false) + insert_linked_list_node(c, b, nil) + d := NewLeaf(4, false) + insert_linked_list_node(d, a, b) + if a.prev != nil { + t.Errorf("expected a.prev == nil") + } + if a.next != d { + t.Errorf("expected a.next == d") + } + if d.prev != a { + t.Errorf("expected d.prev == a") + } + if d.next != b { + t.Errorf("expected d.next == b") + } + if b.prev != d { + t.Errorf("expected b.prev == d") + } + if b.next != c { + t.Errorf("expected b.next == c") + } + if c.prev != b { + t.Errorf("expected c.prev == b") + } + if c.next != nil { + t.Errorf("expected c.next == nil") + } +} + +func Test_remove_linked_list_node(t *testing.T) { + a := NewLeaf(1, false) + insert_linked_list_node(a, nil, nil) + b := NewLeaf(2, false) + insert_linked_list_node(b, a, nil) + c := NewLeaf(3, false) + insert_linked_list_node(c, b, nil) + d := NewLeaf(4, false) + insert_linked_list_node(d, a, b) + if a.prev != nil { + t.Errorf("expected a.prev == nil") + } + if a.next != d { + t.Errorf("expected a.next == d") + } + if d.prev != a { + t.Errorf("expected d.prev == a") + } + if d.next != b { + t.Errorf("expected d.next == b") + } + if b.prev != d { + t.Errorf("expected b.prev == d") + } + if b.next != c { + t.Errorf("expected b.next == c") + } + if c.prev != b { + t.Errorf("expected c.prev == b") + } + if c.next != nil { + t.Errorf("expected c.next == nil") + } + remove_linked_list_node(d) + if a.prev != nil { + t.Errorf("expected a.prev == nil") + } + if a.next != b { + t.Errorf("expected a.next == b") + } + if b.prev != a { + t.Errorf("expected b.prev == a") + } + if b.next != c { + t.Errorf("expected b.next == c") + } + if c.prev != b { + t.Errorf("expected c.prev == b") + } + if c.next != nil { + t.Errorf("expected c.next == nil") + } + remove_linked_list_node(a) + if b.prev != nil { + t.Errorf("expected b.prev == nil") + } + if b.next != c { + t.Errorf("expected b.next == c") + } + if c.prev != b { + t.Errorf("expected c.prev == b") + } + if c.next != nil { + t.Errorf("expected c.next == nil") + } + remove_linked_list_node(c) + if b.prev != nil { + t.Errorf("expected b.prev == nil") + } + if b.next != nil { + t.Errorf("expected b.next == nil") + } + remove_linked_list_node(b) +} + +func Test_balance_leaf_nodes_with_dup(t *testing.T) { + a := NewLeaf(3, false) + b := NewLeaf(3, false) + if err := a.put_kv(Int(1), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(1), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(2), 1); err != nil { + t.Error(err) + } + balance_nodes(a, b) + if !a.has(Int(1)) || a.count(Int(1)) != 2 || a.has(Int(2)) { + t.Error("a had wrong items", a) + } + if !b.has(Int(2)) || b.count(Int(2)) != 1 || b.has(Int(1)) { + t.Error("a had wrong items", b) + } +} + +func Test_balance_leaf_nodes(t *testing.T) { + a := NewLeaf(7, false) + b := NewLeaf(7, false) + if err := a.put_kv(Int(1), 1); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(2), 2); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(3), 3); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(4), 4); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(5), 5); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(6), 6); err != nil { + t.Error(err) + } + if err := a.put_kv(Int(7), 7); err != nil { + t.Error(err) + } + balance_nodes(a, b) + for i, k := range a.keys { + if int(k.(Int)) != i+1 { + t.Errorf("k != %d", i+1) + } + } + for i, k := range b.keys { + if int(k.(Int)) != 3+i+1 { + t.Errorf("k != %d", 3+i+1) + } + } + for i, v := range a.values { + if v.(int) != i+1 { + t.Errorf("k != %d", i+1) + } + } + for i, v := range b.values { + if v.(int) != 3+i+1 { + t.Errorf("v != %d", 3+i+1) + } + } + t.Log(a) + t.Log(b) +} + +func Test_balance_internal_nodes(t *testing.T) { + a := NewInternal(6) + b := NewInternal(6) + if err := a.put_kp(Int(1), nil); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(2), nil); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(3), nil); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(4), nil); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(5), nil); err != nil { + t.Error(err) + } + if err := a.put_kp(Int(6), nil); err != nil { + t.Error(err) + } + balance_nodes(a, b) + for i, k := range a.keys { + if int(k.(Int)) != i+1 { + t.Errorf("k != %d", i+1) + } + } + for i, k := range b.keys { + if int(k.(Int)) != 3+i+1 { + t.Errorf("k != %d", 3+i+1) + } + } + t.Log(a) + t.Log(b) +} + + +// copied from + +// ThreadSafeRand provides a thread safe version of math/rand.Rand using +// the same technique used in the math/rand package to make the top level +// functions thread safe. +func ThreadSafeRand(seed int64) *mrand.Rand { + return mrand.New(&lockedSource{src: mrand.NewSource(seed).(mrand.Source64)}) +} + +// from: https://golang.org/src/math/rand/rand.go?s=8161:8175#L317 +type lockedSource struct { + lk sync.Mutex + src mrand.Source64 +} + +func (r *lockedSource) Int63() (n int64) { + r.lk.Lock() + n = r.src.Int63() + r.lk.Unlock() + return +} + +func (r *lockedSource) Uint64() (n uint64) { + r.lk.Lock() + n = r.src.Uint64() + r.lk.Unlock() + return +} + +func (r *lockedSource) Seed(seed int64) { + r.lk.Lock() + r.src.Seed(seed) + r.lk.Unlock() +} + +// seedPos implements Seed for a lockedSource without a race condiiton. +func (r *lockedSource) seedPos(seed int64, readPos *int8) { + r.lk.Lock() + r.src.Seed(seed) + *readPos = 0 + r.lk.Unlock() +} + +// read implements Read for a lockedSource without a race condition. +func (r *lockedSource) read(p []byte, readVal *int64, readPos *int8) (n int, err error) { + r.lk.Lock() + n, err = read(p, r.src.Int63, readVal, readPos) + r.lk.Unlock() + return +} + +func read(p []byte, int63 func() int64, readVal *int64, readPos *int8) (n int, err error) { + pos := *readPos + val := *readVal + for n = 0; n < len(p); n++ { + if pos == 0 { + val = int63() + pos = 7 + } + p[n] = byte(val) + val >>= 8 + pos-- + } + *readPos = pos + *readVal = val + return +} + +// copied from https://sourcegraph.com/github.com/timtadh/data-structures@master/-/blob/test/support.go + +type T testing.T + +func (t *T) Assert(ok bool, msg string, vars ...interface{}) { + if !ok { + t.Log("\n" + string(debug.Stack())) + t.Fatalf(msg, vars...) + } +} + +func (t *T) AssertNil(errors ...error) { + any := false + for _, err := range errors { + if err != nil { + any = true + t.Log("\n" + string(debug.Stack())) + t.Error(err) + } + } + if any { + t.Fatal("assert failed") + } +} + +func RandSlice(length int) []byte { + slice := make([]byte, length) + if _, err := crand.Read(slice); err != nil { + panic(err) + } + return slice +} + +func RandHex(length int) string { + return hex.EncodeToString(RandSlice(length / 2)) +} + +func RandStr(length int) string { + return string(RandSlice(length)) +} \ No newline at end of file diff --git a/weed/util/bptree/int.go b/weed/util/bptree/int.go new file mode 100644 index 000000000..e8fd9511c --- /dev/null +++ b/weed/util/bptree/int.go @@ -0,0 +1,357 @@ +package bptree + +import ( + "encoding/binary" + "fmt" +) + +type Int8 int8 +type UInt8 uint8 +type Int16 int16 +type UInt16 uint16 +type Int32 int32 +type UInt32 uint32 +type Int64 int64 +type UInt64 uint64 +type Int int +type UInt uint + +func (self *Int8) MarshalBinary() ([]byte, error) { + bytes := make([]byte, 0) + bytes[0] = uint8(*self) + return bytes, nil +} + +func (self *Int8) UnmarshalBinary(data []byte) error { + if len(data) != 1 { + return fmt.Errorf("data wrong size") + } + *self = Int8(data[0]) + return nil +} + +func (self Int8) Equals(other Equatable) bool { + if o, ok := other.(Int8); ok { + return self == o + } else { + return false + } +} + +func (self Int8) Less(other Sortable) bool { + if o, ok := other.(Int8); ok { + return self < o + } else { + return false + } +} + +func (self Int8) Hash() int { + return int(self) +} + +func (self *UInt8) MarshalBinary() ([]byte, error) { + bytes := make([]byte, 0) + bytes[0] = uint8(*self) + return bytes, nil +} + +func (self *UInt8) UnmarshalBinary(data []byte) error { + if len(data) != 1 { + return fmt.Errorf("data wrong size") + } + *self = UInt8(data[0]) + return nil +} + +func (self UInt8) Equals(other Equatable) bool { + if o, ok := other.(UInt8); ok { + return self == o + } else { + return false + } +} + +func (self UInt8) Less(other Sortable) bool { + if o, ok := other.(UInt8); ok { + return self < o + } else { + return false + } +} + +func (self UInt8) Hash() int { + return int(self) +} + +func (self *Int16) MarshalBinary() ([]byte, error) { + bytes := make([]byte, 2) + binary.BigEndian.PutUint16(bytes, uint16(*self)) + return bytes, nil +} + +func (self *Int16) UnmarshalBinary(data []byte) error { + if len(data) != 2 { + return fmt.Errorf("data wrong size") + } + *self = Int16(binary.BigEndian.Uint16(data)) + return nil +} + +func (self Int16) Equals(other Equatable) bool { + if o, ok := other.(Int16); ok { + return self == o + } else { + return false + } +} + +func (self Int16) Less(other Sortable) bool { + if o, ok := other.(Int16); ok { + return self < o + } else { + return false + } +} + +func (self Int16) Hash() int { + return int(self) +} + +func (self *UInt16) MarshalBinary() ([]byte, error) { + bytes := make([]byte, 2) + binary.BigEndian.PutUint16(bytes, uint16(*self)) + return bytes, nil +} + +func (self *UInt16) UnmarshalBinary(data []byte) error { + if len(data) != 2 { + return fmt.Errorf("data wrong size") + } + *self = UInt16(binary.BigEndian.Uint16(data)) + return nil +} + +func (self UInt16) Equals(other Equatable) bool { + if o, ok := other.(UInt16); ok { + return self == o + } else { + return false + } +} + +func (self UInt16) Less(other Sortable) bool { + if o, ok := other.(UInt16); ok { + return self < o + } else { + return false + } +} + +func (self UInt16) Hash() int { + return int(self) +} + +func (self *Int32) MarshalBinary() ([]byte, error) { + bytes := make([]byte, 4) + binary.BigEndian.PutUint32(bytes, uint32(*self)) + return bytes, nil +} + +func (self *Int32) UnmarshalBinary(data []byte) error { + if len(data) != 4 { + return fmt.Errorf("data wrong size") + } + *self = Int32(binary.BigEndian.Uint32(data)) + return nil +} + +func (self Int32) Equals(other Equatable) bool { + if o, ok := other.(Int32); ok { + return self == o + } else { + return false + } +} + +func (self Int32) Less(other Sortable) bool { + if o, ok := other.(Int32); ok { + return self < o + } else { + return false + } +} + +func (self *UInt32) MarshalBinary() ([]byte, error) { + bytes := make([]byte, 4) + binary.BigEndian.PutUint32(bytes, uint32(*self)) + return bytes, nil +} + +func (self *UInt32) UnmarshalBinary(data []byte) error { + if len(data) != 4 { + return fmt.Errorf("data wrong size") + } + *self = UInt32(binary.BigEndian.Uint32(data)) + return nil +} + +func (self Int32) Hash() int { + return int(self) +} + +func (self UInt32) Equals(other Equatable) bool { + if o, ok := other.(UInt32); ok { + return self == o + } else { + return false + } +} + +func (self UInt32) Less(other Sortable) bool { + if o, ok := other.(UInt32); ok { + return self < o + } else { + return false + } +} + +func (self UInt32) Hash() int { + return int(self) +} + +func (self *Int64) MarshalBinary() ([]byte, error) { + bytes := make([]byte, 8) + binary.BigEndian.PutUint64(bytes, uint64(*self)) + return bytes, nil +} + +func (self *Int64) UnmarshalBinary(data []byte) error { + if len(data) != 8 { + return fmt.Errorf("data wrong size") + } + *self = Int64(binary.BigEndian.Uint64(data)) + return nil +} + +func (self Int64) Equals(other Equatable) bool { + if o, ok := other.(Int64); ok { + return self == o + } else { + return false + } +} + +func (self Int64) Less(other Sortable) bool { + if o, ok := other.(Int64); ok { + return self < o + } else { + return false + } +} + +func (self Int64) Hash() int { + return int(self>>32) ^ int(self) +} + +func (self *UInt64) MarshalBinary() ([]byte, error) { + bytes := make([]byte, 8) + binary.BigEndian.PutUint64(bytes, uint64(*self)) + return bytes, nil +} + +func (self *UInt64) UnmarshalBinary(data []byte) error { + if len(data) != 8 { + return fmt.Errorf("data wrong size") + } + *self = UInt64(binary.BigEndian.Uint64(data)) + return nil +} + +func (self UInt64) Equals(other Equatable) bool { + if o, ok := other.(UInt64); ok { + return self == o + } else { + return false + } +} + +func (self UInt64) Less(other Sortable) bool { + if o, ok := other.(UInt64); ok { + return self < o + } else { + return false + } +} + +func (self UInt64) Hash() int { + return int(self>>32) ^ int(self) +} + +func (self *Int) MarshalBinary() ([]byte, error) { + bytes := make([]byte, 4) + binary.BigEndian.PutUint32(bytes, uint32(*self)) + return bytes, nil +} + +func (self *Int) UnmarshalBinary(data []byte) error { + if len(data) != 4 { + return fmt.Errorf("data wrong size") + } + *self = Int(binary.BigEndian.Uint32(data)) + return nil +} + +func (self Int) Equals(other Equatable) bool { + if o, ok := other.(Int); ok { + return self == o + } else { + return false + } +} + +func (self Int) Less(other Sortable) bool { + if o, ok := other.(Int); ok { + return self < o + } else { + return false + } +} + +func (self Int) Hash() int { + return int(self) +} + +func (self *UInt) MarshalBinary() ([]byte, error) { + bytes := make([]byte, 4) + binary.BigEndian.PutUint32(bytes, uint32(*self)) + return bytes, nil +} + +func (self *UInt) UnmarshalBinary(data []byte) error { + if len(data) != 4 { + return fmt.Errorf("data wrong size") + } + *self = UInt(binary.BigEndian.Uint32(data)) + return nil +} + +func (self UInt) Equals(other Equatable) bool { + if o, ok := other.(UInt); ok { + return self == o + } else { + return false + } +} + +func (self UInt) Less(other Sortable) bool { + if o, ok := other.(UInt); ok { + return self < o + } else { + return false + } +} + +func (self UInt) Hash() int { + return int(self) +} diff --git a/weed/util/bptree/rand.go b/weed/util/bptree/rand.go new file mode 100644 index 000000000..08b2e50ab --- /dev/null +++ b/weed/util/bptree/rand.go @@ -0,0 +1,2 @@ +package bptree + diff --git a/weed/util/bptree/string.go b/weed/util/bptree/string.go new file mode 100644 index 000000000..262220878 --- /dev/null +++ b/weed/util/bptree/string.go @@ -0,0 +1,71 @@ +package bptree + +import ( + "bytes" + "hash/fnv" +) + +type String string +type ByteSlice []byte + +func (self *String) MarshalBinary() ([]byte, error) { + return []byte(*self), nil +} + +func (self *String) UnmarshalBinary(data []byte) error { + *self = String(data) + return nil +} + +func (self String) Equals(other Equatable) bool { + if o, ok := other.(String); ok { + return self == o + } else { + return false + } +} + +func (self String) Less(other Sortable) bool { + if o, ok := other.(String); ok { + return self < o + } else { + return false + } +} + +func (self String) Hash() int { + h := fnv.New32a() + h.Write([]byte(string(self))) + return int(h.Sum32()) +} + +func (self *ByteSlice) MarshalBinary() ([]byte, error) { + return []byte(*self), nil +} + +func (self *ByteSlice) UnmarshalBinary(data []byte) error { + *self = ByteSlice(data) + return nil +} + +func (self ByteSlice) Equals(other Equatable) bool { + if o, ok := other.(ByteSlice); ok { + return bytes.Equal(self, o) + } else { + return false + } +} + +func (self ByteSlice) Less(other Sortable) bool { + if o, ok := other.(ByteSlice); ok { + return bytes.Compare(self, o) < 0 // -1 if a < b + } else { + return false + } +} + +func (self ByteSlice) Hash() int { + h := fnv.New32a() + h.Write([]byte(self)) + return int(h.Sum32()) +} diff --git a/weed/util/bptree/types.go b/weed/util/bptree/types.go new file mode 100644 index 000000000..6a1d83098 --- /dev/null +++ b/weed/util/bptree/types.go @@ -0,0 +1,103 @@ +package bptree + +import ( + "errors" + "fmt" +) + +type Equatable interface { + Equals(b Equatable) bool +} + +type Sortable interface { + Equatable + Less(b Sortable) bool +} + +type Hashable interface { + Sortable + Hash() int +} + +var BpTreeError = fmt.Errorf + +func NegativeSize() error { + return errors.New("negative size") +} + +type Iterator func() (item interface{}, next Iterator) +type KIterator func() (key Hashable, next KIterator) +type KVIterator func() (key Hashable, value interface{}, next KVIterator) +type KVIterable interface { + Iterate() KVIterator +} + +type Sized interface { + Size() int +} + +type MapOperable interface { + Sized + Has(key Hashable) bool + Put(key Hashable, value interface{}) (err error) + Get(key Hashable) (value interface{}, err error) + Remove(key Hashable) (value interface{}, err error) +} + +type WhereFunc func(value interface{}) bool + +func MakeValuesIterator(obj KVIterable) Iterator { + kv_iterator := obj.Iterate() + var v_iterator Iterator + v_iterator = func() (value interface{}, next Iterator) { + _, value, kv_iterator = kv_iterator() + if kv_iterator == nil { + return nil, nil + } + return value, v_iterator + } + return v_iterator +} + +func MakeItemsIterator(obj KVIterable) (kit KIterator) { + kv_iterator := obj.Iterate() + kit = func() (item Hashable, next KIterator) { + var key Hashable + var value interface{} + key, value, kv_iterator = kv_iterator() + if kv_iterator == nil { + return nil, nil + } + return &MapEntry{key, value}, kit + } + return kit +} + +type MapEntry struct { + Key Hashable + Value interface{} +} + +func (m *MapEntry) Equals(other Equatable) bool { + if o, ok := other.(*MapEntry); ok { + return m.Key.Equals(o.Key) + } else { + return m.Key.Equals(other) + } +} + +func (m *MapEntry) Less(other Sortable) bool { + if o, ok := other.(*MapEntry); ok { + return m.Key.Less(o.Key) + } else { + return m.Key.Less(other) + } +} + +func (m *MapEntry) Hash() int { + return m.Key.Hash() +} + +func (m *MapEntry) String() string { + return fmt.Sprintf("", m.Key, m.Value) +}