gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

remove bptree

Browse source
This commit is contained in:
Chris Lu 2021-10-02 14:03:54 -07:00
parent 4c1741fdbb
commit 57e2fd3f9b
18 changed files with 0 additions and 3421 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
weed/util/bptree/Makefile
View file

@ -1,6 +0,0 @@
all: gen
.PHONY : gen
gen:
protoc bptree.proto --go_out=plugins=grpc:. --go_opt=paths=source_relative

60
weed/util/bptree/README.md
View file

@ -1,60 +0,0 @@
This adapts one b+ tree implementation
https://sourcegraph.com/github.com/timtadh/data-structures@master/-/tree/tree/bptree
to persist changes to on disk.
# When a node needs to persist itself?
* A node changed its key or value
* When an item is added.
* When an item is updated.
* When an item is deleted.
* When a node is split.
* 2 new nodes are created (they shoud persist themselves).
* Parent node need to point to the new nodes.
* When a node is merged.
* delete one node
* persist the merged node
In general, if one node is returned from a function, the node should have already been persisted.
The parent node may need to delete the old node.
BpTree
Add(key ItemKey, value ItemValue)
new_root = self.getRoot().put(key,value)
a, b, err := self.insert(key, value)
self.internal_insert(key, value)
self.internal_split(q.keys[0], q)
persist(a,b)
self.persist() // child add q node
self.maybePersist(child == p)
self.leaf_insert(key, value)
self.persist() // if dedup
self.leaf_split(key, value)
self.pure_leaf_split(key, value)
persist(a,b)
a.persist()
persist(a,b)
self.put_kv(key, value)
new_root.persist()
self.setRoot(new_root)
oldroot.destroy()
// maybe persist BpTree new root
Replace(key ItemKey, where WhereFunc, value ItemValue)
leaf.persist()
RemoveWhere(key ItemKey, where WhereFunc)
self.getRoot().remove(key, where)
self.internal_remove(key, nil, where)
child.leaf_remove(key, nil, where)
child.leaf_remove(key, sibling.keys[0], where)
l.destroy() // when the node is empty
a.maybePersist(hasChange)
self.destroy() // when no keys left
self.persist() // when some keys are left
self.leaf_remove(key, self.keys[len(self.keys)-1], where)
new_root.persist() // when new root is added
// maybe persist BpTree new root

69
weed/util/bptree/bpmap.go
View file

@ -1,69 +0,0 @@
package bptree
import (
"fmt"
)
/* A BpMap is a B+Tree with support for duplicate keys disabled. This makes it
* behave like a regular Map rather than a MultiMap.
*/
type BpMap BpTree
func NewBpMap(node_size int, nodeStore NodeStore) *BpMap {
return &BpMap{
root: NewLeaf(node_size, nodeStore),
}
}
func (self *BpMap) Has(key ItemKey) bool {
return (*BpTree)(self).Has(key)
}
func (self *BpMap) Put(key ItemKey, value ItemValue) (err error) {
new_root, err := self.getRoot().put(key, value)
if err != nil {
return err
}
self.setRoot(new_root)
return nil
}
func (self *BpMap) Get(key ItemKey) (value ItemValue, err error) {
j, l := self.getRoot().get_start(key)
if l.keys[j].Equals(key) {
return l.values[j], nil
}
return nil, fmt.Errorf("key not found: %s", key)
}
func (self *BpMap) Remove(key ItemKey) (value ItemValue, err error) {
value, err = self.Get(key)
if err != nil {
return nil, err
}
ns := self.getRoot().Capacity()
new_root, err := self.getRoot().remove(key, func(value ItemValue) bool { return true })
if err != nil {
return nil, err
}
if new_root == nil {
new_root = NewLeaf(ns, self.root.nodeStore)
err = new_root.persist()
self.setRoot(new_root)
} else {
self.setRoot(new_root)
}
return value, nil
}
func (self *BpMap) Keys() (ki KIterator) {
return (*BpTree)(self).Keys()
}
func (self *BpMap) Values() (vi Iterator) {
return (*BpTree)(self).Values()
}
func (self *BpMap) Iterate() (kvi KVIterator) {
return (*BpTree)(self).Iterate()
}

158
weed/util/bptree/bptree.go
View file

@ -1,158 +0,0 @@
package bptree
// started by copying from https://sourcegraph.com/github.com/timtadh/data-structures@master/-/tree/tree/bptree
/* A BpTree is a B+Tree with support for duplicate keys. This makes it behave as
* a MultiMap. Additionally you can use the Range operator to select k/v in a
* range. If from > to it will iterate backwards.
*/
type BpTree struct {
root *BpNode
}
type loc_iterator func() (i int, leaf *BpNode, li loc_iterator)
func NewBpTree(node_size int, nodeStore NodeStore) *BpTree {
return &BpTree{
root: NewLeaf(node_size, nodeStore),
}
}
func (self *BpTree) Has(key ItemKey) bool {
if len(self.getRoot().keys) == 0 {
return false
}
j, l := self.getRoot().get_start(key)
return l.keys[j].Equals(key)
}
func (self *BpTree) Count(key ItemKey) int {
if len(self.root.keys) == 0 {
return 0
}
j, l := self.root.get_start(key)
count := 0
end := false
for !end && l.keys[j].Equals(key) {
count++
j, l, end = next_location(j, l)
}
return count
}
func (self *BpTree) Add(key ItemKey, value ItemValue) (err error) {
new_root, err := self.getRoot().put(key, value)
if err != nil {
return err
}
self.setRoot(new_root)
return nil
}
func (self *BpTree) Replace(key ItemKey, where WhereFunc, value ItemValue) (err error) {
li := self.getRoot().forward(key, key)
for i, leaf, next := li(); next != nil; i, leaf, next = next() {
if where(leaf.values[i]) {
leaf.values[i] = value
if persistErr := leaf.persist(); persistErr != nil && err == nil {
err = persistErr
break
}
}
}
return err
}
func (self *BpTree) Find(key ItemKey) (kvi KVIterator) {
return self.Range(key, key)
}
func (self *BpTree) Range(from, to ItemKey) (kvi KVIterator) {
var li loc_iterator
if !to.Less(from) {
li = self.getRoot().forward(from, to)
} else {
li = self.getRoot().backward(from, to)
}
kvi = func() (key ItemKey, value ItemValue, next KVIterator) {
var i int
var leaf *BpNode
i, leaf, li = li()
if li == nil {
return nil, nil, nil
}
return leaf.keys[i], leaf.values[i], kvi
}
return kvi
}
func (self *BpTree) RemoveWhere(key ItemKey, where WhereFunc) (err error) {
ns := self.getRoot().Capacity()
new_root, err := self.getRoot().remove(key, where)
if err != nil {
return err
}
if new_root == nil {
new_root = NewLeaf(ns, self.root.nodeStore)
err = new_root.persist()
self.setRoot(new_root)
} else {
self.setRoot(new_root)
}
return err
}
func (self *BpTree) Keys() (ki KIterator) {
li := self.getRoot().all()
var prev Equatable
ki = func() (key ItemKey, next KIterator) {
var i int
var leaf *BpNode
i, leaf, li = li()
if li == nil {
return nil, nil
}
if leaf.keys[i].Equals(prev) {
return ki()
}
prev = leaf.keys[i]
return leaf.keys[i], ki
}
return ki
}
func (self *BpTree) Values() (vi Iterator) {
return MakeValuesIterator(self)
}
func (self *BpTree) Items() (vi KIterator) {
return MakeItemsIterator(self)
}
func (self *BpTree) Iterate() (kvi KVIterator) {
li := self.getRoot().all()
kvi = func() (key ItemKey, value ItemValue, next KVIterator) {
var i int
var leaf *BpNode
i, leaf, li = li()
if li == nil {
return nil, nil, nil
}
return leaf.keys[i], leaf.values[i], kvi
}
return kvi
}
func (self *BpTree) Backward() (kvi KVIterator) {
li := self.getRoot().all_backward()
kvi = func() (key ItemKey, value ItemValue, next KVIterator) {
var i int
var leaf *BpNode
i, leaf, li = li()
if li == nil {
return nil, nil, nil
}
return leaf.keys[i], leaf.values[i], kvi
}
return kvi
}

195
weed/util/bptree/bptree.pb.go
View file

@ -1,195 +0,0 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.25.0
// protoc v3.12.3
// source: bptree.proto
package bptree
import (
proto "github.com/golang/protobuf/proto"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
// This is a compile-time assertion that a sufficiently up-to-date version
// of the legacy proto package is being used.
const _ = proto.ProtoPackageIsVersion4
type ProtoNode struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Keys [][]byte `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"`
Values [][]byte `protobuf:"bytes,2,rep,name=values,proto3" json:"values,omitempty"`
Pointers []int64 `protobuf:"varint,3,rep,packed,name=pointers,proto3" json:"pointers,omitempty"`
Next int64 `protobuf:"varint,4,opt,name=next,proto3" json:"next,omitempty"`
Prev int64 `protobuf:"varint,5,opt,name=prev,proto3" json:"prev,omitempty"`
Id int64 `protobuf:"varint,6,opt,name=id,proto3" json:"id,omitempty"`
}
func (x *ProtoNode) Reset() {
*x = ProtoNode{}
if protoimpl.UnsafeEnabled {
mi := &file_bptree_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *ProtoNode) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ProtoNode) ProtoMessage() {}
func (x *ProtoNode) ProtoReflect() protoreflect.Message {
mi := &file_bptree_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ProtoNode.ProtoReflect.Descriptor instead.
func (*ProtoNode) Descriptor() ([]byte, []int) {
return file_bptree_proto_rawDescGZIP(), []int{0}
}
func (x *ProtoNode) GetKeys() [][]byte {
if x != nil {
return x.Keys
}
return nil
}
func (x *ProtoNode) GetValues() [][]byte {
if x != nil {
return x.Values
}
return nil
}
func (x *ProtoNode) GetPointers() []int64 {
if x != nil {
return x.Pointers
}
return nil
}
func (x *ProtoNode) GetNext() int64 {
if x != nil {
return x.Next
}
return 0
}
func (x *ProtoNode) GetPrev() int64 {
if x != nil {
return x.Prev
}
return 0
}
func (x *ProtoNode) GetId() int64 {
if x != nil {
return x.Id
}
return 0
}
var File_bptree_proto protoreflect.FileDescriptor
var file_bptree_proto_rawDesc = []byte{
0x0a, 0x0c, 0x62, 0x70, 0x74, 0x72, 0x65, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06,
0x62, 0x70, 0x74, 0x72, 0x65, 0x65, 0x22, 0x8b, 0x01, 0x0a, 0x09, 0x50, 0x72, 0x6f, 0x74, 0x6f,
0x4e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03,
0x28, 0x0c, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75,
0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73,
0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03,
0x28, 0x03, 0x52, 0x08, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x12, 0x12, 0x0a, 0x04,
0x6e, 0x65, 0x78, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6e, 0x65, 0x78, 0x74,
0x12, 0x12, 0x0a, 0x04, 0x70, 0x72, 0x65, 0x76, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04,
0x70, 0x72, 0x65, 0x76, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03,
0x52, 0x02, 0x69, 0x64, 0x42, 0x31, 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65, 0x61,
0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x75, 0x74, 0x69, 0x6c,
0x2f, 0x62, 0x70, 0x74, 0x72, 0x65, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
file_bptree_proto_rawDescOnce sync.Once
file_bptree_proto_rawDescData = file_bptree_proto_rawDesc
)
func file_bptree_proto_rawDescGZIP() []byte {
file_bptree_proto_rawDescOnce.Do(func() {
file_bptree_proto_rawDescData = protoimpl.X.CompressGZIP(file_bptree_proto_rawDescData)
})
return file_bptree_proto_rawDescData
}
var file_bptree_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
var file_bptree_proto_goTypes = []interface{}{
(*ProtoNode)(nil), // 0: bptree.ProtoNode
}
var file_bptree_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_bptree_proto_init() }
func file_bptree_proto_init() {
if File_bptree_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_bptree_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*ProtoNode); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_bptree_proto_rawDesc,
NumEnums: 0,
NumMessages: 1,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_bptree_proto_goTypes,
DependencyIndexes: file_bptree_proto_depIdxs,
MessageInfos: file_bptree_proto_msgTypes,
}.Build()
File_bptree_proto = out.File
file_bptree_proto_rawDesc = nil
file_bptree_proto_goTypes = nil
file_bptree_proto_depIdxs = nil
}

14
weed/util/bptree/bptree.proto
View file

@ -1,14 +0,0 @@
syntax = "proto3";
package bptree;
option go_package = "github.com/chrislusf/seaweedfs/weed/util/bptree";
message ProtoNode {
repeated bytes keys = 1;
repeated bytes values = 2;
repeated int64 pointers = 3;
int64 next = 4;
int64 prev = 5;
int64 id = 6;
}

767
weed/util/bptree/bptree_node.go
View file

@ -1,767 +0,0 @@
package bptree
type ItemKey Hashable
type ItemValue Equatable
type NodeStore interface {
PersistFunc(node *BpNode) error
DestroyFunc(node *BpNode) error
}
type BpNode struct {
keys []ItemKey
values []ItemValue
pointers []*BpNode
next *BpNode
prev *BpNode
protoNodeId int64
protoNode *ProtoNode
nodeStore NodeStore
}
func NewInternal(size int, nodeStore NodeStore) *BpNode {
if size < 0 {
panic(NegativeSize())
}
return &BpNode{
keys: make([]ItemKey, 0, size),
pointers: make([]*BpNode, 0, size),
protoNodeId: GetProtoNodeId(),
nodeStore: nodeStore,
}
}
func NewLeaf(size int, nodeStore NodeStore) *BpNode {
if size < 0 {
panic(NegativeSize())
}
return &BpNode{
keys: make([]ItemKey, 0, size),
values: make([]ItemValue, 0, size),
protoNodeId: GetProtoNodeId(),
nodeStore: nodeStore,
}
}
func (self *BpNode) Full() bool {
return len(self.keys) == cap(self.keys)
}
func (self *BpNode) Pure() bool {
if len(self.keys) == 0 {
return true
}
k0 := self.keys[0]
for _, k := range self.keys {
if !k0.Equals(k) {
return false
}
}
return true
}
func (self *BpNode) Internal() bool {
return cap(self.pointers) > 0
}
func (self *BpNode) Len() int {
return len(self.keys)
}
func (self *BpNode) Capacity() int {
return cap(self.keys)
}
func (self *BpNode) Height() int {
if !self.Internal() {
return 1
} else if len(self.pointers) == 0 {
panic(BpTreeError("Internal node has no pointers but asked for height"))
}
return self.pointers[0].Height() + 1
}
func (self *BpNode) has(key ItemKey) bool {
_, has := self.find(key)
return has
}
func (self *BpNode) left_most_leaf() *BpNode {
if self.Internal() {
return self.pointers[0].left_most_leaf()
}
return self
}
func (self *BpNode) right_most_leaf() *BpNode {
if self.Internal() {
return self.pointers[len(self.pointers)-1].right_most_leaf()
}
return self
}
/* returns the index and leaf-block of the first key greater than or equal to
* the search key. (unless the search key is greater than all the keys in the
* tree, in that case it will be the last key in the tree)
*/
func (self *BpNode) get_start(key ItemKey) (i int, leaf *BpNode) {
if self.Internal() {
return self.internal_get_start(key)
} else {
return self.leaf_get_start(key)
}
}
func next_location(i int, leaf *BpNode) (int, *BpNode, bool) {
j := i + 1
for j >= len(leaf.keys) && leaf.getNext() != nil {
j = 0
leaf = leaf.getNext()
}
if j >= len(leaf.keys) {
return -1, nil, true
}
return j, leaf, false
}
func prev_location(i int, leaf *BpNode) (int, *BpNode, bool) {
j := i - 1
for j < 0 && leaf.getPrev() != nil {
leaf = leaf.getPrev()
j = len(leaf.keys) - 1
}
if j < 0 {
return -1, nil, true
}
return j, leaf, false
}
/* returns the index and leaf-block of the last key equal to the search key or
* the first key greater than the search key. (unless the search key is greater
* than all the keys in the tree, in that case it will be the last key in the
* tree)
*/
func (self *BpNode) get_end(key ItemKey) (i int, leaf *BpNode) {
end := false
i, leaf = self.get_start(key)
pi, pleaf := i, leaf
for !end && leaf.keys[i].Equals(key) {
pi, pleaf = i, leaf
i, leaf, end = next_location(i, leaf)
}
return pi, pleaf
}
func (self *BpNode) internal_get_start(key ItemKey) (i int, leaf *BpNode) {
if !self.Internal() {
panic(BpTreeError("Expected a internal node"))
}
i, has := self.find(key)
if !has && i > 0 {
// if it doesn't have it and the index > 0 then we have the next block
// so we have to subtract one from the index.
i--
}
child := self.pointers[i]
return child.get_start(key)
}
func (self *BpNode) leaf_get_start(key ItemKey) (i int, leaf *BpNode) {
i, has := self.find(key)
if i >= len(self.keys) && i > 0 {
i = len(self.keys) - 1
}
if !has && (len(self.keys) == 0 || self.keys[i].Less(key)) && self.getNext() != nil {
return self.getNext().leaf_get_start(key)
}
return i, self
}
/* This puts the k/v pair into the B+Tree rooted at this node and returns the
* (possibly) new root of the tree.
*/
func (self *BpNode) put(key ItemKey, value ItemValue) (root *BpNode, err error) {
a, b, err := self.insert(key, value)
if err != nil {
return nil, err
} else if b == nil {
return a, nil
}
// else we have root split
root = NewInternal(self.Capacity(), self.nodeStore)
root.put_kp(a.keys[0], a)
root.put_kp(b.keys[0], b)
return root, root.persist()
}
// right is only set on split
// left is always set. When split is false left is the pointer to block
// When split is true left is the pointer to the new left
// block
func (self *BpNode) insert(key ItemKey, value ItemValue) (a, b *BpNode, err error) {
if self.Internal() {
return self.internal_insert(key, value)
} else { // leaf node
return self.leaf_insert(key, value)
}
}
/* - first find the child to insert into
* - do the child insert
* - if there was a split:
* - if the block is full, split this block
* - else insert the new key/pointer into this block
*/
func (self *BpNode) internal_insert(key ItemKey, value ItemValue) (a, b *BpNode, err error) {
if !self.Internal() {
return nil, nil, BpTreeError("Expected a internal node")
}
i, has := self.find(key)
if !has && i > 0 {
// if it doesn't have it and the index > 0 then we have the next block
// so we have to subtract one from the index.
i--
}
child := self.pointers[i]
p, q, err := child.insert(key, value)
if err != nil {
return nil, nil, err
}
self.keys[i] = p.keys[0]
self.pointers[i] = p
if q != nil {
// we had a split
if self.Full() {
return self.internal_split(q.keys[0], q)
} else {
if err := self.put_kp(q.keys[0], q); err != nil {
return nil, nil, err
}
return self, nil, self.persist()
}
}
return self, nil, self.maybePersist(child != p)
}
/* On split
* - first assert that the key to be inserted is not already in the block.
* - Make a new block
* - balance the two blocks.
* - insert the new key/pointer combo into the correct block
*/
func (self *BpNode) internal_split(key ItemKey, ptr *BpNode) (a, b *BpNode, err error) {
if !self.Internal() {
return nil, nil, BpTreeError("Expected a internal node")
}
if self.has(key) {
return nil, nil, BpTreeError("Tried to split an internal block on duplicate key")
}
a = self
b = NewInternal(self.Capacity(), self.nodeStore)
balance_nodes(a, b, key)
if b.Len() > 0 && key.Less(b.keys[0]) {
if err := a.put_kp(key, ptr); err != nil {
return nil, nil, err
}
} else {
if err := b.put_kp(key, ptr); err != nil {
return nil, nil, err
}
}
return a, b, persist(a, b)
}
/* if the leaf is full then it will defer to a leaf_split
* (but in one case that will not actually split in the case of a insert into
* a pure block with a matching key)
* else this leaf will get a new entry.
*/
func (self *BpNode) leaf_insert(key ItemKey, value ItemValue) (a, b *BpNode, err error) {
if self.Internal() {
return nil, nil, BpTreeError("Expected a leaf node")
}
if true { // no_dup = true
i, has := self.find(key)
if has {
self.values[i] = value
return self, nil, self.persist()
}
}
if self.Full() {
return self.leaf_split(key, value)
} else {
if err := self.put_kv(key, value); err != nil {
return nil, nil, err
}
return self, nil, self.persist()
}
}
/* on leaf split if the block is pure then it will defer to pure_leaf_split
* else
* - a new block will be made and inserted after this one
* - the two blocks will be balanced with balanced_nodes
* - if the key is less than b.keys[0] it will go in a else b
*/
func (self *BpNode) leaf_split(key ItemKey, value ItemValue) (a, b *BpNode, err error) {
if self.Internal() {
return nil, nil, BpTreeError("Expected a leaf node")
}
if self.Pure() {
return self.pure_leaf_split(key, value)
}
a = self
b = NewLeaf(self.Capacity(), self.nodeStore)
insert_linked_list_node(b, a, a.getNext())
balance_nodes(a, b, key)
if b.Len() > 0 && key.Less(b.keys[0]) {
if err := a.put_kv(key, value); err != nil {
return nil, nil, err
}
} else {
if err := b.put_kv(key, value); err != nil {
return nil, nil, err
}
}
return a, b, persist(a, b)
}
/* a pure leaf split has two cases:
* 1) the inserted key is less than the current pure block.
* - a new block should be created before the current block
* - the key should be put in it
* 2) the inserted key is greater than or equal to the pure block.
* - the end of run of pure blocks should be found
* - if the key is equal to pure block and the last block is not full insert
* the new kv
* - else split by making a new block after the last block in the run
* and putting the new key there.
* - always return the current block as "a" and the new block as "b"
*/
func (self *BpNode) pure_leaf_split(key ItemKey, value ItemValue) (a, b *BpNode, err error) {
if self.Internal() || !self.Pure() {
return nil, nil, BpTreeError("Expected a pure leaf node")
}
if key.Less(self.keys[0]) {
a = NewLeaf(self.Capacity(), self.nodeStore)
b = self
if err := a.put_kv(key, value); err != nil {
return nil, nil, err
}
insert_linked_list_node(a, b.getPrev(), b)
return a, b, persist(a, b)
} else {
a = self
e := self.find_end_of_pure_run()
if e.keys[0].Equals(key) && !e.Full() {
if err := e.put_kv(key, value); err != nil {
return nil, nil, err
}
return a, nil, a.persist()
} else {
b = NewLeaf(self.Capacity(), self.nodeStore)
if err := b.put_kv(key, value); err != nil {
return nil, nil, err
}
insert_linked_list_node(b, e, e.getNext())
if e.keys[0].Equals(key) {
return a, nil, nil
}
return a, b, persist(a, b)
}
}
}
func (self *BpNode) put_kp(key ItemKey, ptr *BpNode) error {
if self.Full() {
return BpTreeError("Block is full.")
}
if !self.Internal() {
return BpTreeError("Expected a internal node")
}
i, has := self.find(key)
if has {
return BpTreeError("Tried to insert a duplicate key into an internal node")
} else if i < 0 {
panic(BpTreeError("find returned a negative int"))
} else if i >= cap(self.keys) {
panic(BpTreeError("find returned a int > than cap(keys)"))
}
if err := self.put_key_at(i, key); err != nil {
return err
}
if err := self.put_pointer_at(i, ptr); err != nil {
return err
}
return nil
}
func (self *BpNode) put_kv(key ItemKey, value ItemValue) error {
if self.Full() {
return BpTreeError("Block is full.")
}
if self.Internal() {
return BpTreeError("Expected a leaf node")
}
i, _ := self.find(key)
if i < 0 {
panic(BpTreeError("find returned a negative int"))
} else if i >= cap(self.keys) {
panic(BpTreeError("find returned a int > than cap(keys)"))
}
if err := self.put_key_at(i, key); err != nil {
return err
}
if err := self.put_value_at(i, value); err != nil {
return err
}
return nil
}
func (self *BpNode) put_key_at(i int, key ItemKey) error {
if self.Full() {
return BpTreeError("Block is full.")
}
self.keys = self.keys[:len(self.keys)+1]
for j := len(self.keys) - 1; j > i; j-- {
self.keys[j] = self.keys[j-1]
}
self.keys[i] = key
return nil
}
func (self *BpNode) put_value_at(i int, value ItemValue) error {
if len(self.values) == cap(self.values) {
return BpTreeError("Block is full.")
}
if self.Internal() {
return BpTreeError("Expected a leaf node")
}
self.values = self.values[:len(self.values)+1]
for j := len(self.values) - 1; j > i; j-- {
self.values[j] = self.values[j-1]
}
self.values[i] = value
return nil
}
func (self *BpNode) put_pointer_at(i int, pointer *BpNode) error {
if len(self.pointers) == cap(self.pointers) {
return BpTreeError("Block is full.")
}
if !self.Internal() {
return BpTreeError("Expected a internal node")
}
self.pointers = self.pointers[:len(self.pointers)+1]
for j := len(self.pointers) - 1; j > i; j-- {
self.pointers[j] = self.pointers[j-1]
}
self.pointers[i] = pointer
return nil
}
func (self *BpNode) remove(key ItemKey, where WhereFunc) (a *BpNode, err error) {
if self.Internal() {
return self.internal_remove(key, nil, where)
} else {
return self.leaf_remove(key, self.keys[len(self.keys)-1], where)
}
}
func (self *BpNode) internal_remove(key ItemKey, sibling *BpNode, where WhereFunc) (a *BpNode, err error) {
if !self.Internal() {
panic(BpTreeError("Expected a internal node"))
}
i, has := self.find(key)
if !has && i > 0 {
// if it doesn't have it and the index > 0 then we have the next block
// so we have to subtract one from the index.
i--
}
if i+1 < len(self.keys) {
sibling = self.pointers[i+1]
} else if sibling != nil {
sibling = sibling.left_most_leaf()
}
child := self.pointers[i]
oldChild := child
if child.Internal() {
child, err = child.internal_remove(key, sibling, where)
} else {
if sibling == nil {
child, err = child.leaf_remove(key, nil, where)
} else {
child, err = child.leaf_remove(key, sibling.keys[0], where)
}
}
if err != nil {
return nil, err
}
if child == nil {
if err := self.remove_key_at(i); err != nil {
return nil, err
}
if err := self.remove_ptr_at(i); err != nil {
return nil, err
}
} else {
self.keys[i] = child.keys[0]
self.pointers[i] = child
}
if len(self.keys) == 0 {
return nil, self.destroy()
}
return self, self.maybePersist(oldChild != child)
}
func (self *BpNode) leaf_remove(key, stop ItemKey, where WhereFunc) (a *BpNode, err error) {
if self.Internal() {
return nil, BpTreeError("Expected a leaf node")
}
a = self
hasChange := false
for j, l, next := self.forward(key, key)(); next != nil; j, l, next = next() {
if where(l.values[j]) {
hasChange = true
if err := l.remove_key_at(j); err != nil {
return nil, err
}
if err := l.remove_value_at(j); err != nil {
return nil, err
}
}
if len(l.keys) == 0 {
remove_linked_list_node(l)
if l.getNext() == nil {
a = nil
} else if stop == nil {
a = nil
} else if !l.getNext().keys[0].Equals(stop) {
a = l.getNext()
} else {
a = nil
}
if err := l.destroy(); err != nil {
return nil, err
}
}
}
if a != nil {
return a, a.maybePersist(hasChange)
}
return a, nil
}
func (self *BpNode) remove_key_at(i int) error {
if i >= len(self.keys) || i < 0 {
return BpTreeError("i, %v, is out of bounds, %v, %v %v.", i, len(self.keys), len(self.values), self)
}
for j := i; j < len(self.keys)-1; j++ {
self.keys[j] = self.keys[j+1]
}
self.keys = self.keys[:len(self.keys)-1]
return nil
}
func (self *BpNode) remove_value_at(i int) error {
if i >= len(self.values) || i < 0 {
return BpTreeError("i, %v, is out of bounds, %v.", i, len(self.values))
}
for j := i; j < len(self.values)-1; j++ {
self.values[j] = self.values[j+1]
}
self.values = self.values[:len(self.values)-1]
return nil
}
func (self *BpNode) remove_ptr_at(i int) error {
if i >= len(self.pointers) || i < 0 {
return BpTreeError("i, %v, is out of bounds, %v.", i, len(self.pointers))
}
for j := i; j < len(self.pointers)-1; j++ {
self.pointers[j] = self.pointers[j+1]
}
self.pointers = self.pointers[:len(self.pointers)-1]
return nil
}
func (self *BpNode) find(key ItemKey) (int, bool) {
var l = 0
var r = len(self.keys) - 1
var m int
for l <= r {
m = ((r - l) >> 1) + l
if key.Less(self.keys[m]) {
r = m - 1
} else if key.Equals(self.keys[m]) {
return m, true
} else {
l = m + 1
}
}
return l, false
}
func (self *BpNode) find_end_of_pure_run() *BpNode {
k := self.keys[0]
p := self
n := self.getNext()
for n != nil && n.Pure() && k.Equals(n.keys[0]) {
p = n
n = n.getNext()
}
return p
}
func (self *BpNode) all() (li loc_iterator) {
j := -1
l := self.left_most_leaf()
end := false
j, l, end = next_location(j, l)
li = func() (i int, leaf *BpNode, next loc_iterator) {
if end {
return -1, nil, nil
}
i = j
leaf = l
j, l, end = next_location(j, l)
return i, leaf, li
}
return li
}
func (self *BpNode) all_backward() (li loc_iterator) {
l := self.right_most_leaf()
j := len(l.keys)
end := false
j, l, end = prev_location(j, l)
li = func() (i int, leaf *BpNode, next loc_iterator) {
if end {
return -1, nil, nil
}
i = j
leaf = l
j, l, end = prev_location(j, l)
return i, leaf, li
}
return li
}
func (self *BpNode) forward(from, to ItemKey) (li loc_iterator) {
j, l := self.get_start(from)
end := false
j--
li = func() (i int, leaf *BpNode, next loc_iterator) {
j, l, end = next_location(j, l)
if end || to.Less(l.keys[j]) {
return -1, nil, nil
}
return j, l, li
}
return li
}
func (self *BpNode) backward(from, to ItemKey) (li loc_iterator) {
j, l := self.get_end(from)
end := false
li = func() (i int, leaf *BpNode, next loc_iterator) {
if end || l.keys[j].Less(to) {
return -1, nil, nil
}
i = j
leaf = l
j, l, end = prev_location(i, l)
return i, leaf, li
}
return li
}
func insert_linked_list_node(n, prev, next *BpNode) {
if (prev != nil && prev.getNext() != next) || (next != nil && next.getPrev() != prev) {
panic(BpTreeError("prev and next not hooked up"))
}
n.setPrev(prev)
n.setNext(next)
if prev != nil {
prev.setNext(n)
}
if next != nil {
next.setPrev(n)
}
}
func remove_linked_list_node(n *BpNode) {
if n.getPrev() != nil {
n.getPrev().setNext(n.getNext())
}
if n.getNext() != nil {
n.getNext().setPrev(n.getPrev())
}
}
/**
* a must be full and b must be empty else there will be a panic
*
* Different from common btree implementation, this splits the nodes by the inserted key.
* Items less than the splitKey stays in a, or moved to b if otherwise.
* This should help for monotonically increasing inserts.
*
*/
func balance_nodes(a, b *BpNode, splitKey ItemKey) {
if len(b.keys) != 0 {
panic(BpTreeError("b was not empty"))
}
if !a.Full() {
panic(BpTreeError("a was not full", a))
}
if cap(a.keys) != cap(b.keys) {
panic(BpTreeError("cap(a.keys) != cap(b.keys)"))
}
if cap(a.values) != cap(b.values) {
panic(BpTreeError("cap(a.values) != cap(b.values)"))
}
if cap(a.pointers) != cap(b.pointers) {
panic(BpTreeError("cap(a.pointers) != cap(b.pointers)"))
}
m := find_split_index(a, b, splitKey)
var lim = len(a.keys) - m
b.keys = b.keys[:lim]
if cap(a.values) > 0 {
if cap(a.values) != cap(a.keys) {
panic(BpTreeError("cap(a.values) != cap(a.keys)"))
}
b.values = b.values[:lim]
}
if cap(a.pointers) > 0 {
if cap(a.pointers) != cap(a.keys) {
panic(BpTreeError("cap(a.pointers) != cap(a.keys)"))
}
b.pointers = b.pointers[:lim]
}
for i := 0; i < lim; i++ {
j := m + i
b.keys[i] = a.keys[j]
if cap(a.values) > 0 {
b.values[i] = a.values[j]
}
if cap(a.pointers) > 0 {
b.pointers[i] = a.pointers[j]
}
}
a.keys = a.keys[:m]
if cap(a.values) > 0 {
a.values = a.values[:m]
}
if cap(a.pointers) > 0 {
a.pointers = a.pointers[:m]
}
}
func find_split_index(a, b *BpNode, splitKey ItemKey) int {
m := len(a.keys)
for m > 0 && !a.keys[m-1].Less(splitKey) {
m--
}
return m
}

53
weed/util/bptree/bptree_store_test.go
View file

@ -1,53 +0,0 @@
package bptree
import (
"fmt"
"testing"
)
type nodeStorePrintlnImpl struct {
}
func (n *nodeStorePrintlnImpl) PersistFunc(node *BpNode) error {
println("saving node", node.protoNodeId)
return nil
}
func (n *nodeStorePrintlnImpl) DestroyFunc(node *BpNode) error {
println("delete node", node.protoNodeId)
return nil
}
func TestAddRemove(t *testing.T) {
tree := NewBpTree(3, &nodeStorePrintlnImpl{})
for i:=0;i<9;i++{
println("++++++++++", i)
tree.Add(String(fmt.Sprintf("%02d", i)), nil)
printTree(tree.root, "")
}
if !tree.Has(String("08")) {
t.Errorf("lookup error")
}
for i:=5;i<9;i++{
println("----------", i)
tree.RemoveWhere(String(fmt.Sprintf("%02d", i)), func(value ItemValue) bool {
return true
})
printTree(tree.root, "")
}
if tree.Has(String("08")) {
t.Errorf("remove error")
}
}
func printTree(node *BpNode, prefix string) {
fmt.Printf("%sNode %d\n", prefix, node.protoNodeId)
prefix += " "
for i:=0;i<len(node.keys);i++{
fmt.Printf("%skey %v\n", prefix, node.keys[i])
if i < len(node.pointers) && node.pointers[i] != nil {
printTree(node.pointers[i], prefix+" ")
}
}
}

1408
weed/util/bptree/bptree_test.go
View file

File diff suppressed because it is too large Load diff

72
weed/util/bptree/getter_setter.go
View file

@ -1,72 +0,0 @@
package bptree
var (
protoNodeId = int64(0)
)
func GetProtoNodeId() int64 {
protoNodeId++
return protoNodeId
}
func (self *BpMap) getRoot() *BpNode {
return self.root
}
func (self *BpMap) setRoot(root *BpNode) {
self.root = root
}
func (self *BpTree) getRoot() *BpNode {
return self.root
}
func (self *BpTree) setRoot(root *BpNode) {
self.root = root
}
func (self *BpNode) getNext() *BpNode {
return self.next
}
func (self *BpNode) setNext(next *BpNode) {
self.next = next
}
func (self *BpNode) getPrev() *BpNode {
return self.prev
}
func (self *BpNode) setPrev(prev *BpNode) {
self.prev = prev
}
func (self *BpNode) getNode(x int)(*BpNode) {
return self.pointers[x]
}
func (self *BpNode) maybePersist(shouldPersist bool) error {
if !shouldPersist {
return nil
}
return self.persist()
}
func (self *BpNode) persist() error {
if self.nodeStore != nil {
return self.nodeStore.PersistFunc(self)
}
return nil
}
func (self *BpNode) destroy() error {
if self.nodeStore != nil {
return self.nodeStore.DestroyFunc(self)
}
return nil
}
func persist(a, b *BpNode) error {
if a != nil {
if err := a.persist(); err != nil {
return err
}
}
if b != nil {
if err := b.persist(); err != nil {
return err
}
}
return nil
}

357
weed/util/bptree/int.go
View file

@ -1,357 +0,0 @@
package bptree
import (
"encoding/binary"
"fmt"
)
type Int8 int8
type UInt8 uint8
type Int16 int16
type UInt16 uint16
type Int32 int32
type UInt32 uint32
type Int64 int64
type UInt64 uint64
type Int int
type UInt uint
func (self *Int8) MarshalBinary() ([]byte, error) {
bytes := make([]byte, 0)
bytes[0] = uint8(*self)
return bytes, nil
}
func (self *Int8) UnmarshalBinary(data []byte) error {
if len(data) != 1 {
return fmt.Errorf("data wrong size")
}
*self = Int8(data[0])
return nil
}
func (self Int8) Equals(other Equatable) bool {
if o, ok := other.(Int8); ok {
return self == o
} else {
return false
}
}
func (self Int8) Less(other Sortable) bool {
if o, ok := other.(Int8); ok {
return self < o
} else {
return false
}
}
func (self Int8) Hash() int {
return int(self)
}
func (self *UInt8) MarshalBinary() ([]byte, error) {
bytes := make([]byte, 0)
bytes[0] = uint8(*self)
return bytes, nil
}
func (self *UInt8) UnmarshalBinary(data []byte) error {
if len(data) != 1 {
return fmt.Errorf("data wrong size")
}
*self = UInt8(data[0])
return nil
}
func (self UInt8) Equals(other Equatable) bool {
if o, ok := other.(UInt8); ok {
return self == o
} else {
return false
}
}
func (self UInt8) Less(other Sortable) bool {
if o, ok := other.(UInt8); ok {
return self < o
} else {
return false
}
}
func (self UInt8) Hash() int {
return int(self)
}
func (self *Int16) MarshalBinary() ([]byte, error) {
bytes := make([]byte, 2)
binary.BigEndian.PutUint16(bytes, uint16(*self))
return bytes, nil
}
func (self *Int16) UnmarshalBinary(data []byte) error {
if len(data) != 2 {
return fmt.Errorf("data wrong size")
}
*self = Int16(binary.BigEndian.Uint16(data))
return nil
}
func (self Int16) Equals(other Equatable) bool {
if o, ok := other.(Int16); ok {
return self == o
} else {
return false
}
}
func (self Int16) Less(other Sortable) bool {
if o, ok := other.(Int16); ok {
return self < o
} else {
return false
}
}
func (self Int16) Hash() int {
return int(self)
}
func (self *UInt16) MarshalBinary() ([]byte, error) {
bytes := make([]byte, 2)
binary.BigEndian.PutUint16(bytes, uint16(*self))
return bytes, nil
}
func (self *UInt16) UnmarshalBinary(data []byte) error {
if len(data) != 2 {
return fmt.Errorf("data wrong size")
}
*self = UInt16(binary.BigEndian.Uint16(data))
return nil
}
func (self UInt16) Equals(other Equatable) bool {
if o, ok := other.(UInt16); ok {
return self == o
} else {
return false
}
}
func (self UInt16) Less(other Sortable) bool {
if o, ok := other.(UInt16); ok {
return self < o
} else {
return false
}
}
func (self UInt16) Hash() int {
return int(self)
}
func (self *Int32) MarshalBinary() ([]byte, error) {
bytes := make([]byte, 4)
binary.BigEndian.PutUint32(bytes, uint32(*self))
return bytes, nil
}
func (self *Int32) UnmarshalBinary(data []byte) error {
if len(data) != 4 {
return fmt.Errorf("data wrong size")
}
*self = Int32(binary.BigEndian.Uint32(data))
return nil
}
func (self Int32) Equals(other Equatable) bool {
if o, ok := other.(Int32); ok {
return self == o
} else {
return false
}
}
func (self Int32) Less(other Sortable) bool {
if o, ok := other.(Int32); ok {
return self < o
} else {
return false
}
}
func (self *UInt32) MarshalBinary() ([]byte, error) {
bytes := make([]byte, 4)
binary.BigEndian.PutUint32(bytes, uint32(*self))
return bytes, nil
}
func (self *UInt32) UnmarshalBinary(data []byte) error {
if len(data) != 4 {
return fmt.Errorf("data wrong size")
}
*self = UInt32(binary.BigEndian.Uint32(data))
return nil
}
func (self Int32) Hash() int {
return int(self)
}
func (self UInt32) Equals(other Equatable) bool {
if o, ok := other.(UInt32); ok {
return self == o
} else {
return false
}
}
func (self UInt32) Less(other Sortable) bool {
if o, ok := other.(UInt32); ok {
return self < o
} else {
return false
}
}
func (self UInt32) Hash() int {
return int(self)
}
func (self *Int64) MarshalBinary() ([]byte, error) {
bytes := make([]byte, 8)
binary.BigEndian.PutUint64(bytes, uint64(*self))
return bytes, nil
}
func (self *Int64) UnmarshalBinary(data []byte) error {
if len(data) != 8 {
return fmt.Errorf("data wrong size")
}
*self = Int64(binary.BigEndian.Uint64(data))
return nil
}
func (self Int64) Equals(other Equatable) bool {
if o, ok := other.(Int64); ok {
return self == o
} else {
return false
}
}
func (self Int64) Less(other Sortable) bool {
if o, ok := other.(Int64); ok {
return self < o
} else {
return false
}
}
func (self Int64) Hash() int {
return int(self>>32) ^ int(self)
}
func (self *UInt64) MarshalBinary() ([]byte, error) {
bytes := make([]byte, 8)
binary.BigEndian.PutUint64(bytes, uint64(*self))
return bytes, nil
}
func (self *UInt64) UnmarshalBinary(data []byte) error {
if len(data) != 8 {
return fmt.Errorf("data wrong size")
}
*self = UInt64(binary.BigEndian.Uint64(data))
return nil
}
func (self UInt64) Equals(other Equatable) bool {
if o, ok := other.(UInt64); ok {
return self == o
} else {
return false
}
}
func (self UInt64) Less(other Sortable) bool {
if o, ok := other.(UInt64); ok {
return self < o
} else {
return false
}
}
func (self UInt64) Hash() int {
return int(self>>32) ^ int(self)
}
func (self *Int) MarshalBinary() ([]byte, error) {
bytes := make([]byte, 4)
binary.BigEndian.PutUint32(bytes, uint32(*self))
return bytes, nil
}
func (self *Int) UnmarshalBinary(data []byte) error {
if len(data) != 4 {
return fmt.Errorf("data wrong size")
}
*self = Int(binary.BigEndian.Uint32(data))
return nil
}
func (self Int) Equals(other Equatable) bool {
if o, ok := other.(Int); ok {
return self == o
} else {
return false
}
}
func (self Int) Less(other Sortable) bool {
if o, ok := other.(Int); ok {
return self < o
} else {
return false
}
}
func (self Int) Hash() int {
return int(self)
}
func (self *UInt) MarshalBinary() ([]byte, error) {
bytes := make([]byte, 4)
binary.BigEndian.PutUint32(bytes, uint32(*self))
return bytes, nil
}
func (self *UInt) UnmarshalBinary(data []byte) error {
if len(data) != 4 {
return fmt.Errorf("data wrong size")
}
*self = UInt(binary.BigEndian.Uint32(data))
return nil
}
func (self UInt) Equals(other Equatable) bool {
if o, ok := other.(UInt); ok {
return self == o
} else {
return false
}
}
func (self UInt) Less(other Sortable) bool {
if o, ok := other.(UInt); ok {
return self < o
} else {
return false
}
}
func (self UInt) Hash() int {
return int(self)
}

2
weed/util/bptree/rand.go
View file

@ -1,2 +0,0 @@
package bptree

10
weed/util/bptree/serde.go
View file

@ -1,10 +0,0 @@
package bptree
func (protoNode *ProtoNode) ToBpTree() *BpTree {
node := protoNode.ToBpNode()
return &BpTree{root: node}
}
func (protoNode *ProtoNode) ToBpNode() *BpNode {
return nil
}

46
weed/util/bptree/serde_test.go
View file

@ -1,46 +0,0 @@
package bptree
import (
"fmt"
"testing"
)
type nodeStoreMapImpl struct {
m map[int64]*ProtoNode
}
func (n *nodeStoreMapImpl) PersistFunc(node *BpNode) error {
println("saving node", node.protoNodeId)
n.m[node.protoNodeId] = node.protoNode
return nil
}
func (n *nodeStoreMapImpl) DestroyFunc(node *BpNode) error {
println("delete node", node.protoNodeId)
delete(n.m, node.protoNodeId)
return nil
}
func TestSerDe(t *testing.T) {
nodeStore := &nodeStoreMapImpl{
m: make(map[int64]*ProtoNode),
}
tree := NewBpTree(3, nodeStore)
for i:=0;i<32;i++{
println("add", i)
tree.Add(String(fmt.Sprintf("%02d", i)), nil)
}
for i:=5;i<9;i++{
println("----------", i)
tree.RemoveWhere(String(fmt.Sprintf("%02d", i)), func(value ItemValue) bool {
return true
})
printTree(tree.root, "")
}
}

71
weed/util/bptree/string.go
View file

@ -1,71 +0,0 @@
package bptree
import (
"bytes"
"hash/fnv"
)
type String string
type ByteSlice []byte
func (self *String) MarshalBinary() ([]byte, error) {
return []byte(*self), nil
}
func (self *String) UnmarshalBinary(data []byte) error {
*self = String(data)
return nil
}
func (self String) Equals(other Equatable) bool {
if o, ok := other.(String); ok {
return self == o
} else {
return false
}
}
func (self String) Less(other Sortable) bool {
if o, ok := other.(String); ok {
return self < o
} else {
return false
}
}
func (self String) Hash() int {
h := fnv.New32a()
h.Write([]byte(string(self)))
return int(h.Sum32())
}
func (self *ByteSlice) MarshalBinary() ([]byte, error) {
return []byte(*self), nil
}
func (self *ByteSlice) UnmarshalBinary(data []byte) error {
*self = ByteSlice(data)
return nil
}
func (self ByteSlice) Equals(other Equatable) bool {
if o, ok := other.(ByteSlice); ok {
return bytes.Equal(self, o)
} else {
return false
}
}
func (self ByteSlice) Less(other Sortable) bool {
if o, ok := other.(ByteSlice); ok {
return bytes.Compare(self, o) < 0 // -1 if a < b
} else {
return false
}
}
func (self ByteSlice) Hash() int {
h := fnv.New32a()
h.Write([]byte(self))
return int(h.Sum32())
}

29
weed/util/bptree/tree_store/memory_store.go
View file

@ -1,29 +0,0 @@
package tree_store
import "errors"
var (
NotFound = errors.New("not found")
)
type MemoryTreeStore struct {
m map[int64][]byte
}
func NewMemoryTreeStore() *MemoryTreeStore{
return &MemoryTreeStore{
m: make(map[int64][]byte),
}
}
func (m *MemoryTreeStore) Put(k int64, v []byte) error {
m.m[k] = v
return nil
}
func (m *MemoryTreeStore) Get(k int64) ([]byte, error) {
if v, found := m.m[k]; found {
return v, nil
}
return nil, NotFound
}

6
weed/util/bptree/tree_store/tree_store.go.go
View file

@ -1,6 +0,0 @@
package tree_store
type TreeStore interface {
Put(k int64, v []byte) error
Get(k int64) ([]byte, error)
}

98
weed/util/bptree/types.go
View file

@ -1,98 +0,0 @@
package bptree
import (
"errors"
"fmt"
)
type Equatable interface {
Equals(b Equatable) bool
}
type Sortable interface {
Equatable
Less(b Sortable) bool
}
type Hashable interface {
Sortable
Hash() int
}
var BpTreeError = fmt.Errorf
func NegativeSize() error {
return errors.New("negative size")
}
type Iterator func() (item ItemValue, next Iterator)
type KIterator func() (key ItemKey, next KIterator)
type KVIterator func() (key ItemKey, value ItemValue, next KVIterator)
type KVIterable interface {
Iterate() KVIterator
}
type MapOperable interface {
Has(key ItemKey) bool
Put(key ItemKey, value ItemValue) (err error)
Get(key ItemKey) (value ItemValue, err error)
Remove(key ItemKey) (value ItemValue, err error)
}
type WhereFunc func(value ItemValue) bool
func MakeValuesIterator(obj KVIterable) Iterator {
kv_iterator := obj.Iterate()
var v_iterator Iterator
v_iterator = func() (value ItemValue, next Iterator) {
_, value, kv_iterator = kv_iterator()
if kv_iterator == nil {
return nil, nil
}
return value, v_iterator
}
return v_iterator
}
func MakeItemsIterator(obj KVIterable) (kit KIterator) {
kv_iterator := obj.Iterate()
kit = func() (item ItemKey, next KIterator) {
var key ItemKey
var value ItemValue
key, value, kv_iterator = kv_iterator()
if kv_iterator == nil {
return nil, nil
}
return &MapEntry{key, value}, kit
}
return kit
}
type MapEntry struct {
Key ItemKey
Value ItemValue
}
func (m *MapEntry) Equals(other Equatable) bool {
if o, ok := other.(*MapEntry); ok {
return m.Key.Equals(o.Key)
} else {
return m.Key.Equals(other)
}
}
func (m *MapEntry) Less(other Sortable) bool {
if o, ok := other.(*MapEntry); ok {
return m.Key.Less(o.Key)
} else {
return m.Key.Less(other)
}
}
func (m *MapEntry) Hash() int {
return m.Key.Hash()
}
func (m *MapEntry) String() string {
return fmt.Sprintf("<MapEntry %v: %v>", m.Key, m.Value)
}