started by copying from https://sourcegraph.com/github.com/timtadh/data-structures@master/-/tree/tree/bptree

weed/util/bptree/bpmap.go

@@ -0,0 +1,77 @@
+package bptree
+
+import (
+	"fmt"
+)
+
+/* A BpMap is a B+Tree with support for duplicate keys disabled. This makes it
+ * behave like a regular Map rather than a MultiMap.
+ */
+type BpMap BpTree
+
+func NewBpMap(node_size int) *BpMap {
+	return &BpMap{
+		root: NewLeaf(node_size, true),
+		size: 0,
+	}
+}
+
+func (self *BpMap) Size() int {
+	return (*BpTree)(self).Size()
+}
+
+func (self *BpMap) Has(key Hashable) bool {
+	return (*BpTree)(self).Has(key)
+}
+
+func (self *BpMap) Put(key Hashable, value interface{}) (err error) {
+	had := self.Has(key)
+	new_root, err := self.root.put(key, value)
+	if err != nil {
+		return err
+	}
+	self.root = new_root
+	if !had {
+		self.size += 1
+	}
+	return nil
+}
+
+func (self *BpMap) Get(key Hashable) (value interface{}, err error) {
+	j, l := self.root.get_start(key)
+	if l.keys[j].Equals(key) {
+		return l.values[j], nil
+	}
+	return nil, fmt.Errorf("key not found: %s", key)
+}
+
+func (self *BpMap) Remove(key Hashable) (value interface{}, err error) {
+	value, err = self.Get(key)
+	if err != nil {
+		return nil, err
+	}
+	ns := self.root.NodeSize()
+	new_root, err := self.root.remove(key, func(value interface{}) bool { return true })
+	if err != nil {
+		return nil, err
+	}
+	if new_root == nil {
+		self.root = NewLeaf(ns, true)
+	} else {
+		self.root = new_root
+	}
+	self.size--
+	return value, nil
+}
+
+func (self *BpMap) Keys() (ki KIterator) {
+	return (*BpTree)(self).Keys()
+}
+
+func (self *BpMap) Values() (vi Iterator) {
+	return (*BpTree)(self).Values()
+}
+
+func (self *BpMap) Iterate() (kvi KVIterator) {
+	return (*BpTree)(self).Iterate()
+}

weed/util/bptree/bptree.go

@@ -0,0 +1,160 @@
+package bptree
+
+// started by copying from https://sourcegraph.com/github.com/timtadh/data-structures@master/-/tree/tree/bptree
+
+/* A BpTree is a B+Tree with support for duplicate keys. This makes it behave as
+ * a MultiMap. Additionally you can use the Range operator to select k/v in a
+ * range. If from > to it will iterate backwards.
+ */
+type BpTree struct {
+	root *BpNode
+	size int
+}
+
+type loc_iterator func() (i int, leaf *BpNode, li loc_iterator)
+
+func NewBpTree(node_size int) *BpTree {
+	return &BpTree{
+		root: NewLeaf(node_size, false),
+		size: 0,
+	}
+}
+
+func (self *BpTree) Size() int {
+	return self.size
+}
+
+func (self *BpTree) Has(key Hashable) bool {
+	if len(self.root.keys) == 0 {
+		return false
+	}
+	j, l := self.root.get_start(key)
+	return l.keys[j].Equals(key)
+}
+
+func (self *BpTree) Count(key Hashable) int {
+	if len(self.root.keys) == 0 {
+		return 0
+	}
+	j, l := self.root.get_start(key)
+	count := 0
+	end := false
+	for !end && l.keys[j].Equals(key) {
+		count++
+		j, l, end = next_location(j, l)
+	}
+	return count
+}
+
+func (self *BpTree) Add(key Hashable, value interface{}) (err error) {
+	new_root, err := self.root.put(key, value)
+	if err != nil {
+		return err
+	}
+	self.root = new_root
+	self.size += 1
+	return nil
+}
+
+func (self *BpTree) Replace(key Hashable, where WhereFunc, value interface{}) (err error) {
+	li := self.root.forward(key, key)
+	for i, leaf, next := li(); next != nil; i, leaf, next = next() {
+		if where(leaf.values[i]) {
+			leaf.values[i] = value
+		}
+	}
+	return nil
+}
+
+func (self *BpTree) Find(key Hashable) (kvi KVIterator) {
+	return self.Range(key, key)
+}
+
+func (self *BpTree) Range(from, to Hashable) (kvi KVIterator) {
+	var li loc_iterator
+	if !to.Less(from) {
+		li = self.root.forward(from, to)
+	} else {
+		li = self.root.backward(from, to)
+	}
+	kvi = func() (key Hashable, value interface{}, next KVIterator) {
+		var i int
+		var leaf *BpNode
+		i, leaf, li = li()
+		if li == nil {
+			return nil, nil, nil
+		}
+		return leaf.keys[i], leaf.values[i], kvi
+	}
+	return kvi
+}
+
+func (self *BpTree) RemoveWhere(key Hashable, where WhereFunc) (err error) {
+	ns := self.root.NodeSize()
+	new_root, err := self.root.remove(key, where)
+	if err != nil {
+		return err
+	}
+	if new_root == nil {
+		self.root = NewLeaf(ns, false)
+	} else {
+		self.root = new_root
+	}
+	self.size -= 1
+	return nil
+}
+
+func (self *BpTree) Keys() (ki KIterator) {
+	li := self.root.all()
+	var prev Equatable
+	ki = func() (key Hashable, next KIterator) {
+		var i int
+		var leaf *BpNode
+		i, leaf, li = li()
+		if li == nil {
+			return nil, nil
+		}
+		if leaf.keys[i].Equals(prev) {
+			return ki()
+		}
+		prev = leaf.keys[i]
+		return leaf.keys[i], ki
+	}
+	return ki
+}
+
+func (self *BpTree) Values() (vi Iterator) {
+	return MakeValuesIterator(self)
+}
+
+func (self *BpTree) Items() (vi KIterator) {
+	return MakeItemsIterator(self)
+}
+
+func (self *BpTree) Iterate() (kvi KVIterator) {
+	li := self.root.all()
+	kvi = func() (key Hashable, value interface{}, next KVIterator) {
+		var i int
+		var leaf *BpNode
+		i, leaf, li = li()
+		if li == nil {
+			return nil, nil, nil
+		}
+		return leaf.keys[i], leaf.values[i], kvi
+	}
+	return kvi
+}
+
+func (self *BpTree) Backward() (kvi KVIterator) {
+	li := self.root.all_backward()
+	kvi = func() (key Hashable, value interface{}, next KVIterator) {
+		var i int
+		var leaf *BpNode
+		i, leaf, li = li()
+		if li == nil {
+			return nil, nil, nil
+		}
+		return leaf.keys[i], leaf.values[i], kvi
+	}
+	return kvi
+}

weed/util/bptree/bptree_node.go

@@ -0,0 +1,752 @@
+package bptree
+
+type BpNode struct {
+	keys     []Hashable
+	values   []interface{}
+	pointers []*BpNode
+	next     *BpNode
+	prev     *BpNode
+	no_dup   bool
+}
+
+func NewInternal(size int) *BpNode {
+	if size < 0 {
+		panic(NegativeSize())
+	}
+	return &BpNode{
+		keys:     make([]Hashable, 0, size),
+		pointers: make([]*BpNode, 0, size),
+	}
+}
+
+func NewLeaf(size int, no_dup bool) *BpNode {
+	if size < 0 {
+		panic(NegativeSize())
+	}
+	return &BpNode{
+		keys:   make([]Hashable, 0, size),
+		values: make([]interface{}, 0, size),
+		no_dup: no_dup,
+	}
+}
+
+func (self *BpNode) Full() bool {
+	return len(self.keys) == cap(self.keys)
+}
+
+func (self *BpNode) Pure() bool {
+	if len(self.keys) == 0 {
+		return true
+	}
+	k0 := self.keys[0]
+	for _, k := range self.keys {
+		if !k0.Equals(k) {
+			return false
+		}
+	}
+	return true
+}
+
+func (self *BpNode) Internal() bool {
+	return cap(self.pointers) > 0
+}
+
+func (self *BpNode) NodeSize() int {
+	return cap(self.keys)
+}
+
+func (self *BpNode) Height() int {
+	if !self.Internal() {
+		return 1
+	} else if len(self.pointers) == 0 {
+		panic(BpTreeError("Internal node has no pointers but asked for height"))
+	}
+	return self.pointers[0].Height() + 1
+}
+
+func (self *BpNode) count(key Hashable) int {
+	i, _ := self.find(key)
+	count := 0
+	for ; i < len(self.keys); i++ {
+		if self.keys[i].Equals(key) {
+			count++
+		} else {
+			break
+		}
+	}
+	return count
+}
+
+func (self *BpNode) has(key Hashable) bool {
+	_, has := self.find(key)
+	return has
+}
+
+func (self *BpNode) left_most_leaf() *BpNode {
+	if self.Internal() {
+		return self.pointers[0].left_most_leaf()
+	}
+	return self
+}
+
+func (self *BpNode) right_most_leaf() *BpNode {
+	if self.Internal() {
+		return self.pointers[len(self.pointers)-1].right_most_leaf()
+	}
+	return self
+}
+
+/* returns the index and leaf-block of the first key greater than or equal to
+ * the search key. (unless the search key is greater than all the keys in the
+ * tree, in that case it will be the last key in the tree)
+ */
+func (self *BpNode) get_start(key Hashable) (i int, leaf *BpNode) {
+	if self.Internal() {
+		return self.internal_get_start(key)
+	} else {
+		return self.leaf_get_start(key)
+	}
+}
+
+func next_location(i int, leaf *BpNode) (int, *BpNode, bool) {
+	j := i + 1
+	for j >= len(leaf.keys) && leaf.next != nil {
+		j = 0
+		leaf = leaf.next
+	}
+	if j >= len(leaf.keys) {
+		return -1, nil, true
+	}
+	return j, leaf, false
+}
+
+func prev_location(i int, leaf *BpNode) (int, *BpNode, bool) {
+	j := i - 1
+	for j < 0 && leaf.prev != nil {
+		leaf = leaf.prev
+		j = len(leaf.keys) - 1
+	}
+	if j < 0 {
+		return -1, nil, true
+	}
+	return j, leaf, false
+}
+
+/* returns the index and leaf-block of the last key equal to the search key or
+ * the first key greater than the search key. (unless the search key is greater
+ * than all the keys in the tree, in that case it will be the last key in the
+ * tree)
+ */
+func (self *BpNode) get_end(key Hashable) (i int, leaf *BpNode) {
+	end := false
+	i, leaf = self.get_start(key)
+	pi, pleaf := i, leaf
+	for !end && leaf.keys[i].Equals(key) {
+		pi, pleaf = i, leaf
+		i, leaf, end = next_location(i, leaf)
+	}
+	return pi, pleaf
+}
+
+func (self *BpNode) internal_get_start(key Hashable) (i int, leaf *BpNode) {
+	if !self.Internal() {
+		panic(BpTreeError("Expected a internal node"))
+	}
+	i, has := self.find(key)
+	if !has && i > 0 {
+		// if it doesn't have it and the index > 0 then we have the next block
+		// so we have to subtract one from the index.
+		i--
+	}
+	child := self.pointers[i]
+	return child.get_start(key)
+}
+
+func (self *BpNode) leaf_get_start(key Hashable) (i int, leaf *BpNode) {
+	i, has := self.find(key)
+	if i >= len(self.keys) && i > 0 {
+		i = len(self.keys) - 1
+	}
+	if !has && (len(self.keys) == 0 || self.keys[i].Less(key)) && self.next != nil {
+		return self.next.leaf_get_start(key)
+	}
+	return i, self
+}
+
+/* This puts the k/v pair into the B+Tree rooted at this node and returns the
+ * (possibly) new root of the tree.
+ */
+func (self *BpNode) put(key Hashable, value interface{}) (root *BpNode, err error) {
+	a, b, err := self.insert(key, value)
+	if err != nil {
+		return nil, err
+	} else if b == nil {
+		return a, nil
+	}
+	// else we have root split
+	root = NewInternal(self.NodeSize())
+	root.put_kp(a.keys[0], a)
+	root.put_kp(b.keys[0], b)
+	return root, nil
+}
+
+// right is only set on split
+// left is always set. When split is false left is the pointer to block
+//                     When split is true left is the pointer to the new left
+//                     block
+func (self *BpNode) insert(key Hashable, value interface{}) (a, b *BpNode, err error) {
+	if self.Internal() {
+		return self.internal_insert(key, value)
+	} else { // leaf node
+		return self.leaf_insert(key, value)
+	}
+}
+
+/* - first find the child to insert into
+ * - do the child insert
+ * - if there was a split:
+ *    - if the block is full, split this block
+ *    - else insert the new key/pointer into this block
+ */
+func (self *BpNode) internal_insert(key Hashable, value interface{}) (a, b *BpNode, err error) {
+	if !self.Internal() {
+		return nil, nil, BpTreeError("Expected a internal node")
+	}
+	i, has := self.find(key)
+	if !has && i > 0 {
+		// if it doesn't have it and the index > 0 then we have the next block
+		// so we have to subtract one from the index.
+		i--
+	}
+	child := self.pointers[i]
+	p, q, err := child.insert(key, value)
+	if err != nil {
+		return nil, nil, err
+	}
+	self.keys[i] = p.keys[0]
+	self.pointers[i] = p
+	if q != nil {
+		// we had a split
+		if self.Full() {
+			return self.internal_split(q.keys[0], q)
+		} else {
+			if err := self.put_kp(q.keys[0], q); err != nil {
+				return nil, nil, err
+			}
+			return self, nil, nil
+		}
+	}
+	return self, nil, nil
+}
+
+/* On split
+ * - first assert that the key to be inserted is not already in the block.
+ * - Make a new block
+ * - balance the two blocks.
+ * - insert the new key/pointer combo into the correct block
+ */
+func (self *BpNode) internal_split(key Hashable, ptr *BpNode) (a, b *BpNode, err error) {
+	if !self.Internal() {
+		return nil, nil, BpTreeError("Expected a internal node")
+	}
+	if self.has(key) {
+		return nil, nil, BpTreeError("Tried to split an internal block on duplicate key")
+	}
+	a = self
+	b = NewInternal(self.NodeSize())
+	balance_nodes(a, b)
+	if key.Less(b.keys[0]) {
+		if err := a.put_kp(key, ptr); err != nil {
+			return nil, nil, err
+		}
+	} else {
+		if err := b.put_kp(key, ptr); err != nil {
+			return nil, nil, err
+		}
+	}
+	return a, b, nil
+}
+
+/* if the leaf is full then it will defer to a leaf_split
+ *    (but in one case that will not actually split in the case of a insert into
+ *    a pure block with a matching key)
+ * else this leaf will get a new entry.
+ */
+func (self *BpNode) leaf_insert(key Hashable, value interface{}) (a, b *BpNode, err error) {
+	if self.Internal() {
+		return nil, nil, BpTreeError("Expected a leaf node")
+	}
+	if self.no_dup {
+		i, has := self.find(key)
+		if has {
+			self.values[i] = value
+			return self, nil, nil
+		}
+	}
+	if self.Full() {
+		return self.leaf_split(key, value)
+	} else {
+		if err := self.put_kv(key, value); err != nil {
+			return nil, nil, err
+		}
+		return self, nil, nil
+	}
+}
+
+/* on leaf split if the block is pure then it will defer to pure_leaf_split
+ * else
+ *    - a new block will be made and inserted after this one
+ *    - the two blocks will be balanced with balanced_nodes
+ *    - if the key is less than b.keys[0] it will go in a else b
+ */
+func (self *BpNode) leaf_split(key Hashable, value interface{}) (a, b *BpNode, err error) {
+	if self.Internal() {
+		return nil, nil, BpTreeError("Expected a leaf node")
+	}
+	if self.Pure() {
+		return self.pure_leaf_split(key, value)
+	}
+	a = self
+	b = NewLeaf(self.NodeSize(), self.no_dup)
+	insert_linked_list_node(b, a, a.next)
+	balance_nodes(a, b)
+	if key.Less(b.keys[0]) {
+		if err := a.put_kv(key, value); err != nil {
+			return nil, nil, err
+		}
+	} else {
+		if err := b.put_kv(key, value); err != nil {
+			return nil, nil, err
+		}
+	}
+	return a, b, nil
+}
+
+/* a pure leaf split has two cases:
+ *  1) the inserted key is less than the current pure block.
+ *     - a new block should be created before the current block
+ *     - the key should be put in it
+ *  2) the inserted key is greater than or equal to the pure block.
+ *     - the end of run of pure blocks should be found
+ *     - if the key is equal to pure block and the last block is not full insert
+ *       the new kv
+ *     - else split by making a new block after the last block in the run
+ *       and putting the new key there.
+ *     - always return the current block as "a" and the new block as "b"
+ */
+func (self *BpNode) pure_leaf_split(key Hashable, value interface{}) (a, b *BpNode, err error) {
+	if self.Internal() || !self.Pure() {
+		return nil, nil, BpTreeError("Expected a pure leaf node")
+	}
+	if key.Less(self.keys[0]) {
+		a = NewLeaf(self.NodeSize(), self.no_dup)
+		b = self
+		if err := a.put_kv(key, value); err != nil {
+			return nil, nil, err
+		}
+		insert_linked_list_node(a, b.prev, b)
+		return a, b, nil
+	} else {
+		a = self
+		e := self.find_end_of_pure_run()
+		if e.keys[0].Equals(key) && !e.Full() {
+			if err := e.put_kv(key, value); err != nil {
+				return nil, nil, err
+			}
+			return a, nil, nil
+		} else {
+			b = NewLeaf(self.NodeSize(), self.no_dup)
+			if err := b.put_kv(key, value); err != nil {
+				return nil, nil, err
+			}
+			insert_linked_list_node(b, e, e.next)
+			if e.keys[0].Equals(key) {
+				return a, nil, nil
+			}
+			return a, b, nil
+		}
+	}
+}
+
+func (self *BpNode) put_kp(key Hashable, ptr *BpNode) error {
+	if self.Full() {
+		return BpTreeError("Block is full.")
+	}
+	if !self.Internal() {
+		return BpTreeError("Expected a internal node")
+	}
+	i, has := self.find(key)
+	if has {
+		return BpTreeError("Tried to insert a duplicate key into an internal node")
+	} else if i < 0 {
+		panic(BpTreeError("find returned a negative int"))
+	} else if i >= cap(self.keys) {
+		panic(BpTreeError("find returned a int > than cap(keys)"))
+	}
+	if err := self.put_key_at(i, key); err != nil {
+		return err
+	}
+	if err := self.put_pointer_at(i, ptr); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (self *BpNode) put_kv(key Hashable, value interface{}) error {
+	if self.Full() {
+		return BpTreeError("Block is full.")
+	}
+	if self.Internal() {
+		return BpTreeError("Expected a leaf node")
+	}
+	i, _ := self.find(key)
+	if i < 0 {
+		panic(BpTreeError("find returned a negative int"))
+	} else if i >= cap(self.keys) {
+		panic(BpTreeError("find returned a int > than cap(keys)"))
+	}
+	if err := self.put_key_at(i, key); err != nil {
+		return err
+	}
+	if err := self.put_value_at(i, value); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (self *BpNode) put_key_at(i int, key Hashable) error {
+	if self.Full() {
+		return BpTreeError("Block is full.")
+	}
+	self.keys = self.keys[:len(self.keys)+1]
+	for j := len(self.keys) - 1; j > i; j-- {
+		self.keys[j] = self.keys[j-1]
+	}
+	self.keys[i] = key
+	return nil
+}
+
+func (self *BpNode) put_value_at(i int, value interface{}) error {
+	if len(self.values) == cap(self.values) {
+		return BpTreeError("Block is full.")
+	}
+	if self.Internal() {
+		return BpTreeError("Expected a leaf node")
+	}
+	self.values = self.values[:len(self.values)+1]
+	for j := len(self.values) - 1; j > i; j-- {
+		self.values[j] = self.values[j-1]
+	}
+	self.values[i] = value
+	return nil
+}
+
+func (self *BpNode) put_pointer_at(i int, pointer *BpNode) error {
+	if len(self.pointers) == cap(self.pointers) {
+		return BpTreeError("Block is full.")
+	}
+	if !self.Internal() {
+		return BpTreeError("Expected a internal node")
+	}
+	self.pointers = self.pointers[:len(self.pointers)+1]
+	for j := len(self.pointers) - 1; j > i; j-- {
+		self.pointers[j] = self.pointers[j-1]
+	}
+	self.pointers[i] = pointer
+	return nil
+}
+
+func (self *BpNode) remove(key Hashable, where WhereFunc) (a *BpNode, err error) {
+	if self.Internal() {
+		return self.internal_remove(key, nil, where)
+	} else {
+		return self.leaf_remove(key, self.keys[len(self.keys)-1], where)
+	}
+}
+
+func (self *BpNode) internal_remove(key Hashable, sibling *BpNode, where WhereFunc) (a *BpNode, err error) {
+	if !self.Internal() {
+		panic(BpTreeError("Expected a internal node"))
+	}
+	i, has := self.find(key)
+	if !has && i > 0 {
+		// if it doesn't have it and the index > 0 then we have the next block
+		// so we have to subtract one from the index.
+		i--
+	}
+	if i+1 < len(self.keys) {
+		sibling = self.pointers[i+1]
+	} else if sibling != nil {
+		sibling = sibling.left_most_leaf()
+	}
+	child := self.pointers[i]
+	if child.Internal() {
+		child, err = child.internal_remove(key, sibling, where)
+	} else {
+		if sibling == nil {
+			child, err = child.leaf_remove(key, nil, where)
+		} else {
+			child, err = child.leaf_remove(key, sibling.keys[0], where)
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	if child == nil {
+		if err := self.remove_key_at(i); err != nil {
+			return nil, err
+		}
+		if err := self.remove_ptr_at(i); err != nil {
+			return nil, err
+		}
+	} else {
+		self.keys[i] = child.keys[0]
+		self.pointers[i] = child
+	}
+	if len(self.keys) == 0 {
+		return nil, nil
+	}
+	return self, nil
+}
+
+func (self *BpNode) leaf_remove(key, stop Hashable, where WhereFunc) (a *BpNode, err error) {
+	if self.Internal() {
+		return nil, BpTreeError("Expected a leaf node")
+	}
+	a = self
+	for j, l, next := self.forward(key, key)(); next != nil; j, l, next = next() {
+		if where(l.values[j]) {
+			if err := l.remove_key_at(j); err != nil {
+				return nil, err
+			}
+			if err := l.remove_value_at(j); err != nil {
+				return nil, err
+			}
+		}
+		if len(l.keys) == 0 {
+			remove_linked_list_node(l)
+			if l.next == nil {
+				a = nil
+			} else if stop == nil {
+				a = nil
+			} else if !l.next.keys[0].Equals(stop) {
+				a = l.next
+			} else {
+				a = nil
+			}
+		}
+	}
+	return a, nil
+}
+
+func (self *BpNode) remove_key_at(i int) error {
+	if i >= len(self.keys) || i < 0 {
+		return BpTreeError("i, %v, is out of bounds, %v, %v %v.", i, len(self.keys), len(self.values), self)
+	}
+	for j := i; j < len(self.keys)-1; j++ {
+		self.keys[j] = self.keys[j+1]
+	}
+	self.keys = self.keys[:len(self.keys)-1]
+	return nil
+}
+
+func (self *BpNode) remove_value_at(i int) error {
+	if i >= len(self.values) || i < 0 {
+		return BpTreeError("i, %v, is out of bounds, %v.", i, len(self.values))
+	}
+	for j := i; j < len(self.values)-1; j++ {
+		self.values[j] = self.values[j+1]
+	}
+	self.values = self.values[:len(self.values)-1]
+	return nil
+}
+
+func (self *BpNode) remove_ptr_at(i int) error {
+	if i >= len(self.pointers) || i < 0 {
+		return BpTreeError("i, %v, is out of bounds, %v.", i, len(self.pointers))
+	}
+	for j := i; j < len(self.pointers)-1; j++ {
+		self.pointers[j] = self.pointers[j+1]
+	}
+	self.pointers = self.pointers[:len(self.pointers)-1]
+	return nil
+}
+
+func (self *BpNode) find(key Hashable) (int, bool) {
+	var l int = 0
+	var r int = len(self.keys) - 1
+	var m int
+	for l <= r {
+		m = ((r - l) >> 1) + l
+		if key.Less(self.keys[m]) {
+			r = m - 1
+		} else if key.Equals(self.keys[m]) {
+			for j := m; j >= 0; j-- {
+				if j == 0 || !key.Equals(self.keys[j-1]) {
+					return j, true
+				}
+			}
+		} else {
+			l = m + 1
+		}
+	}
+	return l, false
+}
+
+func (self *BpNode) find_end_of_pure_run() *BpNode {
+	k := self.keys[0]
+	p := self
+	n := self.next
+	for n != nil && n.Pure() && k.Equals(n.keys[0]) {
+		p = n
+		n = n.next
+	}
+	return p
+}
+
+func (self *BpNode) all() (li loc_iterator) {
+	j := -1
+	l := self.left_most_leaf()
+	end := false
+	j, l, end = next_location(j, l)
+	li = func() (i int, leaf *BpNode, next loc_iterator) {
+		if end {
+			return -1, nil, nil
+		}
+		i = j
+		leaf = l
+		j, l, end = next_location(j, l)
+		return i, leaf, li
+	}
+	return li
+}
+
+func (self *BpNode) all_backward() (li loc_iterator) {
+	l := self.right_most_leaf()
+	j := len(l.keys)
+	end := false
+	j, l, end = prev_location(j, l)
+	li = func() (i int, leaf *BpNode, next loc_iterator) {
+		if end {
+			return -1, nil, nil
+		}
+		i = j
+		leaf = l
+		j, l, end = prev_location(j, l)
+		return i, leaf, li
+	}
+	return li
+}
+
+func (self *BpNode) forward(from, to Hashable) (li loc_iterator) {
+	j, l := self.get_start(from)
+	end := false
+	j--
+	li = func() (i int, leaf *BpNode, next loc_iterator) {
+		j, l, end = next_location(j, l)
+		if end || to.Less(l.keys[j]) {
+			return -1, nil, nil
+		}
+		return j, l, li
+	}
+	return li
+}
+
+func (self *BpNode) backward(from, to Hashable) (li loc_iterator) {
+	j, l := self.get_end(from)
+	end := false
+	li = func() (i int, leaf *BpNode, next loc_iterator) {
+		if end || l.keys[j].Less(to) {
+			return -1, nil, nil
+		}
+		i = j
+		leaf = l
+		j, l, end = prev_location(i, l)
+		return i, leaf, li
+	}
+	return li
+}
+
+func insert_linked_list_node(n, prev, next *BpNode) {
+	if (prev != nil && prev.next != next) || (next != nil && next.prev != prev) {
+		panic(BpTreeError("prev and next not hooked up"))
+	}
+	n.prev = prev
+	n.next = next
+	if prev != nil {
+		prev.next = n
+	}
+	if next != nil {
+		next.prev = n
+	}
+}
+
+func remove_linked_list_node(n *BpNode) {
+	if n.prev != nil {
+		n.prev.next = n.next
+	}
+	if n.next != nil {
+		n.next.prev = n.prev
+	}
+}
+
+/* a must be full and b must be empty else there will be a panic
+ */
+func balance_nodes(a, b *BpNode) {
+	if len(b.keys) != 0 {
+		panic(BpTreeError("b was not empty"))
+	}
+	if !a.Full() {
+		panic(BpTreeError("a was not full", a))
+	}
+	if cap(a.keys) != cap(b.keys) {
+		panic(BpTreeError("cap(a.keys) != cap(b.keys)"))
+	}
+	if cap(a.values) != cap(b.values) {
+		panic(BpTreeError("cap(a.values) != cap(b.values)"))
+	}
+	if cap(a.pointers) != cap(b.pointers) {
+		panic(BpTreeError("cap(a.pointers) != cap(b.pointers)"))
+	}
+	m := len(a.keys) / 2
+	for m < len(a.keys) && a.keys[m-1].Equals(a.keys[m]) {
+		m++
+	}
+	if m == len(a.keys) {
+		m--
+		for m > 0 && a.keys[m-1].Equals(a.keys[m]) {
+			m--
+		}
+	}
+	var lim int = len(a.keys) - m
+	b.keys = b.keys[:lim]
+	if cap(a.values) > 0 {
+		if cap(a.values) != cap(a.keys) {
+			panic(BpTreeError("cap(a.values) != cap(a.keys)"))
+		}
+		b.values = b.values[:lim]
+	}
+	if cap(a.pointers) > 0 {
+		if cap(a.pointers) != cap(a.keys) {
+			panic(BpTreeError("cap(a.pointers) != cap(a.keys)"))
+		}
+		b.pointers = b.pointers[:lim]
+	}
+	for i := 0; i < lim; i++ {
+		j := m + i
+		b.keys[i] = a.keys[j]
+		if cap(a.values) > 0 {
+			b.values[i] = a.values[j]
+		}
+		if cap(a.pointers) > 0 {
+			b.pointers[i] = a.pointers[j]
+		}
+	}
+	a.keys = a.keys[:m]
+	if cap(a.values) > 0 {
+		a.values = a.values[:m]
+	}
+	if cap(a.pointers) > 0 {
+		a.pointers = a.pointers[:m]
+	}
+}

weed/util/bptree/int.go

@@ -0,0 +1,357 @@
+package bptree
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+type Int8 int8
+type UInt8 uint8
+type Int16 int16
+type UInt16 uint16
+type Int32 int32
+type UInt32 uint32
+type Int64 int64
+type UInt64 uint64
+type Int int
+type UInt uint
+
+func (self *Int8) MarshalBinary() ([]byte, error) {
+	bytes := make([]byte, 0)
+	bytes[0] = uint8(*self)
+	return bytes, nil
+}
+
+func (self *Int8) UnmarshalBinary(data []byte) error {
+	if len(data) != 1 {
+		return fmt.Errorf("data wrong size")
+	}
+	*self = Int8(data[0])
+	return nil
+}
+
+func (self Int8) Equals(other Equatable) bool {
+	if o, ok := other.(Int8); ok {
+		return self == o
+	} else {
+		return false
+	}
+}
+
+func (self Int8) Less(other Sortable) bool {
+	if o, ok := other.(Int8); ok {
+		return self < o
+	} else {
+		return false
+	}
+}
+
+func (self Int8) Hash() int {
+	return int(self)
+}
+
+func (self *UInt8) MarshalBinary() ([]byte, error) {
+	bytes := make([]byte, 0)
+	bytes[0] = uint8(*self)
+	return bytes, nil
+}
+
+func (self *UInt8) UnmarshalBinary(data []byte) error {
+	if len(data) != 1 {
+		return fmt.Errorf("data wrong size")
+	}
+	*self = UInt8(data[0])
+	return nil
+}
+
+func (self UInt8) Equals(other Equatable) bool {
+	if o, ok := other.(UInt8); ok {
+		return self == o
+	} else {
+		return false
+	}
+}
+
+func (self UInt8) Less(other Sortable) bool {
+	if o, ok := other.(UInt8); ok {
+		return self < o
+	} else {
+		return false
+	}
+}
+
+func (self UInt8) Hash() int {
+	return int(self)
+}
+
+func (self *Int16) MarshalBinary() ([]byte, error) {
+	bytes := make([]byte, 2)
+	binary.BigEndian.PutUint16(bytes, uint16(*self))
+	return bytes, nil
+}
+
+func (self *Int16) UnmarshalBinary(data []byte) error {
+	if len(data) != 2 {
+		return fmt.Errorf("data wrong size")
+	}
+	*self = Int16(binary.BigEndian.Uint16(data))
+	return nil
+}
+
+func (self Int16) Equals(other Equatable) bool {
+	if o, ok := other.(Int16); ok {
+		return self == o
+	} else {
+		return false
+	}
+}
+
+func (self Int16) Less(other Sortable) bool {
+	if o, ok := other.(Int16); ok {
+		return self < o
+	} else {
+		return false
+	}
+}
+
+func (self Int16) Hash() int {
+	return int(self)
+}
+
+func (self *UInt16) MarshalBinary() ([]byte, error) {
+	bytes := make([]byte, 2)
+	binary.BigEndian.PutUint16(bytes, uint16(*self))
+	return bytes, nil
+}
+
+func (self *UInt16) UnmarshalBinary(data []byte) error {
+	if len(data) != 2 {
+		return fmt.Errorf("data wrong size")
+	}
+	*self = UInt16(binary.BigEndian.Uint16(data))
+	return nil
+}
+
+func (self UInt16) Equals(other Equatable) bool {
+	if o, ok := other.(UInt16); ok {
+		return self == o
+	} else {
+		return false
+	}
+}
+
+func (self UInt16) Less(other Sortable) bool {
+	if o, ok := other.(UInt16); ok {
+		return self < o
+	} else {
+		return false
+	}
+}
+
+func (self UInt16) Hash() int {
+	return int(self)
+}
+
+func (self *Int32) MarshalBinary() ([]byte, error) {
+	bytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(bytes, uint32(*self))
+	return bytes, nil
+}
+
+func (self *Int32) UnmarshalBinary(data []byte) error {
+	if len(data) != 4 {
+		return fmt.Errorf("data wrong size")
+	}
+	*self = Int32(binary.BigEndian.Uint32(data))
+	return nil
+}
+
+func (self Int32) Equals(other Equatable) bool {
+	if o, ok := other.(Int32); ok {
+		return self == o
+	} else {
+		return false
+	}
+}
+
+func (self Int32) Less(other Sortable) bool {
+	if o, ok := other.(Int32); ok {
+		return self < o
+	} else {
+		return false
+	}
+}
+
+func (self *UInt32) MarshalBinary() ([]byte, error) {
+	bytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(bytes, uint32(*self))
+	return bytes, nil
+}
+
+func (self *UInt32) UnmarshalBinary(data []byte) error {
+	if len(data) != 4 {
+		return fmt.Errorf("data wrong size")
+	}
+	*self = UInt32(binary.BigEndian.Uint32(data))
+	return nil
+}
+
+func (self Int32) Hash() int {
+	return int(self)
+}
+
+func (self UInt32) Equals(other Equatable) bool {
+	if o, ok := other.(UInt32); ok {
+		return self == o
+	} else {
+		return false
+	}
+}
+
+func (self UInt32) Less(other Sortable) bool {
+	if o, ok := other.(UInt32); ok {
+		return self < o
+	} else {
+		return false
+	}
+}
+
+func (self UInt32) Hash() int {
+	return int(self)
+}
+
+func (self *Int64) MarshalBinary() ([]byte, error) {
+	bytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(bytes, uint64(*self))
+	return bytes, nil
+}
+
+func (self *Int64) UnmarshalBinary(data []byte) error {
+	if len(data) != 8 {
+		return fmt.Errorf("data wrong size")
+	}
+	*self = Int64(binary.BigEndian.Uint64(data))
+	return nil
+}
+
+func (self Int64) Equals(other Equatable) bool {
+	if o, ok := other.(Int64); ok {
+		return self == o
+	} else {
+		return false
+	}
+}
+
+func (self Int64) Less(other Sortable) bool {
+	if o, ok := other.(Int64); ok {
+		return self < o
+	} else {
+		return false
+	}
+}
+
+func (self Int64) Hash() int {
+	return int(self>>32) ^ int(self)
+}
+
+func (self *UInt64) MarshalBinary() ([]byte, error) {
+	bytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(bytes, uint64(*self))
+	return bytes, nil
+}
+
+func (self *UInt64) UnmarshalBinary(data []byte) error {
+	if len(data) != 8 {
+		return fmt.Errorf("data wrong size")
+	}
+	*self = UInt64(binary.BigEndian.Uint64(data))
+	return nil
+}
+
+func (self UInt64) Equals(other Equatable) bool {
+	if o, ok := other.(UInt64); ok {
+		return self == o
+	} else {
+		return false
+	}
+}
+
+func (self UInt64) Less(other Sortable) bool {
+	if o, ok := other.(UInt64); ok {
+		return self < o
+	} else {
+		return false
+	}
+}
+
+func (self UInt64) Hash() int {
+	return int(self>>32) ^ int(self)
+}
+
+func (self *Int) MarshalBinary() ([]byte, error) {
+	bytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(bytes, uint32(*self))
+	return bytes, nil
+}
+
+func (self *Int) UnmarshalBinary(data []byte) error {
+	if len(data) != 4 {
+		return fmt.Errorf("data wrong size")
+	}
+	*self = Int(binary.BigEndian.Uint32(data))
+	return nil
+}
+
+func (self Int) Equals(other Equatable) bool {
+	if o, ok := other.(Int); ok {
+		return self == o
+	} else {
+		return false
+	}
+}
+
+func (self Int) Less(other Sortable) bool {
+	if o, ok := other.(Int); ok {
+		return self < o
+	} else {
+		return false
+	}
+}
+
+func (self Int) Hash() int {
+	return int(self)
+}
+
+func (self *UInt) MarshalBinary() ([]byte, error) {
+	bytes := make([]byte, 4)
+	binary.BigEndian.PutUint32(bytes, uint32(*self))
+	return bytes, nil
+}
+
+func (self *UInt) UnmarshalBinary(data []byte) error {
+	if len(data) != 4 {
+		return fmt.Errorf("data wrong size")
+	}
+	*self = UInt(binary.BigEndian.Uint32(data))
+	return nil
+}
+
+func (self UInt) Equals(other Equatable) bool {
+	if o, ok := other.(UInt); ok {
+		return self == o
+	} else {
+		return false
+	}
+}
+
+func (self UInt) Less(other Sortable) bool {
+	if o, ok := other.(UInt); ok {
+		return self < o
+	} else {
+		return false
+	}
+}
+
+func (self UInt) Hash() int {
+	return int(self)
+}

weed/util/bptree/rand.go

@@ -0,0 +1,2 @@
+package bptree
+

weed/util/bptree/string.go

@@ -0,0 +1,71 @@
+package bptree
+
+import (
+	"bytes"
+	"hash/fnv"
+)
+
+type String string
+type ByteSlice []byte
+
+func (self *String) MarshalBinary() ([]byte, error) {
+	return []byte(*self), nil
+}
+
+func (self *String) UnmarshalBinary(data []byte) error {
+	*self = String(data)
+	return nil
+}
+
+func (self String) Equals(other Equatable) bool {
+	if o, ok := other.(String); ok {
+		return self == o
+	} else {
+		return false
+	}
+}
+
+func (self String) Less(other Sortable) bool {
+	if o, ok := other.(String); ok {
+		return self < o
+	} else {
+		return false
+	}
+}
+
+func (self String) Hash() int {
+	h := fnv.New32a()
+	h.Write([]byte(string(self)))
+	return int(h.Sum32())
+}
+
+func (self *ByteSlice) MarshalBinary() ([]byte, error) {
+	return []byte(*self), nil
+}
+
+func (self *ByteSlice) UnmarshalBinary(data []byte) error {
+	*self = ByteSlice(data)
+	return nil
+}
+
+func (self ByteSlice) Equals(other Equatable) bool {
+	if o, ok := other.(ByteSlice); ok {
+		return bytes.Equal(self, o)
+	} else {
+		return false
+	}
+}
+
+func (self ByteSlice) Less(other Sortable) bool {
+	if o, ok := other.(ByteSlice); ok {
+		return bytes.Compare(self, o) < 0 // -1 if a < b
+	} else {
+		return false
+	}
+}
+
+func (self ByteSlice) Hash() int {
+	h := fnv.New32a()
+	h.Write([]byte(self))
+	return int(h.Sum32())
+}

weed/util/bptree/types.go

@@ -0,0 +1,103 @@
+package bptree
+
+import (
+	"errors"
+	"fmt"
+)
+
+type Equatable interface {
+	Equals(b Equatable) bool
+}
+
+type Sortable interface {
+	Equatable
+	Less(b Sortable) bool
+}
+
+type Hashable interface {
+	Sortable
+	Hash() int
+}
+
+var BpTreeError = fmt.Errorf
+
+func NegativeSize() error {
+	return errors.New("negative size")
+}
+
+type Iterator func() (item interface{}, next Iterator)
+type KIterator func() (key Hashable, next KIterator)
+type KVIterator func() (key Hashable, value interface{}, next KVIterator)
+type KVIterable interface {
+	Iterate() KVIterator
+}
+
+type Sized interface {
+	Size() int
+}
+
+type MapOperable interface {
+	Sized
+	Has(key Hashable) bool
+	Put(key Hashable, value interface{}) (err error)
+	Get(key Hashable) (value interface{}, err error)
+	Remove(key Hashable) (value interface{}, err error)
+}
+
+type WhereFunc func(value interface{}) bool
+
+func MakeValuesIterator(obj KVIterable) Iterator {
+	kv_iterator := obj.Iterate()
+	var v_iterator Iterator
+	v_iterator = func() (value interface{}, next Iterator) {
+		_, value, kv_iterator = kv_iterator()
+		if kv_iterator == nil {
+			return nil, nil
+		}
+		return value, v_iterator
+	}
+	return v_iterator
+}
+
+func MakeItemsIterator(obj KVIterable) (kit KIterator) {
+	kv_iterator := obj.Iterate()
+	kit = func() (item Hashable, next KIterator) {
+		var key Hashable
+		var value interface{}
+		key, value, kv_iterator = kv_iterator()
+		if kv_iterator == nil {
+			return nil, nil
+		}
+		return &MapEntry{key, value}, kit
+	}
+	return kit
+}
+
+type MapEntry struct {
+	Key   Hashable
+	Value interface{}
+}
+
+func (m *MapEntry) Equals(other Equatable) bool {
+	if o, ok := other.(*MapEntry); ok {
+		return m.Key.Equals(o.Key)
+	} else {
+		return m.Key.Equals(other)
+	}
+}
+
+func (m *MapEntry) Less(other Sortable) bool {
+	if o, ok := other.(*MapEntry); ok {
+		return m.Key.Less(o.Key)
+	} else {
+		return m.Key.Less(other)
+	}
+}
+
+func (m *MapEntry) Hash() int {
+	return m.Key.Hash()
+}
+
+func (m *MapEntry) String() string {
+	return fmt.Sprintf("<MapEntry %v: %v>", m.Key, m.Value)
+}