remove dedup

weed/util/bptree/bpmap.go

@@ -11,7 +11,7 @@ type BpMap BpTree
 
 func NewBpMap(node_size int) *BpMap {
 	return &BpMap{
-		root: NewLeaf(node_size, true),
+		root: NewLeaf(node_size),
 	}
 }
 
@@ -47,7 +47,7 @@ func (self *BpMap) Remove(key Hashable) (value interface{}, err error) {
 		return nil, err
 	}
 	if new_root == nil {
-		self.root = NewLeaf(ns, true)
+		self.root = NewLeaf(ns)
 	} else {
 		self.root = new_root
 	}

weed/util/bptree/bptree.go

@@ -14,7 +14,7 @@ type loc_iterator func() (i int, leaf *BpNode, li loc_iterator)
 
 func NewBpTree(node_size int) *BpTree {
 	return &BpTree{
-		root: NewLeaf(node_size, false),
+		root: NewLeaf(node_size),
 	}
 }
 
@@ -26,20 +26,6 @@ func (self *BpTree) Has(key Hashable) bool {
 	return l.keys[j].Equals(key)
 }
 
-func (self *BpTree) Count(key Hashable) int {
-	if len(self.root.keys) == 0 {
-		return 0
-	}
-	j, l := self.root.get_start(key)
-	count := 0
-	end := false
-	for !end && l.keys[j].Equals(key) {
-		count++
-		j, l, end = next_location(j, l)
-	}
-	return count
-}
-
 func (self *BpTree) Add(key Hashable, value interface{}) (err error) {
 	new_root, err := self.root.put(key, value)
 	if err != nil {
@@ -89,7 +75,7 @@ func (self *BpTree) RemoveWhere(key Hashable, where WhereFunc) (err error) {
 		return err
 	}
 	if new_root == nil {
-		self.root = NewLeaf(ns, false)
+		self.root = NewLeaf(ns)
 	} else {
 		self.root = new_root
 	}

weed/util/bptree/bptree_node.go

@@ -6,7 +6,6 @@ type BpNode struct {
 	pointers []*BpNode
 	next     *BpNode
 	prev     *BpNode
-	no_dup   bool
 }
 
 func NewInternal(size int) *BpNode {
@@ -19,14 +18,13 @@ func NewInternal(size int) *BpNode {
 	}
 }
 
-func NewLeaf(size int, no_dup bool) *BpNode {
+func NewLeaf(size int) *BpNode {
 	if size < 0 {
 		panic(NegativeSize())
 	}
 	return &BpNode{
 		keys:   make([]Hashable, 0, size),
 		values: make([]interface{}, 0, size),
-		no_dup: no_dup,
 	}
 }
 
@@ -276,13 +274,6 @@ func (self *BpNode) leaf_insert(key Hashable, value interface{}) (a, b *BpNode,
 	if self.Internal() {
 		return nil, nil, BpTreeError("Expected a leaf node")
 	}
-	if self.no_dup {
-		i, has := self.find(key)
-		if has {
-			self.values[i] = value
-			return self, nil, nil
-		}
-	}
 	if self.Full() {
 		return self.leaf_split(key, value)
 	} else {
@@ -307,7 +298,7 @@ func (self *BpNode) leaf_split(key Hashable, value interface{}) (a, b *BpNode, e
 		return self.pure_leaf_split(key, value)
 	}
 	a = self
-	b = NewLeaf(self.NodeSize(), self.no_dup)
+	b = NewLeaf(self.NodeSize())
 	insert_linked_list_node(b, a, a.next)
 	balance_nodes(a, b)
 	if key.Less(b.keys[0]) {
@@ -339,7 +330,7 @@ func (self *BpNode) pure_leaf_split(key Hashable, value interface{}) (a, b *BpNo
 		return nil, nil, BpTreeError("Expected a pure leaf node")
 	}
 	if key.Less(self.keys[0]) {
-		a = NewLeaf(self.NodeSize(), self.no_dup)
+		a = NewLeaf(self.NodeSize())
 		b = self
 		if err := a.put_kv(key, value); err != nil {
 			return nil, nil, err
@@ -355,7 +346,7 @@ func (self *BpNode) pure_leaf_split(key Hashable, value interface{}) (a, b *BpNo
 			}
 			return a, nil, nil
 		} else {
-			b = NewLeaf(self.NodeSize(), self.no_dup)
+			b = NewLeaf(self.NodeSize())
 			if err := b.put_kv(key, value); err != nil {
 				return nil, nil, err
 			}

weed/util/bptree/bptree_test.go

@@ -119,12 +119,6 @@ func TestAddHasCountFindIterateRemove(t *testing.T) {
 			if has := bpt.Has(randstr(10)); has {
 				t.Error("Table has extra key")
 			}
-			if count := bpt.Count(r.key); count != 1 {
-				t.Error(bpt, "Missing key")
-			}
-			if count := bpt.Count(randstr(10)); count != 0 {
-				t.Error("Table has extra key")
-			}
 			for k, v, next := bpt.Find(r.key)(); next != nil; k, v, next = next() {
 				if !k.Equals(r.key) {
 					t.Error(bpt, "Find Failed Key Error")
@@ -190,9 +184,6 @@ func TestAddHasCountFindIterateRemove(t *testing.T) {
 			if has := bpt.Has(r.key); !has {
 				t.Error(bpt, "Missing key")
 			}
-			if count := bpt.Count(r.key); count != 1 {
-				t.Error(bpt, "Missing key")
-			}
 			if err := bpt.RemoveWhere(r.key, func(value interface{}) bool { return true }); err != nil {
 				t.Fatal(bpt, err)
 			}
@@ -275,7 +266,7 @@ func TestBpMap(t *testing.T) {
 }
 
 func Test_get_start(t *testing.T) {
-	root := NewLeaf(2, false)
+	root := NewLeaf(2)
 	root, err := root.put(Int(1), 1)
 	if err != nil {
 		t.Error(err)
@@ -344,7 +335,7 @@ func Test_get_start(t *testing.T) {
 }
 
 func Test_get_end(t *testing.T) {
-	root := NewLeaf(3, false)
+	root := NewLeaf(3)
 	root, err := root.put(Int(1), -1)
 	if err != nil {
 		t.Fatal(err)
@@ -436,7 +427,7 @@ func Test_get_end(t *testing.T) {
 }
 
 func Test_put_no_root_split(t *testing.T) {
-	a := NewLeaf(2, false)
+	a := NewLeaf(2)
 	if err := a.put_kv(Int(1), 1); err != nil {
 		t.Error(err)
 	}
@@ -470,7 +461,7 @@ func Test_put_no_root_split(t *testing.T) {
 }
 
 func Test_put_root_split(t *testing.T) {
-	a := NewLeaf(2, false)
+	a := NewLeaf(2)
 	p, err := a.put(Int(1), 1)
 	if err != nil {
 		t.Error(err)
@@ -520,7 +511,7 @@ func Test_put_root_split(t *testing.T) {
 
 func Test_internal_insert_no_split(t *testing.T) {
 	a := NewInternal(3)
-	leaf := NewLeaf(1, false)
+	leaf := NewLeaf(1)
 	if err := leaf.put_kv(Int(1), 1); err != nil {
 		t.Error(err)
 	}
@@ -548,7 +539,7 @@ func Test_internal_insert_no_split(t *testing.T) {
 
 func Test_internal_insert_split_less(t *testing.T) {
 	a := NewInternal(3)
-	leaf := NewLeaf(1, false)
+	leaf := NewLeaf(1)
 	if err := leaf.put_kv(Int(1), 1); err != nil {
 		t.Error(err)
 	}
@@ -658,7 +649,7 @@ func Test_internal_split_greater(t *testing.T) {
 }
 
 func Test_leaf_insert_no_split(t *testing.T) {
-	a := NewLeaf(3, false)
+	a := NewLeaf(3)
 	insert_linked_list_node(a, nil, nil)
 	if err := a.put_kv(Int(1), 1); err != nil {
 		t.Error(err)
@@ -684,7 +675,7 @@ func Test_leaf_insert_no_split(t *testing.T) {
 
 // tests the defer to split logic
 func Test_leaf_insert_split_less(t *testing.T) {
-	a := NewLeaf(3, false)
+	a := NewLeaf(3)
 	insert_linked_list_node(a, nil, nil)
 	if err := a.put_kv(Int(1), 1); err != nil {
 		t.Error(err)
@@ -715,7 +706,7 @@ func Test_leaf_insert_split_less(t *testing.T) {
 }
 
 func Test_leaf_split_less(t *testing.T) {
-	a := NewLeaf(3, false)
+	a := NewLeaf(3)
 	insert_linked_list_node(a, nil, nil)
 	if err := a.put_kv(Int(1), 1); err != nil {
 		t.Error(err)
@@ -746,7 +737,7 @@ func Test_leaf_split_less(t *testing.T) {
 }
 
 func Test_leaf_split_equal(t *testing.T) {
-	a := NewLeaf(3, false)
+	a := NewLeaf(3)
 	insert_linked_list_node(a, nil, nil)
 	if err := a.put_kv(Int(1), 1); err != nil {
 		t.Error(err)
@@ -777,7 +768,7 @@ func Test_leaf_split_equal(t *testing.T) {
 }
 
 func Test_leaf_split_greater(t *testing.T) {
-	a := NewLeaf(3, false)
+	a := NewLeaf(3)
 	insert_linked_list_node(a, nil, nil)
 	if err := a.put_kv(Int(1), 1); err != nil {
 		t.Error(err)
@@ -809,13 +800,13 @@ func Test_leaf_split_greater(t *testing.T) {
 
 // tests the defer logic
 func Test_pure_leaf_insert_split_less(t *testing.T) {
-	a := NewLeaf(2, false)
+	a := NewLeaf(2)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2, false)
+	b := NewLeaf(2)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(2, false)
+	c := NewLeaf(2)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(2, false)
+	d := NewLeaf(2)
 	insert_linked_list_node(d, c, nil)
 	if err := a.put_kv(Int(3), 1); err != nil {
 		t.Error(err)
@@ -882,13 +873,13 @@ func Test_pure_leaf_insert_split_less(t *testing.T) {
 }
 
 func Test_pure_leaf_split_less(t *testing.T) {
-	a := NewLeaf(2, false)
+	a := NewLeaf(2)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2, false)
+	b := NewLeaf(2)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(2, false)
+	c := NewLeaf(2)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(2, false)
+	d := NewLeaf(2)
 	insert_linked_list_node(d, c, nil)
 	if err := a.put_kv(Int(3), 1); err != nil {
 		t.Error(err)
@@ -955,13 +946,13 @@ func Test_pure_leaf_split_less(t *testing.T) {
 }
 
 func Test_pure_leaf_split_equal(t *testing.T) {
-	a := NewLeaf(2, false)
+	a := NewLeaf(2)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2, false)
+	b := NewLeaf(2)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(2, false)
+	c := NewLeaf(2)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(2, false)
+	d := NewLeaf(2)
 	insert_linked_list_node(d, c, nil)
 	if err := a.put_kv(Int(3), 1); err != nil {
 		t.Error(err)
@@ -1019,13 +1010,13 @@ func Test_pure_leaf_split_equal(t *testing.T) {
 }
 
 func Test_pure_leaf_split_greater(t *testing.T) {
-	a := NewLeaf(2, false)
+	a := NewLeaf(2)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2, false)
+	b := NewLeaf(2)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(2, false)
+	c := NewLeaf(2)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(2, false)
+	d := NewLeaf(2)
 	insert_linked_list_node(d, c, nil)
 	if err := a.put_kv(Int(3), 1); err != nil {
 		t.Error(err)
@@ -1089,13 +1080,13 @@ func Test_pure_leaf_split_greater(t *testing.T) {
 }
 
 func Test_find_end_of_pure_run(t *testing.T) {
-	a := NewLeaf(2, false)
+	a := NewLeaf(2)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2, false)
+	b := NewLeaf(2)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(2, false)
+	c := NewLeaf(2)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(2, false)
+	d := NewLeaf(2)
 	insert_linked_list_node(d, c, nil)
 	if err := a.put_kv(Int(3), 1); err != nil {
 		t.Error(err)
@@ -1125,13 +1116,13 @@ func Test_find_end_of_pure_run(t *testing.T) {
 }
 
 func Test_insert_linked_list_node(t *testing.T) {
-	a := NewLeaf(1, false)
+	a := NewLeaf(1)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2, false)
+	b := NewLeaf(2)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(3, false)
+	c := NewLeaf(3)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(4, false)
+	d := NewLeaf(4)
 	insert_linked_list_node(d, a, b)
 	if a.prev != nil {
 		t.Errorf("expected a.prev == nil")
@@ -1160,13 +1151,13 @@ func Test_insert_linked_list_node(t *testing.T) {
 }
 
 func Test_remove_linked_list_node(t *testing.T) {
-	a := NewLeaf(1, false)
+	a := NewLeaf(1)
 	insert_linked_list_node(a, nil, nil)
-	b := NewLeaf(2, false)
+	b := NewLeaf(2)
 	insert_linked_list_node(b, a, nil)
-	c := NewLeaf(3, false)
+	c := NewLeaf(3)
 	insert_linked_list_node(c, b, nil)
-	d := NewLeaf(4, false)
+	d := NewLeaf(4)
 	insert_linked_list_node(d, a, b)
 	if a.prev != nil {
 		t.Errorf("expected a.prev == nil")
@@ -1235,8 +1226,8 @@ func Test_remove_linked_list_node(t *testing.T) {
 }
 
 func Test_balance_leaf_nodes_with_dup(t *testing.T) {
-	a := NewLeaf(3, false)
-	b := NewLeaf(3, false)
+	a := NewLeaf(3)
+	b := NewLeaf(3)
 	if err := a.put_kv(Int(1), 1); err != nil {
 		t.Error(err)
 	}
@@ -1256,8 +1247,8 @@ func Test_balance_leaf_nodes_with_dup(t *testing.T) {
 }
 
 func Test_balance_leaf_nodes(t *testing.T) {
-	a := NewLeaf(7, false)
-	b := NewLeaf(7, false)
+	a := NewLeaf(7)
+	b := NewLeaf(7)
 	if err := a.put_kv(Int(1), 1); err != nil {
 		t.Error(err)
 	}