From 88d68cad872e6f0d10fadaa14949937b365d8545 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 20 Aug 2021 04:14:52 -0700 Subject: [PATCH] remove dedup --- weed/util/bptree/bpmap.go | 4 +- weed/util/bptree/bptree.go | 18 +------ weed/util/bptree/bptree_node.go | 17 ++---- weed/util/bptree/bptree_test.go | 95 +++++++++++++++------------------ 4 files changed, 51 insertions(+), 83 deletions(-) diff --git a/weed/util/bptree/bpmap.go b/weed/util/bptree/bpmap.go index 43c07d71f..1dff8643a 100644 --- a/weed/util/bptree/bpmap.go +++ b/weed/util/bptree/bpmap.go @@ -11,7 +11,7 @@ type BpMap BpTree func NewBpMap(node_size int) *BpMap { return &BpMap{ - root: NewLeaf(node_size, true), + root: NewLeaf(node_size), } } @@ -47,7 +47,7 @@ func (self *BpMap) Remove(key Hashable) (value interface{}, err error) { return nil, err } if new_root == nil { - self.root = NewLeaf(ns, true) + self.root = NewLeaf(ns) } else { self.root = new_root } diff --git a/weed/util/bptree/bptree.go b/weed/util/bptree/bptree.go index 95f53ab29..eff4f3238 100644 --- a/weed/util/bptree/bptree.go +++ b/weed/util/bptree/bptree.go @@ -14,7 +14,7 @@ type loc_iterator func() (i int, leaf *BpNode, li loc_iterator) func NewBpTree(node_size int) *BpTree { return &BpTree{ - root: NewLeaf(node_size, false), + root: NewLeaf(node_size), } } @@ -26,20 +26,6 @@ func (self *BpTree) Has(key Hashable) bool { return l.keys[j].Equals(key) } -func (self *BpTree) Count(key Hashable) int { - if len(self.root.keys) == 0 { - return 0 - } - j, l := self.root.get_start(key) - count := 0 - end := false - for !end && l.keys[j].Equals(key) { - count++ - j, l, end = next_location(j, l) - } - return count -} - func (self *BpTree) Add(key Hashable, value interface{}) (err error) { new_root, err := self.root.put(key, value) if err != nil { @@ -89,7 +75,7 @@ func (self *BpTree) RemoveWhere(key Hashable, where WhereFunc) (err error) { return err } if new_root == nil { - self.root = NewLeaf(ns, false) + self.root = NewLeaf(ns) } else { self.root = new_root } diff --git a/weed/util/bptree/bptree_node.go b/weed/util/bptree/bptree_node.go index 3574371f5..bd74e6327 100644 --- a/weed/util/bptree/bptree_node.go +++ b/weed/util/bptree/bptree_node.go @@ -6,7 +6,6 @@ type BpNode struct { pointers []*BpNode next *BpNode prev *BpNode - no_dup bool } func NewInternal(size int) *BpNode { @@ -19,14 +18,13 @@ func NewInternal(size int) *BpNode { } } -func NewLeaf(size int, no_dup bool) *BpNode { +func NewLeaf(size int) *BpNode { if size < 0 { panic(NegativeSize()) } return &BpNode{ keys: make([]Hashable, 0, size), values: make([]interface{}, 0, size), - no_dup: no_dup, } } @@ -276,13 +274,6 @@ func (self *BpNode) leaf_insert(key Hashable, value interface{}) (a, b *BpNode, if self.Internal() { return nil, nil, BpTreeError("Expected a leaf node") } - if self.no_dup { - i, has := self.find(key) - if has { - self.values[i] = value - return self, nil, nil - } - } if self.Full() { return self.leaf_split(key, value) } else { @@ -307,7 +298,7 @@ func (self *BpNode) leaf_split(key Hashable, value interface{}) (a, b *BpNode, e return self.pure_leaf_split(key, value) } a = self - b = NewLeaf(self.NodeSize(), self.no_dup) + b = NewLeaf(self.NodeSize()) insert_linked_list_node(b, a, a.next) balance_nodes(a, b) if key.Less(b.keys[0]) { @@ -339,7 +330,7 @@ func (self *BpNode) pure_leaf_split(key Hashable, value interface{}) (a, b *BpNo return nil, nil, BpTreeError("Expected a pure leaf node") } if key.Less(self.keys[0]) { - a = NewLeaf(self.NodeSize(), self.no_dup) + a = NewLeaf(self.NodeSize()) b = self if err := a.put_kv(key, value); err != nil { return nil, nil, err @@ -355,7 +346,7 @@ func (self *BpNode) pure_leaf_split(key Hashable, value interface{}) (a, b *BpNo } return a, nil, nil } else { - b = NewLeaf(self.NodeSize(), self.no_dup) + b = NewLeaf(self.NodeSize()) if err := b.put_kv(key, value); err != nil { return nil, nil, err } diff --git a/weed/util/bptree/bptree_test.go b/weed/util/bptree/bptree_test.go index 38663c543..ef73e862d 100644 --- a/weed/util/bptree/bptree_test.go +++ b/weed/util/bptree/bptree_test.go @@ -119,12 +119,6 @@ func TestAddHasCountFindIterateRemove(t *testing.T) { if has := bpt.Has(randstr(10)); has { t.Error("Table has extra key") } - if count := bpt.Count(r.key); count != 1 { - t.Error(bpt, "Missing key") - } - if count := bpt.Count(randstr(10)); count != 0 { - t.Error("Table has extra key") - } for k, v, next := bpt.Find(r.key)(); next != nil; k, v, next = next() { if !k.Equals(r.key) { t.Error(bpt, "Find Failed Key Error") @@ -190,9 +184,6 @@ func TestAddHasCountFindIterateRemove(t *testing.T) { if has := bpt.Has(r.key); !has { t.Error(bpt, "Missing key") } - if count := bpt.Count(r.key); count != 1 { - t.Error(bpt, "Missing key") - } if err := bpt.RemoveWhere(r.key, func(value interface{}) bool { return true }); err != nil { t.Fatal(bpt, err) } @@ -275,7 +266,7 @@ func TestBpMap(t *testing.T) { } func Test_get_start(t *testing.T) { - root := NewLeaf(2, false) + root := NewLeaf(2) root, err := root.put(Int(1), 1) if err != nil { t.Error(err) @@ -344,7 +335,7 @@ func Test_get_start(t *testing.T) { } func Test_get_end(t *testing.T) { - root := NewLeaf(3, false) + root := NewLeaf(3) root, err := root.put(Int(1), -1) if err != nil { t.Fatal(err) @@ -436,7 +427,7 @@ func Test_get_end(t *testing.T) { } func Test_put_no_root_split(t *testing.T) { - a := NewLeaf(2, false) + a := NewLeaf(2) if err := a.put_kv(Int(1), 1); err != nil { t.Error(err) } @@ -470,7 +461,7 @@ func Test_put_no_root_split(t *testing.T) { } func Test_put_root_split(t *testing.T) { - a := NewLeaf(2, false) + a := NewLeaf(2) p, err := a.put(Int(1), 1) if err != nil { t.Error(err) @@ -520,7 +511,7 @@ func Test_put_root_split(t *testing.T) { func Test_internal_insert_no_split(t *testing.T) { a := NewInternal(3) - leaf := NewLeaf(1, false) + leaf := NewLeaf(1) if err := leaf.put_kv(Int(1), 1); err != nil { t.Error(err) } @@ -548,7 +539,7 @@ func Test_internal_insert_no_split(t *testing.T) { func Test_internal_insert_split_less(t *testing.T) { a := NewInternal(3) - leaf := NewLeaf(1, false) + leaf := NewLeaf(1) if err := leaf.put_kv(Int(1), 1); err != nil { t.Error(err) } @@ -658,7 +649,7 @@ func Test_internal_split_greater(t *testing.T) { } func Test_leaf_insert_no_split(t *testing.T) { - a := NewLeaf(3, false) + a := NewLeaf(3) insert_linked_list_node(a, nil, nil) if err := a.put_kv(Int(1), 1); err != nil { t.Error(err) @@ -684,7 +675,7 @@ func Test_leaf_insert_no_split(t *testing.T) { // tests the defer to split logic func Test_leaf_insert_split_less(t *testing.T) { - a := NewLeaf(3, false) + a := NewLeaf(3) insert_linked_list_node(a, nil, nil) if err := a.put_kv(Int(1), 1); err != nil { t.Error(err) @@ -715,7 +706,7 @@ func Test_leaf_insert_split_less(t *testing.T) { } func Test_leaf_split_less(t *testing.T) { - a := NewLeaf(3, false) + a := NewLeaf(3) insert_linked_list_node(a, nil, nil) if err := a.put_kv(Int(1), 1); err != nil { t.Error(err) @@ -746,7 +737,7 @@ func Test_leaf_split_less(t *testing.T) { } func Test_leaf_split_equal(t *testing.T) { - a := NewLeaf(3, false) + a := NewLeaf(3) insert_linked_list_node(a, nil, nil) if err := a.put_kv(Int(1), 1); err != nil { t.Error(err) @@ -777,7 +768,7 @@ func Test_leaf_split_equal(t *testing.T) { } func Test_leaf_split_greater(t *testing.T) { - a := NewLeaf(3, false) + a := NewLeaf(3) insert_linked_list_node(a, nil, nil) if err := a.put_kv(Int(1), 1); err != nil { t.Error(err) @@ -809,13 +800,13 @@ func Test_leaf_split_greater(t *testing.T) { // tests the defer logic func Test_pure_leaf_insert_split_less(t *testing.T) { - a := NewLeaf(2, false) + a := NewLeaf(2) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2, false) + b := NewLeaf(2) insert_linked_list_node(b, a, nil) - c := NewLeaf(2, false) + c := NewLeaf(2) insert_linked_list_node(c, b, nil) - d := NewLeaf(2, false) + d := NewLeaf(2) insert_linked_list_node(d, c, nil) if err := a.put_kv(Int(3), 1); err != nil { t.Error(err) @@ -882,13 +873,13 @@ func Test_pure_leaf_insert_split_less(t *testing.T) { } func Test_pure_leaf_split_less(t *testing.T) { - a := NewLeaf(2, false) + a := NewLeaf(2) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2, false) + b := NewLeaf(2) insert_linked_list_node(b, a, nil) - c := NewLeaf(2, false) + c := NewLeaf(2) insert_linked_list_node(c, b, nil) - d := NewLeaf(2, false) + d := NewLeaf(2) insert_linked_list_node(d, c, nil) if err := a.put_kv(Int(3), 1); err != nil { t.Error(err) @@ -955,13 +946,13 @@ func Test_pure_leaf_split_less(t *testing.T) { } func Test_pure_leaf_split_equal(t *testing.T) { - a := NewLeaf(2, false) + a := NewLeaf(2) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2, false) + b := NewLeaf(2) insert_linked_list_node(b, a, nil) - c := NewLeaf(2, false) + c := NewLeaf(2) insert_linked_list_node(c, b, nil) - d := NewLeaf(2, false) + d := NewLeaf(2) insert_linked_list_node(d, c, nil) if err := a.put_kv(Int(3), 1); err != nil { t.Error(err) @@ -1019,13 +1010,13 @@ func Test_pure_leaf_split_equal(t *testing.T) { } func Test_pure_leaf_split_greater(t *testing.T) { - a := NewLeaf(2, false) + a := NewLeaf(2) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2, false) + b := NewLeaf(2) insert_linked_list_node(b, a, nil) - c := NewLeaf(2, false) + c := NewLeaf(2) insert_linked_list_node(c, b, nil) - d := NewLeaf(2, false) + d := NewLeaf(2) insert_linked_list_node(d, c, nil) if err := a.put_kv(Int(3), 1); err != nil { t.Error(err) @@ -1089,13 +1080,13 @@ func Test_pure_leaf_split_greater(t *testing.T) { } func Test_find_end_of_pure_run(t *testing.T) { - a := NewLeaf(2, false) + a := NewLeaf(2) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2, false) + b := NewLeaf(2) insert_linked_list_node(b, a, nil) - c := NewLeaf(2, false) + c := NewLeaf(2) insert_linked_list_node(c, b, nil) - d := NewLeaf(2, false) + d := NewLeaf(2) insert_linked_list_node(d, c, nil) if err := a.put_kv(Int(3), 1); err != nil { t.Error(err) @@ -1125,13 +1116,13 @@ func Test_find_end_of_pure_run(t *testing.T) { } func Test_insert_linked_list_node(t *testing.T) { - a := NewLeaf(1, false) + a := NewLeaf(1) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2, false) + b := NewLeaf(2) insert_linked_list_node(b, a, nil) - c := NewLeaf(3, false) + c := NewLeaf(3) insert_linked_list_node(c, b, nil) - d := NewLeaf(4, false) + d := NewLeaf(4) insert_linked_list_node(d, a, b) if a.prev != nil { t.Errorf("expected a.prev == nil") @@ -1160,13 +1151,13 @@ func Test_insert_linked_list_node(t *testing.T) { } func Test_remove_linked_list_node(t *testing.T) { - a := NewLeaf(1, false) + a := NewLeaf(1) insert_linked_list_node(a, nil, nil) - b := NewLeaf(2, false) + b := NewLeaf(2) insert_linked_list_node(b, a, nil) - c := NewLeaf(3, false) + c := NewLeaf(3) insert_linked_list_node(c, b, nil) - d := NewLeaf(4, false) + d := NewLeaf(4) insert_linked_list_node(d, a, b) if a.prev != nil { t.Errorf("expected a.prev == nil") @@ -1235,8 +1226,8 @@ func Test_remove_linked_list_node(t *testing.T) { } func Test_balance_leaf_nodes_with_dup(t *testing.T) { - a := NewLeaf(3, false) - b := NewLeaf(3, false) + a := NewLeaf(3) + b := NewLeaf(3) if err := a.put_kv(Int(1), 1); err != nil { t.Error(err) } @@ -1256,8 +1247,8 @@ func Test_balance_leaf_nodes_with_dup(t *testing.T) { } func Test_balance_leaf_nodes(t *testing.T) { - a := NewLeaf(7, false) - b := NewLeaf(7, false) + a := NewLeaf(7) + b := NewLeaf(7) if err := a.put_kv(Int(1), 1); err != nil { t.Error(err) }