Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

add back non_dedup

pull/2354/head
Chris Lu 3 years ago
parent
849f185a20
commit
38c8470d1d
4 changed files with 83 additions and 51 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      weed/util/bptree/bpmap.go
  2. 18
      weed/util/bptree/bptree.go
  3. 17
      weed/util/bptree/bptree_node.go
  4. 95
      weed/util/bptree/bptree_test.go

4
weed/util/bptree/bpmap.go
View File

@ -11,7 +11,7 @@ type BpMap BpTree
func NewBpMap(node_size int) *BpMap {
return &BpMap{
root: NewLeaf(node_size),
root: NewLeaf(node_size, true),
}
}
@ -47,7 +47,7 @@ func (self *BpMap) Remove(key Hashable) (value interface{}, err error) {
return nil, err
}
if new_root == nil {
self.setRoot(NewLeaf(ns))
self.setRoot(NewLeaf(ns, true))
} else {
self.setRoot(new_root)
}

18
weed/util/bptree/bptree.go
View File

@ -14,7 +14,7 @@ type loc_iterator func() (i int, leaf *BpNode, li loc_iterator)
func NewBpTree(node_size int) *BpTree {
return &BpTree{
root: NewLeaf(node_size),
root: NewLeaf(node_size, false),
}
}
@ -26,6 +26,20 @@ func (self *BpTree) Has(key Hashable) bool {
return l.keys[j].Equals(key)
}
func (self *BpTree) Count(key Hashable) int {
if len(self.root.keys) == 0 {
return 0
}
j, l := self.root.get_start(key)
count := 0
end := false
for !end && l.keys[j].Equals(key) {
count++
j, l, end = next_location(j, l)
}
return count
}
func (self *BpTree) Add(key Hashable, value interface{}) (err error) {
new_root, err := self.getRoot().put(key, value)
if err != nil {
@ -75,7 +89,7 @@ func (self *BpTree) RemoveWhere(key Hashable, where WhereFunc) (err error) {
return err
}
if new_root == nil {
self.setRoot(NewLeaf(ns))
self.setRoot(NewLeaf(ns, false))
} else {
self.setRoot(new_root)
}

17
weed/util/bptree/bptree_node.go
View File

@ -6,6 +6,7 @@ type BpNode struct {
pointers []*BpNode
next *BpNode
prev *BpNode
no_dup bool
}
func NewInternal(size int) *BpNode {
@ -18,13 +19,14 @@ func NewInternal(size int) *BpNode {
}
}
func NewLeaf(size int) *BpNode {
func NewLeaf(size int, no_dup bool) *BpNode {
if size < 0 {
panic(NegativeSize())
}
return &BpNode{
keys: make([]Hashable, 0, size),
values: make([]interface{}, 0, size),
no_dup: no_dup,
}
}
@ -274,6 +276,13 @@ func (self *BpNode) leaf_insert(key Hashable, value interface{}) (a, b *BpNode,
if self.Internal() {
return nil, nil, BpTreeError("Expected a leaf node")
}
if self.no_dup {
i, has := self.find(key)
if has {
self.values[i] = value
return self, nil, nil
}
}
if self.Full() {
return self.leaf_split(key, value)
} else {
@ -298,7 +307,7 @@ func (self *BpNode) leaf_split(key Hashable, value interface{}) (a, b *BpNode, e
return self.pure_leaf_split(key, value)
}
a = self
b = NewLeaf(self.NodeSize())
b = NewLeaf(self.NodeSize(), self.no_dup)
insert_linked_list_node(b, a, a.getNext())
balance_nodes(a, b)
if key.Less(b.keys[0]) {
@ -330,7 +339,7 @@ func (self *BpNode) pure_leaf_split(key Hashable, value interface{}) (a, b *BpNo
return nil, nil, BpTreeError("Expected a pure leaf node")
}
if key.Less(self.keys[0]) {
a = NewLeaf(self.NodeSize())
a = NewLeaf(self.NodeSize(), self.no_dup)
b = self
if err := a.put_kv(key, value); err != nil {
return nil, nil, err
@ -346,7 +355,7 @@ func (self *BpNode) pure_leaf_split(key Hashable, value interface{}) (a, b *BpNo
}
return a, nil, nil
} else {
b = NewLeaf(self.NodeSize())
b = NewLeaf(self.NodeSize(), self.no_dup)
if err := b.put_kv(key, value); err != nil {
return nil, nil, err
}

95
weed/util/bptree/bptree_test.go
View File

@ -119,6 +119,12 @@ func TestAddHasCountFindIterateRemove(t *testing.T) {
if has := bpt.Has(randstr(10)); has {
t.Error("Table has extra key")
}
if count := bpt.Count(r.key); count != 1 {
t.Error(bpt, "Missing key")
}
if count := bpt.Count(randstr(10)); count != 0 {
t.Error("Table has extra key")
}
for k, v, next := bpt.Find(r.key)(); next != nil; k, v, next = next() {
if !k.Equals(r.key) {
t.Error(bpt, "Find Failed Key Error")
@ -184,6 +190,9 @@ func TestAddHasCountFindIterateRemove(t *testing.T) {
if has := bpt.Has(r.key); !has {
t.Error(bpt, "Missing key")
}
if count := bpt.Count(r.key); count != 1 {
t.Error(bpt, "Missing key")
}
if err := bpt.RemoveWhere(r.key, func(value interface{}) bool { return true }); err != nil {
t.Fatal(bpt, err)
}
@ -266,7 +275,7 @@ func TestBpMap(t *testing.T) {
}
func Test_get_start(t *testing.T) {
root := NewLeaf(2)
root := NewLeaf(2, false)
root, err := root.put(Int(1), 1)
if err != nil {
t.Error(err)
@ -335,7 +344,7 @@ func Test_get_start(t *testing.T) {
}
func Test_get_end(t *testing.T) {
root := NewLeaf(3)
root := NewLeaf(3, false)
root, err := root.put(Int(1), -1)
if err != nil {
t.Fatal(err)
@ -427,7 +436,7 @@ func Test_get_end(t *testing.T) {
}
func Test_put_no_root_split(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, false)
if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err)
}
@ -461,7 +470,7 @@ func Test_put_no_root_split(t *testing.T) {
}
func Test_put_root_split(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, false)
p, err := a.put(Int(1), 1)
if err != nil {
t.Error(err)
@ -511,7 +520,7 @@ func Test_put_root_split(t *testing.T) {
func Test_internal_insert_no_split(t *testing.T) {
a := NewInternal(3)
leaf := NewLeaf(1)
leaf := NewLeaf(1, false)
if err := leaf.put_kv(Int(1), 1); err != nil {
t.Error(err)
}
@ -539,7 +548,7 @@ func Test_internal_insert_no_split(t *testing.T) {
func Test_internal_insert_split_less(t *testing.T) {
a := NewInternal(3)
leaf := NewLeaf(1)
leaf := NewLeaf(1, false)
if err := leaf.put_kv(Int(1), 1); err != nil {
t.Error(err)
}
@ -649,7 +658,7 @@ func Test_internal_split_greater(t *testing.T) {
}
func Test_leaf_insert_no_split(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, false)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err)
@ -675,7 +684,7 @@ func Test_leaf_insert_no_split(t *testing.T) {
// tests the defer to split logic
func Test_leaf_insert_split_less(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, false)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err)
@ -706,7 +715,7 @@ func Test_leaf_insert_split_less(t *testing.T) {
}
func Test_leaf_split_less(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, false)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err)
@ -737,7 +746,7 @@ func Test_leaf_split_less(t *testing.T) {
}
func Test_leaf_split_equal(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, false)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err)
@ -768,7 +777,7 @@ func Test_leaf_split_equal(t *testing.T) {
}
func Test_leaf_split_greater(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, false)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err)
@ -800,13 +809,13 @@ func Test_leaf_split_greater(t *testing.T) {
// tests the defer logic
func Test_pure_leaf_insert_split_less(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, false)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, false)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, false)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), 1); err != nil {
t.Error(err)
@ -873,13 +882,13 @@ func Test_pure_leaf_insert_split_less(t *testing.T) {
}
func Test_pure_leaf_split_less(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, false)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, false)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, false)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), 1); err != nil {
t.Error(err)
@ -946,13 +955,13 @@ func Test_pure_leaf_split_less(t *testing.T) {
}
func Test_pure_leaf_split_equal(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, false)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, false)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, false)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), 1); err != nil {
t.Error(err)
@ -1010,13 +1019,13 @@ func Test_pure_leaf_split_equal(t *testing.T) {
}
func Test_pure_leaf_split_greater(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, false)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, false)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, false)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), 1); err != nil {
t.Error(err)
@ -1080,13 +1089,13 @@ func Test_pure_leaf_split_greater(t *testing.T) {
}
func Test_find_end_of_pure_run(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, false)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, false)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, false)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), 1); err != nil {
t.Error(err)
@ -1116,13 +1125,13 @@ func Test_find_end_of_pure_run(t *testing.T) {
}
func Test_insert_linked_list_node(t *testing.T) {
a := NewLeaf(1)
a := NewLeaf(1, false)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil)
c := NewLeaf(3)
c := NewLeaf(3, false)
insert_linked_list_node(c, b, nil)
d := NewLeaf(4)
d := NewLeaf(4, false)
insert_linked_list_node(d, a, b)
if a.getPrev() != nil {
t.Errorf("expected a.prev == nil")
@ -1151,13 +1160,13 @@ func Test_insert_linked_list_node(t *testing.T) {
}
func Test_remove_linked_list_node(t *testing.T) {
a := NewLeaf(1)
a := NewLeaf(1, false)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, false)
insert_linked_list_node(b, a, nil)
c := NewLeaf(3)
c := NewLeaf(3, false)
insert_linked_list_node(c, b, nil)
d := NewLeaf(4)
d := NewLeaf(4, false)
insert_linked_list_node(d, a, b)
if a.getPrev() != nil {
t.Errorf("expected a.prev == nil")
@ -1226,8 +1235,8 @@ func Test_remove_linked_list_node(t *testing.T) {
}
func Test_balance_leaf_nodes_with_dup(t *testing.T) {
a := NewLeaf(3)
b := NewLeaf(3)
a := NewLeaf(3, false)
b := NewLeaf(3, false)
if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err)
}
@ -1247,8 +1256,8 @@ func Test_balance_leaf_nodes_with_dup(t *testing.T) {
}
func Test_balance_leaf_nodes(t *testing.T) {
a := NewLeaf(7)
b := NewLeaf(7)
a := NewLeaf(7, false)
b := NewLeaf(7, false)
if err := a.put_kv(Int(1), 1); err != nil {
t.Error(err)
}

Write Preview
Loading…
Cancel
Save