Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

add NodeStore

pull/2354/head
Chris Lu 3 years ago
parent
198fa58e3c
commit
e066e2642c
7 changed files with 147 additions and 91 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      weed/util/bptree/bpmap.go
  2. 6
      weed/util/bptree/bptree.go
  3. 27
      weed/util/bptree/bptree_node.go
  4. 39
      weed/util/bptree/bptree_store_test.go
  5. 106
      weed/util/bptree/bptree_test.go
  6. 8
      weed/util/bptree/getter_setter.go
  7. 46
      weed/util/bptree/serde_test.go

6
weed/util/bptree/bpmap.go
View File

@ -9,9 +9,9 @@ import (
*/
type BpMap BpTree
func NewBpMap(node_size int) *BpMap {
func NewBpMap(node_size int, nodeStore NodeStore) *BpMap {
return &BpMap{
root: NewLeaf(node_size),
root: NewLeaf(node_size, nodeStore),
}
}
@ -47,7 +47,7 @@ func (self *BpMap) Remove(key ItemKey) (value ItemValue, err error) {
return nil, err
}
if new_root == nil {
new_root = NewLeaf(ns)
new_root = NewLeaf(ns, self.root.nodeStore)
err = new_root.persist()
self.setRoot(new_root)
} else {

6
weed/util/bptree/bptree.go
View File

@ -12,9 +12,9 @@ type BpTree struct {
type loc_iterator func() (i int, leaf *BpNode, li loc_iterator)
func NewBpTree(node_size int) *BpTree {
func NewBpTree(node_size int, nodeStore NodeStore) *BpTree {
return &BpTree{
root: NewLeaf(node_size),
root: NewLeaf(node_size, nodeStore),
}
}
@ -93,7 +93,7 @@ func (self *BpTree) RemoveWhere(key ItemKey, where WhereFunc) (err error) {
return err
}
if new_root == nil {
new_root = NewLeaf(ns)
new_root = NewLeaf(ns, self.root.nodeStore)
err = new_root.persist()
self.setRoot(new_root)
} else {

27
weed/util/bptree/bptree_node.go
View File

@ -2,13 +2,11 @@ package bptree
type ItemKey Hashable
type ItemValue Equatable
type PersistFunc func(node *BpNode) error
type DestroyFunc func(node *BpNode) error
var (
PersistFn PersistFunc
DestroyFn DestroyFunc
)
type NodeStore interface {
PersistFunc(node *BpNode) error
DestroyFunc(node *BpNode) error
}
type BpNode struct {
keys []ItemKey
@ -18,9 +16,10 @@ type BpNode struct {
prev *BpNode
protoNodeId int64
protoNode *ProtoNode
nodeStore NodeStore
}
func NewInternal(size int) *BpNode {
func NewInternal(size int, nodeStore NodeStore) *BpNode {
if size < 0 {
panic(NegativeSize())
}
@ -28,10 +27,11 @@ func NewInternal(size int) *BpNode {
keys: make([]ItemKey, 0, size),
pointers: make([]*BpNode, 0, size),
protoNodeId: GetProtoNodeId(),
nodeStore: nodeStore,
}
}
func NewLeaf(size int) *BpNode {
func NewLeaf(size int, nodeStore NodeStore) *BpNode {
if size < 0 {
panic(NegativeSize())
}
@ -39,6 +39,7 @@ func NewLeaf(size int) *BpNode {
keys: make([]ItemKey, 0, size),
values: make([]ItemValue, 0, size),
protoNodeId: GetProtoNodeId(),
nodeStore: nodeStore,
}
}
@ -187,7 +188,7 @@ func (self *BpNode) put(key ItemKey, value ItemValue) (root *BpNode, err error)
return a, nil
}
// else we have root split
root = NewInternal(self.Capacity())
root = NewInternal(self.Capacity(), self.nodeStore)
root.put_kp(a.keys[0], a)
root.put_kp(b.keys[0], b)
return root, root.persist()
@ -256,7 +257,7 @@ func (self *BpNode) internal_split(key ItemKey, ptr *BpNode) (a, b *BpNode, err
return nil, nil, BpTreeError("Tried to split an internal block on duplicate key")
}
a = self
b = NewInternal(self.Capacity())
b = NewInternal(self.Capacity(), self.nodeStore)
balance_nodes(a, b, key)
if b.Len() > 0 && key.Less(b.keys[0]) {
if err := a.put_kp(key, ptr); err != nil {
@ -310,7 +311,7 @@ func (self *BpNode) leaf_split(key ItemKey, value ItemValue) (a, b *BpNode, err
return self.pure_leaf_split(key, value)
}
a = self
b = NewLeaf(self.Capacity())
b = NewLeaf(self.Capacity(), self.nodeStore)
insert_linked_list_node(b, a, a.getNext())
balance_nodes(a, b, key)
if b.Len() > 0 && key.Less(b.keys[0]) {
@ -342,7 +343,7 @@ func (self *BpNode) pure_leaf_split(key ItemKey, value ItemValue) (a, b *BpNode,
return nil, nil, BpTreeError("Expected a pure leaf node")
}
if key.Less(self.keys[0]) {
a = NewLeaf(self.Capacity())
a = NewLeaf(self.Capacity(), self.nodeStore)
b = self
if err := a.put_kv(key, value); err != nil {
return nil, nil, err
@ -358,7 +359,7 @@ func (self *BpNode) pure_leaf_split(key ItemKey, value ItemValue) (a, b *BpNode,
}
return a, nil, a.persist()
} else {
b = NewLeaf(self.Capacity())
b = NewLeaf(self.Capacity(), self.nodeStore)
if err := b.put_kv(key, value); err != nil {
return nil, nil, err
}

39
weed/util/bptree/bptree_store_test.go
View File

@ -5,29 +5,38 @@ import (
"testing"
)
type nodeStorePrintlnImpl struct {
}
func (n *nodeStorePrintlnImpl) PersistFunc(node *BpNode) error {
println("saving node", node.protoNodeId)
return nil
}
func (n *nodeStorePrintlnImpl) DestroyFunc(node *BpNode) error {
println("delete node", node.protoNodeId)
return nil
}
func TestAddRemove(t *testing.T) {
tree := NewBpTree(5)
PersistFn = func(node *BpNode) error {
println("saving", node.protoNodeId)
return nil
}
DestroyFn = func(node *BpNode) error {
println("delete", node.protoNodeId)
return nil
}
for i:=0;i<32;i++{
tree := NewBpTree(3, &nodeStorePrintlnImpl{})
for i:=0;i<9;i++{
println("++++++++++", i)
tree.Add(String(fmt.Sprintf("%02d", i)), nil)
printTree(tree.root, "")
}
if !tree.Has(String("30")) {
if !tree.Has(String("08")) {
t.Errorf("lookup error")
}
tree.RemoveWhere(String("30"), func(value ItemValue) bool {
return true
})
if tree.Has(String("30")) {
for i:=5;i<9;i++{
println("----------", i)
tree.RemoveWhere(String(fmt.Sprintf("%02d", i)), func(value ItemValue) bool {
return true
})
printTree(tree.root, "")
}
if tree.Has(String("08")) {
t.Errorf("remove error")
}
}

106
weed/util/bptree/bptree_test.go
View File

@ -79,7 +79,7 @@ func BenchmarkBpTree(b *testing.B) {
b.StartTimer()
for i := 0; i < b.N; i++ {
t := NewBpTree(23)
t := NewBpTree(23, nil)
for _, r := range recs {
t.Add(r.key, r.value)
}
@ -207,7 +207,7 @@ func TestAddHasCountFindIterateRemove(t *testing.T) {
}
}
for i := 2; i < 64; i++ {
test(NewBpTree(i))
test(NewBpTree(i, nil))
}
}
@ -271,11 +271,11 @@ func TestBpMap(t *testing.T) {
}
}
test(NewBpMap(23))
test(NewBpMap(23, nil))
}
func Test_get_start(t *testing.T) {
root := NewLeaf(2)
root := NewLeaf(2, nil)
root, err := root.put(Int(1), Int(1))
if err != nil {
t.Error(err)
@ -344,7 +344,7 @@ func Test_get_start(t *testing.T) {
}
func Test_get_end(t *testing.T) {
root := NewLeaf(3)
root := NewLeaf(3, nil)
root, err := root.put(Int(1), Int(1))
if err != nil {
t.Fatal(err)
@ -388,7 +388,7 @@ func Test_get_end(t *testing.T) {
}
func Test_put_no_root_split(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
}
@ -423,7 +423,7 @@ func Test_put_no_root_split(t *testing.T) {
}
func Test_put_root_split(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
p, err := a.put(Int(1), Int(1))
if err != nil {
t.Error(err)
@ -472,8 +472,8 @@ func Test_put_root_split(t *testing.T) {
}
func Test_internal_insert_no_split(t *testing.T) {
a := NewInternal(3)
leaf := NewLeaf(1)
a := NewInternal(3, nil)
leaf := NewLeaf(1, nil)
if err := leaf.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
}
@ -500,8 +500,8 @@ func Test_internal_insert_no_split(t *testing.T) {
}
func Test_internal_insert_split_less(t *testing.T) {
a := NewInternal(3)
leaf := NewLeaf(1)
a := NewInternal(3, nil)
leaf := NewLeaf(1, nil)
if err := leaf.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
}
@ -534,7 +534,7 @@ func Test_internal_insert_split_less(t *testing.T) {
}
func Test_internal_split_less(t *testing.T) {
a := NewInternal(3)
a := NewInternal(3, nil)
if err := a.put_kp(Int(1), nil); err != nil {
t.Error(err)
}
@ -564,7 +564,7 @@ func Test_internal_split_less(t *testing.T) {
}
func Test_internal_split_equal(t *testing.T) {
a := NewInternal(3)
a := NewInternal(3, nil)
if err := a.put_kp(Int(1), nil); err != nil {
t.Error(err)
}
@ -581,7 +581,7 @@ func Test_internal_split_equal(t *testing.T) {
}
func Test_internal_split_greater(t *testing.T) {
a := NewInternal(3)
a := NewInternal(3, nil)
if err := a.put_kp(Int(1), nil); err != nil {
t.Error(err)
}
@ -611,7 +611,7 @@ func Test_internal_split_greater(t *testing.T) {
}
func Test_leaf_insert_no_split(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, nil)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
@ -637,7 +637,7 @@ func Test_leaf_insert_no_split(t *testing.T) {
// tests the defer to split logic
func Test_leaf_insert_split_less(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, nil)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
@ -668,7 +668,7 @@ func Test_leaf_insert_split_less(t *testing.T) {
}
func Test_leaf_split_less(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, nil)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
@ -699,7 +699,7 @@ func Test_leaf_split_less(t *testing.T) {
}
func Test_leaf_split_equal(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, nil)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
@ -730,7 +730,7 @@ func Test_leaf_split_equal(t *testing.T) {
}
func Test_leaf_split_greater(t *testing.T) {
a := NewLeaf(3)
a := NewLeaf(3, nil)
insert_linked_list_node(a, nil, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
@ -762,13 +762,13 @@ func Test_leaf_split_greater(t *testing.T) {
// tests the defer logic
func Test_pure_leaf_insert_split_less(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, nil)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), Int(1)); err != nil {
t.Error(err)
@ -835,13 +835,13 @@ func Test_pure_leaf_insert_split_less(t *testing.T) {
}
func Test_pure_leaf_split_less(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, nil)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), Int(1)); err != nil {
t.Error(err)
@ -908,13 +908,13 @@ func Test_pure_leaf_split_less(t *testing.T) {
}
func Test_pure_leaf_split_equal(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, nil)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), Int(1)); err != nil {
t.Error(err)
@ -972,13 +972,13 @@ func Test_pure_leaf_split_equal(t *testing.T) {
}
func Test_pure_leaf_split_greater(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, nil)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), Int(1)); err != nil {
t.Error(err)
@ -1042,13 +1042,13 @@ func Test_pure_leaf_split_greater(t *testing.T) {
}
func Test_find_end_of_pure_run(t *testing.T) {
a := NewLeaf(2)
a := NewLeaf(2, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(2)
c := NewLeaf(2, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(2)
d := NewLeaf(2, nil)
insert_linked_list_node(d, c, nil)
if err := a.put_kv(Int(3), Int(1)); err != nil {
t.Error(err)
@ -1078,13 +1078,13 @@ func Test_find_end_of_pure_run(t *testing.T) {
}
func Test_insert_linked_list_node(t *testing.T) {
a := NewLeaf(1)
a := NewLeaf(1, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(3)
c := NewLeaf(3, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(4)
d := NewLeaf(4, nil)
insert_linked_list_node(d, a, b)
if a.getPrev() != nil {
t.Errorf("expected a.prev == nil")
@ -1113,13 +1113,13 @@ func Test_insert_linked_list_node(t *testing.T) {
}
func Test_remove_linked_list_node(t *testing.T) {
a := NewLeaf(1)
a := NewLeaf(1, nil)
insert_linked_list_node(a, nil, nil)
b := NewLeaf(2)
b := NewLeaf(2, nil)
insert_linked_list_node(b, a, nil)
c := NewLeaf(3)
c := NewLeaf(3, nil)
insert_linked_list_node(c, b, nil)
d := NewLeaf(4)
d := NewLeaf(4, nil)
insert_linked_list_node(d, a, b)
if a.getPrev() != nil {
t.Errorf("expected a.prev == nil")
@ -1188,8 +1188,8 @@ func Test_remove_linked_list_node(t *testing.T) {
}
func Test_balance_leaf_nodes_with_dup(t *testing.T) {
a := NewLeaf(3)
b := NewLeaf(3)
a := NewLeaf(3, nil)
b := NewLeaf(3, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
}
@ -1209,8 +1209,8 @@ func Test_balance_leaf_nodes_with_dup(t *testing.T) {
}
func Test_balance_leaf_nodes(t *testing.T) {
a := NewLeaf(7)
b := NewLeaf(7)
a := NewLeaf(7, nil)
b := NewLeaf(7, nil)
if err := a.put_kv(Int(1), Int(1)); err != nil {
t.Error(err)
}
@ -1258,8 +1258,8 @@ func Test_balance_leaf_nodes(t *testing.T) {
}
func Test_balance_internal_nodes(t *testing.T) {
a := NewInternal(6)
b := NewInternal(6)
a := NewInternal(6, nil)
b := NewInternal(6, nil)
if err := a.put_kp(Int(1), nil); err != nil {
t.Error(err)
}

8
weed/util/bptree/getter_setter.go
View File

@ -45,14 +45,14 @@ func (self *BpNode) maybePersist(shouldPersist bool) error {
return self.persist()
}
func (self *BpNode) persist() error {
if PersistFn != nil {
return PersistFn(self)
if self.nodeStore != nil {
return self.nodeStore.PersistFunc(self)
}
return nil
}
func (self *BpNode) destroy() error {
if DestroyFn != nil {
return DestroyFn(self)
if self.nodeStore != nil {
return self.nodeStore.DestroyFunc(self)
}
return nil
}

46
weed/util/bptree/serde_test.go
View File

@ -0,0 +1,46 @@
package bptree
import (
"fmt"
"testing"
)
type nodeStoreMapImpl struct {
m map[int64]*ProtoNode
}
func (n *nodeStoreMapImpl) PersistFunc(node *BpNode) error {
println("saving node", node.protoNodeId)
n.m[node.protoNodeId] = node.protoNode
return nil
}
func (n *nodeStoreMapImpl) DestroyFunc(node *BpNode) error {
println("delete node", node.protoNodeId)
delete(n.m, node.protoNodeId)
return nil
}
func TestSerDe(t *testing.T) {
nodeStore := &nodeStoreMapImpl{
m: make(map[int64]*ProtoNode),
}
tree := NewBpTree(3, nodeStore)
for i:=0;i<32;i++{
println("add", i)
tree.Add(String(fmt.Sprintf("%02d", i)), nil)
}
for i:=5;i<9;i++{
println("----------", i)
tree.RemoveWhere(String(fmt.Sprintf("%02d", i)), func(value ItemValue) bool {
return true
})
printTree(tree.root, "")
}
}
Write Preview
Loading…
Cancel
Save