Chris Lu
8 years ago
17 changed files with 361 additions and 48 deletions
-
2weed/command/export.go
-
2weed/command/fix.go
-
4weed/command/server.go
-
4weed/command/volume.go
-
52weed/storage/needle/btree_map.go
-
194weed/storage/needle/compact_map.go
-
9weed/storage/needle/compact_map_perf_test.go
-
2weed/storage/needle/compact_map_test.go
-
28weed/storage/needle/needle_value.go
-
8weed/storage/needle/needle_value_map.go
-
4weed/storage/needle_map.go
-
7weed/storage/needle_map_boltdb.go
-
7weed/storage/needle_map_leveldb.go
-
40weed/storage/needle_map_memory.go
-
15weed/storage/volume_loading.go
-
27weed/storage/volume_sync.go
-
4weed/storage/volume_vacuum.go
@ -0,0 +1,52 @@ |
|||||
|
package needle |
||||
|
|
||||
|
import ( |
||||
|
"github.com/google/btree" |
||||
|
) |
||||
|
|
||||
|
//This map assumes mostly inserting increasing keys
|
||||
|
type BtreeMap struct { |
||||
|
tree *btree.BTree |
||||
|
} |
||||
|
|
||||
|
func NewBtreeMap() *BtreeMap { |
||||
|
return &BtreeMap{ |
||||
|
tree: btree.New(32), |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func (cm *BtreeMap) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32) { |
||||
|
found := cm.tree.ReplaceOrInsert(NeedleValue{key, offset, size}) |
||||
|
if found != nil { |
||||
|
old := found.(NeedleValue) |
||||
|
return old.Offset, old.Size |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
func (cm *BtreeMap) Delete(key Key) (oldSize uint32) { |
||||
|
found := cm.tree.Delete(NeedleValue{key, 0, 0}) |
||||
|
if found != nil { |
||||
|
old := found.(NeedleValue) |
||||
|
return old.Size |
||||
|
} |
||||
|
return |
||||
|
} |
||||
|
func (cm *BtreeMap) Get(key Key) (*NeedleValue, bool) { |
||||
|
found := cm.tree.Get(NeedleValue{key, 0, 0}) |
||||
|
if found != nil { |
||||
|
old := found.(NeedleValue) |
||||
|
return &old, true |
||||
|
} |
||||
|
return nil, false |
||||
|
} |
||||
|
|
||||
|
// Visit visits all entries or stop if any error when visiting
|
||||
|
func (cm *BtreeMap) Visit(visit func(NeedleValue) error) (ret error) { |
||||
|
cm.tree.Ascend(func(item btree.Item) bool { |
||||
|
needle := item.(NeedleValue) |
||||
|
ret = visit(needle) |
||||
|
return ret == nil |
||||
|
}) |
||||
|
return ret |
||||
|
} |
@ -0,0 +1,194 @@ |
|||||
|
package needle |
||||
|
|
||||
|
import ( |
||||
|
"sync" |
||||
|
) |
||||
|
|
||||
|
type CompactSection struct { |
||||
|
sync.RWMutex |
||||
|
values []NeedleValue |
||||
|
overflow map[Key]NeedleValue |
||||
|
start Key |
||||
|
end Key |
||||
|
counter int |
||||
|
} |
||||
|
|
||||
|
func NewCompactSection(start Key) *CompactSection { |
||||
|
return &CompactSection{ |
||||
|
values: make([]NeedleValue, batch), |
||||
|
overflow: make(map[Key]NeedleValue), |
||||
|
start: start, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
//return old entry size
|
||||
|
func (cs *CompactSection) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32) { |
||||
|
cs.Lock() |
||||
|
if key > cs.end { |
||||
|
cs.end = key |
||||
|
} |
||||
|
if i := cs.binarySearchValues(key); i >= 0 { |
||||
|
oldOffset, oldSize = cs.values[i].Offset, cs.values[i].Size |
||||
|
//println("key", key, "old size", ret)
|
||||
|
cs.values[i].Offset, cs.values[i].Size = offset, size |
||||
|
} else { |
||||
|
needOverflow := cs.counter >= batch |
||||
|
needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > key |
||||
|
if needOverflow { |
||||
|
//println("start", cs.start, "counter", cs.counter, "key", key)
|
||||
|
if oldValue, found := cs.overflow[key]; found { |
||||
|
oldOffset, oldSize = oldValue.Offset, oldValue.Size |
||||
|
} |
||||
|
cs.overflow[key] = NeedleValue{Key: key, Offset: offset, Size: size} |
||||
|
} else { |
||||
|
p := &cs.values[cs.counter] |
||||
|
p.Key, p.Offset, p.Size = key, offset, size |
||||
|
//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
|
||||
|
cs.counter++ |
||||
|
} |
||||
|
} |
||||
|
cs.Unlock() |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
//return old entry size
|
||||
|
func (cs *CompactSection) Delete(key Key) uint32 { |
||||
|
cs.Lock() |
||||
|
ret := uint32(0) |
||||
|
if i := cs.binarySearchValues(key); i >= 0 { |
||||
|
if cs.values[i].Size > 0 { |
||||
|
ret = cs.values[i].Size |
||||
|
cs.values[i].Size = 0 |
||||
|
} |
||||
|
} |
||||
|
if v, found := cs.overflow[key]; found { |
||||
|
delete(cs.overflow, key) |
||||
|
ret = v.Size |
||||
|
} |
||||
|
cs.Unlock() |
||||
|
return ret |
||||
|
} |
||||
|
func (cs *CompactSection) Get(key Key) (*NeedleValue, bool) { |
||||
|
cs.RLock() |
||||
|
if v, ok := cs.overflow[key]; ok { |
||||
|
cs.RUnlock() |
||||
|
return &v, true |
||||
|
} |
||||
|
if i := cs.binarySearchValues(key); i >= 0 { |
||||
|
cs.RUnlock() |
||||
|
return &cs.values[i], true |
||||
|
} |
||||
|
cs.RUnlock() |
||||
|
return nil, false |
||||
|
} |
||||
|
func (cs *CompactSection) binarySearchValues(key Key) int { |
||||
|
l, h := 0, cs.counter-1 |
||||
|
if h >= 0 && cs.values[h].Key < key { |
||||
|
return -2 |
||||
|
} |
||||
|
//println("looking for key", key)
|
||||
|
for l <= h { |
||||
|
m := (l + h) / 2 |
||||
|
//println("mid", m, "key", cs.values[m].Key, cs.values[m].Offset, cs.values[m].Size)
|
||||
|
if cs.values[m].Key < key { |
||||
|
l = m + 1 |
||||
|
} else if key < cs.values[m].Key { |
||||
|
h = m - 1 |
||||
|
} else { |
||||
|
//println("found", m)
|
||||
|
return m |
||||
|
} |
||||
|
} |
||||
|
return -1 |
||||
|
} |
||||
|
|
||||
|
//This map assumes mostly inserting increasing keys
|
||||
|
//This map assumes mostly inserting increasing keys
|
||||
|
type CompactMap struct { |
||||
|
list []*CompactSection |
||||
|
} |
||||
|
|
||||
|
func NewCompactMap() *CompactMap { |
||||
|
return &CompactMap{} |
||||
|
} |
||||
|
|
||||
|
func (cm *CompactMap) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32) { |
||||
|
x := cm.binarySearchCompactSection(key) |
||||
|
if x < 0 { |
||||
|
//println(x, "creating", len(cm.list), "section, starting", key)
|
||||
|
cm.list = append(cm.list, NewCompactSection(key)) |
||||
|
x = len(cm.list) - 1 |
||||
|
//keep compact section sorted by start
|
||||
|
for x > 0 { |
||||
|
if cm.list[x-1].start > cm.list[x].start { |
||||
|
cm.list[x-1], cm.list[x] = cm.list[x], cm.list[x-1] |
||||
|
x = x - 1 |
||||
|
} else { |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return cm.list[x].Set(key, offset, size) |
||||
|
} |
||||
|
func (cm *CompactMap) Delete(key Key) uint32 { |
||||
|
x := cm.binarySearchCompactSection(key) |
||||
|
if x < 0 { |
||||
|
return uint32(0) |
||||
|
} |
||||
|
return cm.list[x].Delete(key) |
||||
|
} |
||||
|
func (cm *CompactMap) Get(key Key) (*NeedleValue, bool) { |
||||
|
x := cm.binarySearchCompactSection(key) |
||||
|
if x < 0 { |
||||
|
return nil, false |
||||
|
} |
||||
|
return cm.list[x].Get(key) |
||||
|
} |
||||
|
func (cm *CompactMap) binarySearchCompactSection(key Key) int { |
||||
|
l, h := 0, len(cm.list)-1 |
||||
|
if h < 0 { |
||||
|
return -5 |
||||
|
} |
||||
|
if cm.list[h].start <= key { |
||||
|
if cm.list[h].counter < batch || key <= cm.list[h].end { |
||||
|
return h |
||||
|
} |
||||
|
return -4 |
||||
|
} |
||||
|
for l <= h { |
||||
|
m := (l + h) / 2 |
||||
|
if key < cm.list[m].start { |
||||
|
h = m - 1 |
||||
|
} else { // cm.list[m].start <= key
|
||||
|
if cm.list[m+1].start <= key { |
||||
|
l = m + 1 |
||||
|
} else { |
||||
|
return m |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return -3 |
||||
|
} |
||||
|
|
||||
|
// Visit visits all entries or stop if any error when visiting
|
||||
|
func (cm *CompactMap) Visit(visit func(NeedleValue) error) error { |
||||
|
for _, cs := range cm.list { |
||||
|
cs.RLock() |
||||
|
for _, v := range cs.overflow { |
||||
|
if err := visit(v); err != nil { |
||||
|
cs.RUnlock() |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
for _, v := range cs.values { |
||||
|
if _, found := cs.overflow[v.Key]; !found { |
||||
|
if err := visit(v); err != nil { |
||||
|
cs.RUnlock() |
||||
|
return err |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
cs.RUnlock() |
||||
|
} |
||||
|
return nil |
||||
|
} |
@ -1,4 +1,4 @@ |
|||||
package storage |
|
||||
|
package needle |
||||
|
|
||||
import ( |
import ( |
||||
"testing" |
"testing" |
@ -0,0 +1,28 @@ |
|||||
|
package needle |
||||
|
|
||||
|
import ( |
||||
|
"strconv" |
||||
|
|
||||
|
"github.com/google/btree" |
||||
|
) |
||||
|
|
||||
|
const ( |
||||
|
batch = 100000 |
||||
|
) |
||||
|
|
||||
|
type NeedleValue struct { |
||||
|
Key Key |
||||
|
Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
|
||||
|
Size uint32 `comment:"Size of the data portion"` |
||||
|
} |
||||
|
|
||||
|
func (this NeedleValue) Less(than btree.Item) bool { |
||||
|
that := than.(NeedleValue) |
||||
|
return this.Key < that.Key |
||||
|
} |
||||
|
|
||||
|
type Key uint64 |
||||
|
|
||||
|
func (k Key) String() string { |
||||
|
return strconv.FormatUint(uint64(k), 10) |
||||
|
} |
@ -0,0 +1,8 @@ |
|||||
|
package needle |
||||
|
|
||||
|
type NeedleValueMap interface { |
||||
|
Set(key Key, offset, size uint32) (oldOffset, oldSize uint32) |
||||
|
Delete(key Key) uint32 |
||||
|
Get(key Key) (*NeedleValue, bool) |
||||
|
Visit(visit func(NeedleValue) error) error |
||||
|
} |
Write
Preview
Loading…
Cancel
Save
Reference in new issue