You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
477 lines
12 KiB
477 lines
12 KiB
package needle_map
|
|
|
|
import (
|
|
"fmt"
|
|
"math/rand"
|
|
"reflect"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
)
|
|
|
|
func TestSegmentBsearchKey(t *testing.T) {
|
|
testSegment := &CompactMapSegment{
|
|
list: []CompactNeedleValue{
|
|
CompactNeedleValue{key: 10},
|
|
CompactNeedleValue{key: 20},
|
|
CompactNeedleValue{key: 21},
|
|
CompactNeedleValue{key: 26},
|
|
CompactNeedleValue{key: 30},
|
|
},
|
|
firstKey: 10,
|
|
lastKey: 30,
|
|
}
|
|
|
|
testCases := []struct {
|
|
name string
|
|
cs *CompactMapSegment
|
|
key types.NeedleId
|
|
wantIndex int
|
|
wantFound bool
|
|
}{
|
|
{
|
|
name: "empty segment",
|
|
cs: newCompactMapSegment(0),
|
|
key: 123,
|
|
wantIndex: 0,
|
|
wantFound: false,
|
|
},
|
|
{
|
|
name: "new key, insert at beggining",
|
|
cs: testSegment,
|
|
key: 5,
|
|
wantIndex: 0,
|
|
wantFound: false,
|
|
},
|
|
{
|
|
name: "new key, insert at end",
|
|
cs: testSegment,
|
|
key: 100,
|
|
wantIndex: 5,
|
|
wantFound: false,
|
|
},
|
|
{
|
|
name: "new key, insert second",
|
|
cs: testSegment,
|
|
key: 12,
|
|
wantIndex: 1,
|
|
wantFound: false,
|
|
},
|
|
{
|
|
name: "new key, insert in middle",
|
|
cs: testSegment,
|
|
key: 23,
|
|
wantIndex: 3,
|
|
wantFound: false,
|
|
},
|
|
{
|
|
name: "key #1",
|
|
cs: testSegment,
|
|
key: 10,
|
|
wantIndex: 0,
|
|
wantFound: true,
|
|
},
|
|
{
|
|
name: "key #2",
|
|
cs: testSegment,
|
|
key: 20,
|
|
wantIndex: 1,
|
|
wantFound: true,
|
|
},
|
|
{
|
|
name: "key #3",
|
|
cs: testSegment,
|
|
key: 21,
|
|
wantIndex: 2,
|
|
wantFound: true,
|
|
},
|
|
{
|
|
name: "key #4",
|
|
cs: testSegment,
|
|
key: 26,
|
|
wantIndex: 3,
|
|
wantFound: true,
|
|
},
|
|
{
|
|
name: "key #5",
|
|
cs: testSegment,
|
|
key: 30,
|
|
wantIndex: 4,
|
|
wantFound: true,
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
index, found := tc.cs.bsearchKey(tc.key)
|
|
if got, want := index, tc.wantIndex; got != want {
|
|
t.Errorf("expected %v, got %v", want, got)
|
|
}
|
|
if got, want := found, tc.wantFound; got != want {
|
|
t.Errorf("expected %v, got %v", want, got)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSegmentSet(t *testing.T) {
|
|
testSegment := &CompactMapSegment{
|
|
list: []CompactNeedleValue{
|
|
CompactNeedleValue{key: 10, offset: OffsetToCompact(types.Uint32ToOffset(0)), size: 100},
|
|
CompactNeedleValue{key: 20, offset: OffsetToCompact(types.Uint32ToOffset(100)), size: 200},
|
|
CompactNeedleValue{key: 30, offset: OffsetToCompact(types.Uint32ToOffset(300)), size: 300},
|
|
},
|
|
firstKey: 10,
|
|
lastKey: 30,
|
|
}
|
|
|
|
if got, want := testSegment.len(), 3; got != want {
|
|
t.Errorf("got starting size %d, want %d", got, want)
|
|
}
|
|
if got, want := testSegment.cap(), 3; got != want {
|
|
t.Errorf("got starting capacity %d, want %d", got, want)
|
|
}
|
|
|
|
testSets := []struct {
|
|
name string
|
|
key types.NeedleId
|
|
offset types.Offset
|
|
size types.Size
|
|
wantOffset types.Offset
|
|
wantSize types.Size
|
|
}{
|
|
{
|
|
name: "insert at beggining",
|
|
key: 5, offset: types.Uint32ToOffset(1000), size: 123,
|
|
wantOffset: types.Uint32ToOffset(0), wantSize: 0,
|
|
},
|
|
{
|
|
name: "insert at end",
|
|
key: 51, offset: types.Uint32ToOffset(7000), size: 456,
|
|
wantOffset: types.Uint32ToOffset(0), wantSize: 0,
|
|
},
|
|
{
|
|
name: "insert in middle",
|
|
key: 25, offset: types.Uint32ToOffset(8000), size: 789,
|
|
wantOffset: types.Uint32ToOffset(0), wantSize: 0,
|
|
},
|
|
{
|
|
name: "update existing",
|
|
key: 30, offset: types.Uint32ToOffset(9000), size: 999,
|
|
wantOffset: types.Uint32ToOffset(300), wantSize: 300,
|
|
},
|
|
}
|
|
|
|
for _, ts := range testSets {
|
|
offset, size := testSegment.set(ts.key, ts.offset, ts.size)
|
|
if offset != ts.wantOffset {
|
|
t.Errorf("%s: got offset %v, want %v", ts.name, offset, ts.wantOffset)
|
|
}
|
|
if size != ts.wantSize {
|
|
t.Errorf("%s: got size %v, want %v", ts.name, size, ts.wantSize)
|
|
}
|
|
}
|
|
|
|
wantSegment := &CompactMapSegment{
|
|
list: []CompactNeedleValue{
|
|
CompactNeedleValue{key: 5, offset: OffsetToCompact(types.Uint32ToOffset(1000)), size: 123},
|
|
CompactNeedleValue{key: 10, offset: OffsetToCompact(types.Uint32ToOffset(0)), size: 100},
|
|
CompactNeedleValue{key: 20, offset: OffsetToCompact(types.Uint32ToOffset(100)), size: 200},
|
|
CompactNeedleValue{key: 25, offset: OffsetToCompact(types.Uint32ToOffset(8000)), size: 789},
|
|
CompactNeedleValue{key: 30, offset: OffsetToCompact(types.Uint32ToOffset(9000)), size: 999},
|
|
CompactNeedleValue{key: 51, offset: OffsetToCompact(types.Uint32ToOffset(7000)), size: 456},
|
|
},
|
|
firstKey: 5,
|
|
lastKey: 51,
|
|
}
|
|
if !reflect.DeepEqual(testSegment, wantSegment) {
|
|
t.Errorf("got result segment %v, want %v", testSegment, wantSegment)
|
|
}
|
|
|
|
if got, want := testSegment.len(), 6; got != want {
|
|
t.Errorf("got result size %d, want %d", got, want)
|
|
}
|
|
if got, want := testSegment.cap(), 6; got != want {
|
|
t.Errorf("got result capacity %d, want %d", got, want)
|
|
}
|
|
}
|
|
|
|
func TestSegmentSetOrdering(t *testing.T) {
|
|
keys := []types.NeedleId{}
|
|
for i := 0; i < SegmentChunkSize; i++ {
|
|
keys = append(keys, types.NeedleId(i))
|
|
}
|
|
|
|
r := rand.New(rand.NewSource(123456789))
|
|
r.Shuffle(len(keys), func(i, j int) { keys[i], keys[j] = keys[j], keys[i] })
|
|
|
|
cs := newCompactMapSegment(0)
|
|
for _, k := range keys {
|
|
_, _ = cs.set(k, types.Uint32ToOffset(123), 456)
|
|
}
|
|
if got, want := cs.len(), SegmentChunkSize; got != want {
|
|
t.Errorf("expected size %d, got %d", want, got)
|
|
}
|
|
for i := 1; i < cs.len(); i++ {
|
|
if ka, kb := cs.list[i-1].key, cs.list[i].key; ka >= kb {
|
|
t.Errorf("found out of order entries at (%d, %d) = (%d, %d)", i-1, i, ka, kb)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSegmentGet(t *testing.T) {
|
|
testSegment := &CompactMapSegment{
|
|
list: []CompactNeedleValue{
|
|
CompactNeedleValue{key: 10, offset: OffsetToCompact(types.Uint32ToOffset(0)), size: 100},
|
|
CompactNeedleValue{key: 20, offset: OffsetToCompact(types.Uint32ToOffset(100)), size: 200},
|
|
CompactNeedleValue{key: 30, offset: OffsetToCompact(types.Uint32ToOffset(300)), size: 300},
|
|
},
|
|
firstKey: 10,
|
|
lastKey: 30,
|
|
}
|
|
|
|
testCases := []struct {
|
|
name string
|
|
key types.NeedleId
|
|
wantValue *CompactNeedleValue
|
|
wantFound bool
|
|
}{
|
|
{
|
|
name: "invalid key",
|
|
key: 99,
|
|
wantValue: nil,
|
|
wantFound: false,
|
|
},
|
|
{
|
|
name: "key #1",
|
|
key: 10,
|
|
wantValue: &testSegment.list[0],
|
|
wantFound: true,
|
|
},
|
|
{
|
|
name: "key #2",
|
|
key: 20,
|
|
wantValue: &testSegment.list[1],
|
|
wantFound: true,
|
|
},
|
|
{
|
|
name: "key #3",
|
|
key: 30,
|
|
wantValue: &testSegment.list[2],
|
|
wantFound: true,
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
value, found := testSegment.get(tc.key)
|
|
if got, want := value, tc.wantValue; got != want {
|
|
t.Errorf("got %v, want %v", got, want)
|
|
}
|
|
if got, want := found, tc.wantFound; got != want {
|
|
t.Errorf("got %v, want %v", got, want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSegmentDelete(t *testing.T) {
|
|
testSegment := &CompactMapSegment{
|
|
list: []CompactNeedleValue{
|
|
CompactNeedleValue{key: 10, offset: OffsetToCompact(types.Uint32ToOffset(0)), size: 100},
|
|
CompactNeedleValue{key: 20, offset: OffsetToCompact(types.Uint32ToOffset(100)), size: 200},
|
|
CompactNeedleValue{key: 30, offset: OffsetToCompact(types.Uint32ToOffset(300)), size: 300},
|
|
CompactNeedleValue{key: 40, offset: OffsetToCompact(types.Uint32ToOffset(600)), size: 400},
|
|
},
|
|
firstKey: 10,
|
|
lastKey: 40,
|
|
}
|
|
|
|
testDeletes := []struct {
|
|
name string
|
|
key types.NeedleId
|
|
want types.Size
|
|
}{
|
|
{
|
|
name: "invalid key",
|
|
key: 99,
|
|
want: 0,
|
|
},
|
|
{
|
|
name: "delete key #2",
|
|
key: 20,
|
|
want: 200,
|
|
},
|
|
{
|
|
name: "delete key #4",
|
|
key: 40,
|
|
want: 400,
|
|
},
|
|
}
|
|
|
|
for _, td := range testDeletes {
|
|
size := testSegment.delete(td.key)
|
|
if got, want := size, td.want; got != want {
|
|
t.Errorf("%s: got %v, want %v", td.name, got, want)
|
|
}
|
|
}
|
|
|
|
wantSegment := &CompactMapSegment{
|
|
list: []CompactNeedleValue{
|
|
CompactNeedleValue{key: 10, offset: OffsetToCompact(types.Uint32ToOffset(0)), size: 100},
|
|
CompactNeedleValue{key: 20, offset: OffsetToCompact(types.Uint32ToOffset(100)), size: -200},
|
|
CompactNeedleValue{key: 30, offset: OffsetToCompact(types.Uint32ToOffset(300)), size: 300},
|
|
CompactNeedleValue{key: 40, offset: OffsetToCompact(types.Uint32ToOffset(600)), size: -400},
|
|
},
|
|
firstKey: 10,
|
|
lastKey: 40,
|
|
}
|
|
if !reflect.DeepEqual(testSegment, wantSegment) {
|
|
t.Errorf("got result segment %v, want %v", testSegment, wantSegment)
|
|
}
|
|
}
|
|
|
|
func TestSegmentForKey(t *testing.T) {
|
|
testMap := NewCompactMap()
|
|
|
|
tests := []struct {
|
|
name string
|
|
key types.NeedleId
|
|
want *CompactMapSegment
|
|
}{
|
|
{
|
|
name: "first segment",
|
|
key: 12,
|
|
want: &CompactMapSegment{
|
|
list: []CompactNeedleValue{},
|
|
chunk: 0,
|
|
firstKey: MaxCompactKey,
|
|
lastKey: 0,
|
|
},
|
|
},
|
|
{
|
|
name: "second segment, gapless",
|
|
key: SegmentChunkSize + 34,
|
|
want: &CompactMapSegment{
|
|
list: []CompactNeedleValue{},
|
|
chunk: 1,
|
|
firstKey: MaxCompactKey,
|
|
lastKey: 0,
|
|
},
|
|
},
|
|
{
|
|
name: "gapped segment",
|
|
key: (5 * SegmentChunkSize) + 56,
|
|
want: &CompactMapSegment{
|
|
list: []CompactNeedleValue{},
|
|
chunk: 5,
|
|
firstKey: MaxCompactKey,
|
|
lastKey: 0,
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
cs := testMap.segmentForKey(tc.key)
|
|
if !reflect.DeepEqual(cs, tc.want) {
|
|
t.Errorf("got segment %v, want %v", cs, tc.want)
|
|
}
|
|
})
|
|
}
|
|
|
|
wantMap := &CompactMap{
|
|
segments: map[Chunk]*CompactMapSegment{
|
|
0: &CompactMapSegment{
|
|
list: []CompactNeedleValue{},
|
|
chunk: 0,
|
|
firstKey: MaxCompactKey,
|
|
lastKey: 0,
|
|
},
|
|
1: &CompactMapSegment{
|
|
list: []CompactNeedleValue{},
|
|
chunk: 1,
|
|
firstKey: MaxCompactKey,
|
|
lastKey: 0,
|
|
},
|
|
5: &CompactMapSegment{
|
|
list: []CompactNeedleValue{},
|
|
chunk: 5,
|
|
firstKey: MaxCompactKey,
|
|
lastKey: 0,
|
|
},
|
|
},
|
|
}
|
|
if !reflect.DeepEqual(testMap, wantMap) {
|
|
t.Errorf("got map %v, want %v", testMap, wantMap)
|
|
}
|
|
}
|
|
|
|
func TestAscendingVisit(t *testing.T) {
|
|
cm := NewCompactMap()
|
|
for _, nid := range []types.NeedleId{20, 7, 40000, 300000, 0, 100, 500, 10000, 200000} {
|
|
cm.Set(nid, types.Uint32ToOffset(123), 456)
|
|
}
|
|
|
|
got := []NeedleValue{}
|
|
err := cm.AscendingVisit(func(nv NeedleValue) error {
|
|
got = append(got, nv)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Errorf("got error %v, expected none", err)
|
|
}
|
|
|
|
want := []NeedleValue{
|
|
NeedleValue{Key: 0, Offset: types.Uint32ToOffset(123), Size: 456},
|
|
NeedleValue{Key: 7, Offset: types.Uint32ToOffset(123), Size: 456},
|
|
NeedleValue{Key: 20, Offset: types.Uint32ToOffset(123), Size: 456},
|
|
NeedleValue{Key: 100, Offset: types.Uint32ToOffset(123), Size: 456},
|
|
NeedleValue{Key: 500, Offset: types.Uint32ToOffset(123), Size: 456},
|
|
NeedleValue{Key: 10000, Offset: types.Uint32ToOffset(123), Size: 456},
|
|
NeedleValue{Key: 40000, Offset: types.Uint32ToOffset(123), Size: 456},
|
|
NeedleValue{Key: 200000, Offset: types.Uint32ToOffset(123), Size: 456},
|
|
NeedleValue{Key: 300000, Offset: types.Uint32ToOffset(123), Size: 456},
|
|
}
|
|
if !reflect.DeepEqual(got, want) {
|
|
t.Errorf("got values %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestRandomInsert(t *testing.T) {
|
|
count := 8 * SegmentChunkSize
|
|
keys := []types.NeedleId{}
|
|
for i := 0; i < count; i++ {
|
|
keys = append(keys, types.NeedleId(i))
|
|
}
|
|
|
|
r := rand.New(rand.NewSource(123456789))
|
|
r.Shuffle(len(keys), func(i, j int) { keys[i], keys[j] = keys[j], keys[i] })
|
|
|
|
cm := NewCompactMap()
|
|
for _, k := range keys {
|
|
_, _ = cm.Set(k, types.Uint32ToOffset(123), 456)
|
|
}
|
|
if got, want := cm.Len(), count; got != want {
|
|
t.Errorf("expected size %d, got %d", want, got)
|
|
}
|
|
|
|
last := -1
|
|
err := cm.AscendingVisit(func(nv NeedleValue) error {
|
|
key := int(nv.Key)
|
|
if key <= last {
|
|
return fmt.Errorf("found out of order entries (%d vs %d)", key, last)
|
|
}
|
|
last = key
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Errorf("got error %v, expected none", err)
|
|
}
|
|
|
|
// Given that we've written a integer multiple of SegmentChunkSize, all
|
|
// segments should be fully utilized and capacity-adjusted.
|
|
if l, c := cm.Len(), cm.Cap(); l != c {
|
|
t.Errorf("map length (%d) doesn't match capacity (%d)", l, c)
|
|
}
|
|
}
|