From 56fbd2c2119520702915b07dda419dae97cdd943 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 17 Aug 2020 21:17:32 -0700 Subject: [PATCH] fix reading --- weed/filer2/reader_at.go | 5 +- weed/filer2/reader_at_test.go | 122 ++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 weed/filer2/reader_at_test.go diff --git a/weed/filer2/reader_at.go b/weed/filer2/reader_at.go index b036e64fe..09d99dfb8 100644 --- a/weed/filer2/reader_at.go +++ b/weed/filer2/reader_at.go @@ -101,7 +101,7 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) { return } bufferOffset := chunkStart - chunk.LogicOffset + chunk.Offset - copied := copy(p[chunkStart-startOffset:chunkStop-startOffset], buffer[bufferOffset:bufferOffset+chunkStop-chunkStart]) + copied := copy(p[startOffset-offset:chunkStop-chunkStart+startOffset-offset], buffer[bufferOffset:bufferOffset+chunkStop-chunkStart]) n += copied startOffset, remaining = startOffset+int64(copied), remaining-int64(copied) } @@ -111,6 +111,9 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) { if remaining > 0 { glog.V(4).Infof("zero2 [%d,%d)", n, n+int(remaining)) n += int(remaining) + if n > int(c.fileSize - offset){ + n = int(c.fileSize - offset) + } } if offset+int64(n) >= c.fileSize { diff --git a/weed/filer2/reader_at_test.go b/weed/filer2/reader_at_test.go new file mode 100644 index 000000000..2a42cfd49 --- /dev/null +++ b/weed/filer2/reader_at_test.go @@ -0,0 +1,122 @@ +package filer2 + +import ( + "fmt" + "io" + "math" + "strconv" + "sync" + "testing" +) + +type mockChunkCache struct { +} + +func (m *mockChunkCache) GetChunk(fileId string, minSize uint64) (data []byte) { + x, _ := strconv.Atoi(fileId) + data = make([]byte, minSize) + for i := 0; i < int(minSize); i++ { + data[i] = byte(x) + } + return data +} +func (m *mockChunkCache) SetChunk(fileId string, data []byte) { +} + +func TestReaderAt(t *testing.T) { + + visibles := []VisibleInterval{ + { + start: 1, + stop: 2, + fileId: "1", + chunkSize: 9, + }, + { + start: 3, + stop: 4, + fileId: "3", + chunkSize: 1, + }, + { + start: 5, + stop: 6, + fileId: "5", + chunkSize: 2, + }, + { + start: 7, + stop: 9, + fileId: "7", + chunkSize: 2, + }, + { + start: 9, + stop: 10, + fileId: "9", + chunkSize: 2, + }, + } + + readerAt := &ChunkReadAt{ + chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64), + lookupFileId: nil, + readerLock: sync.Mutex{}, + fileSize: 10, + chunkCache: &mockChunkCache{}, + } + + testReadAt(t, readerAt, 0, 10, 10, io.EOF) + testReadAt(t, readerAt, 0, 12, 10, io.EOF) + testReadAt(t, readerAt, 2, 8, 8, io.EOF) + testReadAt(t, readerAt, 3, 6, 6, nil) + +} + +func testReadAt(t *testing.T, readerAt *ChunkReadAt, offset int64, size int, expected int, expectedErr error) { + data := make([]byte, size) + n, err := readerAt.ReadAt(data, offset) + + if expected != n { + t.Errorf("unexpected read size: %d, expect: %d", n, expected) + } + if err != expectedErr { + t.Errorf("unexpected read error: %v, expect: %v", err, expectedErr) + } + + for _, d := range data { + fmt.Printf("%x", d) + } + fmt.Println() +} + +func TestReaderAt0(t *testing.T) { + + visibles := []VisibleInterval{ + { + start: 2, + stop: 5, + fileId: "1", + chunkSize: 9, + }, + { + start: 7, + stop: 9, + fileId: "2", + chunkSize: 9, + }, + } + + readerAt := &ChunkReadAt{ + chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64), + lookupFileId: nil, + readerLock: sync.Mutex{}, + fileSize: 10, + chunkCache: &mockChunkCache{}, + } + + testReadAt(t, readerAt, 0, 10, 10, io.EOF) + testReadAt(t, readerAt, 3, 16, 7, io.EOF) + testReadAt(t, readerAt, 3, 5, 5, nil) + +}