Merge branch 'master' into add_remote_storage

other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java

@@ -32,10 +32,10 @@ public class SeaweedOutputStream extends OutputStream {
     private long lastTotalAppendOffset = 0;
     private ByteBuffer buffer;
     private long outputIndex;
-    private String replication = "000";
+    private String replication = "";
 
     public SeaweedOutputStream(FilerClient filerClient, final String fullpath) {
-        this(filerClient, fullpath, "000");
+        this(filerClient, fullpath, "");
     }
 
     public SeaweedOutputStream(FilerClient filerClient, final String fullpath, final String replication) {

weed/filer/stream.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"sort"
 	"strings"
 	"time"
 
@@ -89,51 +90,69 @@ func ReadAll(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk)
 // ----------------  ChunkStreamReader ----------------------------------
 type ChunkStreamReader struct {
 	chunkViews         []*ChunkView
-	logicOffset  int64
+	totalSize          int64
 	buffer             []byte
 	bufferOffset       int64
 	bufferPos          int
-	chunkIndex   int
+	nextChunkViewIndex int
 	lookupFileId       wdclient.LookupFileIdFunctionType
 }
 
 var _ = io.ReadSeeker(&ChunkStreamReader{})
+var _ = io.ReaderAt(&ChunkStreamReader{})
 
-func NewChunkStreamReaderFromFiler(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
-
-	lookupFileIdFn := func(fileId string) (targetUrl []string, err error) {
-		return masterClient.LookupFileId(fileId)
-	}
+func doNewChunkStreamReader(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
 
 	chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
+	sort.Slice(chunkViews, func(i, j int) bool {
+		return chunkViews[i].LogicOffset < chunkViews[j].LogicOffset
+	})
+
+	var totalSize int64
+	for _, chunk := range chunkViews {
+		totalSize += int64(chunk.Size)
+	}
 
 	return &ChunkStreamReader{
 		chunkViews:   chunkViews,
 		lookupFileId: lookupFileIdFn,
+		totalSize:    totalSize,
 	}
 }
 
+func NewChunkStreamReaderFromFiler(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
+
+	lookupFileIdFn := func(fileId string) (targetUrl []string, err error) {
+		return masterClient.LookupFileId(fileId)
+	}
+
+	return doNewChunkStreamReader(lookupFileIdFn, chunks)
+}
+
 func NewChunkStreamReader(filerClient filer_pb.FilerClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader {
 
 	lookupFileIdFn := LookupFn(filerClient)
 
-	chunkViews := ViewFromChunks(lookupFileIdFn, chunks, 0, math.MaxInt64)
+	return doNewChunkStreamReader(lookupFileIdFn, chunks)
+}
 
-	return &ChunkStreamReader{
-		chunkViews:   chunkViews,
-		lookupFileId: lookupFileIdFn,
+func (c *ChunkStreamReader) ReadAt(p []byte, off int64) (n int, err error) {
+	_, err = c.Seek(off, io.SeekStart)
+	if err != nil {
+		return
 	}
+	return c.Read(p)
 }
 
 func (c *ChunkStreamReader) Read(p []byte) (n int, err error) {
 	for n < len(p) {
 		if c.isBufferEmpty() {
-			if c.chunkIndex >= len(c.chunkViews) {
+			if c.nextChunkViewIndex >= len(c.chunkViews) {
 				return n, io.EOF
 			}
-			chunkView := c.chunkViews[c.chunkIndex]
+			chunkView := c.chunkViews[c.nextChunkViewIndex]
 			c.fetchChunkToBuffer(chunkView)
-			c.chunkIndex++
+			c.nextChunkViewIndex++
 		}
 		t := copy(p[n:], c.buffer[c.bufferPos:])
 		c.bufferPos += t
@@ -148,33 +167,45 @@ func (c *ChunkStreamReader) isBufferEmpty() bool {
 
 func (c *ChunkStreamReader) Seek(offset int64, whence int) (int64, error) {
 
-	var totalSize int64
-	for _, chunk := range c.chunkViews {
-		totalSize += int64(chunk.Size)
-	}
-
 	var err error
 	switch whence {
 	case io.SeekStart:
 	case io.SeekCurrent:
 		offset += c.bufferOffset + int64(c.bufferPos)
 	case io.SeekEnd:
-		offset = totalSize + offset
+		offset = c.totalSize + offset
 	}
-	if offset > totalSize {
+	if offset > c.totalSize {
 		err = io.ErrUnexpectedEOF
 	}
 
-	for i, chunk := range c.chunkViews {
+	// stay in the same chunk
+	if !c.isBufferEmpty() {
+		if c.bufferOffset <= offset && offset < c.bufferOffset+int64(len(c.buffer)) {
+			c.bufferPos = int(offset - c.bufferOffset)
+			return offset, nil
+		}
+	}
+
+	// need to seek to a different chunk
+	currentChunkIndex := sort.Search(len(c.chunkViews), func(i int) bool {
+		return c.chunkViews[i].LogicOffset <= offset
+	})
+	if currentChunkIndex == len(c.chunkViews) {
+		return 0, io.EOF
+	}
+
+	// positioning within the new chunk
+	chunk := c.chunkViews[currentChunkIndex]
 	if chunk.LogicOffset <= offset && offset < chunk.LogicOffset+int64(chunk.Size) {
 		if c.isBufferEmpty() || c.bufferOffset != chunk.LogicOffset {
 			c.fetchChunkToBuffer(chunk)
-				c.chunkIndex = i + 1
-				break
-			}
-		}
+			c.nextChunkViewIndex = currentChunkIndex + 1
 		}
 		c.bufferPos = int(offset - c.bufferOffset)
+	} else {
+		return 0, io.ErrUnexpectedEOF
+	}
 
 	return offset, err