fix needle Append return offset to avoid uint32 overflow

weed/storage/needle_read_write.go

@@ -3,6 +3,7 @@ package storage
 import (
 	"errors"
 	"fmt"
+	"io"
 	"os"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -27,8 +28,8 @@ func (n *Needle) DiskSize(version Version) int64 {
 	return getActualSize(n.Size, version)
 }
 
-func (n *Needle) Append(w *os.File, version Version) (offset Offset, size uint32, actualSize int64, err error) {
-	if end, e := w.Seek(0, 2); e == nil {
+func (n *Needle) Append(w *os.File, version Version) (offset uint64, size uint32, actualSize int64, err error) {
+	if end, e := w.Seek(0, io.SeekEnd); e == nil {
 		defer func(w *os.File, off int64) {
 			if err != nil {
 				if te := w.Truncate(end); te != nil {
@@ -36,7 +37,7 @@ func (n *Needle) Append(w *os.File, version Version) (offset Offset, size uint32
 				}
 			}
 		}(w, end)
-		offset = Offset(end)
+		offset = uint64(end)
 	} else {
 		err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
 		return
@@ -175,7 +176,7 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
 	}
 	n.ParseNeedleHeader(bytes)
 	if n.Size != size {
-		return fmt.Errorf("File Entry Not Found. Needle id %d expected size %d Memory %d", n.Id, n.Size, size)
+		return fmt.Errorf("File Entry Not Found. offset %d, Needle id %d expected size %d Memory %d", offset, n.Id, n.Size, size)
 	}
 	switch version {
 	case Version1:

weed/storage/needle_read_write_test.go

@@ -0,0 +1,62 @@
+package storage
+
+import (
+	"crypto/rand"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestAppend(t *testing.T) {
+	n := &Needle{
+
+		Cookie:       types.Cookie(123),   // Cookie Cookie   `comment:"random number to mitigate brute force lookups"`
+		Id:           types.NeedleId(123), // Id     NeedleId `comment:"needle id"`
+		Size:         8,                   // Size   uint32   `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
+		DataSize:     4,                   // DataSize     uint32 `comment:"Data size"` //version2
+		Data:         []byte("abcd"),      // Data         []byte `comment:"The actual file data"`
+		Flags:        0,                   // Flags        byte   `comment:"boolean flags"`          //version2
+		NameSize:     0,                   // NameSize     uint8                                     //version2
+		Name:         nil,                 // Name         []byte `comment:"maximum 256 characters"` //version2
+		MimeSize:     0,                   // MimeSize     uint8                                     //version2
+		Mime:         nil,                 // Mime         []byte `comment:"maximum 256 characters"` //version2
+		PairsSize:    0,                   // PairsSize    uint16                                    //version2
+		Pairs:        nil,                 // Pairs        []byte `comment:"additional name value pairs, json format, maximum 6
+		LastModified: 123,                 // LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes
+		Ttl:          nil,                 // Ttl          *TTL
+		Checksum:     123,                 // Checksum   CRC    `comment:"CRC32 to check integrity"`
+		AppendAtNs:   123,                 // AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
+		Padding:      nil,                 // Padding    []byte `comment:"Aligned to 8 bytes"`
+	}
+
+	tempFile, err := ioutil.TempFile("", ".dat")
+	if err != nil {
+		t.Errorf("Fail TempFile. %v", err)
+		return
+	}
+
+	/*
+		uint8  : 0 to 255
+		uint16 : 0 to 65535
+		uint32 : 0 to 4294967295
+		uint64 : 0 to 18446744073709551615
+		int8   : -128 to 127
+		int16  : -32768 to 32767
+		int32  : -2147483648 to 2147483647
+		int64  : -9223372036854775808 to 9223372036854775807
+	*/
+
+	fileSize := int64(4294967295) + 10000
+	io.CopyN(tempFile, rand.Reader, fileSize)
+	defer func() {
+		tempFile.Close()
+		os.Remove(tempFile.Name())
+	}()
+
+	offset, _, _, _ := n.Append(tempFile, CurrentVersion)
+	if offset != uint64(fileSize) {
+		t.Errorf("Fail to Append Needle.")
+	}
+}

weed/storage/volume_read_write.go

@@ -76,7 +76,7 @@ func (v *Volume) AppendBlob(b []byte) (offset int64, err error) {
 	return
 }
 
-func (v *Volume) writeNeedle(n *Needle) (offset Offset, size uint32, err error) {
+func (v *Volume) writeNeedle(n *Needle) (offset uint64, size uint32, err error) {
 	glog.V(4).Infof("writing needle %s", NewFileIdFromNeedle(v.Id, n).String())
 	if v.readOnly {
 		err = fmt.Errorf("%s is read-only", v.dataFile.Name())
@@ -96,8 +96,8 @@ func (v *Volume) writeNeedle(n *Needle) (offset Offset, size uint32, err error)
 	}
 
 	nv, ok := v.nm.Get(n.Id)
-	if !ok || Offset(nv.Offset)*NeedlePaddingSize < offset {
-		if err = v.nm.Put(n.Id, offset/NeedlePaddingSize, n.Size); err != nil {
+	if !ok || uint64(nv.Offset)*NeedlePaddingSize < offset {
+		if err = v.nm.Put(n.Id, Offset(offset/NeedlePaddingSize), n.Size); err != nil {
 			glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err)
 		}
 	}
@@ -124,7 +124,7 @@ func (v *Volume) deleteNeedle(n *Needle) (uint32, error) {
 		if err != nil {
 			return size, err
 		}
-		if err = v.nm.Delete(n.Id, offset/NeedlePaddingSize); err != nil {
+		if err = v.nm.Delete(n.Id, Offset(offset/NeedlePaddingSize)); err != nil {
 			return size, err
 		}
 		return size, err