From 69ac6b6bf6e5608322884ed6d01b9163a656797b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 19 Sep 2013 11:06:14 -0700 Subject: [PATCH] Issue 45 in weed-fs: [Compact issue] Offset overflow New issue 45 by hieu.hcmus@gmail.com: [Compact issue] Offset overflow http://code.google.com/p/weed-fs/issues/detail?id=45 You are using uint32(Maximum 4Gb) to store needle offset(Maximum 32Gb) when compacting. Currently It is ok if the volume size is < 4gb Change variable "offset" in ScanVolumeFile function to uint64 to fix the issue. --- go/storage/needle.go | 3 +++ go/storage/needle_read_write.go | 6 +++--- go/storage/volume.go | 14 +++++++------- go/weed/export.go | 2 +- go/weed/fix.go | 4 ++-- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/go/storage/needle.go b/go/storage/needle.go index e23f4e67b..24b34c32e 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -19,6 +19,9 @@ const ( NeedleChecksumSize = 4 ) +/* +* Needle file size is limited to 4GB for now. + */ type Needle struct { Cookie uint32 `comment:"random number to mitigate brute force lookups"` Id uint64 `comment:"needle id"` diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index d525a10d5..c414ac299 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -17,9 +17,9 @@ const ( LastModifiedBytesLength = 5 ) -func (n *Needle) DiskSize() uint32 { - padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize) - return NeedleHeaderSize + n.Size + padding + NeedleChecksumSize +func (n *Needle) DiskSize() int64 { + padding := NeedlePaddingSize - ((NeedleHeaderSize + int64(n.Size) + NeedleChecksumSize) % NeedlePaddingSize) + return NeedleHeaderSize + int64(n.Size) + padding + NeedleChecksumSize } func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { if s, ok := w.(io.Seeker); ok { diff --git a/go/storage/volume.go b/go/storage/volume.go index 094134356..2466113c5 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -307,7 +307,7 @@ func (v *Volume) freeze() error { func ScanVolumeFile(dirname string, id VolumeId, visitSuperBlock func(SuperBlock) error, - visitNeedle func(n *Needle, offset uint32) error) (err error) { + visitNeedle func(n *Needle, offset int64) error) (err error) { var v *Volume if v, err = loadVolumeWithoutIndex(dirname, id); err != nil { return @@ -318,7 +318,7 @@ func ScanVolumeFile(dirname string, id VolumeId, version := v.Version() - offset := uint32(SuperBlockSize) + offset := int64(SuperBlockSize) n, rest, e := ReadNeedleHeader(v.dataFile, version) if e != nil { err = fmt.Errorf("cannot read needle header: %s", e) @@ -332,7 +332,7 @@ func ScanVolumeFile(dirname string, id VolumeId, if err = visitNeedle(n, offset); err != nil { return } - offset += NeedleHeaderSize + rest + offset += int64(NeedleHeaderSize + rest) if n, rest, err = ReadNeedleHeader(v.dataFile, version); err != nil { if err == io.EOF { return nil @@ -359,17 +359,17 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err erro defer idx.Close() nm := NewNeedleMap(idx) - new_offset := uint32(SuperBlockSize) + new_offset := int64(SuperBlockSize) err = ScanVolumeFile(v.dir, v.Id, func(superBlock SuperBlock) error { _, err = dst.Write(superBlock.Bytes()) return err - }, func(n *Needle, offset uint32) error { + }, func(n *Needle, offset int64) error { nv, ok := v.nm.Get(n.Id) //glog.V(0).Infoln("file size is", n.Size, "rest", rest) - if ok && nv.Offset*NeedlePaddingSize == offset { + if ok && int64(nv.Offset)*NeedlePaddingSize == offset { if nv.Size > 0 { - if _, err = nm.Put(n.Id, new_offset/NeedlePaddingSize, n.Size); err != nil { + if _, err = nm.Put(n.Id, uint32(new_offset/NeedlePaddingSize), n.Size); err != nil { return fmt.Errorf("cannot put needle: %s", err) } if _, err = n.Append(dst, v.Version()); err != nil { diff --git a/go/weed/export.go b/go/weed/export.go index 965a17699..671068a82 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -98,7 +98,7 @@ func runExport(cmd *Command, args []string) bool { err = storage.ScanVolumeFile(*exportVolumePath, vid, func(superBlock storage.SuperBlock) error { version = superBlock.Version return nil - }, func(n *storage.Needle, offset uint32) error { + }, func(n *storage.Needle, offset int64) error { debug("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped()) nv, ok := nm.Get(n.Id) if ok && nv.Size > 0 { diff --git a/go/weed/fix.go b/go/weed/fix.go index e1cc63943..c97fd60d3 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -45,10 +45,10 @@ func runFix(cmd *Command, args []string) bool { vid := storage.VolumeId(*fixVolumeId) err = storage.ScanVolumeFile(*fixVolumePath, vid, func(superBlock storage.SuperBlock) error { return nil - }, func(n *storage.Needle, offset uint32) error { + }, func(n *storage.Needle, offset int64) error { debug("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped()) if n.Size > 0 { - count, pe := nm.Put(n.Id, offset/storage.NeedlePaddingSize, n.Size) + count, pe := nm.Put(n.Id, uint32(offset/storage.NeedlePaddingSize), n.Size) debug("saved", count, "with error", pe) } else { debug("skipping deleted file ...")