From 1943647860ad0f6e85dc2acd2a5a68053b3770b5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 21 Dec 2012 02:13:02 -0800 Subject: [PATCH] can read/write files in format version 2 --- weed-fs/src/cmd/weed/version.go | 2 +- weed-fs/src/pkg/storage/needle.go | 6 +- weed-fs/src/pkg/storage/needle_read_write.go | 92 +++++++++++++++++--- weed-fs/src/pkg/storage/volume.go | 2 +- weed-fs/src/pkg/storage/volume_version.go | 2 +- weed-fs/src/pkg/util/bytes.go | 3 + 6 files changed, 91 insertions(+), 16 deletions(-) diff --git a/weed-fs/src/cmd/weed/version.go b/weed-fs/src/cmd/weed/version.go index 571bbe260..c11dd29d7 100644 --- a/weed-fs/src/cmd/weed/version.go +++ b/weed-fs/src/cmd/weed/version.go @@ -6,7 +6,7 @@ import ( ) const ( - VERSION = "0.25" + VERSION = "0.26" ) var cmdVersion = &Command{ diff --git a/weed-fs/src/pkg/storage/needle.go b/weed-fs/src/pkg/storage/needle.go index 0363fcb13..aaea521a0 100644 --- a/weed-fs/src/pkg/storage/needle.go +++ b/weed-fs/src/pkg/storage/needle.go @@ -20,13 +20,15 @@ type Needle struct { Cookie uint32 "random number to mitigate brute force lookups" Id uint64 "needle id" Size uint32 "sum of DataSize,Data,NameSize,Name,MimeSize,Mime" - Flags byte "boolean flags" //version2 - DataSize uint32 "Data size" //version2 + + DataSize uint32 "Data size" //version2 Data []byte "The actual file data" + Flags byte "boolean flags" //version2 NameSize uint8 //version2 Name []byte "maximum 256 characters" //version2 MimeSize uint8 //version2 Mime []byte "maximum 256 characters" //version2 + Checksum CRC "CRC32 to check integrity" Padding []byte "Aligned to 8 bytes" } diff --git a/weed-fs/src/pkg/storage/needle_read_write.go b/weed-fs/src/pkg/storage/needle_read_write.go index 72096edf7..be15b9d84 100644 --- a/weed-fs/src/pkg/storage/needle_read_write.go +++ b/weed-fs/src/pkg/storage/needle_read_write.go @@ -19,7 +19,44 @@ func (n *Needle) Append(w io.Writer, version Version) uint32 { rest := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + 4) % NeedlePaddingSize) util.Uint32toBytes(header[0:4], n.Checksum.Value()) w.Write(header[0 : 4+rest]) + return n.Size } else if version == Version2 { + header := make([]byte, NeedleHeaderSize) + util.Uint32toBytes(header[0:4], n.Cookie) + util.Uint64toBytes(header[4:12], n.Id) + n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime)) + if n.DataSize > 0 { + n.Size = 4 + n.DataSize + 1 + if n.NameSize > 0 { + n.Size = n.Size + 1 + uint32(n.NameSize) + } + if n.NameSize > 0 { + n.Size = n.Size + 1 + uint32(n.MimeSize) + } + } + util.Uint32toBytes(header[12:16], n.Size) + w.Write(header) + if n.DataSize > 0 { + util.Uint32toBytes(header[0:4], n.DataSize) + w.Write(header[0:4]) + w.Write(n.Data) + util.Uint8toBytes(header[0:1], n.Flags) + w.Write(header[0:1]) + } + if n.NameSize > 0 { + util.Uint8toBytes(header[0:1], n.NameSize) + w.Write(header[0:1]) + w.Write(n.Name) + } + if n.MimeSize > 0 { + util.Uint8toBytes(header[0:1], n.MimeSize) + w.Write(header[0:1]) + w.Write(n.Mime) + } + rest := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + 4) % NeedlePaddingSize) + util.Uint32toBytes(header[0:4], n.Checksum.Value()) + w.Write(header[0 : 4+rest]) + return n.DataSize } return n.Size } @@ -27,33 +64,62 @@ func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) { if version == Version1 { bytes := make([]byte, NeedleHeaderSize+size+4) ret, e := r.Read(bytes) - n.Cookie = util.BytesToUint32(bytes[0:4]) - n.Id = util.BytesToUint64(bytes[4:12]) - n.Size = util.BytesToUint32(bytes[12:16]) - n.Data = bytes[16 : 16+size] - checksum := util.BytesToUint32(bytes[16+size : 16+size+4]) + n.readNeedleHeader(bytes) + n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size] + checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+4]) if checksum != NewCRC(n.Data).Value() { return 0, errors.New("CRC error! Data On Disk Corrupted!") } return ret, e } else if version == Version2 { + bytes := make([]byte, NeedleHeaderSize+size+4) + ret, e := r.Read(bytes) + n.readNeedleHeader(bytes) + n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)]) + checksum := util.BytesToUint32(bytes[NeedleHeaderSize+n.Size : NeedleHeaderSize+n.Size+4]) + if checksum != NewCRC(n.Data).Value() { + return 0, errors.New("CRC error! Data On Disk Corrupted!") + } + return ret, e } return 0, errors.New("Unsupported Version!") } +func (n *Needle) readNeedleHeader(bytes []byte) { + n.Cookie = util.BytesToUint32(bytes[0:4]) + n.Id = util.BytesToUint64(bytes[4:12]) + n.Size = util.BytesToUint32(bytes[12:NeedleHeaderSize]) +} +func (n *Needle) readNeedleDataVersion2(bytes []byte) { + index := 0 + n.DataSize = util.BytesToUint32(bytes[index : index+4]) + index = index + 4 + n.Data = bytes[index : index+int(n.DataSize)] + index = index + int(n.DataSize) + n.Flags = bytes[index] + index = index + 1 + if index < len(bytes) { + n.NameSize = uint8(bytes[index]) + index = index + 1 + n.Name = bytes[index : index+int(n.NameSize)] + index = index + int(n.NameSize) + } + if index < len(bytes) { + n.MimeSize = uint8(bytes[index]) + index = index + 1 + n.Mime = bytes[index : index+int(n.MimeSize)] + } +} func ReadNeedleHeader(r *os.File, version Version) (n *Needle, bodyLength uint32) { n = new(Needle) - if version == Version1 { + if version == Version1 || version == Version2 { bytes := make([]byte, NeedleHeaderSize) count, e := r.Read(bytes) if count <= 0 || e != nil { return nil, 0 } - n.Cookie = util.BytesToUint32(bytes[0:4]) - n.Id = util.BytesToUint64(bytes[4:12]) - n.Size = util.BytesToUint32(bytes[12:16]) - rest := NeedlePaddingSize - ((n.Size + 16 + 4) % NeedlePaddingSize) + n.readNeedleHeader(bytes) + rest := NeedlePaddingSize - ((n.Size + NeedleHeaderSize + 4) % NeedlePaddingSize) bodyLength = n.Size + 4 + rest - } else if version == Version2 { } return } @@ -67,6 +133,10 @@ func (n *Needle) ReadNeedleBody(r *os.File, version Version, bodyLength uint32) n.Data = bytes[:n.Size] n.Checksum = NewCRC(n.Data) } else if version == Version2 { + bytes := make([]byte, bodyLength) + r.Read(bytes) + n.readNeedleDataVersion2(bytes[0:n.Size]) + n.Checksum = NewCRC(n.Data) } return } diff --git a/weed-fs/src/pkg/storage/volume.go b/weed-fs/src/pkg/storage/volume.go index d12c9d889..d983e1213 100644 --- a/weed-fs/src/pkg/storage/volume.go +++ b/weed-fs/src/pkg/storage/volume.go @@ -51,7 +51,7 @@ func (v *Volume) load() error { return nil } func (v *Volume) Version() Version { - return CurrentVersion + return v.version } func (v *Volume) Size() int64 { v.accessLock.Lock() diff --git a/weed-fs/src/pkg/storage/volume_version.go b/weed-fs/src/pkg/storage/volume_version.go index 974a8079c..da91ad038 100644 --- a/weed-fs/src/pkg/storage/volume_version.go +++ b/weed-fs/src/pkg/storage/volume_version.go @@ -6,7 +6,7 @@ import ( type Version uint8 const ( - CurrentVersion = Version(1) Version1 = Version(1) Version2 = Version(2) + CurrentVersion = Version2 ) diff --git a/weed-fs/src/pkg/util/bytes.go b/weed-fs/src/pkg/util/bytes.go index 24fdf47af..177da06db 100644 --- a/weed-fs/src/pkg/util/bytes.go +++ b/weed-fs/src/pkg/util/bytes.go @@ -28,4 +28,7 @@ func Uint32toBytes(b []byte, v uint32){ b[3-i] = byte(v>>(i*8)) } } +func Uint8toBytes(b []byte, v uint8){ + b[0] = byte(v) +}