diff --git a/weed-fs/src/pkg/storage/needle.go b/weed-fs/src/pkg/storage/needle.go index 346d476e5..8f014f73a 100644 --- a/weed-fs/src/pkg/storage/needle.go +++ b/weed-fs/src/pkg/storage/needle.go @@ -10,9 +10,8 @@ import ( ) type Needle struct { - Cookie uint8 "random number to mitigate brute force lookups" + Cookie uint32 "random number to mitigate brute force lookups" Key uint64 "file id" - AlternateKey uint32 "supplemental id" Size uint32 "Data size" Data []byte "The actual file data" Checksum int32 "CRC32 to check integrity" @@ -36,32 +35,28 @@ func NewNeedle(r *http.Request) (n *Needle) { func (n *Needle) ParsePath(path string) { a := strings.Split(path, "_") log.Println("cookie", a[0], "key", a[1], "altKey", a[2]) - cookie, _ := strconv.Atoi(a[0]) - n.Cookie = uint8(cookie) + cookie, _ := strconv.Atoui(a[0]) + n.Cookie = uint32(cookie) n.Key, _ = strconv.Atoui64(a[1]) - altKey, _ := strconv.Atoui64(a[2]) - n.AlternateKey = uint32(altKey) } func (n *Needle) Append(w io.Writer) { - header := make([]byte, 17) - header[0] = n.Cookie - uint64toBytes(header[1:9], n.Key) - uint32toBytes(header[9:13], n.AlternateKey) + header := make([]byte, 16) + uint32toBytes(header[0:4], n.Cookie) + uint64toBytes(header[4:12], n.Key) n.Size = uint32(len(n.Data)) - uint32toBytes(header[13:17], n.Size) + uint32toBytes(header[12:16], n.Size) w.Write(header) w.Write(n.Data) - rest := 8 - ((n.Size + 17 + 4) % 8) + rest := 8 - ((n.Size + 16 + 4) % 8) uint32toBytes(header[0:4], uint32(n.Checksum)) w.Write(header[0 : rest+4]) } func (n *Needle) Read(r io.Reader, size uint32) { - bytes := make([]byte, size+17+4) + bytes := make([]byte, size+16+4) r.Read(bytes) - n.Cookie = bytes[0] - n.Key = bytesToUint64(bytes[1:9]) - n.AlternateKey = bytesToUint32(bytes[9:13]) - n.Size = bytesToUint32(bytes[13:17]) - n.Data = bytes[17 : 17+size] - n.Checksum = int32(bytesToUint32(bytes[17+size : 17+size+4])) + n.Cookie = bytesToUint32(bytes[0:4]) + n.Key = bytesToUint64(bytes[4:12]) + n.Size = bytesToUint32(bytes[12:16]) + n.Data = bytes[16 : 16+size] + n.Checksum = int32(bytesToUint32(bytes[16+size : 16+size+4])) } diff --git a/weed-fs/src/pkg/storage/needle_map.go b/weed-fs/src/pkg/storage/needle_map.go index 6d0eecb74..3fbf1f6f5 100644 --- a/weed-fs/src/pkg/storage/needle_map.go +++ b/weed-fs/src/pkg/storage/needle_map.go @@ -6,7 +6,6 @@ import ( type NeedleKey struct { Key uint64 "file id" - AlternateKey uint32 "supplemental id" } func (k *NeedleKey) String() string { @@ -14,9 +13,6 @@ func (k *NeedleKey) String() string { for i := uint(0); i < 8; i++ { tmp[i] = byte(k.Key >> (8 * i)) } - for i := uint(0); i < 4; i++ { - tmp[i+8] = byte(k.AlternateKey >> (8 * i)) - } return string(tmp[:]) } @@ -26,28 +22,21 @@ type NeedleValue struct { } type NeedleMap struct { - m map[string]*NeedleValue //mapping NeedleKey(Key,AlternateKey) to NeedleValue + m map[uint64]*NeedleValue //mapping NeedleKey(Key,AlternateKey) to NeedleValue } func NewNeedleMap() *NeedleMap { - return &NeedleMap{m: make(map[string]*NeedleValue)} + return &NeedleMap{m: make(map[uint64]*NeedleValue)} } func (nm *NeedleMap) load(file *os.File) { } -func makeKey(key uint64, altKey uint32) string { - var tmp [12]byte - for i := uint(0); i < 8; i++ { - tmp[i] = byte(key >> (8 * i)) - } - for i := uint(0); i < 4; i++ { - tmp[i+8] = byte(altKey >> (8 * i)) - } - return string(tmp[:]) +func makeKey(key uint64) uint64 { + return key } -func (nm *NeedleMap) put(key uint64, altKey uint32, offset uint32, size uint32) { - nm.m[makeKey(key, altKey)] = &NeedleValue{Offset: offset, Size: size} +func (nm *NeedleMap) put(key uint64, offset uint32, size uint32) { + nm.m[makeKey(key)] = &NeedleValue{Offset: offset, Size: size} } -func (nm *NeedleMap) get(key uint64, altKey uint32) (element *NeedleValue, ok bool) { - element, ok = nm.m[makeKey(key, altKey)] +func (nm *NeedleMap) get(key uint64) (element *NeedleValue, ok bool) { + element, ok = nm.m[makeKey(key)] return } diff --git a/weed-fs/src/pkg/storage/volume.go b/weed-fs/src/pkg/storage/volume.go index f065233fe..dd1dff54e 100644 --- a/weed-fs/src/pkg/storage/volume.go +++ b/weed-fs/src/pkg/storage/volume.go @@ -52,15 +52,15 @@ func (v *Volume) write(n *Needle) { counter := <-v.accessChannel offset, _ := v.dataFile.Seek(0, 2) n.Append(v.dataFile) - nv, ok := v.nm.get(n.Key, n.AlternateKey) + nv, ok := v.nm.get(n.Key) if !ok || int64(nv.Offset)*8 < offset { - v.nm.put(n.Key, n.AlternateKey, uint32(offset/8), n.Size) + v.nm.put(n.Key, uint32(offset/8), n.Size) } v.accessChannel <- counter + 1 } func (v *Volume) read(n *Needle) { counter := <-v.accessChannel - nv, ok := v.nm.get(n.Key, n.AlternateKey) + nv, ok := v.nm.get(n.Key) if ok && nv.Offset > 0 { v.dataFile.Seek(int64(nv.Offset)*8, 0) n.Read(v.dataFile, nv.Size)