Browse Source

add build option support 5-byte offset

pull/925/head
Chris Lu 6 years ago
parent
commit
784c5bb73a
  1. 53
      weed/storage/types/needle_types.go
  2. 63
      weed/storage/types/offset_4bytes.go
  3. 80
      weed/storage/types/offset_5bytes.go

53
weed/storage/types/needle_types.go

@ -12,10 +12,6 @@ type Offset struct {
OffsetLower OffsetLower
} }
type OffsetHigher struct {
// b4 byte
}
type OffsetLower struct { type OffsetLower struct {
b3 byte b3 byte
b2 byte b2 byte
@ -26,12 +22,10 @@ type OffsetLower struct {
type Cookie uint32 type Cookie uint32
const ( const (
OffsetSize = 4 // + 1
SizeSize = 4 // uint32 size SizeSize = 4 // uint32 size
NeedleEntrySize = CookieSize + NeedleIdSize + SizeSize NeedleEntrySize = CookieSize + NeedleIdSize + SizeSize
TimestampSize = 8 // int64 size TimestampSize = 8 // int64 size
NeedlePaddingSize = 8 NeedlePaddingSize = 8
MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8
TombstoneFileSize = math.MaxUint32 TombstoneFileSize = math.MaxUint32
CookieSize = 4 CookieSize = 4
) )
@ -54,50 +48,3 @@ func ParseCookie(cookieString string) (Cookie, error) {
} }
return Cookie(cookie), nil return Cookie(cookie), nil
} }
func OffsetToBytes(bytes []byte, offset Offset) {
bytes[3] = offset.b0
bytes[2] = offset.b1
bytes[1] = offset.b2
bytes[0] = offset.b3
}
// only for testing, will be removed later.
func Uint32ToOffset(offset uint32) Offset {
return Offset{
OffsetLower: OffsetLower{
b0: byte(offset),
b1: byte(offset >> 8),
b2: byte(offset >> 16),
b3: byte(offset >> 24),
},
}
}
func BytesToOffset(bytes []byte) Offset {
return Offset{
OffsetLower: OffsetLower{
b0: bytes[3],
b1: bytes[2],
b2: bytes[1],
b3: bytes[0],
},
}
}
func (offset Offset) IsZero() bool {
return offset.b0 == 0 && offset.b1 == 0 && offset.b2 == 0 && offset.b3 == 0
}
func ToOffset(offset int64) Offset {
smaller := uint32(offset / int64(NeedlePaddingSize))
return Uint32ToOffset(smaller)
}
func (offset Offset) ToAcutalOffset() (actualOffset int64) {
return (int64(offset.b0) + int64(offset.b1)<<8 + int64(offset.b2)<<16 + int64(offset.b3)<<24) * int64(NeedlePaddingSize)
}
func (offset Offset) String() string {
return fmt.Sprintf("%d", int64(offset.b0)+int64(offset.b1)<<8+int64(offset.b2)<<16+int64(offset.b3)<<24)
}

63
weed/storage/types/offset_4bytes.go

@ -0,0 +1,63 @@
// +build !5BytesOffset
package types
import (
"fmt"
)
type OffsetHigher struct {
// b4 byte
}
const (
OffsetSize = 4
MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8 // 32GB
)
func OffsetToBytes(bytes []byte, offset Offset) {
bytes[3] = offset.b0
bytes[2] = offset.b1
bytes[1] = offset.b2
bytes[0] = offset.b3
}
// only for testing, will be removed later.
func Uint32ToOffset(offset uint32) Offset {
return Offset{
OffsetLower: OffsetLower{
b0: byte(offset),
b1: byte(offset >> 8),
b2: byte(offset >> 16),
b3: byte(offset >> 24),
},
}
}
func BytesToOffset(bytes []byte) Offset {
return Offset{
OffsetLower: OffsetLower{
b0: bytes[3],
b1: bytes[2],
b2: bytes[1],
b3: bytes[0],
},
}
}
func (offset Offset) IsZero() bool {
return offset.b0 == 0 && offset.b1 == 0 && offset.b2 == 0 && offset.b3 == 0
}
func ToOffset(offset int64) Offset {
smaller := uint32(offset / int64(NeedlePaddingSize))
return Uint32ToOffset(smaller)
}
func (offset Offset) ToAcutalOffset() (actualOffset int64) {
return (int64(offset.b0) + int64(offset.b1)<<8 + int64(offset.b2)<<16 + int64(offset.b3)<<24) * int64(NeedlePaddingSize)
}
func (offset Offset) String() string {
return fmt.Sprintf("%d", int64(offset.b0)+int64(offset.b1)<<8+int64(offset.b2)<<16+int64(offset.b3)<<24)
}

80
weed/storage/types/offset_5bytes.go

@ -0,0 +1,80 @@
// +build 5BytesOffset
package types
import (
"fmt"
)
type OffsetHigher struct {
b4 byte
}
const (
OffsetSize = 4 + 1
MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8 * 256 /* 256 is from the extra byte */ // 8TB
)
func OffsetToBytes(bytes []byte, offset Offset) {
bytes[4] = offset.b4
bytes[3] = offset.b0
bytes[2] = offset.b1
bytes[1] = offset.b2
bytes[0] = offset.b3
}
// only for testing, will be removed later.
func Uint32ToOffset(offset uint32) Offset {
return Offset{
OffsetHigher: OffsetHigher{
b4: byte(offset >> 32),
},
OffsetLower: OffsetLower{
b0: byte(offset),
b1: byte(offset >> 8),
b2: byte(offset >> 16),
b3: byte(offset >> 24),
},
}
}
func BytesToOffset(bytes []byte) Offset {
return Offset{
OffsetHigher: OffsetHigher{
b4: bytes[4],
},
OffsetLower: OffsetLower{
b0: bytes[3],
b1: bytes[2],
b2: bytes[1],
b3: bytes[0],
},
}
}
func (offset Offset) IsZero() bool {
return offset.b0 == 0 && offset.b1 == 0 && offset.b2 == 0 && offset.b3 == 0 && offset.b4 == 0
}
func ToOffset(offset int64) Offset {
smaller := offset / int64(NeedlePaddingSize)
return Offset{
OffsetHigher: OffsetHigher{
b4: byte(smaller >> 32),
},
OffsetLower: OffsetLower{
b0: byte(smaller),
b1: byte(smaller >> 8),
b2: byte(smaller >> 16),
b3: byte(smaller >> 24),
},
}
}
func (offset Offset) ToAcutalOffset() (actualOffset int64) {
return (int64(offset.b0) + int64(offset.b1)<<8 + int64(offset.b2)<<16 + int64(offset.b3)<<24 + int64(offset.b4)<<32) * int64(NeedlePaddingSize)
}
func (offset Offset) String() string {
return fmt.Sprintf("%d", int64(offset.b0)+int64(offset.b1)<<8+int64(offset.b2)<<16+int64(offset.b3)<<24+int64(offset.b4)<<32)
}
Loading…
Cancel
Save