diff --git a/go/operation/sync_volume.go b/go/operation/sync_volume.go index 626b2a7ba..54944a64e 100644 --- a/go/operation/sync_volume.go +++ b/go/operation/sync_volume.go @@ -1,7 +1,6 @@ package operation import ( - "encoding/binary" "encoding/json" "fmt" "net/url" @@ -43,9 +42,9 @@ func GetVolumeIdxEntries(server string, vid string, eachEntryFn func(key uint64, values.Add("volume", vid) line := make([]byte, 16) err := util.GetBufferStream("http://"+server+"/admin/sync/index", values, line, func(bytes []byte) { - key := binary.BigEndian.Uint64(bytes[:8]) - offset := binary.BigEndian.Uint32(bytes[8:12]) - size := binary.BigEndian.Uint32(bytes[12:16]) + key := util.BytesToUint64(bytes[:8]) + offset := util.BytesToUint32(bytes[8:12]) + size := util.BytesToUint32(bytes[12:16]) eachEntryFn(key, offset, size) }) if err != nil { diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index 7fd4c2dfb..1b429f263 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -1,12 +1,12 @@ package storage import ( - "encoding/binary" "log" "os" "testing" "github.com/chrislusf/seaweedfs/go/glog" + "github.com/chrislusf/seaweedfs/go/util" ) func TestMemoryUsage(t *testing.T) { @@ -29,9 +29,9 @@ func LoadNewNeedleMap(file *os.File) CompactMap { } for count > 0 && e == nil { for i := 0; i < count; i += 16 { - key := binary.BigEndian.Uint64(bytes[i : i+8]) - offset := binary.BigEndian.Uint32(bytes[i+8 : i+12]) - size := binary.BigEndian.Uint32(bytes[i+12 : i+16]) + key := util.BytesToUint64(bytes[i : i+8]) + offset := util.BytesToUint32(bytes[i+8 : i+12]) + size := util.BytesToUint32(bytes[i+12 : i+16]) if offset > 0 { m.Set(Key(key), offset, size) } else { diff --git a/go/storage/crc.go b/go/storage/crc.go index 3150b786c..21e384854 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -1,10 +1,10 @@ package storage import ( - "encoding/binary" "fmt" - "github.com/klauspost/crc32" + + "github.com/chrislusf/seaweedfs/go/util" ) var table = crc32.MakeTable(crc32.Castagnoli) @@ -25,6 +25,6 @@ func (c CRC) Value() uint32 { func (n *Needle) Etag() string { bits := make([]byte, 4) - binary.BigEndian.PutUint32(bits, uint32(n.Checksum)) + util.Uint32toBytes(bits, uint32(n.Checksum)) return fmt.Sprintf("\"%x\"", bits) } diff --git a/go/storage/file_id.go b/go/storage/file_id.go index 3d2944403..64b61ba89 100644 --- a/go/storage/file_id.go +++ b/go/storage/file_id.go @@ -1,12 +1,12 @@ package storage import ( - "encoding/binary" "encoding/hex" "errors" "strings" "github.com/chrislusf/seaweedfs/go/glog" + "github.com/chrislusf/seaweedfs/go/util" ) type FileId struct { @@ -34,8 +34,8 @@ func ParseFileId(fid string) (*FileId, error) { } func (n *FileId) String() string { bytes := make([]byte, 12) - binary.BigEndian.PutUint64(bytes[0:8], n.Key) - binary.BigEndian.PutUint32(bytes[8:12], n.Hashcode) + util.Uint64toBytes(bytes[0:8], n.Key) + util.Uint32toBytes(bytes[8:12], n.Hashcode) nonzero_index := 0 for ; bytes[nonzero_index] == 0; nonzero_index++ { } diff --git a/go/storage/needle.go b/go/storage/needle.go index 6636c342c..e49368820 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -1,7 +1,6 @@ package storage import ( - "encoding/binary" "encoding/hex" "errors" "fmt" @@ -16,6 +15,7 @@ import ( "github.com/chrislusf/seaweedfs/go/glog" "github.com/chrislusf/seaweedfs/go/images" "github.com/chrislusf/seaweedfs/go/operation" + "github.com/chrislusf/seaweedfs/go/util" ) const ( @@ -222,7 +222,7 @@ func ParseKeyHash(key_hash_string string) (uint64, uint32, error) { glog.V(0).Infoln("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe) return 0, 0, errors.New("Invalid key and hash:" + key_hash_string) } - key := binary.BigEndian.Uint64(key_hash_bytes[0 : key_hash_len-4]) - hash := binary.BigEndian.Uint32(key_hash_bytes[key_hash_len-4 : key_hash_len]) + key := util.BytesToUint64(key_hash_bytes[0 : key_hash_len-4]) + hash := util.BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len]) return key, hash, nil } diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index 41f53b90b..814789616 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -1,11 +1,12 @@ package storage import ( - "encoding/binary" "fmt" "io/ioutil" "os" "sync" + + "github.com/chrislusf/seaweedfs/go/util" ) type NeedleMapType int @@ -52,16 +53,16 @@ func (nm baseNeedleMapper) IndexFileName() string { } func idxFileEntry(bytes []byte) (key uint64, offset uint32, size uint32) { - key = binary.BigEndian.Uint64(bytes[:8]) - offset = binary.BigEndian.Uint32(bytes[8:12]) - size = binary.BigEndian.Uint32(bytes[12:16]) + key = util.BytesToUint64(bytes[:8]) + offset = util.BytesToUint32(bytes[8:12]) + size = util.BytesToUint32(bytes[12:16]) return } func (nm baseNeedleMapper) appendToIndexFile(key uint64, offset uint32, size uint32) error { bytes := make([]byte, 16) - binary.BigEndian.PutUint64(bytes[0:8], key) - binary.BigEndian.PutUint32(bytes[8:12], offset) - binary.BigEndian.PutUint32(bytes[12:16], size) + util.Uint64toBytes(bytes[0:8], key) + util.Uint32toBytes(bytes[8:12], offset) + util.Uint32toBytes(bytes[12:16], size) nm.indexFileAccessLock.Lock() defer nm.indexFileAccessLock.Unlock() diff --git a/go/storage/needle_map_boltdb.go b/go/storage/needle_map_boltdb.go index a17e6ccf6..e95c016bb 100644 --- a/go/storage/needle_map_boltdb.go +++ b/go/storage/needle_map_boltdb.go @@ -1,12 +1,13 @@ package storage import ( - "encoding/binary" "fmt" "os" "github.com/boltdb/bolt" + "github.com/chrislusf/seaweedfs/go/glog" + "github.com/chrislusf/seaweedfs/go/util" ) type BoltDbNeedleMap struct { @@ -74,7 +75,7 @@ func generateBoltDbFile(dbFileName string, indexFile *os.File) error { func (m *BoltDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { bytes := make([]byte, 8) var data []byte - binary.BigEndian.PutUint64(bytes, key) + util.Uint64toBytes(bytes, key) err := m.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket(boltdbBucket) if bucket == nil { @@ -88,8 +89,8 @@ func (m *BoltDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { if err != nil || len(data) != 8 { return nil, false } - offset := binary.BigEndian.Uint32(data[0:4]) - size := binary.BigEndian.Uint32(data[4:8]) + offset := util.BytesToUint32(data[0:4]) + size := util.BytesToUint32(data[4:8]) return &NeedleValue{Key: Key(key), Offset: offset, Size: size}, true } @@ -109,9 +110,9 @@ func (m *BoltDbNeedleMap) Put(key uint64, offset uint32, size uint32) error { func boltDbWrite(db *bolt.DB, key uint64, offset uint32, size uint32) error { bytes := make([]byte, 16) - binary.BigEndian.PutUint64(bytes[0:8], key) - binary.BigEndian.PutUint32(bytes[8:12], offset) - binary.BigEndian.PutUint32(bytes[12:16], size) + util.Uint64toBytes(bytes[0:8], key) + util.Uint32toBytes(bytes[8:12], offset) + util.Uint32toBytes(bytes[12:16], size) return db.Update(func(tx *bolt.Tx) error { bucket, err := tx.CreateBucketIfNotExists(boltdbBucket) if err != nil { @@ -127,7 +128,7 @@ func boltDbWrite(db *bolt.DB, } func boltDbDelete(db *bolt.DB, key uint64) error { bytes := make([]byte, 8) - binary.BigEndian.PutUint64(bytes, key) + util.Uint64toBytes(bytes, key) return db.Update(func(tx *bolt.Tx) error { bucket, err := tx.CreateBucketIfNotExists(boltdbBucket) if err != nil { diff --git a/go/storage/needle_map_leveldb.go b/go/storage/needle_map_leveldb.go index f9e580f6a..47f63e3ae 100644 --- a/go/storage/needle_map_leveldb.go +++ b/go/storage/needle_map_leveldb.go @@ -1,12 +1,12 @@ package storage import ( - "encoding/binary" "fmt" "os" "path/filepath" "github.com/chrislusf/seaweedfs/go/glog" + "github.com/chrislusf/seaweedfs/go/util" "github.com/syndtr/goleveldb/leveldb" ) @@ -72,13 +72,13 @@ func generateLevelDbFile(dbFileName string, indexFile *os.File) error { func (m *LevelDbNeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { bytes := make([]byte, 8) - binary.BigEndian.PutUint64(bytes, key) + util.Uint64toBytes(bytes, key) data, err := m.db.Get(bytes, nil) if err != nil || len(data) != 8 { return nil, false } - offset := binary.BigEndian.Uint32(data[0:4]) - size := binary.BigEndian.Uint32(data[4:8]) + offset := util.BytesToUint32(data[0:4]) + size := util.BytesToUint32(data[4:8]) return &NeedleValue{Key: Key(key), Offset: offset, Size: size}, true } @@ -98,9 +98,9 @@ func (m *LevelDbNeedleMap) Put(key uint64, offset uint32, size uint32) error { func levelDbWrite(db *leveldb.DB, key uint64, offset uint32, size uint32) error { bytes := make([]byte, 16) - binary.BigEndian.PutUint64(bytes[0:8], key) - binary.BigEndian.PutUint32(bytes[8:12], offset) - binary.BigEndian.PutUint32(bytes[12:16], size) + util.Uint64toBytes(bytes[0:8], key) + util.Uint32toBytes(bytes[8:12], offset) + util.Uint32toBytes(bytes[12:16], size) if err := db.Put(bytes[0:8], bytes[8:16], nil); err != nil { return fmt.Errorf("failed to write leveldb: %v", err) } @@ -108,7 +108,7 @@ func levelDbWrite(db *leveldb.DB, } func levelDbDelete(db *leveldb.DB, key uint64) error { bytes := make([]byte, 8) - binary.BigEndian.PutUint64(bytes, key) + util.Uint64toBytes(bytes, key) return db.Delete(bytes, nil) } diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index daab6121f..ccfe1d498 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -1,13 +1,13 @@ package storage import ( - "encoding/binary" "errors" "fmt" "io" "os" "github.com/chrislusf/seaweedfs/go/glog" + "github.com/chrislusf/seaweedfs/go/util" ) const ( @@ -43,11 +43,11 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { switch version { case Version1: header := make([]byte, NeedleHeaderSize) - binary.BigEndian.PutUint32(header[0:4], n.Cookie) - binary.BigEndian.PutUint64(header[4:12], n.Id) + util.Uint32toBytes(header[0:4], n.Cookie) + util.Uint64toBytes(header[4:12], n.Id) n.Size = uint32(len(n.Data)) size = n.Size - binary.BigEndian.PutUint32(header[12:16], n.Size) + util.Uint32toBytes(header[12:16], n.Size) if _, err = w.Write(header); err != nil { return } @@ -55,13 +55,13 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { return } padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize) - binary.BigEndian.PutUint32(header[0:NeedleChecksumSize], n.Checksum.Value()) + util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) _, err = w.Write(header[0 : NeedleChecksumSize+padding]) return case Version2: header := make([]byte, NeedleHeaderSize) - binary.BigEndian.PutUint32(header[0:4], n.Cookie) - binary.BigEndian.PutUint64(header[4:12], n.Id) + util.Uint32toBytes(header[0:4], n.Cookie) + util.Uint64toBytes(header[4:12], n.Id) n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime)) if n.DataSize > 0 { n.Size = 4 + n.DataSize + 1 @@ -81,24 +81,24 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { n.Size = 0 } size = n.DataSize - binary.BigEndian.PutUint32(header[12:16], n.Size) + util.Uint32toBytes(header[12:16], n.Size) if _, err = w.Write(header); err != nil { return } if n.DataSize > 0 { - binary.BigEndian.PutUint32(header[0:4], n.DataSize) + util.Uint32toBytes(header[0:4], n.DataSize) if _, err = w.Write(header[0:4]); err != nil { return } if _, err = w.Write(n.Data); err != nil { return } - header[0] = n.Flags + util.Uint8toBytes(header[0:1], n.Flags) if _, err = w.Write(header[0:1]); err != nil { return } if n.HasName() { - header[0] = n.NameSize + util.Uint8toBytes(header[0:1], n.NameSize) if _, err = w.Write(header[0:1]); err != nil { return } @@ -107,7 +107,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { } } if n.HasMime() { - header[0] = n.MimeSize + util.Uint8toBytes(header[0:1], n.MimeSize) if _, err = w.Write(header[0:1]); err != nil { return } @@ -116,7 +116,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { } } if n.HasLastModifiedDate() { - binary.BigEndian.PutUint64(header[0:8], n.LastModified) + util.Uint64toBytes(header[0:8], n.LastModified) if _, err = w.Write(header[8-LastModifiedBytesLength : 8]); err != nil { return } @@ -129,7 +129,7 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { } } padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize) - binary.BigEndian.PutUint32(header[0:NeedleChecksumSize], n.Checksum.Value()) + util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) _, err = w.Write(header[0 : NeedleChecksumSize+padding]) return n.DataSize, err } @@ -158,7 +158,7 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version case Version2: n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)]) } - checksum := binary.BigEndian.Uint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize]) + checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize]) newChecksum := NewCRC(n.Data) if checksum != newChecksum.Value() { return errors.New("CRC error! Data On Disk Corrupted") @@ -167,14 +167,14 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version return nil } func (n *Needle) ParseNeedleHeader(bytes []byte) { - n.Cookie = binary.BigEndian.Uint32(bytes[0:4]) - n.Id = binary.BigEndian.Uint64(bytes[4:12]) - n.Size = binary.BigEndian.Uint32(bytes[12:NeedleHeaderSize]) + n.Cookie = util.BytesToUint32(bytes[0:4]) + n.Id = util.BytesToUint64(bytes[4:12]) + n.Size = util.BytesToUint32(bytes[12:NeedleHeaderSize]) } func (n *Needle) readNeedleDataVersion2(bytes []byte) { index, lenBytes := 0, len(bytes) if index < lenBytes { - n.DataSize = binary.BigEndian.Uint32(bytes[index : index+4]) + n.DataSize = util.BytesToUint32(bytes[index : index+4]) index = index + 4 if int(n.DataSize)+index > lenBytes { // this if clause is due to bug #87 and #93, fixed in v0.69 @@ -199,7 +199,7 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) { index = index + int(n.MimeSize) } if index < lenBytes && n.HasLastModifiedDate() { - n.LastModified = binary.BigEndian.Uint64(bytes[index : index+LastModifiedBytesLength]) + n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength]) index = index + LastModifiedBytesLength } if index < lenBytes && n.HasTtl() { diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index 76116090d..e37360075 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -1,11 +1,11 @@ package storage import ( - "encoding/binary" "fmt" "os" "github.com/chrislusf/seaweedfs/go/glog" + "github.com/chrislusf/seaweedfs/go/util" ) const ( @@ -35,7 +35,7 @@ func (s *SuperBlock) Bytes() []byte { header[0] = byte(s.version) header[1] = s.ReplicaPlacement.Byte() s.Ttl.ToBytes(header[2:4]) - binary.BigEndian.PutUint16(header[4:6], s.CompactRevision) + util.Uint16toBytes(header[4:6], s.CompactRevision) return header } @@ -76,6 +76,6 @@ func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) { err = fmt.Errorf("cannot read replica type: %s", err.Error()) } superBlock.Ttl = LoadTTLFromBytes(header[2:4]) - superBlock.CompactRevision = binary.BigEndian.Uint16(header[4:6]) + superBlock.CompactRevision = util.BytesToUint16(header[4:6]) return } diff --git a/go/util/bytes.go b/go/util/bytes.go new file mode 100644 index 000000000..dfa4ae665 --- /dev/null +++ b/go/util/bytes.go @@ -0,0 +1,45 @@ +package util + +// big endian + +func BytesToUint64(b []byte) (v uint64) { + length := uint(len(b)) + for i := uint(0); i < length-1; i++ { + v += uint64(b[i]) + v <<= 8 + } + v += uint64(b[length-1]) + return +} +func BytesToUint32(b []byte) (v uint32) { + length := uint(len(b)) + for i := uint(0); i < length-1; i++ { + v += uint32(b[i]) + v <<= 8 + } + v += uint32(b[length-1]) + return +} +func BytesToUint16(b []byte) (v uint16) { + v += uint16(b[0]) + v <<= 8 + v += uint16(b[1]) + return +} +func Uint64toBytes(b []byte, v uint64) { + for i := uint(0); i < 8; i++ { + b[7-i] = byte(v >> (i * 8)) + } +} +func Uint32toBytes(b []byte, v uint32) { + for i := uint(0); i < 4; i++ { + b[3-i] = byte(v >> (i * 8)) + } +} +func Uint16toBytes(b []byte, v uint16) { + b[0] = byte(v >> 8) + b[1] = byte(v) +} +func Uint8toBytes(b []byte, v uint8) { + b[0] = byte(v) +}