From 6ce0097846b8a22de8d77de16d16f49b9af17a87 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 21 May 2018 10:28:46 -0700 Subject: [PATCH 1/4] add left padding fix https://github.com/chrislusf/seaweedfs/issues/654 --- weed/filer/mysql_store/mysql_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer/mysql_store/mysql_store.go b/weed/filer/mysql_store/mysql_store.go index 4a8e889df..32cf8c567 100644 --- a/weed/filer/mysql_store/mysql_store.go +++ b/weed/filer/mysql_store/mysql_store.go @@ -198,7 +198,7 @@ CREATE TABLE IF NOT EXISTS %s ( func (s *MySqlStore) createTables(db *sql.DB, tableName string, postfix int) error { var realTableName string if s.isSharding { - realTableName = fmt.Sprintf("%s_%4d", tableName, postfix) + realTableName = fmt.Sprintf("%s_%04d", tableName, postfix) } else { realTableName = tableName } From 04c2c5e58d1ea1cd83907e16dbcee979f98371e3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 22 May 2018 00:15:11 -0700 Subject: [PATCH 2/4] fix mysql data store bug --- weed/filer/mysql_store/mysql_store.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/weed/filer/mysql_store/mysql_store.go b/weed/filer/mysql_store/mysql_store.go index 32cf8c567..72a8bb66a 100644 --- a/weed/filer/mysql_store/mysql_store.go +++ b/weed/filer/mysql_store/mysql_store.go @@ -148,10 +148,14 @@ func (s *MySqlStore) Put(fullFilePath string, fid string) (err error) { } else { if len(old_fid) == 0 { err = s.insert(fullFilePath, fid, s.dbs[instance_offset], tableFullName) - err = fmt.Errorf("MySqlStore Put operation failed when inserting path %s with fid %s : err is %v", fullFilePath, fid, err) + if err != nil { + err = fmt.Errorf("MySqlStore Put operation failed when inserting path %s with fid %s : err is %v", fullFilePath, fid, err) + } } else { err = s.update(fullFilePath, fid, s.dbs[instance_offset], tableFullName) - err = fmt.Errorf("MySqlStore Put operation failed when updating path %s with fid %s : err is %v", fullFilePath, fid, err) + if err != nil { + err = fmt.Errorf("MySqlStore Put operation failed when updating path %s with fid %s : err is %v", fullFilePath, fid, err) + } } } return From 88d608b661b092446b8512c2c8c7c74b191f1c25 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 22 May 2018 10:19:44 -0700 Subject: [PATCH 3/4] avoid using the bytes out side of the transaction fix https://github.com/chrislusf/seaweedfs/issues/656 --- weed/storage/needle_map_boltdb.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/weed/storage/needle_map_boltdb.go b/weed/storage/needle_map_boltdb.go index 96c29cab6..cd15607bb 100644 --- a/weed/storage/needle_map_boltdb.go +++ b/weed/storage/needle_map_boltdb.go @@ -75,8 +75,8 @@ func generateBoltDbFile(dbFileName string, indexFile *os.File) error { } func (m *BoltDbNeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool) { + var offset, size uint32 bytes := make([]byte, 8) - var data []byte util.Uint64toBytes(bytes, key) err := m.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket(boltdbBucket) @@ -84,15 +84,22 @@ func (m *BoltDbNeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool) return fmt.Errorf("Bucket %q not found!", boltdbBucket) } - data = bucket.Get(bytes) + data := bucket.Get(bytes) + + if len(data) != 8 { + glog.V(0).Infof("wrong data length: %d", len(data)) + return fmt.Errorf("wrong data length: %d", len(data)) + } + + offset = util.BytesToUint32(data[0:4]) + size = util.BytesToUint32(data[4:8]) + return nil }) - if err != nil || len(data) != 8 { + if err != nil { return nil, false } - offset := util.BytesToUint32(data[0:4]) - size := util.BytesToUint32(data[4:8]) return &needle.NeedleValue{Key: needle.Key(key), Offset: offset, Size: size}, true } From 3a3553dc463a8f5f5739b58e325da4f14650517b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 22 May 2018 11:12:59 -0700 Subject: [PATCH 4/4] remove unused files --- weed/compress/compression_test.go | 45 ---------------------------- weed/compress/delta_binary_pack32.go | 32 -------------------- weed/glide.lock | 6 ---- weed/glide.yaml | 4 --- 4 files changed, 87 deletions(-) delete mode 100644 weed/compress/compression_test.go delete mode 100644 weed/compress/delta_binary_pack32.go diff --git a/weed/compress/compression_test.go b/weed/compress/compression_test.go deleted file mode 100644 index 83b7c0055..000000000 --- a/weed/compress/compression_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package compress - -import ( - "math/rand" - "testing" -) - -func TestSortedData(t *testing.T) { - data := make([]int32, 102400) - for i := 1; i < len(data); i++ { - data[i] = data[i-1] + rand.Int31n(15) - } - testCompressAndUncompress(t, data, "Sorted data") -} - -func TestUnsortedData(t *testing.T) { - data := make([]int32, 102400) - for i := 0; i < len(data); i++ { - data[i] = rand.Int31n(255) - } - testCompressAndUncompress(t, data, "Unsorted data") -} - -func testCompressAndUncompress(t *testing.T, data []int32, desc string) { - - compressed_data, err := Compress32(data) - if err != nil { - t.Fatal("Compress error", err.Error()) - } - uncompressed_data, err := Uncompress32(compressed_data, make([]int32, len(data)*2)) - if err != nil { - t.Fatal("Compress error", err.Error()) - } - if len(uncompressed_data) != len(data) { - t.Fatal("Len differs", len(data), len(uncompressed_data)) - } - for i := 0; i < len(data); i++ { - if data[i] != uncompressed_data[i] { - t.Fatal("Data differs:", i, data[i], uncompressed_data[i]) - } - } - - println(desc, " Data length:", len(data), " => Compressed length:", len(compressed_data)) - -} diff --git a/weed/compress/delta_binary_pack32.go b/weed/compress/delta_binary_pack32.go deleted file mode 100644 index 1da0e427a..000000000 --- a/weed/compress/delta_binary_pack32.go +++ /dev/null @@ -1,32 +0,0 @@ -package compress - -import ( - "github.com/dataence/encoding/cursor" - "github.com/dataence/encoding/delta/bp32" -) - -// Compress compresses in[]int32 to out[]int32 -func Compress32(in []int32) (out []int32, err error) { - out = make([]int32, len(in)*2) - inpos := cursor.New() - outpos := cursor.New() - - if err = bp32.New().Compress(in, inpos, len(in), out, outpos); err != nil { - return nil, err - } - - return out[:outpos.Get()], nil -} - -// Uncompress uncompresses in[]int32 to out[]int32 -func Uncompress32(in []int32, buffer []int32) (out []int32, err error) { - out = buffer - inpos := cursor.New() - outpos := cursor.New() - - if err = bp32.New().Uncompress(in, inpos, len(in), out, outpos); err != nil { - return nil, err - } - - return out[:outpos.Get()], nil -} diff --git a/weed/glide.lock b/weed/glide.lock index 52bdedc54..ad0e1bf8e 100644 --- a/weed/glide.lock +++ b/weed/glide.lock @@ -12,12 +12,6 @@ imports: version: 5f7ddd8f479583daf05879d3d3b174aa202c8fb7 subpackages: - protobuf -- name: github.com/dataence/encoding - version: b90e310a0325f9b765b4be7220df3642ad93ad8d - subpackages: - - bitpacking - - cursor - - delta/bp32 - name: github.com/dgrijalva/jwt-go version: 06ea1031745cb8b3dab3f6a236daf2b0aa468b7e - name: github.com/disintegration/imaging diff --git a/weed/glide.yaml b/weed/glide.yaml index 36737fb21..7903a6728 100644 --- a/weed/glide.yaml +++ b/weed/glide.yaml @@ -6,10 +6,6 @@ import: - package: github.com/boltdb/bolt version: ^1.3.1 - package: github.com/chrislusf/raft -- package: github.com/dataence/encoding - subpackages: - - cursor - - delta/bp32 - package: github.com/dgrijalva/jwt-go version: ^3.2.0 - package: github.com/disintegration/imaging