diff --git a/weed/storage/erasure_coding/1.dat b/weed/storage/erasure_coding/1.dat new file mode 100644 index 000000000..869427926 Binary files /dev/null and b/weed/storage/erasure_coding/1.dat differ diff --git a/weed/storage/erasure_coding/1.idx b/weed/storage/erasure_coding/1.idx new file mode 100644 index 000000000..65a950e64 Binary files /dev/null and b/weed/storage/erasure_coding/1.idx differ diff --git a/weed/storage/erasure_coding/ec_encoder.go b/weed/storage/erasure_coding/ec_encoder.go new file mode 100644 index 000000000..5ff73de69 --- /dev/null +++ b/weed/storage/erasure_coding/ec_encoder.go @@ -0,0 +1,8 @@ +package erasure_coding + +const ( + DataShardsCount = 10 + ParityShardsCount = 4 + ErasureCodingLargeBlockSize = 1024 * 1024 * 1024 // 1GB + ErasureCodingSmallBlockSize = 1024 * 1024 // 1MB +) diff --git a/weed/storage/erasure_coding/ec_test.go b/weed/storage/erasure_coding/ec_test.go new file mode 100644 index 000000000..8f75384c0 --- /dev/null +++ b/weed/storage/erasure_coding/ec_test.go @@ -0,0 +1,77 @@ +package erasure_coding + +import ( + "io" + "os" + "testing" + + "github.com/klauspost/reedsolomon" +) + +func TestEncodingDecoding(t *testing.T) { + largeBlockSize := int64(10000) + smallBlockSize := int64(100) + bufferSize := 50 + + file, err := os.OpenFile("1.dat", os.O_RDONLY, 0) + if err != nil { + t.Logf("failed to open dat file: %v", err) + } + + fi, err := file.Stat() + if err != nil { + t.Logf("failed to stat dat file: %v", err) + } + + remainingSize := fi.Size() + var processedSize int64 + + enc, err := reedsolomon.New(DataShardsCount, ParityShardsCount) + if err != nil { + t.Logf("failed to create encoder: %v", err) + } + + buffers := make([][]byte, DataShardsCount+ParityShardsCount) + + for i, _ := range buffers { + buffers[i] = make([]byte, bufferSize) + } + + for remainingSize > largeBlockSize*DataShardsCount { + encodeData(file, enc, processedSize, largeBlockSize, buffers) + remainingSize -= largeBlockSize * DataShardsCount + processedSize += largeBlockSize * DataShardsCount + } + + for remainingSize > 0 { + encodeData(file, enc, processedSize, smallBlockSize, buffers) + remainingSize -= smallBlockSize * DataShardsCount + processedSize += smallBlockSize * DataShardsCount + } + +} + +func encodeData(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize int64, buffers [][]byte) error { + + // read data into buffers + for i := 0; i < DataShardsCount; i++ { + n, err := file.ReadAt(buffers[i], startOffset+blockSize*int64(i)) + if err != nil { + if err != io.EOF { + return err + } + } + if n < len(buffers[i]) { + for t := len(buffers[i]) - 1; t >= n; t-- { + buffers[i][t] = 0 + } + } + } + + err := enc.Encode(buffers) + if err != nil { + return err + } + + return nil +}