testing RS coding

weed/storage/erasure_coding/ec_encoder.go

@@ -0,0 +1,8 @@
+package erasure_coding
+
+const (
+	DataShardsCount             = 10
+	ParityShardsCount           = 4
+	ErasureCodingLargeBlockSize = 1024 * 1024 * 1024 // 1GB
+	ErasureCodingSmallBlockSize = 1024 * 1024        // 1MB
+)

weed/storage/erasure_coding/ec_test.go

@@ -0,0 +1,77 @@
+package erasure_coding
+
+import (
+	"io"
+	"os"
+	"testing"
+
+	"github.com/klauspost/reedsolomon"
+)
+
+func TestEncodingDecoding(t *testing.T) {
+	largeBlockSize := int64(10000)
+	smallBlockSize := int64(100)
+	bufferSize := 50
+
+	file, err := os.OpenFile("1.dat", os.O_RDONLY, 0)
+	if err != nil {
+		t.Logf("failed to open dat file: %v", err)
+	}
+
+	fi, err := file.Stat()
+	if err != nil {
+		t.Logf("failed to stat dat file: %v", err)
+	}
+
+	remainingSize := fi.Size()
+	var processedSize int64
+
+	enc, err := reedsolomon.New(DataShardsCount, ParityShardsCount)
+	if err != nil {
+		t.Logf("failed to create encoder: %v", err)
+	}
+
+	buffers := make([][]byte, DataShardsCount+ParityShardsCount)
+
+	for i, _ := range buffers {
+		buffers[i] = make([]byte, bufferSize)
+	}
+
+	for remainingSize > largeBlockSize*DataShardsCount {
+		encodeData(file, enc, processedSize, largeBlockSize, buffers)
+		remainingSize -= largeBlockSize * DataShardsCount
+		processedSize += largeBlockSize * DataShardsCount
+	}
+
+	for remainingSize > 0 {
+		encodeData(file, enc, processedSize, smallBlockSize, buffers)
+		remainingSize -= smallBlockSize * DataShardsCount
+		processedSize += smallBlockSize * DataShardsCount
+	}
+
+}
+
+func encodeData(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize int64, buffers [][]byte) error {
+
+	// read data into buffers
+	for i := 0; i < DataShardsCount; i++ {
+		n, err := file.ReadAt(buffers[i], startOffset+blockSize*int64(i))
+		if err != nil {
+			if err != io.EOF {
+				return err
+			}
+		}
+		if n < len(buffers[i]) {
+			for t := len(buffers[i]) - 1; t >= n; t-- {
+				buffers[i][t] = 0
+			}
+		}
+	}
+
+	err := enc.Encode(buffers)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}