terribleplan/file-store
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

CLean up types to use int64 many places, as that is what is used by os.File

Browse source
This commit is contained in:
Kegan Myers 2022-08-24 22:12:12 -05:00
parent 5bf1599611
commit db256918db
8 changed files with 76 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
pkg/erasureencode/decode.go
View file

@ -15,8 +15,8 @@ func decodeFn(inputs []io.ReadSeeker, file io.Writer, meta *EEMeta, getPlan Read
fullPlan := getPlan(meta) fullPlan := getPlan(meta)
// we only need to seek once as the rest of the reads should be linear // we only need to seek once as the rest of the reads should be linear
for _, plan := range fullPlan[0:min(uint64(meta.Params.Shards), uint64(len(fullPlan)))] { for _, plan := range fullPlan[0:min(int64(meta.Params.Shards), int64(len(fullPlan)))] {
if _, err := inputs[plan.Chunk].Seek(int64(plan.ChunkOffset), io.SeekStart); err != nil { if _, err := inputs[plan.Chunk].Seek(plan.ChunkOffset, io.SeekStart); err != nil {
return err return err
} }
} }
@ -49,13 +49,12 @@ func Decode(inputs []io.ReadSeeker, file io.Writer, meta *EEMeta) error {
} }
func DecodeAndValidate(inputs []io.ReadSeeker, file io.Writer, meta *EEMeta) error { func DecodeAndValidate(inputs []io.ReadSeeker, file io.Writer, meta *EEMeta) error {
size := uint64(meta.Params.Size) shards := int64(meta.Params.Shards)
shards := uint64(meta.Params.Shards)
// get set up to read meta including the padding // get set up to read meta including the padding
validateParams := *meta validateParams := *meta
if size%shards > 0 { if meta.Params.Size%shards > 0 {
validateParams.Size = (size / shards) * (shards + 1) validateParams.Size = (meta.Params.Size / shards) * (shards + 1)
} }
return decodeFn(inputs, file, meta, func(_ *EEMeta) []ChunkShardMeta { return decodeFn(inputs, file, meta, func(_ *EEMeta) []ChunkShardMeta {
@ -65,12 +64,12 @@ func DecodeAndValidate(inputs []io.ReadSeeker, file io.Writer, meta *EEMeta) err
if !bytes.Equal(actual, meta.ShardHashes[i]) { if !bytes.Equal(actual, meta.ShardHashes[i]) {
return fmt.Errorf("shard hash mismatch") return fmt.Errorf("shard hash mismatch")
} }
dataLen := uint64(len(data)) dataLen := int64(len(data))
writeData := data writeData := data
if read.GlobalOffset > size { if read.GlobalOffset > meta.Params.Size {
writeData = nil writeData = nil
} else if read.GlobalOffset+dataLen > size { } else if read.GlobalOffset+dataLen > meta.Params.Size {
writeData = data[0 : read.GlobalOffset-size] writeData = data[0 : read.GlobalOffset-meta.Params.Size]
} }
if writeData != nil { if writeData != nil {
if _, err := file.Write(writeData); err != nil { if _, err := file.Write(writeData); err != nil {

12
pkg/erasureencode/decode_test.go
View file

@ -18,7 +18,7 @@ var simpleDecodeMeta = &EEMeta{
func TestDecode_EvenStride_Full(t *testing.T) { func TestDecode_EvenStride_Full(t *testing.T) {
expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11} expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11}
simpleDecodeMeta.Params.Size = uint64(len(expected)) simpleDecodeMeta.Params.Size = int64(len(expected))
inputs := []io.ReadSeeker{ inputs := []io.ReadSeeker{
bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c, 0x0d}), bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c, 0x0d}),
@ -39,7 +39,7 @@ func TestDecode_EvenStride_Full(t *testing.T) {
func TestDecode_EvenStride_Short1(t *testing.T) { func TestDecode_EvenStride_Short1(t *testing.T) {
expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10} expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10}
simpleDecodeMeta.Params.Size = uint64(len(expected)) simpleDecodeMeta.Params.Size = int64(len(expected))
inputs := []io.ReadSeeker{ inputs := []io.ReadSeeker{
bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c, 0x0d}), bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c, 0x0d}),
@ -60,7 +60,7 @@ func TestDecode_EvenStride_Short1(t *testing.T) {
func TestDecode_EvenStride_Short2(t *testing.T) { func TestDecode_EvenStride_Short2(t *testing.T) {
expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f} expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}
simpleDecodeMeta.Params.Size = uint64(len(expected)) simpleDecodeMeta.Params.Size = int64(len(expected))
inputs := []io.ReadSeeker{ inputs := []io.ReadSeeker{
bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c, 0x0d}), bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c, 0x0d}),
@ -81,7 +81,7 @@ func TestDecode_EvenStride_Short2(t *testing.T) {
func TestDecode_OddStride_Full(t *testing.T) { func TestDecode_OddStride_Full(t *testing.T) {
expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e} expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e}
simpleDecodeMeta.Params.Size = uint64(len(expected)) simpleDecodeMeta.Params.Size = int64(len(expected))
inputs := []io.ReadSeeker{ inputs := []io.ReadSeeker{
bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c}), bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c}),
@ -102,7 +102,7 @@ func TestDecode_OddStride_Full(t *testing.T) {
func TestDecode_OddStride_Short1(t *testing.T) { func TestDecode_OddStride_Short1(t *testing.T) {
expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d} expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d}
simpleDecodeMeta.Params.Size = uint64(len(expected)) simpleDecodeMeta.Params.Size = int64(len(expected))
inputs := []io.ReadSeeker{ inputs := []io.ReadSeeker{
bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c}), bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c}),
@ -123,7 +123,7 @@ func TestDecode_OddStride_Short1(t *testing.T) {
func TestDecode_OddStride_Short2(t *testing.T) { func TestDecode_OddStride_Short2(t *testing.T) {
expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c} expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c}
simpleDecodeMeta.Params.Size = uint64(len(expected)) simpleDecodeMeta.Params.Size = int64(len(expected))
inputs := []io.ReadSeeker{ inputs := []io.ReadSeeker{
bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c}), bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c}),

30
pkg/erasureencode/encode.go
View file

@ -13,7 +13,7 @@ func EncodeFile(file *os.File, oututs []io.Writer, stride int32, shards, parity
if err != nil { if err != nil {
return nil, err return nil, err
} }
size := uint64(stats.Size()) size := stats.Size()
meta := &EEMeta{ meta := &EEMeta{
Params: Params{ Params: Params{
@ -32,35 +32,27 @@ func EncodeFile(file *os.File, oututs []io.Writer, stride int32, shards, parity
} }
func Encode(file io.Reader, outputs []io.Writer, meta *EEMeta) error { func Encode(file io.Reader, outputs []io.Writer, meta *EEMeta) error {
// int(uint16) + int(uint16) != int; should be safe
if int(meta.Params.Shards)+int(meta.Params.Parity) != len(outputs) { if int(meta.Params.Shards)+int(meta.Params.Parity) != len(outputs) {
return fmt.Errorf("expected the number of shards+parity to equal the number of output files provided") return fmt.Errorf("expected the number of shards+parity to equal the number of output files provided")
} }
// int(uint16), int(uint16), int(int32)
enc, err := reedsolomon.New(int(meta.Params.Shards), int(meta.Params.Parity), reedsolomon.WithAutoGoroutines(int(meta.Params.Stride))) enc, err := reedsolomon.New(int(meta.Params.Shards), int(meta.Params.Parity), reedsolomon.WithAutoGoroutines(int(meta.Params.Stride)))
if err != nil { if err != nil {
return err return err
} }
shards := uint64(meta.Params.Shards) outputChunkCount := meta.Params.Shards + meta.Params.Parity
parity := uint64(meta.Params.Parity) lastShardChunk := meta.Params.Shards - 1
outputChunkCount := shards + parity
lastShardChunk := shards - 1
endShards := shards + parity
data := make([][]byte, outputChunkCount) data := make([][]byte, outputChunkCount)
data[0] = []byte{} data[0] = []byte{}
written := false // track whether the current stripe has been written written := false // track whether the current stripe has been written
for shard, csm := range meta.Params.Plan(0, meta.Params.Size) { for i, csm := range meta.Params.Plan(0, meta.Params.Size) {
chunk := uint64(shard) % shards
written = false written = false
// prepare data slices, shard size only meaningfuly changes at stripe boundary // prepare data slices, shard size only meaningfuly changes at stripe boundary
if chunk == 0 { if csm.Chunk == 0 || i == 0 {
// if int32(len(data[0])) != csm.Size { // if int32(len(data[0])) != csm.Size {
for i := uint64(0); i < outputChunkCount; i++ { for i := uint16(0); i < outputChunkCount; i++ {
data[i] = make([]byte, csm.Size) data[i] = make([]byte, csm.Size)
} }
// } // }
@ -72,8 +64,8 @@ func Encode(file io.Reader, outputs []io.Writer, meta *EEMeta) error {
meta.ShardHashes = append(meta.ShardHashes, sha256sum(data[csm.Chunk])) meta.ShardHashes = append(meta.ShardHashes, sha256sum(data[csm.Chunk]))
// if we are on the last chunk calculate the parity and write things out // if we are on the last chunk calculate the parity and write things out
if chunk == lastShardChunk { if csm.Chunk == lastShardChunk {
if err := writeChunks(data, outputs, enc, meta, shards, endShards); err != nil { if err := writeChunks(data, outputs, enc, meta, meta.Params.Shards, outputChunkCount); err != nil {
return err return err
} }
written = true written = true
@ -81,7 +73,7 @@ func Encode(file io.Reader, outputs []io.Writer, meta *EEMeta) error {
} }
if !written { if !written {
if err := writeChunks(data, outputs, enc, meta, shards, endShards); err != nil { if err := writeChunks(data, outputs, enc, meta, meta.Params.Shards, outputChunkCount); err != nil {
return err return err
} }
written = true written = true
@ -92,11 +84,11 @@ func Encode(file io.Reader, outputs []io.Writer, meta *EEMeta) error {
return nil return nil
} }
func writeChunks(data [][]byte, files []io.Writer, enc reedsolomon.Encoder, meta *EEMeta, shards, endShards uint64) error { func writeChunks(data [][]byte, files []io.Writer, enc reedsolomon.Encoder, meta *EEMeta, shards, totalShards uint16) error {
if err := enc.Encode(data); err != nil { if err := enc.Encode(data); err != nil {
return err return err
} }
for i := shards; i < endShards; i++ { for i := shards; i < totalShards; i++ {
meta.ParityHashes = append(meta.ParityHashes, sha256sum(data[i])) meta.ParityHashes = append(meta.ParityHashes, sha256sum(data[i]))
} }
for i := 0; i < len(data); i++ { for i := 0; i < len(data); i++ {

12
pkg/erasureencode/encode_test.go
View file

@ -18,7 +18,7 @@ var simpleEncodeMeta = &EEMeta{
func TestEncode_EvenStride_Full(t *testing.T) { func TestEncode_EvenStride_Full(t *testing.T) {
file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11} file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11}
simpleEncodeMeta.Params.Size = uint64(len(file)) simpleEncodeMeta.Params.Size = int64(len(file))
reader := bytes.NewReader(file) reader := bytes.NewReader(file)
@ -61,7 +61,7 @@ func TestEncode_EvenStride_Full(t *testing.T) {
func TestEncode_EvenStride_Short1(t *testing.T) { func TestEncode_EvenStride_Short1(t *testing.T) {
file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10} file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10}
simpleEncodeMeta.Params.Size = uint64(len(file)) simpleEncodeMeta.Params.Size = int64(len(file))
reader := bytes.NewReader(file) reader := bytes.NewReader(file)
@ -104,7 +104,7 @@ func TestEncode_EvenStride_Short1(t *testing.T) {
func TestEncode_EvenStride_Short2(t *testing.T) { func TestEncode_EvenStride_Short2(t *testing.T) {
file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f} file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}
simpleEncodeMeta.Params.Size = uint64(len(file)) simpleEncodeMeta.Params.Size = int64(len(file))
reader := bytes.NewReader(file) reader := bytes.NewReader(file)
@ -147,7 +147,7 @@ func TestEncode_EvenStride_Short2(t *testing.T) {
func TestEncode_OddStride_Full(t *testing.T) { func TestEncode_OddStride_Full(t *testing.T) {
file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e} file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e}
simpleEncodeMeta.Params.Size = uint64(len(file)) simpleEncodeMeta.Params.Size = int64(len(file))
reader := bytes.NewReader(file) reader := bytes.NewReader(file)
@ -196,7 +196,7 @@ func TestEncode_OddStride_Full(t *testing.T) {
func TestEncode_OddStride_Short1(t *testing.T) { func TestEncode_OddStride_Short1(t *testing.T) {
file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d} file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d}
simpleEncodeMeta.Params.Size = uint64(len(file)) simpleEncodeMeta.Params.Size = int64(len(file))
reader := bytes.NewReader(file) reader := bytes.NewReader(file)
@ -245,7 +245,7 @@ func TestEncode_OddStride_Short1(t *testing.T) {
func TestEncode_OddStride_Short2(t *testing.T) { func TestEncode_OddStride_Short2(t *testing.T) {
file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c} file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c}
simpleEncodeMeta.Params.Size = uint64(len(file)) simpleEncodeMeta.Params.Size = int64(len(file))
reader := bytes.NewReader(file) reader := bytes.NewReader(file)

34
pkg/erasureencode/params.go
View file

@ -5,28 +5,30 @@ import (
) )
type Params struct { type Params struct {
Size uint64 `json:"s,string"` Size int64 `json:"s,string"`
Stride int32 `json:"t"` Stride int32 `json:"t"`
Shards uint16 `json:"h"` Shards uint16 `json:"h"`
Parity uint16 `json:"p"` Parity uint16 `json:"p"`
} }
type ChunkShardMeta struct { type ChunkShardMeta struct {
Shard uint16
ShardOffset int64
Chunk uint16 Chunk uint16
ChunkOffset uint64 ChunkOffset int64
GlobalOffset uint64 GlobalOffset int64
Size int32 Size int32
} }
func (params Params) Plan(offset, size uint64) []ChunkShardMeta { func (params Params) Plan(offset, size int64) []ChunkShardMeta {
outputs := []ChunkShardMeta{} outputs := []ChunkShardMeta{}
end := offset + size end := offset + size
if (end - 1) > params.Size { if (end - 1) > params.Size {
panic(fmt.Errorf("attempted read beyond end of file")) panic(fmt.Errorf("attempted read beyond end of file"))
} }
// constant // constant
shards := uint64(params.Shards) shards := int64(params.Shards)
baseStride := uint64(params.Stride) baseStride := int64(params.Stride)
baseStripeWidth := baseStride * shards baseStripeWidth := baseStride * shards
oddStripeOffset := (params.Size / baseStripeWidth) * baseStripeWidth oddStripeOffset := (params.Size / baseStripeWidth) * baseStripeWidth
@ -47,21 +49,23 @@ func (params Params) Plan(offset, size uint64) []ChunkShardMeta {
if offset >= oddStripeOffset { if offset >= oddStripeOffset {
localOffset := offset - oddStripeOffset // the location relative to the odd data at which the desired data begins localOffset := offset - oddStripeOffset // the location relative to the odd data at which the desired data begins
output.Chunk = uint16(localOffset / oddStride) output.Chunk = uint16(localOffset / oddStride)
shardOffset := localOffset % oddStride output.Shard = uint16(oddStripeOffset/baseStride) + output.Chunk
output.ChunkOffset = oddChunkOffset + shardOffset output.ShardOffset = localOffset % oddStride
output.Size = int32(min(end-offset, oddStride-shardOffset)) output.ChunkOffset = oddChunkOffset + output.ShardOffset
output.Size = int32(min(end-offset, oddStride-output.ShardOffset))
} else { } else {
shardNum := offset / baseStride // which shard the data is in shard := offset / baseStride
output.Chunk = uint16(shardNum % shards) output.Shard = uint16(offset / baseStride) // which shard the data is in
shardOffset := offset % baseStride output.Chunk = uint16(shard % shards)
output.ChunkOffset = ((shardNum / shards) * baseStride) + shardOffset output.ShardOffset = offset % baseStride
output.Size = int32(min(end-offset, baseStride-shardOffset)) output.ChunkOffset = ((shard / shards) * baseStride) + output.ShardOffset
output.Size = int32(min(end-offset, baseStride-output.ShardOffset))
} }
if output.Size <= 0 { if output.Size <= 0 {
panic(fmt.Errorf("invalid read size")) panic(fmt.Errorf("invalid read size"))
} }
outputs = append(outputs, output) outputs = append(outputs, output)
offset += uint64(output.Size) offset += int64(output.Size)
} }
return outputs return outputs
} }

78
pkg/erasureencode/params_test.go
View file

@ -5,14 +5,6 @@ import (
"testing" "testing"
) )
// example uses a tiny shard size for testing
// ex: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, a, b, c, d, e, f, g, h] ShardSize(2) RS(3,2)
// c0: [01, 67, cd] [s0, s3, s6]
// c1: [23, 89, ef] [s1, s4, s7]
// c2: [45, ab, gh] [s2, s5, s8]
// read(7, 7)
// desired plan 0,3,1; 1,2,2; 2,2,2; 0,4,2 -> 7; 89; ab; cd
func TestPlan_EvenStride_Full(t *testing.T) { func TestPlan_EvenStride_Full(t *testing.T) {
params := Params{ params := Params{
Size: 18, Size: 18,
@ -22,10 +14,10 @@ func TestPlan_EvenStride_Full(t *testing.T) {
} }
expected := []ChunkShardMeta{ expected := []ChunkShardMeta{
ChunkShardMeta{Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1}, ChunkShardMeta{Shard: 3, ShardOffset: 1, Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
ChunkShardMeta{Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2}, ChunkShardMeta{Shard: 4, ShardOffset: 0, Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
ChunkShardMeta{Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2}, ChunkShardMeta{Shard: 5, ShardOffset: 0, Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
ChunkShardMeta{Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 2}, ChunkShardMeta{Shard: 6, ShardOffset: 0, Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 2},
} }
actual := params.Plan(7, 7) actual := params.Plan(7, 7)
@ -35,21 +27,13 @@ func TestPlan_EvenStride_Full(t *testing.T) {
for i, actualItem := range actual { for i, actualItem := range actual {
if fmt.Sprintf("%#v", actualItem) != fmt.Sprintf("%#v", expected[i]) { if fmt.Sprintf("%#v", actualItem) != fmt.Sprintf("%#v", expected[i]) {
t.Errorf(`Expected %#v to equal %#v at %d`, actualItem, expected[i], i) t.Errorf("Expected \n%#v to equal \n%#v at %d", actualItem, expected[i], i)
} else { } else {
// fmt.Printf("ok: %#v\n", actualItem) // fmt.Printf("ok: %#v\n", actualItem)
} }
} }
} }
// example uses a tiny shard size for testing
// ex: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, a, b, c, d, e, f] ShardSize(2) RS(3,2)
// c0: [01, 67, cd] [s0, s3, s6]
// c1: [23, 89, ef] [s1, s4, s7]
// c2: [45, ab, !!] [s2, s5, s8]
// read(7, 7)
// desired plan 0,3,1; 1,2,2; 2,2,2; 0,4,2 -> 7; 89; ab; cd
func TestPlan_EvenStride_Short2(t *testing.T) { func TestPlan_EvenStride_Short2(t *testing.T) {
params := Params{ params := Params{
Size: 16, Size: 16,
@ -59,10 +43,10 @@ func TestPlan_EvenStride_Short2(t *testing.T) {
} }
expected := []ChunkShardMeta{ expected := []ChunkShardMeta{
ChunkShardMeta{Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1}, ChunkShardMeta{Shard: 3, ShardOffset: 1, Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
ChunkShardMeta{Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2}, ChunkShardMeta{Shard: 4, ShardOffset: 0, Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
ChunkShardMeta{Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2}, ChunkShardMeta{Shard: 5, ShardOffset: 0, Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
ChunkShardMeta{Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 2}, ChunkShardMeta{Shard: 6, ShardOffset: 0, Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 2},
} }
actual := params.Plan(7, 7) actual := params.Plan(7, 7)
@ -79,14 +63,6 @@ func TestPlan_EvenStride_Short2(t *testing.T) {
} }
} }
// example uses a tiny shard size for testing
// ex: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, a, b, c, d, e, f, g] ShardSize(2) RS(3,2)
// c0: [01, 67, cd] [s0, s3, s6]
// c1: [23, 89, ef] [s1, s4, s7]
// c2: [45, ab, g!] [s2, s5, s8]
// read(7, 7)
// desired plan 0,3,1; 1,2,2; 2,2,2; 0,4,2 -> 7; 89; ab; cd
func TestPlan_EvenStride_Short1(t *testing.T) { func TestPlan_EvenStride_Short1(t *testing.T) {
params := Params{ params := Params{
Size: 17, Size: 17,
@ -96,10 +72,10 @@ func TestPlan_EvenStride_Short1(t *testing.T) {
} }
expected := []ChunkShardMeta{ expected := []ChunkShardMeta{
ChunkShardMeta{Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1}, ChunkShardMeta{Shard: 3, ShardOffset: 1, Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
ChunkShardMeta{Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2}, ChunkShardMeta{Shard: 4, ShardOffset: 0, Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
ChunkShardMeta{Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2}, ChunkShardMeta{Shard: 5, ShardOffset: 0, Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
ChunkShardMeta{Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 2}, ChunkShardMeta{Shard: 6, ShardOffset: 0, Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 2},
} }
actual := params.Plan(7, 7) actual := params.Plan(7, 7)
@ -116,14 +92,6 @@ func TestPlan_EvenStride_Short1(t *testing.T) {
} }
} }
// example uses a tiny shard size for testing
// ex: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, a, b, c, d] ShardSize(2) RS(3,2)
// c0: [01, 67, c] [s0, s3, s6]
// c1: [23, 89, d] [s1, s4, s7]
// c2: [45, ab, !] [s2, s5, s8]
// read(7, 7)
// desired plan 0,3,1; 1,2,2; 2,2,2; 0,4,1; 1,4,1 -> 7; 89; ab; c; d
func TestPlan_OddStride_Short1(t *testing.T) { func TestPlan_OddStride_Short1(t *testing.T) {
params := Params{ params := Params{
Size: 13, Size: 13,
@ -133,11 +101,11 @@ func TestPlan_OddStride_Short1(t *testing.T) {
} }
expected := []ChunkShardMeta{ expected := []ChunkShardMeta{
ChunkShardMeta{Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1}, ChunkShardMeta{Shard: 3, ShardOffset: 1, Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
ChunkShardMeta{Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2}, ChunkShardMeta{Shard: 4, ShardOffset: 0, Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
ChunkShardMeta{Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2}, ChunkShardMeta{Shard: 5, ShardOffset: 0, Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
ChunkShardMeta{Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 1}, ChunkShardMeta{Shard: 6, ShardOffset: 0, Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 1},
ChunkShardMeta{Chunk: 1, ChunkOffset: 4, GlobalOffset: 13, Size: 1}, ChunkShardMeta{Shard: 7, ShardOffset: 0, Chunk: 1, ChunkOffset: 4, GlobalOffset: 13, Size: 1},
} }
actual := params.Plan(7, 7) actual := params.Plan(7, 7)
@ -163,11 +131,11 @@ func TestPlan_OddStride_Full(t *testing.T) {
} }
expected := []ChunkShardMeta{ expected := []ChunkShardMeta{
ChunkShardMeta{Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1}, ChunkShardMeta{Shard: 3, ShardOffset: 1, Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
ChunkShardMeta{Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2}, ChunkShardMeta{Shard: 4, ShardOffset: 0, Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
ChunkShardMeta{Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2}, ChunkShardMeta{Shard: 5, ShardOffset: 0, Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
ChunkShardMeta{Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 1}, ChunkShardMeta{Shard: 6, ShardOffset: 0, Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 1},
ChunkShardMeta{Chunk: 1, ChunkOffset: 4, GlobalOffset: 13, Size: 1}, ChunkShardMeta{Shard: 7, ShardOffset: 0, Chunk: 1, ChunkOffset: 4, GlobalOffset: 13, Size: 1},
} }
actual := params.Plan(7, 7) actual := params.Plan(7, 7)

2
pkg/erasureencode/util.go
View file

@ -4,7 +4,7 @@ import (
"crypto/sha256" "crypto/sha256"
) )
func min(input ...uint64) uint64 { func min(input ...int64) int64 {
min := input[0] min := input[0]
for i := 1; i < len(input); i++ { for i := 1; i < len(input); i++ {
if input[i] < min { if input[i] < min {

2
pkg/erasureencode/util_test.go
View file

@ -13,7 +13,7 @@ import (
// desired plan 0,3,1; 1,2,2; 2,2,2; 0,4,2 -> 7; 89; ab; cd // desired plan 0,3,1; 1,2,2; 2,2,2; 0,4,2 -> 7; 89; ab; cd
func TestMin(t *testing.T) { func TestMin(t *testing.T) {
expected := uint64(61) expected := int64(61)
actual := min(91, 111111, 9102, 61) actual := min(91, 111111, 9102, 61)
if actual != expected { if actual != expected {