CLean up types to use int64 many places, as that is what is used by os.File
This commit is contained in:
parent
5bf1599611
commit
db256918db
|
|
@ -15,8 +15,8 @@ func decodeFn(inputs []io.ReadSeeker, file io.Writer, meta *EEMeta, getPlan Read
|
|||
fullPlan := getPlan(meta)
|
||||
|
||||
// we only need to seek once as the rest of the reads should be linear
|
||||
for _, plan := range fullPlan[0:min(uint64(meta.Params.Shards), uint64(len(fullPlan)))] {
|
||||
if _, err := inputs[plan.Chunk].Seek(int64(plan.ChunkOffset), io.SeekStart); err != nil {
|
||||
for _, plan := range fullPlan[0:min(int64(meta.Params.Shards), int64(len(fullPlan)))] {
|
||||
if _, err := inputs[plan.Chunk].Seek(plan.ChunkOffset, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
@ -49,13 +49,12 @@ func Decode(inputs []io.ReadSeeker, file io.Writer, meta *EEMeta) error {
|
|||
}
|
||||
|
||||
func DecodeAndValidate(inputs []io.ReadSeeker, file io.Writer, meta *EEMeta) error {
|
||||
size := uint64(meta.Params.Size)
|
||||
shards := uint64(meta.Params.Shards)
|
||||
shards := int64(meta.Params.Shards)
|
||||
|
||||
// get set up to read meta including the padding
|
||||
validateParams := *meta
|
||||
if size%shards > 0 {
|
||||
validateParams.Size = (size / shards) * (shards + 1)
|
||||
if meta.Params.Size%shards > 0 {
|
||||
validateParams.Size = (meta.Params.Size / shards) * (shards + 1)
|
||||
}
|
||||
|
||||
return decodeFn(inputs, file, meta, func(_ *EEMeta) []ChunkShardMeta {
|
||||
|
|
@ -65,12 +64,12 @@ func DecodeAndValidate(inputs []io.ReadSeeker, file io.Writer, meta *EEMeta) err
|
|||
if !bytes.Equal(actual, meta.ShardHashes[i]) {
|
||||
return fmt.Errorf("shard hash mismatch")
|
||||
}
|
||||
dataLen := uint64(len(data))
|
||||
dataLen := int64(len(data))
|
||||
writeData := data
|
||||
if read.GlobalOffset > size {
|
||||
if read.GlobalOffset > meta.Params.Size {
|
||||
writeData = nil
|
||||
} else if read.GlobalOffset+dataLen > size {
|
||||
writeData = data[0 : read.GlobalOffset-size]
|
||||
} else if read.GlobalOffset+dataLen > meta.Params.Size {
|
||||
writeData = data[0 : read.GlobalOffset-meta.Params.Size]
|
||||
}
|
||||
if writeData != nil {
|
||||
if _, err := file.Write(writeData); err != nil {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ var simpleDecodeMeta = &EEMeta{
|
|||
|
||||
func TestDecode_EvenStride_Full(t *testing.T) {
|
||||
expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11}
|
||||
simpleDecodeMeta.Params.Size = uint64(len(expected))
|
||||
simpleDecodeMeta.Params.Size = int64(len(expected))
|
||||
|
||||
inputs := []io.ReadSeeker{
|
||||
bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c, 0x0d}),
|
||||
|
|
@ -39,7 +39,7 @@ func TestDecode_EvenStride_Full(t *testing.T) {
|
|||
|
||||
func TestDecode_EvenStride_Short1(t *testing.T) {
|
||||
expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10}
|
||||
simpleDecodeMeta.Params.Size = uint64(len(expected))
|
||||
simpleDecodeMeta.Params.Size = int64(len(expected))
|
||||
|
||||
inputs := []io.ReadSeeker{
|
||||
bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c, 0x0d}),
|
||||
|
|
@ -60,7 +60,7 @@ func TestDecode_EvenStride_Short1(t *testing.T) {
|
|||
|
||||
func TestDecode_EvenStride_Short2(t *testing.T) {
|
||||
expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}
|
||||
simpleDecodeMeta.Params.Size = uint64(len(expected))
|
||||
simpleDecodeMeta.Params.Size = int64(len(expected))
|
||||
|
||||
inputs := []io.ReadSeeker{
|
||||
bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c, 0x0d}),
|
||||
|
|
@ -81,7 +81,7 @@ func TestDecode_EvenStride_Short2(t *testing.T) {
|
|||
|
||||
func TestDecode_OddStride_Full(t *testing.T) {
|
||||
expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e}
|
||||
simpleDecodeMeta.Params.Size = uint64(len(expected))
|
||||
simpleDecodeMeta.Params.Size = int64(len(expected))
|
||||
|
||||
inputs := []io.ReadSeeker{
|
||||
bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c}),
|
||||
|
|
@ -102,7 +102,7 @@ func TestDecode_OddStride_Full(t *testing.T) {
|
|||
|
||||
func TestDecode_OddStride_Short1(t *testing.T) {
|
||||
expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d}
|
||||
simpleDecodeMeta.Params.Size = uint64(len(expected))
|
||||
simpleDecodeMeta.Params.Size = int64(len(expected))
|
||||
|
||||
inputs := []io.ReadSeeker{
|
||||
bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c}),
|
||||
|
|
@ -123,7 +123,7 @@ func TestDecode_OddStride_Short1(t *testing.T) {
|
|||
|
||||
func TestDecode_OddStride_Short2(t *testing.T) {
|
||||
expected := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c}
|
||||
simpleDecodeMeta.Params.Size = uint64(len(expected))
|
||||
simpleDecodeMeta.Params.Size = int64(len(expected))
|
||||
|
||||
inputs := []io.ReadSeeker{
|
||||
bytes.NewReader([]byte{0x00, 0x01, 0x06, 0x07, 0x0c}),
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ func EncodeFile(file *os.File, oututs []io.Writer, stride int32, shards, parity
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
size := uint64(stats.Size())
|
||||
size := stats.Size()
|
||||
|
||||
meta := &EEMeta{
|
||||
Params: Params{
|
||||
|
|
@ -32,35 +32,27 @@ func EncodeFile(file *os.File, oututs []io.Writer, stride int32, shards, parity
|
|||
}
|
||||
|
||||
func Encode(file io.Reader, outputs []io.Writer, meta *EEMeta) error {
|
||||
// int(uint16) + int(uint16) != int; should be safe
|
||||
if int(meta.Params.Shards)+int(meta.Params.Parity) != len(outputs) {
|
||||
return fmt.Errorf("expected the number of shards+parity to equal the number of output files provided")
|
||||
}
|
||||
|
||||
// int(uint16), int(uint16), int(int32)
|
||||
enc, err := reedsolomon.New(int(meta.Params.Shards), int(meta.Params.Parity), reedsolomon.WithAutoGoroutines(int(meta.Params.Stride)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
shards := uint64(meta.Params.Shards)
|
||||
parity := uint64(meta.Params.Parity)
|
||||
|
||||
outputChunkCount := shards + parity
|
||||
|
||||
lastShardChunk := shards - 1
|
||||
endShards := shards + parity
|
||||
outputChunkCount := meta.Params.Shards + meta.Params.Parity
|
||||
lastShardChunk := meta.Params.Shards - 1
|
||||
|
||||
data := make([][]byte, outputChunkCount)
|
||||
data[0] = []byte{}
|
||||
written := false // track whether the current stripe has been written
|
||||
for shard, csm := range meta.Params.Plan(0, meta.Params.Size) {
|
||||
chunk := uint64(shard) % shards
|
||||
for i, csm := range meta.Params.Plan(0, meta.Params.Size) {
|
||||
written = false
|
||||
// prepare data slices, shard size only meaningfuly changes at stripe boundary
|
||||
if chunk == 0 {
|
||||
if csm.Chunk == 0 || i == 0 {
|
||||
// if int32(len(data[0])) != csm.Size {
|
||||
for i := uint64(0); i < outputChunkCount; i++ {
|
||||
for i := uint16(0); i < outputChunkCount; i++ {
|
||||
data[i] = make([]byte, csm.Size)
|
||||
}
|
||||
// }
|
||||
|
|
@ -72,8 +64,8 @@ func Encode(file io.Reader, outputs []io.Writer, meta *EEMeta) error {
|
|||
meta.ShardHashes = append(meta.ShardHashes, sha256sum(data[csm.Chunk]))
|
||||
|
||||
// if we are on the last chunk calculate the parity and write things out
|
||||
if chunk == lastShardChunk {
|
||||
if err := writeChunks(data, outputs, enc, meta, shards, endShards); err != nil {
|
||||
if csm.Chunk == lastShardChunk {
|
||||
if err := writeChunks(data, outputs, enc, meta, meta.Params.Shards, outputChunkCount); err != nil {
|
||||
return err
|
||||
}
|
||||
written = true
|
||||
|
|
@ -81,7 +73,7 @@ func Encode(file io.Reader, outputs []io.Writer, meta *EEMeta) error {
|
|||
}
|
||||
|
||||
if !written {
|
||||
if err := writeChunks(data, outputs, enc, meta, shards, endShards); err != nil {
|
||||
if err := writeChunks(data, outputs, enc, meta, meta.Params.Shards, outputChunkCount); err != nil {
|
||||
return err
|
||||
}
|
||||
written = true
|
||||
|
|
@ -92,11 +84,11 @@ func Encode(file io.Reader, outputs []io.Writer, meta *EEMeta) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func writeChunks(data [][]byte, files []io.Writer, enc reedsolomon.Encoder, meta *EEMeta, shards, endShards uint64) error {
|
||||
func writeChunks(data [][]byte, files []io.Writer, enc reedsolomon.Encoder, meta *EEMeta, shards, totalShards uint16) error {
|
||||
if err := enc.Encode(data); err != nil {
|
||||
return err
|
||||
}
|
||||
for i := shards; i < endShards; i++ {
|
||||
for i := shards; i < totalShards; i++ {
|
||||
meta.ParityHashes = append(meta.ParityHashes, sha256sum(data[i]))
|
||||
}
|
||||
for i := 0; i < len(data); i++ {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ var simpleEncodeMeta = &EEMeta{
|
|||
|
||||
func TestEncode_EvenStride_Full(t *testing.T) {
|
||||
file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11}
|
||||
simpleEncodeMeta.Params.Size = uint64(len(file))
|
||||
simpleEncodeMeta.Params.Size = int64(len(file))
|
||||
|
||||
reader := bytes.NewReader(file)
|
||||
|
||||
|
|
@ -61,7 +61,7 @@ func TestEncode_EvenStride_Full(t *testing.T) {
|
|||
|
||||
func TestEncode_EvenStride_Short1(t *testing.T) {
|
||||
file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10}
|
||||
simpleEncodeMeta.Params.Size = uint64(len(file))
|
||||
simpleEncodeMeta.Params.Size = int64(len(file))
|
||||
|
||||
reader := bytes.NewReader(file)
|
||||
|
||||
|
|
@ -104,7 +104,7 @@ func TestEncode_EvenStride_Short1(t *testing.T) {
|
|||
|
||||
func TestEncode_EvenStride_Short2(t *testing.T) {
|
||||
file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}
|
||||
simpleEncodeMeta.Params.Size = uint64(len(file))
|
||||
simpleEncodeMeta.Params.Size = int64(len(file))
|
||||
|
||||
reader := bytes.NewReader(file)
|
||||
|
||||
|
|
@ -147,7 +147,7 @@ func TestEncode_EvenStride_Short2(t *testing.T) {
|
|||
|
||||
func TestEncode_OddStride_Full(t *testing.T) {
|
||||
file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e}
|
||||
simpleEncodeMeta.Params.Size = uint64(len(file))
|
||||
simpleEncodeMeta.Params.Size = int64(len(file))
|
||||
|
||||
reader := bytes.NewReader(file)
|
||||
|
||||
|
|
@ -196,7 +196,7 @@ func TestEncode_OddStride_Full(t *testing.T) {
|
|||
|
||||
func TestEncode_OddStride_Short1(t *testing.T) {
|
||||
file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d}
|
||||
simpleEncodeMeta.Params.Size = uint64(len(file))
|
||||
simpleEncodeMeta.Params.Size = int64(len(file))
|
||||
|
||||
reader := bytes.NewReader(file)
|
||||
|
||||
|
|
@ -245,7 +245,7 @@ func TestEncode_OddStride_Short1(t *testing.T) {
|
|||
|
||||
func TestEncode_OddStride_Short2(t *testing.T) {
|
||||
file := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c}
|
||||
simpleEncodeMeta.Params.Size = uint64(len(file))
|
||||
simpleEncodeMeta.Params.Size = int64(len(file))
|
||||
|
||||
reader := bytes.NewReader(file)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,28 +5,30 @@ import (
|
|||
)
|
||||
|
||||
type Params struct {
|
||||
Size uint64 `json:"s,string"`
|
||||
Size int64 `json:"s,string"`
|
||||
Stride int32 `json:"t"`
|
||||
Shards uint16 `json:"h"`
|
||||
Parity uint16 `json:"p"`
|
||||
}
|
||||
|
||||
type ChunkShardMeta struct {
|
||||
Shard uint16
|
||||
ShardOffset int64
|
||||
Chunk uint16
|
||||
ChunkOffset uint64
|
||||
GlobalOffset uint64
|
||||
ChunkOffset int64
|
||||
GlobalOffset int64
|
||||
Size int32
|
||||
}
|
||||
|
||||
func (params Params) Plan(offset, size uint64) []ChunkShardMeta {
|
||||
func (params Params) Plan(offset, size int64) []ChunkShardMeta {
|
||||
outputs := []ChunkShardMeta{}
|
||||
end := offset + size
|
||||
if (end - 1) > params.Size {
|
||||
panic(fmt.Errorf("attempted read beyond end of file"))
|
||||
}
|
||||
// constant
|
||||
shards := uint64(params.Shards)
|
||||
baseStride := uint64(params.Stride)
|
||||
shards := int64(params.Shards)
|
||||
baseStride := int64(params.Stride)
|
||||
baseStripeWidth := baseStride * shards
|
||||
|
||||
oddStripeOffset := (params.Size / baseStripeWidth) * baseStripeWidth
|
||||
|
|
@ -47,21 +49,23 @@ func (params Params) Plan(offset, size uint64) []ChunkShardMeta {
|
|||
if offset >= oddStripeOffset {
|
||||
localOffset := offset - oddStripeOffset // the location relative to the odd data at which the desired data begins
|
||||
output.Chunk = uint16(localOffset / oddStride)
|
||||
shardOffset := localOffset % oddStride
|
||||
output.ChunkOffset = oddChunkOffset + shardOffset
|
||||
output.Size = int32(min(end-offset, oddStride-shardOffset))
|
||||
output.Shard = uint16(oddStripeOffset/baseStride) + output.Chunk
|
||||
output.ShardOffset = localOffset % oddStride
|
||||
output.ChunkOffset = oddChunkOffset + output.ShardOffset
|
||||
output.Size = int32(min(end-offset, oddStride-output.ShardOffset))
|
||||
} else {
|
||||
shardNum := offset / baseStride // which shard the data is in
|
||||
output.Chunk = uint16(shardNum % shards)
|
||||
shardOffset := offset % baseStride
|
||||
output.ChunkOffset = ((shardNum / shards) * baseStride) + shardOffset
|
||||
output.Size = int32(min(end-offset, baseStride-shardOffset))
|
||||
shard := offset / baseStride
|
||||
output.Shard = uint16(offset / baseStride) // which shard the data is in
|
||||
output.Chunk = uint16(shard % shards)
|
||||
output.ShardOffset = offset % baseStride
|
||||
output.ChunkOffset = ((shard / shards) * baseStride) + output.ShardOffset
|
||||
output.Size = int32(min(end-offset, baseStride-output.ShardOffset))
|
||||
}
|
||||
if output.Size <= 0 {
|
||||
panic(fmt.Errorf("invalid read size"))
|
||||
}
|
||||
outputs = append(outputs, output)
|
||||
offset += uint64(output.Size)
|
||||
offset += int64(output.Size)
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,14 +5,6 @@ import (
|
|||
"testing"
|
||||
)
|
||||
|
||||
// example uses a tiny shard size for testing
|
||||
// ex: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, a, b, c, d, e, f, g, h] ShardSize(2) RS(3,2)
|
||||
// c0: [01, 67, cd] [s0, s3, s6]
|
||||
// c1: [23, 89, ef] [s1, s4, s7]
|
||||
// c2: [45, ab, gh] [s2, s5, s8]
|
||||
// read(7, 7)
|
||||
// desired plan 0,3,1; 1,2,2; 2,2,2; 0,4,2 -> 7; 89; ab; cd
|
||||
|
||||
func TestPlan_EvenStride_Full(t *testing.T) {
|
||||
params := Params{
|
||||
Size: 18,
|
||||
|
|
@ -22,10 +14,10 @@ func TestPlan_EvenStride_Full(t *testing.T) {
|
|||
}
|
||||
|
||||
expected := []ChunkShardMeta{
|
||||
ChunkShardMeta{Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
|
||||
ChunkShardMeta{Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
|
||||
ChunkShardMeta{Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
|
||||
ChunkShardMeta{Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 2},
|
||||
ChunkShardMeta{Shard: 3, ShardOffset: 1, Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
|
||||
ChunkShardMeta{Shard: 4, ShardOffset: 0, Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
|
||||
ChunkShardMeta{Shard: 5, ShardOffset: 0, Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
|
||||
ChunkShardMeta{Shard: 6, ShardOffset: 0, Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 2},
|
||||
}
|
||||
actual := params.Plan(7, 7)
|
||||
|
||||
|
|
@ -35,21 +27,13 @@ func TestPlan_EvenStride_Full(t *testing.T) {
|
|||
|
||||
for i, actualItem := range actual {
|
||||
if fmt.Sprintf("%#v", actualItem) != fmt.Sprintf("%#v", expected[i]) {
|
||||
t.Errorf(`Expected %#v to equal %#v at %d`, actualItem, expected[i], i)
|
||||
t.Errorf("Expected \n%#v to equal \n%#v at %d", actualItem, expected[i], i)
|
||||
} else {
|
||||
// fmt.Printf("ok: %#v\n", actualItem)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// example uses a tiny shard size for testing
|
||||
// ex: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, a, b, c, d, e, f] ShardSize(2) RS(3,2)
|
||||
// c0: [01, 67, cd] [s0, s3, s6]
|
||||
// c1: [23, 89, ef] [s1, s4, s7]
|
||||
// c2: [45, ab, !!] [s2, s5, s8]
|
||||
// read(7, 7)
|
||||
// desired plan 0,3,1; 1,2,2; 2,2,2; 0,4,2 -> 7; 89; ab; cd
|
||||
|
||||
func TestPlan_EvenStride_Short2(t *testing.T) {
|
||||
params := Params{
|
||||
Size: 16,
|
||||
|
|
@ -59,10 +43,10 @@ func TestPlan_EvenStride_Short2(t *testing.T) {
|
|||
}
|
||||
|
||||
expected := []ChunkShardMeta{
|
||||
ChunkShardMeta{Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
|
||||
ChunkShardMeta{Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
|
||||
ChunkShardMeta{Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
|
||||
ChunkShardMeta{Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 2},
|
||||
ChunkShardMeta{Shard: 3, ShardOffset: 1, Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
|
||||
ChunkShardMeta{Shard: 4, ShardOffset: 0, Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
|
||||
ChunkShardMeta{Shard: 5, ShardOffset: 0, Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
|
||||
ChunkShardMeta{Shard: 6, ShardOffset: 0, Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 2},
|
||||
}
|
||||
actual := params.Plan(7, 7)
|
||||
|
||||
|
|
@ -79,14 +63,6 @@ func TestPlan_EvenStride_Short2(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// example uses a tiny shard size for testing
|
||||
// ex: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, a, b, c, d, e, f, g] ShardSize(2) RS(3,2)
|
||||
// c0: [01, 67, cd] [s0, s3, s6]
|
||||
// c1: [23, 89, ef] [s1, s4, s7]
|
||||
// c2: [45, ab, g!] [s2, s5, s8]
|
||||
// read(7, 7)
|
||||
// desired plan 0,3,1; 1,2,2; 2,2,2; 0,4,2 -> 7; 89; ab; cd
|
||||
|
||||
func TestPlan_EvenStride_Short1(t *testing.T) {
|
||||
params := Params{
|
||||
Size: 17,
|
||||
|
|
@ -96,10 +72,10 @@ func TestPlan_EvenStride_Short1(t *testing.T) {
|
|||
}
|
||||
|
||||
expected := []ChunkShardMeta{
|
||||
ChunkShardMeta{Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
|
||||
ChunkShardMeta{Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
|
||||
ChunkShardMeta{Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
|
||||
ChunkShardMeta{Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 2},
|
||||
ChunkShardMeta{Shard: 3, ShardOffset: 1, Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
|
||||
ChunkShardMeta{Shard: 4, ShardOffset: 0, Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
|
||||
ChunkShardMeta{Shard: 5, ShardOffset: 0, Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
|
||||
ChunkShardMeta{Shard: 6, ShardOffset: 0, Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 2},
|
||||
}
|
||||
actual := params.Plan(7, 7)
|
||||
|
||||
|
|
@ -116,14 +92,6 @@ func TestPlan_EvenStride_Short1(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// example uses a tiny shard size for testing
|
||||
// ex: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, a, b, c, d] ShardSize(2) RS(3,2)
|
||||
// c0: [01, 67, c] [s0, s3, s6]
|
||||
// c1: [23, 89, d] [s1, s4, s7]
|
||||
// c2: [45, ab, !] [s2, s5, s8]
|
||||
// read(7, 7)
|
||||
// desired plan 0,3,1; 1,2,2; 2,2,2; 0,4,1; 1,4,1 -> 7; 89; ab; c; d
|
||||
|
||||
func TestPlan_OddStride_Short1(t *testing.T) {
|
||||
params := Params{
|
||||
Size: 13,
|
||||
|
|
@ -133,11 +101,11 @@ func TestPlan_OddStride_Short1(t *testing.T) {
|
|||
}
|
||||
|
||||
expected := []ChunkShardMeta{
|
||||
ChunkShardMeta{Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
|
||||
ChunkShardMeta{Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
|
||||
ChunkShardMeta{Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
|
||||
ChunkShardMeta{Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 1},
|
||||
ChunkShardMeta{Chunk: 1, ChunkOffset: 4, GlobalOffset: 13, Size: 1},
|
||||
ChunkShardMeta{Shard: 3, ShardOffset: 1, Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
|
||||
ChunkShardMeta{Shard: 4, ShardOffset: 0, Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
|
||||
ChunkShardMeta{Shard: 5, ShardOffset: 0, Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
|
||||
ChunkShardMeta{Shard: 6, ShardOffset: 0, Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 1},
|
||||
ChunkShardMeta{Shard: 7, ShardOffset: 0, Chunk: 1, ChunkOffset: 4, GlobalOffset: 13, Size: 1},
|
||||
}
|
||||
actual := params.Plan(7, 7)
|
||||
|
||||
|
|
@ -163,11 +131,11 @@ func TestPlan_OddStride_Full(t *testing.T) {
|
|||
}
|
||||
|
||||
expected := []ChunkShardMeta{
|
||||
ChunkShardMeta{Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
|
||||
ChunkShardMeta{Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
|
||||
ChunkShardMeta{Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
|
||||
ChunkShardMeta{Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 1},
|
||||
ChunkShardMeta{Chunk: 1, ChunkOffset: 4, GlobalOffset: 13, Size: 1},
|
||||
ChunkShardMeta{Shard: 3, ShardOffset: 1, Chunk: 0, ChunkOffset: 3, GlobalOffset: 7, Size: 1},
|
||||
ChunkShardMeta{Shard: 4, ShardOffset: 0, Chunk: 1, ChunkOffset: 2, GlobalOffset: 8, Size: 2},
|
||||
ChunkShardMeta{Shard: 5, ShardOffset: 0, Chunk: 2, ChunkOffset: 2, GlobalOffset: 10, Size: 2},
|
||||
ChunkShardMeta{Shard: 6, ShardOffset: 0, Chunk: 0, ChunkOffset: 4, GlobalOffset: 12, Size: 1},
|
||||
ChunkShardMeta{Shard: 7, ShardOffset: 0, Chunk: 1, ChunkOffset: 4, GlobalOffset: 13, Size: 1},
|
||||
}
|
||||
actual := params.Plan(7, 7)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"crypto/sha256"
|
||||
)
|
||||
|
||||
func min(input ...uint64) uint64 {
|
||||
func min(input ...int64) int64 {
|
||||
min := input[0]
|
||||
for i := 1; i < len(input); i++ {
|
||||
if input[i] < min {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import (
|
|||
// desired plan 0,3,1; 1,2,2; 2,2,2; 0,4,2 -> 7; 89; ab; cd
|
||||
|
||||
func TestMin(t *testing.T) {
|
||||
expected := uint64(61)
|
||||
expected := int64(61)
|
||||
|
||||
actual := min(91, 111111, 9102, 61)
|
||||
if actual != expected {
|
||||
|
|
|
|||
Loading…
Reference in a new issue