gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

fix needle Append return offset to avoid uint32 overflow

Browse source
This commit is contained in:
bingoohuang 2019-01-16 16:48:03 +08:00
parent 7081261fca
commit 27093bc2e5
3 changed files with 71 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
weed/storage/needle_read_write.go
View file

@ -3,6 +3,7 @@ package storage
import (
"errors"
"fmt"
"io"
"os"
"github.com/chrislusf/seaweedfs/weed/glog"
@ -27,8 +28,8 @@ func (n *Needle) DiskSize(version Version) int64 {
return getActualSize(n.Size, version)
}
func (n *Needle) Append(w *os.File, version Version) (offset Offset, size uint32, actualSize int64, err error) {
if end, e := w.Seek(0, 2); e == nil {
func (n *Needle) Append(w *os.File, version Version) (offset uint64, size uint32, actualSize int64, err error) {
if end, e := w.Seek(0, io.SeekEnd); e == nil {
defer func(w *os.File, off int64) {
if err != nil {
if te := w.Truncate(end); te != nil {
@ -36,7 +37,7 @@ func (n *Needle) Append(w *os.File, version Version) (offset Offset, size uint32
}
}
}(w, end)
offset = Offset(end)
offset = uint64(end)
} else {
err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
return
@ -175,7 +176,7 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
}
n.ParseNeedleHeader(bytes)
if n.Size != size {
return fmt.Errorf("File Entry Not Found. Needle id %d expected size %d Memory %d", n.Id, n.Size, size)
return fmt.Errorf("File Entry Not Found. offset %d, Needle id %d expected size %d Memory %d", offset, n.Id, n.Size, size)
}
switch version {
case Version1:

62
weed/storage/needle_read_write_test.go Normal file
View file

@ -0,0 +1,62 @@
package storage
import (
"crypto/rand"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"io"
"io/ioutil"
"os"
"testing"
)
func TestAppend(t *testing.T) {
n := &Needle{
Cookie: types.Cookie(123), // Cookie Cookie `comment:"random number to mitigate brute force lookups"`
Id: types.NeedleId(123), // Id NeedleId `comment:"needle id"`
Size: 8, // Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
DataSize: 4, // DataSize uint32 `comment:"Data size"` //version2
Data: []byte("abcd"), // Data []byte `comment:"The actual file data"`
Flags: 0, // Flags byte `comment:"boolean flags"` //version2
NameSize: 0, // NameSize uint8 //version2
Name: nil, // Name []byte `comment:"maximum 256 characters"` //version2
MimeSize: 0, // MimeSize uint8 //version2
Mime: nil, // Mime []byte `comment:"maximum 256 characters"` //version2
PairsSize: 0, // PairsSize uint16 //version2
Pairs: nil, // Pairs []byte `comment:"additional name value pairs, json format, maximum 6
LastModified: 123, // LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes
Ttl: nil, // Ttl *TTL
Checksum: 123, // Checksum CRC `comment:"CRC32 to check integrity"`
AppendAtNs: 123, // AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
Padding: nil, // Padding []byte `comment:"Aligned to 8 bytes"`
}
tempFile, err := ioutil.TempFile("", ".dat")
if err != nil {
t.Errorf("Fail TempFile. %v", err)
return
}
/*
uint8 : 0 to 255
uint16 : 0 to 65535
uint32 : 0 to 4294967295
uint64 : 0 to 18446744073709551615
int8 : -128 to 127
int16 : -32768 to 32767
int32 : -2147483648 to 2147483647
int64 : -9223372036854775808 to 9223372036854775807
*/
fileSize := int64(4294967295) + 10000
io.CopyN(tempFile, rand.Reader, fileSize)
defer func() {
tempFile.Close()
os.Remove(tempFile.Name())
}()
offset, _, _, _ := n.Append(tempFile, CurrentVersion)
if offset != uint64(fileSize) {
t.Errorf("Fail to Append Needle.")
}
}

8
weed/storage/volume_read_write.go
View file

@ -76,7 +76,7 @@ func (v *Volume) AppendBlob(b []byte) (offset int64, err error) {
return
}
func (v *Volume) writeNeedle(n *Needle) (offset Offset, size uint32, err error) {
func (v *Volume) writeNeedle(n *Needle) (offset uint64, size uint32, err error) {
glog.V(4).Infof("writing needle %s", NewFileIdFromNeedle(v.Id, n).String())
if v.readOnly {
err = fmt.Errorf("%s is read-only", v.dataFile.Name())
@ -96,8 +96,8 @@ func (v *Volume) writeNeedle(n *Needle) (offset Offset, size uint32, err error)
}
nv, ok := v.nm.Get(n.Id)
if !ok || Offset(nv.Offset)*NeedlePaddingSize < offset {
if err = v.nm.Put(n.Id, offset/NeedlePaddingSize, n.Size); err != nil {
if !ok || uint64(nv.Offset)*NeedlePaddingSize < offset {
if err = v.nm.Put(n.Id, Offset(offset/NeedlePaddingSize), n.Size); err != nil {
glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err)
}
}
@ -124,7 +124,7 @@ func (v *Volume) deleteNeedle(n *Needle) (uint32, error) {
if err != nil {
return size, err
}
if err = v.nm.Delete(n.Id, offset/NeedlePaddingSize); err != nil {
if err = v.nm.Delete(n.Id, Offset(offset/NeedlePaddingSize)); err != nil {
return size, err
}
return size, err