gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

can read/write files in format version 2

Browse source
This commit is contained in:
Chris Lu 2012-12-21 02:13:02 -08:00
parent 11cc489ca5
commit 1943647860
6 changed files with 91 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
weed-fs/src/cmd/weed/version.go
View file

@ -6,7 +6,7 @@ import (
)
const (
VERSION = "0.25"
VERSION = "0.26"
)
var cmdVersion = &Command{

6
weed-fs/src/pkg/storage/needle.go
View file

@ -20,13 +20,15 @@ type Needle struct {
Cookie uint32 "random number to mitigate brute force lookups"
Id uint64 "needle id"
Size uint32 "sum of DataSize,Data,NameSize,Name,MimeSize,Mime"
Flags byte "boolean flags" //version2
DataSize uint32 "Data size" //version2
DataSize uint32 "Data size" //version2
Data []byte "The actual file data"
Flags byte "boolean flags" //version2
NameSize uint8 //version2
Name []byte "maximum 256 characters" //version2
MimeSize uint8 //version2
Mime []byte "maximum 256 characters" //version2
Checksum CRC "CRC32 to check integrity"
Padding []byte "Aligned to 8 bytes"
}

92
weed-fs/src/pkg/storage/needle_read_write.go
View file

@ -19,7 +19,44 @@ func (n *Needle) Append(w io.Writer, version Version) uint32 {
rest := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + 4) % NeedlePaddingSize)
util.Uint32toBytes(header[0:4], n.Checksum.Value())
w.Write(header[0 : 4+rest])
return n.Size
} else if version == Version2 {
header := make([]byte, NeedleHeaderSize)
util.Uint32toBytes(header[0:4], n.Cookie)
util.Uint64toBytes(header[4:12], n.Id)
n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime))
if n.DataSize > 0 {
n.Size = 4 + n.DataSize + 1
if n.NameSize > 0 {
n.Size = n.Size + 1 + uint32(n.NameSize)
}
if n.NameSize > 0 {
n.Size = n.Size + 1 + uint32(n.MimeSize)
}
}
util.Uint32toBytes(header[12:16], n.Size)
w.Write(header)
if n.DataSize > 0 {
util.Uint32toBytes(header[0:4], n.DataSize)
w.Write(header[0:4])
w.Write(n.Data)
util.Uint8toBytes(header[0:1], n.Flags)
w.Write(header[0:1])
}
if n.NameSize > 0 {
util.Uint8toBytes(header[0:1], n.NameSize)
w.Write(header[0:1])
w.Write(n.Name)
}
if n.MimeSize > 0 {
util.Uint8toBytes(header[0:1], n.MimeSize)
w.Write(header[0:1])
w.Write(n.Mime)
}
rest := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + 4) % NeedlePaddingSize)
util.Uint32toBytes(header[0:4], n.Checksum.Value())
w.Write(header[0 : 4+rest])
return n.DataSize
}
return n.Size
}
@ -27,33 +64,62 @@ func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) {
if version == Version1 {
bytes := make([]byte, NeedleHeaderSize+size+4)
ret, e := r.Read(bytes)
n.Cookie = util.BytesToUint32(bytes[0:4])
n.Id = util.BytesToUint64(bytes[4:12])
n.Size = util.BytesToUint32(bytes[12:16])
n.Data = bytes[16 : 16+size]
checksum := util.BytesToUint32(bytes[16+size : 16+size+4])
n.readNeedleHeader(bytes)
n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size]
checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+4])
if checksum != NewCRC(n.Data).Value() {
return 0, errors.New("CRC error! Data On Disk Corrupted!")
}
return ret, e
} else if version == Version2 {
bytes := make([]byte, NeedleHeaderSize+size+4)
ret, e := r.Read(bytes)
n.readNeedleHeader(bytes)
n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)])
checksum := util.BytesToUint32(bytes[NeedleHeaderSize+n.Size : NeedleHeaderSize+n.Size+4])
if checksum != NewCRC(n.Data).Value() {
return 0, errors.New("CRC error! Data On Disk Corrupted!")
}
return ret, e
}
return 0, errors.New("Unsupported Version!")
}
func (n *Needle) readNeedleHeader(bytes []byte) {
n.Cookie = util.BytesToUint32(bytes[0:4])
n.Id = util.BytesToUint64(bytes[4:12])
n.Size = util.BytesToUint32(bytes[12:NeedleHeaderSize])
}
func (n *Needle) readNeedleDataVersion2(bytes []byte) {
index := 0
n.DataSize = util.BytesToUint32(bytes[index : index+4])
index = index + 4
n.Data = bytes[index : index+int(n.DataSize)]
index = index + int(n.DataSize)
n.Flags = bytes[index]
index = index + 1
if index < len(bytes) {
n.NameSize = uint8(bytes[index])
index = index + 1
n.Name = bytes[index : index+int(n.NameSize)]
index = index + int(n.NameSize)
}
if index < len(bytes) {
n.MimeSize = uint8(bytes[index])
index = index + 1
n.Mime = bytes[index : index+int(n.MimeSize)]
}
}
func ReadNeedleHeader(r *os.File, version Version) (n *Needle, bodyLength uint32) {
n = new(Needle)
if version == Version1 {
if version == Version1 || version == Version2 {
bytes := make([]byte, NeedleHeaderSize)
count, e := r.Read(bytes)
if count <= 0 || e != nil {
return nil, 0
}
n.Cookie = util.BytesToUint32(bytes[0:4])
n.Id = util.BytesToUint64(bytes[4:12])
n.Size = util.BytesToUint32(bytes[12:16])
rest := NeedlePaddingSize - ((n.Size + 16 + 4) % NeedlePaddingSize)
n.readNeedleHeader(bytes)
rest := NeedlePaddingSize - ((n.Size + NeedleHeaderSize + 4) % NeedlePaddingSize)
bodyLength = n.Size + 4 + rest
} else if version == Version2 {
}
return
}
@ -67,6 +133,10 @@ func (n *Needle) ReadNeedleBody(r *os.File, version Version, bodyLength uint32)
n.Data = bytes[:n.Size]
n.Checksum = NewCRC(n.Data)
} else if version == Version2 {
bytes := make([]byte, bodyLength)
r.Read(bytes)
n.readNeedleDataVersion2(bytes[0:n.Size])
n.Checksum = NewCRC(n.Data)
}
return
}

2
weed-fs/src/pkg/storage/volume.go
View file

@ -51,7 +51,7 @@ func (v *Volume) load() error {
return nil
}
func (v *Volume) Version() Version {
return CurrentVersion
return v.version
}
func (v *Volume) Size() int64 {
v.accessLock.Lock()

2
weed-fs/src/pkg/storage/volume_version.go
View file

@ -6,7 +6,7 @@ import (
type Version uint8
const (
CurrentVersion = Version(1)
Version1 = Version(1)
Version2 = Version(2)
CurrentVersion = Version2
)

3
weed-fs/src/pkg/util/bytes.go
View file

@ -28,4 +28,7 @@ func Uint32toBytes(b []byte, v uint32){
b[3-i] = byte(v>>(i*8))
}
}
func Uint8toBytes(b []byte, v uint8){
b[0] = byte(v)
}