gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

use bits in needle.Flags to mark hasName and hasMime

Browse source
This commit is contained in:
Chris Lu 2012-12-22 12:38:09 -08:00
parent 9b95430e9f
commit 46b8c4cc98
2 changed files with 37 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
weed-fs/src/pkg/storage/needle.go
View file

@ -14,6 +14,7 @@ import (
const ( const (
NeedleHeaderSize = 16 //should never change this NeedleHeaderSize = 16 //should never change this
NeedlePaddingSize = 8 NeedlePaddingSize = 8
NeedleChecksumSize = 4
) )
type Needle struct { type Needle struct {
@ -59,6 +60,7 @@ func NewNeedle(r *http.Request) (n *Needle, fname string, e error) {
contentType := part.Header.Get("Content-Type") contentType := part.Header.Get("Content-Type")
if contentType != "" && mtype != contentType && len(contentType) < 256 { if contentType != "" && mtype != contentType && len(contentType) < 256 {
n.Mime = []byte(contentType) n.Mime = []byte(contentType)
n.SetHasMime()
mtype = contentType mtype = contentType
} }
if IsGzippable(ext, mtype) { if IsGzippable(ext, mtype) {
@ -74,6 +76,7 @@ func NewNeedle(r *http.Request) (n *Needle, fname string, e error) {
} else { } else {
n.Name = []byte(fname) n.Name = []byte(fname)
} }
n.SetHasName()
} }
n.Data = data n.Data = data

48
weed-fs/src/pkg/storage/needle_read_write.go
View file

@ -16,9 +16,9 @@ func (n *Needle) Append(w io.Writer, version Version) uint32 {
util.Uint32toBytes(header[12:16], n.Size) util.Uint32toBytes(header[12:16], n.Size)
w.Write(header) w.Write(header)
w.Write(n.Data) w.Write(n.Data)
rest := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + 4) % NeedlePaddingSize) padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
util.Uint32toBytes(header[0:4], n.Checksum.Value()) util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
w.Write(header[0 : 4+rest]) w.Write(header[0 : NeedleChecksumSize+padding])
return n.Size return n.Size
} else if version == Version2 { } else if version == Version2 {
header := make([]byte, NeedleHeaderSize) header := make([]byte, NeedleHeaderSize)
@ -27,10 +27,10 @@ func (n *Needle) Append(w io.Writer, version Version) uint32 {
n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime)) n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime))
if n.DataSize > 0 { if n.DataSize > 0 {
n.Size = 4 + n.DataSize + 1 n.Size = 4 + n.DataSize + 1
if n.NameSize > 0 { if n.HasName() {
n.Size = n.Size + 1 + uint32(n.NameSize) n.Size = n.Size + 1 + uint32(n.NameSize)
} }
if n.MimeSize > 0 { if n.HasMime() {
n.Size = n.Size + 1 + uint32(n.MimeSize) n.Size = n.Size + 1 + uint32(n.MimeSize)
} }
} }
@ -43,40 +43,40 @@ func (n *Needle) Append(w io.Writer, version Version) uint32 {
util.Uint8toBytes(header[0:1], n.Flags) util.Uint8toBytes(header[0:1], n.Flags)
w.Write(header[0:1]) w.Write(header[0:1])
} }
if n.NameSize > 0 { if n.HasName() {
util.Uint8toBytes(header[0:1], n.NameSize) util.Uint8toBytes(header[0:1], n.NameSize)
w.Write(header[0:1]) w.Write(header[0:1])
w.Write(n.Name) w.Write(n.Name)
} }
if n.MimeSize > 0 { if n.HasMime() {
util.Uint8toBytes(header[0:1], n.MimeSize) util.Uint8toBytes(header[0:1], n.MimeSize)
w.Write(header[0:1]) w.Write(header[0:1])
w.Write(n.Mime) w.Write(n.Mime)
} }
rest := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + 4) % NeedlePaddingSize) padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
util.Uint32toBytes(header[0:4], n.Checksum.Value()) util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
w.Write(header[0 : 4+rest]) w.Write(header[0 : NeedleChecksumSize+padding])
return n.DataSize return n.DataSize
} }
return n.Size return n.Size
} }
func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) { func (n *Needle) Read(r io.Reader, size uint32, version Version) (int, error) {
if version == Version1 { if version == Version1 {
bytes := make([]byte, NeedleHeaderSize+size+4) bytes := make([]byte, NeedleHeaderSize+size+NeedleChecksumSize)
ret, e := r.Read(bytes) ret, e := r.Read(bytes)
n.readNeedleHeader(bytes) n.readNeedleHeader(bytes)
n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size] n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size]
checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+4]) checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize])
if checksum != NewCRC(n.Data).Value() { if checksum != NewCRC(n.Data).Value() {
return 0, errors.New("CRC error! Data On Disk Corrupted!") return 0, errors.New("CRC error! Data On Disk Corrupted!")
} }
return ret, e return ret, e
} else if version == Version2 { } else if version == Version2 {
bytes := make([]byte, NeedleHeaderSize+size+4) bytes := make([]byte, NeedleHeaderSize+size+NeedleChecksumSize)
ret, e := r.Read(bytes) ret, e := r.Read(bytes)
n.readNeedleHeader(bytes) n.readNeedleHeader(bytes)
n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)]) n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)])
checksum := util.BytesToUint32(bytes[NeedleHeaderSize+n.Size : NeedleHeaderSize+n.Size+4]) checksum := util.BytesToUint32(bytes[NeedleHeaderSize+n.Size : NeedleHeaderSize+n.Size+NeedleChecksumSize])
if checksum != NewCRC(n.Data).Value() { if checksum != NewCRC(n.Data).Value() {
return 0, errors.New("CRC error! Data On Disk Corrupted!") return 0, errors.New("CRC error! Data On Disk Corrupted!")
} }
@ -99,13 +99,13 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) {
n.Flags = bytes[index] n.Flags = bytes[index]
index = index + 1 index = index + 1
} }
if index < lenBytes { if index < lenBytes && n.HasName() {
n.NameSize = uint8(bytes[index]) n.NameSize = uint8(bytes[index])
index = index + 1 index = index + 1
n.Name = bytes[index : index+int(n.NameSize)] n.Name = bytes[index : index+int(n.NameSize)]
index = index + int(n.NameSize) index = index + int(n.NameSize)
} }
if index < lenBytes { if index < lenBytes && n.HasMime() {
n.MimeSize = uint8(bytes[index]) n.MimeSize = uint8(bytes[index])
index = index + 1 index = index + 1
n.Mime = bytes[index : index+int(n.MimeSize)] n.Mime = bytes[index : index+int(n.MimeSize)]
@ -120,8 +120,8 @@ func ReadNeedleHeader(r *os.File, version Version) (n *Needle, bodyLength uint32
return nil, 0 return nil, 0
} }
n.readNeedleHeader(bytes) n.readNeedleHeader(bytes)
rest := NeedlePaddingSize - ((n.Size + NeedleHeaderSize + 4) % NeedlePaddingSize) padding := NeedlePaddingSize - ((n.Size + NeedleHeaderSize + NeedleChecksumSize) % NeedlePaddingSize)
bodyLength = n.Size + 4 + rest bodyLength = n.Size + NeedleChecksumSize + padding
} }
return return
} }
@ -149,3 +149,15 @@ func (n *Needle) IsGzipped() bool{
func (n *Needle) SetGzipped() { func (n *Needle) SetGzipped() {
n.Flags = n.Flags | 0x01 n.Flags = n.Flags | 0x01
} }
func (n *Needle) HasName() bool {
return n.Flags&0x02 == 0x02
}
func (n *Needle) SetHasName() {
n.Flags = n.Flags | 0x02
}
func (n *Needle) HasMime() bool {
return n.Flags&0x04 == 0x04
}
func (n *Needle) SetHasMime() {
n.Flags = n.Flags | 0x04
}