gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

use uint64, instead of key+alternativeKey, as file id

Browse source 
git-svn-id: https://weed-fs.googlecode.com/svn/trunk@18 282b0af5-e82d-9cf1-ede4-77906d7719d0
This commit is contained in:
chris.lu@gmail.com 2011-12-20 02:34:53 +00:00
parent d018809830
commit a657eec299
3 changed files with 25 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
weed-fs/src/pkg/storage/needle.go
View file

@ -10,9 +10,8 @@ import (
)
type Needle struct {
Cookie uint8 "random number to mitigate brute force lookups"
Cookie uint32 "random number to mitigate brute force lookups"
Key uint64 "file id"
AlternateKey uint32 "supplemental id"
Size uint32 "Data size"
Data []byte "The actual file data"
Checksum int32 "CRC32 to check integrity"
@ -36,32 +35,28 @@ func NewNeedle(r *http.Request) (n *Needle) {
func (n *Needle) ParsePath(path string) {
a := strings.Split(path, "_")
log.Println("cookie", a[0], "key", a[1], "altKey", a[2])
cookie, _ := strconv.Atoi(a[0])
n.Cookie = uint8(cookie)
cookie, _ := strconv.Atoui(a[0])
n.Cookie = uint32(cookie)
n.Key, _ = strconv.Atoui64(a[1])
altKey, _ := strconv.Atoui64(a[2])
n.AlternateKey = uint32(altKey)
}
func (n *Needle) Append(w io.Writer) {
header := make([]byte, 17)
header[0] = n.Cookie
uint64toBytes(header[1:9], n.Key)
uint32toBytes(header[9:13], n.AlternateKey)
header := make([]byte, 16)
uint32toBytes(header[0:4], n.Cookie)
uint64toBytes(header[4:12], n.Key)
n.Size = uint32(len(n.Data))
uint32toBytes(header[13:17], n.Size)
uint32toBytes(header[12:16], n.Size)
w.Write(header)
w.Write(n.Data)
rest := 8 - ((n.Size + 17 + 4) % 8)
rest := 8 - ((n.Size + 16 + 4) % 8)
uint32toBytes(header[0:4], uint32(n.Checksum))
w.Write(header[0 : rest+4])
}
func (n *Needle) Read(r io.Reader, size uint32) {
bytes := make([]byte, size+17+4)
bytes := make([]byte, size+16+4)
r.Read(bytes)
n.Cookie = bytes[0]
n.Key = bytesToUint64(bytes[1:9])
n.AlternateKey = bytesToUint32(bytes[9:13])
n.Size = bytesToUint32(bytes[13:17])
n.Data = bytes[17 : 17+size]
n.Checksum = int32(bytesToUint32(bytes[17+size : 17+size+4]))
n.Cookie = bytesToUint32(bytes[0:4])
n.Key = bytesToUint64(bytes[4:12])
n.Size = bytesToUint32(bytes[12:16])
n.Data = bytes[16 : 16+size]
n.Checksum = int32(bytesToUint32(bytes[16+size : 16+size+4]))
}

27
weed-fs/src/pkg/storage/needle_map.go
View file

@ -6,7 +6,6 @@ import (
type NeedleKey struct {
Key uint64 "file id"
AlternateKey uint32 "supplemental id"
}
func (k *NeedleKey) String() string {
@ -14,9 +13,6 @@ func (k *NeedleKey) String() string {
for i := uint(0); i < 8; i++ {
tmp[i] = byte(k.Key >> (8 * i))
}
for i := uint(0); i < 4; i++ {
tmp[i+8] = byte(k.AlternateKey >> (8 * i))
}
return string(tmp[:])
}
@ -26,28 +22,21 @@ type NeedleValue struct {
}
type NeedleMap struct {
m map[string]*NeedleValue //mapping NeedleKey(Key,AlternateKey) to NeedleValue
m map[uint64]*NeedleValue //mapping NeedleKey(Key,AlternateKey) to NeedleValue
}
func NewNeedleMap() *NeedleMap {
return &NeedleMap{m: make(map[string]*NeedleValue)}
return &NeedleMap{m: make(map[uint64]*NeedleValue)}
}
func (nm *NeedleMap) load(file *os.File) {
}
func makeKey(key uint64, altKey uint32) string {
var tmp [12]byte
for i := uint(0); i < 8; i++ {
tmp[i] = byte(key >> (8 * i))
}
for i := uint(0); i < 4; i++ {
tmp[i+8] = byte(altKey >> (8 * i))
}
return string(tmp[:])
func makeKey(key uint64) uint64 {
return key
}
func (nm *NeedleMap) put(key uint64, altKey uint32, offset uint32, size uint32) {
nm.m[makeKey(key, altKey)] = &NeedleValue{Offset: offset, Size: size}
func (nm *NeedleMap) put(key uint64, offset uint32, size uint32) {
nm.m[makeKey(key)] = &NeedleValue{Offset: offset, Size: size}
}
func (nm *NeedleMap) get(key uint64, altKey uint32) (element *NeedleValue, ok bool) {
element, ok = nm.m[makeKey(key, altKey)]
func (nm *NeedleMap) get(key uint64) (element *NeedleValue, ok bool) {
element, ok = nm.m[makeKey(key)]
return
}

6
weed-fs/src/pkg/storage/volume.go
View file

@ -52,15 +52,15 @@ func (v *Volume) write(n *Needle) {
counter := <-v.accessChannel
offset, _ := v.dataFile.Seek(0, 2)
n.Append(v.dataFile)
nv, ok := v.nm.get(n.Key, n.AlternateKey)
nv, ok := v.nm.get(n.Key)
if !ok || int64(nv.Offset)*8 < offset {
v.nm.put(n.Key, n.AlternateKey, uint32(offset/8), n.Size)
v.nm.put(n.Key, uint32(offset/8), n.Size)
}
v.accessChannel <- counter + 1
}
func (v *Volume) read(n *Needle) {
counter := <-v.accessChannel
nv, ok := v.nm.get(n.Key, n.AlternateKey)
nv, ok := v.nm.get(n.Key)
if ok && nv.Offset > 0 {
v.dataFile.Seek(int64(nv.Offset)*8, 0)
n.Read(v.dataFile, nv.Size)