gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

use uint64, instead of key+alternativeKey, as file id

Browse source 
git-svn-id: https://weed-fs.googlecode.com/svn/trunk@18 282b0af5-e82d-9cf1-ede4-77906d7719d0
This commit is contained in:
chris.lu@gmail.com 2011-12-20 02:34:53 +00:00
parent d018809830
commit a657eec299
3 changed files with 25 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
weed-fs/src/pkg/storage/needle.go
View file

@ -10,9 +10,8 @@ import (
) )
type Needle struct { type Needle struct {
Cookie uint8 "random number to mitigate brute force lookups" Cookie uint32 "random number to mitigate brute force lookups"
Key uint64 "file id" Key uint64 "file id"
AlternateKey uint32 "supplemental id"
Size uint32 "Data size" Size uint32 "Data size"
Data []byte "The actual file data" Data []byte "The actual file data"
Checksum int32 "CRC32 to check integrity" Checksum int32 "CRC32 to check integrity"
@ -36,32 +35,28 @@ func NewNeedle(r *http.Request) (n *Needle) {
func (n *Needle) ParsePath(path string) { func (n *Needle) ParsePath(path string) {
a := strings.Split(path, "_") a := strings.Split(path, "_")
log.Println("cookie", a[0], "key", a[1], "altKey", a[2]) log.Println("cookie", a[0], "key", a[1], "altKey", a[2])
cookie, _ := strconv.Atoi(a[0]) cookie, _ := strconv.Atoui(a[0])
n.Cookie = uint8(cookie) n.Cookie = uint32(cookie)
n.Key, _ = strconv.Atoui64(a[1]) n.Key, _ = strconv.Atoui64(a[1])
altKey, _ := strconv.Atoui64(a[2])
n.AlternateKey = uint32(altKey)
} }
func (n *Needle) Append(w io.Writer) { func (n *Needle) Append(w io.Writer) {
header := make([]byte, 17) header := make([]byte, 16)
header[0] = n.Cookie uint32toBytes(header[0:4], n.Cookie)
uint64toBytes(header[1:9], n.Key) uint64toBytes(header[4:12], n.Key)
uint32toBytes(header[9:13], n.AlternateKey)
n.Size = uint32(len(n.Data)) n.Size = uint32(len(n.Data))
uint32toBytes(header[13:17], n.Size) uint32toBytes(header[12:16], n.Size)
w.Write(header) w.Write(header)
w.Write(n.Data) w.Write(n.Data)
rest := 8 - ((n.Size + 17 + 4) % 8) rest := 8 - ((n.Size + 16 + 4) % 8)
uint32toBytes(header[0:4], uint32(n.Checksum)) uint32toBytes(header[0:4], uint32(n.Checksum))
w.Write(header[0 : rest+4]) w.Write(header[0 : rest+4])
} }
func (n *Needle) Read(r io.Reader, size uint32) { func (n *Needle) Read(r io.Reader, size uint32) {
bytes := make([]byte, size+17+4) bytes := make([]byte, size+16+4)
r.Read(bytes) r.Read(bytes)
n.Cookie = bytes[0] n.Cookie = bytesToUint32(bytes[0:4])
n.Key = bytesToUint64(bytes[1:9]) n.Key = bytesToUint64(bytes[4:12])
n.AlternateKey = bytesToUint32(bytes[9:13]) n.Size = bytesToUint32(bytes[12:16])
n.Size = bytesToUint32(bytes[13:17]) n.Data = bytes[16 : 16+size]
n.Data = bytes[17 : 17+size] n.Checksum = int32(bytesToUint32(bytes[16+size : 16+size+4]))
n.Checksum = int32(bytesToUint32(bytes[17+size : 17+size+4]))
} }

27
weed-fs/src/pkg/storage/needle_map.go
View file

@ -6,7 +6,6 @@ import (
type NeedleKey struct { type NeedleKey struct {
Key uint64 "file id" Key uint64 "file id"
AlternateKey uint32 "supplemental id"
} }
func (k *NeedleKey) String() string { func (k *NeedleKey) String() string {
@ -14,9 +13,6 @@ func (k *NeedleKey) String() string {
for i := uint(0); i < 8; i++ { for i := uint(0); i < 8; i++ {
tmp[i] = byte(k.Key >> (8 * i)) tmp[i] = byte(k.Key >> (8 * i))
} }
for i := uint(0); i < 4; i++ {
tmp[i+8] = byte(k.AlternateKey >> (8 * i))
}
return string(tmp[:]) return string(tmp[:])
} }
@ -26,28 +22,21 @@ type NeedleValue struct {
} }
type NeedleMap struct { type NeedleMap struct {
m map[string]*NeedleValue //mapping NeedleKey(Key,AlternateKey) to NeedleValue m map[uint64]*NeedleValue //mapping NeedleKey(Key,AlternateKey) to NeedleValue
} }
func NewNeedleMap() *NeedleMap { func NewNeedleMap() *NeedleMap {
return &NeedleMap{m: make(map[string]*NeedleValue)} return &NeedleMap{m: make(map[uint64]*NeedleValue)}
} }
func (nm *NeedleMap) load(file *os.File) { func (nm *NeedleMap) load(file *os.File) {
} }
func makeKey(key uint64, altKey uint32) string { func makeKey(key uint64) uint64 {
var tmp [12]byte return key
for i := uint(0); i < 8; i++ {
tmp[i] = byte(key >> (8 * i))
}
for i := uint(0); i < 4; i++ {
tmp[i+8] = byte(altKey >> (8 * i))
}
return string(tmp[:])
} }
func (nm *NeedleMap) put(key uint64, altKey uint32, offset uint32, size uint32) { func (nm *NeedleMap) put(key uint64, offset uint32, size uint32) {
nm.m[makeKey(key, altKey)] = &NeedleValue{Offset: offset, Size: size} nm.m[makeKey(key)] = &NeedleValue{Offset: offset, Size: size}
} }
func (nm *NeedleMap) get(key uint64, altKey uint32) (element *NeedleValue, ok bool) { func (nm *NeedleMap) get(key uint64) (element *NeedleValue, ok bool) {
element, ok = nm.m[makeKey(key, altKey)] element, ok = nm.m[makeKey(key)]
return return
} }

6
weed-fs/src/pkg/storage/volume.go
View file

@ -52,15 +52,15 @@ func (v *Volume) write(n *Needle) {
counter := <-v.accessChannel counter := <-v.accessChannel
offset, _ := v.dataFile.Seek(0, 2) offset, _ := v.dataFile.Seek(0, 2)
n.Append(v.dataFile) n.Append(v.dataFile)
nv, ok := v.nm.get(n.Key, n.AlternateKey) nv, ok := v.nm.get(n.Key)
if !ok || int64(nv.Offset)*8 < offset { if !ok || int64(nv.Offset)*8 < offset {
v.nm.put(n.Key, n.AlternateKey, uint32(offset/8), n.Size) v.nm.put(n.Key, uint32(offset/8), n.Size)
} }
v.accessChannel <- counter + 1 v.accessChannel <- counter + 1
} }
func (v *Volume) read(n *Needle) { func (v *Volume) read(n *Needle) {
counter := <-v.accessChannel counter := <-v.accessChannel
nv, ok := v.nm.get(n.Key, n.AlternateKey) nv, ok := v.nm.get(n.Key)
if ok && nv.Offset > 0 { if ok && nv.Offset > 0 {
v.dataFile.Seek(int64(nv.Offset)*8, 0) v.dataFile.Seek(int64(nv.Offset)*8, 0)
n.Read(v.dataFile, nv.Size) n.Read(v.dataFile, nv.Size)