package storage import ( "code.google.com/p/weed-fs/go/util" "fmt" "io" "os" ) type NeedleMap struct { indexFile *os.File m CompactMap //transient bytes []byte deletionCounter int fileCounter int deletionByteCounter uint64 fileByteCounter uint64 } func NewNeedleMap(file *os.File) *NeedleMap { nm := &NeedleMap{ m: NewCompactMap(), bytes: make([]byte, 16), indexFile: file, } return nm } const ( RowsToRead = 1024 ) func LoadNeedleMap(file *os.File) (*NeedleMap, error) { nm := NewNeedleMap(file) bytes := make([]byte, 16*RowsToRead) count, e := nm.indexFile.Read(bytes) for count > 0 && e == nil { for i := 0; i < count; i += 16 { key := util.BytesToUint64(bytes[i : i+8]) offset := util.BytesToUint32(bytes[i+8 : i+12]) size := util.BytesToUint32(bytes[i+12 : i+16]) nm.fileCounter++ nm.fileByteCounter = nm.fileByteCounter + uint64(size) if offset > 0 { oldSize := nm.m.Set(Key(key), offset, size) //log.Println("reading key", key, "offset", offset, "size", size, "oldSize", oldSize) if oldSize > 0 { nm.deletionCounter++ nm.deletionByteCounter = nm.deletionByteCounter + uint64(oldSize) } } else { oldSize := nm.m.Delete(Key(key)) //log.Println("removing key", key, "offset", offset, "size", size, "oldSize", oldSize) nm.deletionCounter++ nm.deletionByteCounter = nm.deletionByteCounter + uint64(oldSize) } } count, e = nm.indexFile.Read(bytes) } if e == io.EOF { e = nil } return nm, e } func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) (int, error) { oldSize := nm.m.Set(Key(key), offset, size) util.Uint64toBytes(nm.bytes[0:8], key) util.Uint32toBytes(nm.bytes[8:12], offset) util.Uint32toBytes(nm.bytes[12:16], size) nm.fileCounter++ nm.fileByteCounter = nm.fileByteCounter + uint64(size) if oldSize > 0 { nm.deletionCounter++ nm.deletionByteCounter = nm.deletionByteCounter + uint64(oldSize) } return nm.indexFile.Write(nm.bytes) } func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) { element, ok = nm.m.Get(Key(key)) return } func (nm *NeedleMap) Delete(key uint64) error { nm.deletionByteCounter = nm.deletionByteCounter + uint64(nm.m.Delete(Key(key))) offset, err := nm.indexFile.Seek(0, 1) if err != nil { return fmt.Errorf("cannot get position of indexfile: %s", err) } util.Uint64toBytes(nm.bytes[0:8], key) util.Uint32toBytes(nm.bytes[8:12], 0) util.Uint32toBytes(nm.bytes[12:16], 0) if _, err = nm.indexFile.Write(nm.bytes); err != nil { nm.indexFile.Truncate(offset) return fmt.Errorf("error writing to indexfile %s: %s", nm.indexFile, err) } nm.deletionCounter++ return nil } func (nm *NeedleMap) Close() { nm.indexFile.Close() } func (nm *NeedleMap) ContentSize() uint64 { return nm.fileByteCounter } func (nm *NeedleMap) Visit(visit func(NeedleValue) error) (err error) { return nm.m.Visit(visit) }