skip seek() when index file writes

This commit is contained in:
Chris Lu 2021-02-20 12:39:25 -08:00
parent 98c93ca465
commit 1ad3200094
4 changed files with 23 additions and 6 deletions

View file

@ -161,7 +161,15 @@ func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size Size, versi
dataSize := GetActualSize(size, version)
dataSlice = make([]byte, int(dataSize))
_, err = r.ReadAt(dataSlice, offset)
var n int
n, err = r.ReadAt(dataSlice, offset)
if err != nil && int64(n) == dataSize {
err = nil
}
if err != nil {
fileSize, _, _ := r.GetStat()
println("n",n, "dataSize", dataSize, "offset", offset, "fileSize", fileSize)
}
return dataSlice, err
}

View file

@ -1,7 +1,6 @@
package storage
import (
"fmt"
"io"
"os"
"sync"
@ -41,6 +40,7 @@ type baseNeedleMapper struct {
indexFile *os.File
indexFileAccessLock sync.Mutex
indexFileOffset int64
}
func (nm *baseNeedleMapper) IndexFileSize() uint64 {
@ -56,11 +56,10 @@ func (nm *baseNeedleMapper) appendToIndexFile(key NeedleId, offset Offset, size
nm.indexFileAccessLock.Lock()
defer nm.indexFileAccessLock.Unlock()
if _, err := nm.indexFile.Seek(0, 2); err != nil {
return fmt.Errorf("cannot seek end of indexfile %s: %v",
nm.indexFile.Name(), err)
written, err := nm.indexFile.WriteAt(bytes, nm.indexFileOffset)
if err == nil {
nm.indexFileOffset += int64(written)
}
_, err := nm.indexFile.Write(bytes)
return err
}

View file

@ -31,6 +31,11 @@ func NewLevelDbNeedleMap(dbFileName string, indexFile *os.File, opts *opt.Option
generateLevelDbFile(dbFileName, indexFile)
glog.V(1).Infof("Finished Generating %s from %s", dbFileName, indexFile.Name())
}
if stat, err := indexFile.Stat(); err != nil {
glog.Fatalf("stat file %s: %v", indexFile.Name(), err)
} else {
m.indexFileOffset = stat.Size()
}
glog.V(1).Infof("Opening %s...", dbFileName)
if m.db, err = leveldb.OpenFile(dbFileName, opts); err != nil {

View file

@ -19,6 +19,11 @@ func NewCompactNeedleMap(file *os.File) *NeedleMap {
m: needle_map.NewCompactMap(),
}
nm.indexFile = file
stat, err := file.Stat()
if err != nil {
glog.Fatalf("stat file %s: %v", file.Name(), err)
}
nm.indexFileOffset = stat.Size()
return nm
}