2019-12-18 09:21:21 +00:00
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
|
|
|
"os"
|
|
|
|
|
2022-07-29 07:17:28 +00:00
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
|
|
|
. "github.com/seaweedfs/seaweedfs/weed/storage/types"
|
2019-12-18 09:21:21 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type SortedFileNeedleMap struct {
|
|
|
|
baseNeedleMapper
|
|
|
|
baseFileName string
|
|
|
|
dbFile *os.File
|
|
|
|
dbFileSize int64
|
|
|
|
}
|
|
|
|
|
2020-11-27 11:17:10 +00:00
|
|
|
func NewSortedFileNeedleMap(indexBaseFileName string, indexFile *os.File) (m *SortedFileNeedleMap, err error) {
|
|
|
|
m = &SortedFileNeedleMap{baseFileName: indexBaseFileName}
|
2019-12-18 09:21:21 +00:00
|
|
|
m.indexFile = indexFile
|
2020-11-27 11:17:10 +00:00
|
|
|
fileName := indexBaseFileName + ".sdx"
|
2019-12-18 09:21:21 +00:00
|
|
|
if !isSortedFileFresh(fileName, indexFile) {
|
|
|
|
glog.V(0).Infof("Start to Generate %s from %s", fileName, indexFile.Name())
|
2020-11-27 11:17:10 +00:00
|
|
|
erasure_coding.WriteSortedFileFromIdx(indexBaseFileName, ".sdx")
|
2019-12-18 09:21:21 +00:00
|
|
|
glog.V(0).Infof("Finished Generating %s from %s", fileName, indexFile.Name())
|
|
|
|
}
|
|
|
|
glog.V(1).Infof("Opening %s...", fileName)
|
|
|
|
|
2020-11-27 11:17:10 +00:00
|
|
|
if m.dbFile, err = os.Open(indexBaseFileName + ".sdx"); err != nil {
|
2019-12-18 09:21:21 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
dbStat, _ := m.dbFile.Stat()
|
|
|
|
m.dbFileSize = dbStat.Size()
|
|
|
|
glog.V(1).Infof("Loading %s...", indexFile.Name())
|
|
|
|
mm, indexLoadError := newNeedleMapMetricFromIndexFile(indexFile)
|
|
|
|
if indexLoadError != nil {
|
2023-08-09 22:30:36 +00:00
|
|
|
_ = m.dbFile.Close()
|
2019-12-18 09:21:21 +00:00
|
|
|
return nil, indexLoadError
|
|
|
|
}
|
|
|
|
m.mapMetric = *mm
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
func isSortedFileFresh(dbFileName string, indexFile *os.File) bool {
|
|
|
|
// normally we always write to index file first
|
|
|
|
dbFile, err := os.Open(dbFileName)
|
|
|
|
if err != nil {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
defer dbFile.Close()
|
|
|
|
dbStat, dbStatErr := dbFile.Stat()
|
|
|
|
indexStat, indexStatErr := indexFile.Stat()
|
|
|
|
if dbStatErr != nil || indexStatErr != nil {
|
|
|
|
glog.V(0).Infof("Can not stat file: %v and %v", dbStatErr, indexStatErr)
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
return dbStat.ModTime().After(indexStat.ModTime())
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *SortedFileNeedleMap) Get(key NeedleId) (element *needle_map.NeedleValue, ok bool) {
|
|
|
|
offset, size, err := erasure_coding.SearchNeedleFromSortedIndex(m.dbFile, m.dbFileSize, key, nil)
|
|
|
|
ok = err == nil
|
|
|
|
return &needle_map.NeedleValue{Key: key, Offset: offset, Size: size}, ok
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2020-08-19 00:04:28 +00:00
|
|
|
func (m *SortedFileNeedleMap) Put(key NeedleId, offset Offset, size Size) error {
|
2019-12-18 09:21:21 +00:00
|
|
|
return os.ErrInvalid
|
|
|
|
}
|
|
|
|
|
2020-09-12 19:42:36 +00:00
|
|
|
func (m *SortedFileNeedleMap) Delete(key NeedleId, offset Offset) error {
|
2019-12-18 09:21:21 +00:00
|
|
|
|
2020-09-12 19:42:36 +00:00
|
|
|
_, size, err := erasure_coding.SearchNeedleFromSortedIndex(m.dbFile, m.dbFileSize, key, nil)
|
2019-12-18 09:21:21 +00:00
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
if err == erasure_coding.NotFoundError {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2020-08-19 00:35:19 +00:00
|
|
|
if size.IsDeleted() {
|
2019-12-18 09:21:21 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// write to index file first
|
2020-09-12 19:42:36 +00:00
|
|
|
if err := m.appendToIndexFile(key, offset, TombstoneFileSize); err != nil {
|
2019-12-18 09:21:21 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
_, _, err = erasure_coding.SearchNeedleFromSortedIndex(m.dbFile, m.dbFileSize, key, erasure_coding.MarkNeedleDeleted)
|
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *SortedFileNeedleMap) Close() {
|
avoid data race of TraverseBfs (#3856)
* avoid data race of TraverseBfs
* close is enough
avoid panic
I1014 12:29:59.207120 volume_loading.go:131 loading sorted db /tmp/sw/test2_19.sdx error: unexpected file /tmp/sw/test2_19.idx size: 255
I1014 12:29:59.207125 volume_loading.go:119 open to write file /tmp/sw/test4_26.idx
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x5260a4c]
goroutine 287 [running]:
github.com/seaweedfs/seaweedfs/weed/storage.(*SortedFileNeedleMap).Close(0x0)
/Users/tochka/GolandProjects/seaweedfs/weed/storage/needle_map_sorted_file.go:97 +0x2c
github.com/seaweedfs/seaweedfs/weed/storage.(*Volume).load.func1()
/Users/tochka/GolandProjects/seaweedfs/weed/storage/volume_loading.go:32 +0x8e
github.com/seaweedfs/seaweedfs/weed/storage.(*Volume).load(0xc001b36280, 0x1, 0x1, 0x0, 0x69228c0?)
/Users/tochka/GolandProjects/seaweedfs/weed/storage/volume_loading.go:205 +0x256c
github.com/seaweedfs/seaweedfs/weed/storage.NewVolume({0x7ffeefbff6e0, 0x7}, {0x7ffeefbff6e0, 0x7}, {0xc0009a9284, 0x5}, 0x13, 0x0, 0x0, 0x0, ...)
/Users/tochka/GolandProjects/seaweedfs/weed/storage/volume.go:62 +0x30f
github.com/seaweedfs/seaweedfs/weed/storage.(*DiskLocation).loadExistingVolume(0xc0006f40c0, {0x846c8d0, 0xc0009ce600}, 0x0?, 0x1)
/Users/tochka/GolandProjects/seaweedfs/weed/storage/disk_location.go:161 +0x4da
github.com/seaweedfs/seaweedfs/weed/storage.(*DiskLocation).concurrentLoadingVolumes.func2()
/Users/tochka/GolandProjects/seaweedfs/weed/storage/disk_location.go:201 +0xf9
created by github.com/seaweedfs/seaweedfs/weed/storage.(*DiskLocation).concurrentLoadingVolumes
/Users/tochka/GolandProjects/seaweedfs/weed/storage/disk_location.go:198 +0x150
2022-10-16 18:30:41 +00:00
|
|
|
if m == nil {
|
|
|
|
return
|
|
|
|
}
|
2021-03-07 19:03:09 +00:00
|
|
|
if m.indexFile != nil {
|
|
|
|
m.indexFile.Close()
|
|
|
|
}
|
|
|
|
if m.dbFile != nil {
|
|
|
|
m.dbFile.Close()
|
|
|
|
}
|
2019-12-18 09:21:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (m *SortedFileNeedleMap) Destroy() error {
|
|
|
|
m.Close()
|
|
|
|
os.Remove(m.indexFile.Name())
|
2019-12-24 18:19:12 +00:00
|
|
|
return os.Remove(m.baseFileName + ".sdx")
|
2019-12-18 09:21:21 +00:00
|
|
|
}
|