2015-03-27 23:34:58 +00:00
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
|
|
|
"os"
|
|
|
|
|
2016-06-03 01:09:14 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
2019-05-22 05:41:20 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/storage/idx"
|
2019-04-19 04:43:36 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
|
2018-07-08 09:28:04 +00:00
|
|
|
. "github.com/chrislusf/seaweedfs/weed/storage/types"
|
2015-03-27 23:34:58 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type NeedleMap struct {
|
2015-05-26 07:58:41 +00:00
|
|
|
baseNeedleMapper
|
2019-05-04 15:47:11 +00:00
|
|
|
m needle_map.NeedleValueMap
|
2015-03-27 23:34:58 +00:00
|
|
|
}
|
|
|
|
|
2017-05-27 05:51:25 +00:00
|
|
|
func NewCompactNeedleMap(file *os.File) *NeedleMap {
|
2015-03-27 23:34:58 +00:00
|
|
|
nm := &NeedleMap{
|
2019-04-19 04:43:36 +00:00
|
|
|
m: needle_map.NewCompactMap(),
|
2017-05-27 05:51:25 +00:00
|
|
|
}
|
|
|
|
nm.indexFile = file
|
|
|
|
return nm
|
|
|
|
}
|
|
|
|
|
|
|
|
func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) {
|
|
|
|
nm := NewCompactNeedleMap(file)
|
|
|
|
return doLoading(file, nm)
|
|
|
|
}
|
|
|
|
|
|
|
|
func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
|
2020-08-19 00:04:28 +00:00
|
|
|
e := idx.WalkIndexFile(file, func(key NeedleId, offset Offset, size Size) error {
|
2019-04-15 06:00:37 +00:00
|
|
|
nm.MaybeSetMaxFileKey(key)
|
2020-09-10 21:42:50 +00:00
|
|
|
if !offset.IsZero() {
|
2017-01-18 16:34:44 +00:00
|
|
|
nm.FileCounter++
|
|
|
|
nm.FileByteCounter = nm.FileByteCounter + uint64(size)
|
2020-09-10 21:42:50 +00:00
|
|
|
|
2018-07-08 09:28:04 +00:00
|
|
|
oldOffset, oldSize := nm.m.Set(NeedleId(key), offset, size)
|
2020-08-19 01:01:37 +00:00
|
|
|
if !oldOffset.IsZero() && oldSize.IsValid() {
|
2015-03-27 23:34:58 +00:00
|
|
|
nm.DeletionCounter++
|
|
|
|
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
|
2020-09-10 21:42:50 +00:00
|
|
|
} else if size < 0 {
|
|
|
|
// deletion
|
|
|
|
nm.DeletionCounter++
|
|
|
|
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(-size)
|
2015-03-27 23:34:58 +00:00
|
|
|
}
|
|
|
|
} else {
|
2018-07-08 09:28:04 +00:00
|
|
|
oldSize := nm.m.Delete(NeedleId(key))
|
2015-03-27 23:34:58 +00:00
|
|
|
nm.DeletionCounter++
|
|
|
|
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
})
|
2019-04-15 06:00:37 +00:00
|
|
|
glog.V(1).Infof("max file key: %d for file: %s", nm.MaxFileKey(), file.Name())
|
2015-03-27 23:34:58 +00:00
|
|
|
return nm, e
|
|
|
|
}
|
|
|
|
|
2020-08-19 00:04:28 +00:00
|
|
|
func (nm *NeedleMap) Put(key NeedleId, offset Offset, size Size) error {
|
2018-07-08 09:28:04 +00:00
|
|
|
_, oldSize := nm.m.Set(NeedleId(key), offset, size)
|
2015-03-27 23:34:58 +00:00
|
|
|
nm.logPut(key, oldSize, size)
|
2015-05-26 07:58:41 +00:00
|
|
|
return nm.appendToIndexFile(key, offset, size)
|
2015-03-27 23:34:58 +00:00
|
|
|
}
|
2020-09-10 21:42:50 +00:00
|
|
|
func (nm *NeedleMap) Get(key NeedleId) (existingValue *needle_map.NeedleValue, ok bool) {
|
|
|
|
existingValue, ok = nm.m.Get(NeedleId(key))
|
2015-03-27 23:34:58 +00:00
|
|
|
return
|
|
|
|
}
|
2020-09-10 21:42:50 +00:00
|
|
|
func (nm *NeedleMap) Delete(key NeedleId) error {
|
|
|
|
existingValue, ok := nm.m.Get(NeedleId(key))
|
2020-09-12 11:08:03 +00:00
|
|
|
if !ok || existingValue.Size.IsDeleted() {
|
2020-09-10 21:42:50 +00:00
|
|
|
return nil
|
|
|
|
}
|
2018-07-08 09:28:04 +00:00
|
|
|
deletedBytes := nm.m.Delete(NeedleId(key))
|
2015-03-27 23:34:58 +00:00
|
|
|
nm.logDelete(deletedBytes)
|
2020-09-10 21:42:50 +00:00
|
|
|
return nm.appendToIndexFile(key, existingValue.Offset, -existingValue.Size)
|
2015-03-27 23:34:58 +00:00
|
|
|
}
|
|
|
|
func (nm *NeedleMap) Close() {
|
2020-02-04 18:38:12 +00:00
|
|
|
indexFileName := nm.indexFile.Name()
|
|
|
|
if err := nm.indexFile.Sync(); err != nil {
|
|
|
|
glog.Warningf("sync file %s failed, %v", indexFileName, err)
|
|
|
|
}
|
2015-03-27 23:34:58 +00:00
|
|
|
_ = nm.indexFile.Close()
|
|
|
|
}
|
|
|
|
func (nm *NeedleMap) Destroy() error {
|
|
|
|
nm.Close()
|
|
|
|
return os.Remove(nm.indexFile.Name())
|
|
|
|
}
|