2016-04-27 03:10:26 +00:00
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
2020-06-05 15:18:15 +00:00
|
|
|
"fmt"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/stats"
|
|
|
|
"io/ioutil"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
"time"
|
2016-04-27 03:10:26 +00:00
|
|
|
|
2019-09-12 13:18:21 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/storage/needle"
|
2016-04-27 03:10:26 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type DiskLocation struct {
|
|
|
|
Directory string
|
|
|
|
MaxVolumeCount int
|
2020-06-05 15:18:15 +00:00
|
|
|
FreeDiskSpaceWatermark float32
|
2019-04-19 04:43:36 +00:00
|
|
|
volumes map[needle.VolumeId]*Volume
|
2019-12-03 04:49:50 +00:00
|
|
|
volumesLock sync.RWMutex
|
2019-05-22 05:41:20 +00:00
|
|
|
|
|
|
|
// erasure coding
|
2019-05-28 04:40:51 +00:00
|
|
|
ecVolumes map[needle.VolumeId]*erasure_coding.EcVolume
|
|
|
|
ecVolumesLock sync.RWMutex
|
2016-04-27 03:10:26 +00:00
|
|
|
}
|
|
|
|
|
2020-06-05 15:18:15 +00:00
|
|
|
func NewDiskLocation(dir string, maxVolumeCount int, freeDiskSpaceWatermark float32) *DiskLocation {
|
|
|
|
location := &DiskLocation{Directory: dir, MaxVolumeCount: maxVolumeCount, FreeDiskSpaceWatermark: freeDiskSpaceWatermark}
|
2019-04-19 04:43:36 +00:00
|
|
|
location.volumes = make(map[needle.VolumeId]*Volume)
|
2019-05-28 04:40:51 +00:00
|
|
|
location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
|
2020-06-05 15:18:15 +00:00
|
|
|
go location.CheckDiskSpace()
|
2016-04-27 03:45:35 +00:00
|
|
|
return location
|
2016-04-27 03:10:26 +00:00
|
|
|
}
|
|
|
|
|
2019-04-19 04:43:36 +00:00
|
|
|
func (l *DiskLocation) volumeIdFromPath(dir os.FileInfo) (needle.VolumeId, string, error) {
|
2016-11-13 05:24:52 +00:00
|
|
|
name := dir.Name()
|
2019-12-02 23:08:28 +00:00
|
|
|
if !dir.IsDir() && strings.HasSuffix(name, ".idx") {
|
|
|
|
base := name[:len(name)-len(".idx")]
|
2019-05-22 05:41:20 +00:00
|
|
|
collection, volumeId, err := parseCollectionVolumeId(base)
|
|
|
|
return volumeId, collection, err
|
2017-01-20 11:49:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0, "", fmt.Errorf("Path is not a volume: %s", name)
|
|
|
|
}
|
|
|
|
|
2019-05-22 05:41:20 +00:00
|
|
|
func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) {
|
|
|
|
i := strings.LastIndex(base, "_")
|
|
|
|
if i > 0 {
|
|
|
|
collection, base = base[0:i], base[i+1:]
|
|
|
|
}
|
|
|
|
vol, err := needle.NewVolumeId(base)
|
|
|
|
return collection, vol, err
|
|
|
|
}
|
|
|
|
|
2020-02-13 07:19:00 +00:00
|
|
|
func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) bool {
|
2019-05-22 05:41:20 +00:00
|
|
|
name := fileInfo.Name()
|
2019-12-02 23:08:28 +00:00
|
|
|
if !fileInfo.IsDir() && strings.HasSuffix(name, ".idx") {
|
2019-05-22 05:41:20 +00:00
|
|
|
vid, collection, err := l.volumeIdFromPath(fileInfo)
|
2020-02-13 07:19:00 +00:00
|
|
|
if err != nil {
|
|
|
|
glog.Warningf("get volume id failed, %s, err : %s", name, err)
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
// void loading one volume more than once
|
|
|
|
l.volumesLock.RLock()
|
|
|
|
_, found := l.volumes[vid]
|
|
|
|
l.volumesLock.RUnlock()
|
|
|
|
if found {
|
|
|
|
glog.V(1).Infof("loaded volume, %v", vid)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0, 0)
|
|
|
|
if e != nil {
|
|
|
|
glog.V(0).Infof("new volume %s error %s", name, e)
|
|
|
|
return false
|
2016-11-13 05:24:52 +00:00
|
|
|
}
|
2020-02-13 07:19:00 +00:00
|
|
|
|
|
|
|
l.volumesLock.Lock()
|
|
|
|
l.volumes[vid] = v
|
|
|
|
l.volumesLock.Unlock()
|
|
|
|
size, _, _ := v.FileStat()
|
|
|
|
glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s",
|
|
|
|
l.Directory+"/"+name, v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
|
|
|
|
return true
|
2016-11-13 05:24:52 +00:00
|
|
|
}
|
2020-02-13 07:19:00 +00:00
|
|
|
return false
|
2016-11-13 05:24:52 +00:00
|
|
|
}
|
2016-04-27 03:45:35 +00:00
|
|
|
|
2019-03-18 05:32:01 +00:00
|
|
|
func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, concurrency int) {
|
2016-11-13 05:24:52 +00:00
|
|
|
|
|
|
|
task_queue := make(chan os.FileInfo, 10*concurrency)
|
2016-11-11 03:53:22 +00:00
|
|
|
go func() {
|
|
|
|
if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
|
|
|
|
for _, dir := range dirs {
|
|
|
|
task_queue <- dir
|
|
|
|
}
|
|
|
|
}
|
|
|
|
close(task_queue)
|
|
|
|
}()
|
|
|
|
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
for workerNum := 0; workerNum < concurrency; workerNum++ {
|
|
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
|
|
|
for dir := range task_queue {
|
2020-02-13 07:19:00 +00:00
|
|
|
_ = l.loadExistingVolume(dir, needleMapKind)
|
2016-04-27 03:10:26 +00:00
|
|
|
}
|
2016-11-11 03:53:22 +00:00
|
|
|
}()
|
2016-04-27 03:10:26 +00:00
|
|
|
}
|
2016-11-11 03:53:22 +00:00
|
|
|
wg.Wait()
|
|
|
|
|
2016-11-13 05:24:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
|
|
|
|
|
2019-03-18 05:32:01 +00:00
|
|
|
l.concurrentLoadingVolumes(needleMapKind, 10)
|
2019-05-22 05:41:20 +00:00
|
|
|
glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount)
|
|
|
|
|
|
|
|
l.loadAllEcShards()
|
2019-05-28 04:40:51 +00:00
|
|
|
glog.V(0).Infof("Store started on dir: %s with %d ec shards", l.Directory, len(l.ecVolumes))
|
2016-11-13 05:24:52 +00:00
|
|
|
|
2016-04-27 03:10:26 +00:00
|
|
|
}
|
2016-04-27 03:45:35 +00:00
|
|
|
|
|
|
|
func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) {
|
2016-11-07 04:55:22 +00:00
|
|
|
|
2019-12-03 04:49:50 +00:00
|
|
|
l.volumesLock.Lock()
|
2019-12-24 09:20:34 +00:00
|
|
|
delVolsMap := l.unmountVolumeByCollection(collection)
|
2019-12-03 04:49:50 +00:00
|
|
|
l.volumesLock.Unlock()
|
2019-05-30 16:47:54 +00:00
|
|
|
|
|
|
|
l.ecVolumesLock.Lock()
|
2019-12-24 09:20:34 +00:00
|
|
|
delEcVolsMap := l.unmountEcVolumeByCollection(collection)
|
|
|
|
l.ecVolumesLock.Unlock()
|
|
|
|
|
|
|
|
errChain := make(chan error, 2)
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
wg.Add(2)
|
|
|
|
go func() {
|
|
|
|
for _, v := range delVolsMap {
|
|
|
|
if err := v.Destroy(); err != nil {
|
|
|
|
errChain <- err
|
2016-04-27 03:45:35 +00:00
|
|
|
}
|
|
|
|
}
|
2019-12-24 09:20:34 +00:00
|
|
|
wg.Done()
|
|
|
|
}()
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
for _, v := range delEcVolsMap {
|
|
|
|
v.Destroy()
|
|
|
|
}
|
|
|
|
wg.Done()
|
|
|
|
}()
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
wg.Wait()
|
|
|
|
close(errChain)
|
|
|
|
}()
|
|
|
|
|
|
|
|
errBuilder := strings.Builder{}
|
|
|
|
for err := range errChain {
|
|
|
|
errBuilder.WriteString(err.Error())
|
|
|
|
errBuilder.WriteString("; ")
|
2016-04-27 03:45:35 +00:00
|
|
|
}
|
2019-12-24 09:20:34 +00:00
|
|
|
if errBuilder.Len() > 0 {
|
|
|
|
e = fmt.Errorf(errBuilder.String())
|
|
|
|
}
|
|
|
|
|
2016-04-27 03:45:35 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-04-21 21:49:58 +00:00
|
|
|
func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (found bool, e error) {
|
2016-04-27 03:45:35 +00:00
|
|
|
v, ok := l.volumes[vid]
|
|
|
|
if !ok {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
e = v.Destroy()
|
|
|
|
if e != nil {
|
|
|
|
return
|
|
|
|
}
|
2020-04-21 21:49:58 +00:00
|
|
|
found = true
|
2016-04-27 03:45:35 +00:00
|
|
|
delete(l.volumes, vid)
|
|
|
|
return
|
|
|
|
}
|
2016-11-07 04:55:22 +00:00
|
|
|
|
2019-04-19 04:43:36 +00:00
|
|
|
func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapType) bool {
|
2020-02-02 23:37:23 +00:00
|
|
|
if fileInfo, found := l.LocateVolume(vid); found {
|
2020-02-13 07:19:00 +00:00
|
|
|
return l.loadExistingVolume(fileInfo, needleMapKind)
|
2017-01-20 11:49:20 +00:00
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2019-04-19 04:43:36 +00:00
|
|
|
func (l *DiskLocation) DeleteVolume(vid needle.VolumeId) error {
|
2019-12-03 04:49:50 +00:00
|
|
|
l.volumesLock.Lock()
|
|
|
|
defer l.volumesLock.Unlock()
|
2017-01-20 18:18:43 +00:00
|
|
|
|
2017-01-20 12:02:37 +00:00
|
|
|
_, ok := l.volumes[vid]
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("Volume not found, VolumeId: %d", vid)
|
|
|
|
}
|
2020-04-21 21:49:58 +00:00
|
|
|
_, err := l.deleteVolumeById(vid)
|
|
|
|
return err
|
2017-01-20 12:02:37 +00:00
|
|
|
}
|
|
|
|
|
2019-04-19 04:43:36 +00:00
|
|
|
func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error {
|
2019-12-03 04:49:50 +00:00
|
|
|
l.volumesLock.Lock()
|
|
|
|
defer l.volumesLock.Unlock()
|
2017-01-20 18:18:43 +00:00
|
|
|
|
2018-05-03 04:13:53 +00:00
|
|
|
v, ok := l.volumes[vid]
|
2017-01-20 11:49:20 +00:00
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("Volume not loaded, VolumeId: %d", vid)
|
|
|
|
}
|
2018-05-03 04:13:53 +00:00
|
|
|
v.Close()
|
2017-01-20 11:49:20 +00:00
|
|
|
delete(l.volumes, vid)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-12-24 09:20:34 +00:00
|
|
|
func (l *DiskLocation) unmountVolumeByCollection(collectionName string) map[needle.VolumeId]*Volume {
|
|
|
|
deltaVols := make(map[needle.VolumeId]*Volume, 0)
|
|
|
|
for k, v := range l.volumes {
|
|
|
|
if v.Collection == collectionName && !v.isCompacting {
|
|
|
|
deltaVols[k] = v
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-02-02 23:37:23 +00:00
|
|
|
for k := range deltaVols {
|
2019-12-24 09:20:34 +00:00
|
|
|
delete(l.volumes, k)
|
|
|
|
}
|
|
|
|
return deltaVols
|
|
|
|
}
|
|
|
|
|
2019-04-19 04:43:36 +00:00
|
|
|
func (l *DiskLocation) SetVolume(vid needle.VolumeId, volume *Volume) {
|
2019-12-03 04:49:50 +00:00
|
|
|
l.volumesLock.Lock()
|
|
|
|
defer l.volumesLock.Unlock()
|
2016-11-07 04:55:22 +00:00
|
|
|
|
|
|
|
l.volumes[vid] = volume
|
|
|
|
}
|
|
|
|
|
2019-04-19 04:43:36 +00:00
|
|
|
func (l *DiskLocation) FindVolume(vid needle.VolumeId) (*Volume, bool) {
|
2019-12-03 04:49:50 +00:00
|
|
|
l.volumesLock.RLock()
|
|
|
|
defer l.volumesLock.RUnlock()
|
2016-11-07 04:55:22 +00:00
|
|
|
|
|
|
|
v, ok := l.volumes[vid]
|
|
|
|
return v, ok
|
|
|
|
}
|
|
|
|
|
|
|
|
func (l *DiskLocation) VolumesLen() int {
|
2019-12-03 04:49:50 +00:00
|
|
|
l.volumesLock.RLock()
|
|
|
|
defer l.volumesLock.RUnlock()
|
2016-11-07 04:55:22 +00:00
|
|
|
|
|
|
|
return len(l.volumes)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (l *DiskLocation) Close() {
|
2019-12-03 04:49:50 +00:00
|
|
|
l.volumesLock.Lock()
|
2016-11-07 04:55:22 +00:00
|
|
|
for _, v := range l.volumes {
|
|
|
|
v.Close()
|
|
|
|
}
|
2019-12-03 04:49:50 +00:00
|
|
|
l.volumesLock.Unlock()
|
2019-05-22 05:41:20 +00:00
|
|
|
|
2019-05-28 04:40:51 +00:00
|
|
|
l.ecVolumesLock.Lock()
|
2019-06-06 06:20:26 +00:00
|
|
|
for _, ecVolume := range l.ecVolumes {
|
|
|
|
ecVolume.Close()
|
2019-05-22 05:41:20 +00:00
|
|
|
}
|
2019-05-28 04:40:51 +00:00
|
|
|
l.ecVolumesLock.Unlock()
|
2019-05-22 05:41:20 +00:00
|
|
|
|
2016-11-07 04:55:22 +00:00
|
|
|
return
|
|
|
|
}
|
2020-02-02 23:37:23 +00:00
|
|
|
|
|
|
|
func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.FileInfo, bool) {
|
|
|
|
if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
|
|
|
|
for _, fileInfo := range fileInfos {
|
|
|
|
volId, _, err := l.volumeIdFromPath(fileInfo)
|
|
|
|
if vid == volId && err == nil {
|
|
|
|
return fileInfo, true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil, false
|
|
|
|
}
|
2020-03-22 23:21:42 +00:00
|
|
|
|
|
|
|
func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) {
|
|
|
|
|
|
|
|
l.volumesLock.RLock()
|
|
|
|
defer l.volumesLock.RUnlock()
|
|
|
|
|
|
|
|
for _, vol := range l.volumes {
|
|
|
|
if vol.IsReadOnly() {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
datSize, idxSize, _ := vol.FileStat()
|
|
|
|
unUsedSpace += volumeSizeLimit - (datSize + idxSize)
|
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
2020-06-05 15:18:15 +00:00
|
|
|
|
|
|
|
func (l *DiskLocation) CheckDiskSpace() {
|
|
|
|
lastStat := false
|
|
|
|
t := time.NewTicker(time.Minute)
|
|
|
|
for _ = range t.C {
|
|
|
|
if dir, e := filepath.Abs(l.Directory); e == nil {
|
|
|
|
s := stats.NewDiskStatus(dir)
|
|
|
|
if (s.PercentFree < l.FreeDiskSpaceWatermark) != lastStat {
|
|
|
|
lastStat = !lastStat
|
|
|
|
for _, v := range l.volumes {
|
|
|
|
v.SetLowDiskSpace(lastStat)
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|