seaweedfs/weed/storage/disk_location.go

376 lines
9.1 KiB
Go
Raw Normal View History

2016-04-27 03:10:26 +00:00
package storage
import (
2020-06-05 22:27:10 +00:00
"fmt"
"io/ioutil"
"os"
"path/filepath"
2020-06-05 22:27:10 +00:00
"strings"
"sync"
"time"
2016-04-27 03:10:26 +00:00
"github.com/chrislusf/seaweedfs/weed/glog"
2020-07-03 23:34:31 +00:00
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util"
2016-04-27 03:10:26 +00:00
)
type DiskLocation struct {
Directory string
IdxDirectory string
2020-12-16 17:14:05 +00:00
DiskType DiskType
MaxVolumeCount int
OriginalMaxVolumeCount int
MinFreeSpacePercent float32
volumes map[needle.VolumeId]*Volume
volumesLock sync.RWMutex
// erasure coding
2019-05-28 04:40:51 +00:00
ecVolumes map[needle.VolumeId]*erasure_coding.EcVolume
ecVolumesLock sync.RWMutex
2020-07-03 23:34:31 +00:00
isDiskSpaceLow bool
2016-04-27 03:10:26 +00:00
}
2020-12-13 19:59:32 +00:00
func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpacePercent float32, idxDir string, diskType DiskType) *DiskLocation {
dir = util.ResolvePath(dir)
if idxDir == "" {
idxDir = dir
} else {
idxDir = util.ResolvePath(idxDir)
}
location := &DiskLocation{
2020-11-26 23:08:05 +00:00
Directory: dir,
IdxDirectory: idxDir,
2020-12-16 17:14:05 +00:00
DiskType: diskType,
2020-11-26 23:08:05 +00:00
MaxVolumeCount: maxVolumeCount,
OriginalMaxVolumeCount: maxVolumeCount,
2020-11-26 23:08:05 +00:00
MinFreeSpacePercent: minFreeSpacePercent,
}
2019-04-19 04:43:36 +00:00
location.volumes = make(map[needle.VolumeId]*Volume)
2019-05-28 04:40:51 +00:00
location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
go location.CheckDiskSpace()
2016-04-27 03:45:35 +00:00
return location
2016-04-27 03:10:26 +00:00
}
2020-11-27 01:21:55 +00:00
func volumeIdFromFileName(filename string) (needle.VolumeId, string, error) {
if isValidVolume(filename) {
base := filename[:len(filename)-4]
collection, volumeId, err := parseCollectionVolumeId(base)
return volumeId, collection, err
}
2020-11-26 23:19:43 +00:00
return 0, "", fmt.Errorf("file is not a volume: %s", filename)
}
func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) {
i := strings.LastIndex(base, "_")
if i > 0 {
collection, base = base[0:i], base[i+1:]
}
vol, err := needle.NewVolumeId(base)
return collection, vol, err
}
func isValidVolume(basename string) bool {
return strings.HasSuffix(basename, ".idx") || strings.HasSuffix(basename, ".vif")
}
func getValidVolumeName(basename string) string {
if isValidVolume(basename) {
return basename[:len(basename)-4]
}
return ""
}
func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapKind) bool {
2020-11-26 23:19:43 +00:00
basename := fileInfo.Name()
2020-11-26 23:08:05 +00:00
if fileInfo.IsDir() {
return false
}
volumeName := getValidVolumeName(basename)
if volumeName == "" {
2020-11-26 23:08:05 +00:00
return false
}
2020-11-27 01:21:55 +00:00
// check for incomplete volume
2020-11-26 23:19:43 +00:00
noteFile := l.Directory + "/" + volumeName + ".note"
2020-11-26 23:08:05 +00:00
if util.FileExists(noteFile) {
note, _ := ioutil.ReadFile(noteFile)
2020-11-26 23:19:43 +00:00
glog.Warningf("volume %s was not completed: %s", volumeName, string(note))
removeVolumeFiles(l.Directory + "/" + volumeName)
removeVolumeFiles(l.IdxDirectory + "/" + volumeName)
2020-11-26 23:08:05 +00:00
return false
}
2020-11-27 01:21:55 +00:00
// parse out collection, volume id
vid, collection, err := volumeIdFromFileName(basename)
2020-11-26 23:08:05 +00:00
if err != nil {
2020-11-26 23:19:43 +00:00
glog.Warningf("get volume id failed, %s, err : %s", volumeName, err)
2020-11-26 23:08:05 +00:00
return false
}
2020-02-13 07:19:00 +00:00
2020-11-27 01:21:55 +00:00
// avoid loading one volume more than once
l.volumesLock.RLock()
_, found := l.volumes[vid]
l.volumesLock.RUnlock()
if found {
2020-11-26 23:08:05 +00:00
glog.V(1).Infof("loaded volume, %v", vid)
return true
}
2020-02-13 07:19:00 +00:00
2020-11-27 01:21:55 +00:00
// load the volume
v, e := NewVolume(l.Directory, l.IdxDirectory, collection, vid, needleMapKind, nil, nil, 0, 0)
2020-11-26 23:08:05 +00:00
if e != nil {
2020-11-26 23:19:43 +00:00
glog.V(0).Infof("new volume %s error %s", volumeName, e)
2020-11-26 23:08:05 +00:00
return false
}
2020-12-16 17:14:05 +00:00
2020-11-26 23:08:05 +00:00
l.SetVolume(vid, v)
2020-07-03 23:34:31 +00:00
2020-11-26 23:08:05 +00:00
size, _, _ := v.FileStat()
glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s",
2020-11-26 23:19:43 +00:00
l.Directory+"/"+volumeName+".dat", v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
2020-11-26 23:08:05 +00:00
return true
2016-11-13 05:24:52 +00:00
}
2016-04-27 03:45:35 +00:00
func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapKind, concurrency int) {
2016-11-13 05:24:52 +00:00
task_queue := make(chan os.FileInfo, 10*concurrency)
2016-11-11 03:53:22 +00:00
go func() {
foundVolumeNames := make(map[string]bool)
2020-11-26 23:08:05 +00:00
if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
for _, fi := range fileInfos {
volumeName := getValidVolumeName(fi.Name())
if volumeName == "" {
continue
}
if _, found := foundVolumeNames[volumeName]; !found {
foundVolumeNames[volumeName] = true
task_queue <- fi
}
2016-11-11 03:53:22 +00:00
}
}
close(task_queue)
}()
var wg sync.WaitGroup
for workerNum := 0; workerNum < concurrency; workerNum++ {
wg.Add(1)
go func() {
defer wg.Done()
2020-11-26 23:08:05 +00:00
for fi := range task_queue {
_ = l.loadExistingVolume(fi, needleMapKind)
2016-04-27 03:10:26 +00:00
}
2016-11-11 03:53:22 +00:00
}()
2016-04-27 03:10:26 +00:00
}
2016-11-11 03:53:22 +00:00
wg.Wait()
2016-11-13 05:24:52 +00:00
}
func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapKind) {
2016-11-13 05:24:52 +00:00
l.concurrentLoadingVolumes(needleMapKind, 10)
glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount)
l.loadAllEcShards()
2019-05-28 04:40:51 +00:00
glog.V(0).Infof("Store started on dir: %s with %d ec shards", l.Directory, len(l.ecVolumes))
2016-11-13 05:24:52 +00:00
2016-04-27 03:10:26 +00:00
}
2016-04-27 03:45:35 +00:00
func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) {
2019-12-03 04:49:50 +00:00
l.volumesLock.Lock()
delVolsMap := l.unmountVolumeByCollection(collection)
2019-12-03 04:49:50 +00:00
l.volumesLock.Unlock()
2019-05-30 16:47:54 +00:00
l.ecVolumesLock.Lock()
delEcVolsMap := l.unmountEcVolumeByCollection(collection)
l.ecVolumesLock.Unlock()
errChain := make(chan error, 2)
var wg sync.WaitGroup
wg.Add(2)
go func() {
for _, v := range delVolsMap {
if err := v.Destroy(); err != nil {
errChain <- err
2016-04-27 03:45:35 +00:00
}
}
wg.Done()
}()
go func() {
for _, v := range delEcVolsMap {
v.Destroy()
}
wg.Done()
}()
go func() {
wg.Wait()
close(errChain)
}()
errBuilder := strings.Builder{}
for err := range errChain {
errBuilder.WriteString(err.Error())
errBuilder.WriteString("; ")
2016-04-27 03:45:35 +00:00
}
if errBuilder.Len() > 0 {
e = fmt.Errorf(errBuilder.String())
}
2016-04-27 03:45:35 +00:00
return
}
func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (found bool, e error) {
2016-04-27 03:45:35 +00:00
v, ok := l.volumes[vid]
if !ok {
return
}
e = v.Destroy()
if e != nil {
return
}
found = true
2016-04-27 03:45:35 +00:00
delete(l.volumes, vid)
return
}
func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapKind) bool {
if fileInfo, found := l.LocateVolume(vid); found {
2020-02-13 07:19:00 +00:00
return l.loadExistingVolume(fileInfo, needleMapKind)
}
return false
}
2019-04-19 04:43:36 +00:00
func (l *DiskLocation) DeleteVolume(vid needle.VolumeId) error {
2019-12-03 04:49:50 +00:00
l.volumesLock.Lock()
defer l.volumesLock.Unlock()
2017-01-20 18:18:43 +00:00
_, ok := l.volumes[vid]
if !ok {
return fmt.Errorf("Volume not found, VolumeId: %d", vid)
}
_, err := l.deleteVolumeById(vid)
return err
}
2019-04-19 04:43:36 +00:00
func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error {
2019-12-03 04:49:50 +00:00
l.volumesLock.Lock()
defer l.volumesLock.Unlock()
2017-01-20 18:18:43 +00:00
v, ok := l.volumes[vid]
if !ok {
return fmt.Errorf("Volume not loaded, VolumeId: %d", vid)
}
v.Close()
delete(l.volumes, vid)
return nil
}
func (l *DiskLocation) unmountVolumeByCollection(collectionName string) map[needle.VolumeId]*Volume {
deltaVols := make(map[needle.VolumeId]*Volume, 0)
for k, v := range l.volumes {
if v.Collection == collectionName && !v.isCompacting {
deltaVols[k] = v
}
}
for k := range deltaVols {
delete(l.volumes, k)
}
return deltaVols
}
2019-04-19 04:43:36 +00:00
func (l *DiskLocation) SetVolume(vid needle.VolumeId, volume *Volume) {
2019-12-03 04:49:50 +00:00
l.volumesLock.Lock()
defer l.volumesLock.Unlock()
l.volumes[vid] = volume
2020-07-03 23:34:31 +00:00
volume.location = l
}
2019-04-19 04:43:36 +00:00
func (l *DiskLocation) FindVolume(vid needle.VolumeId) (*Volume, bool) {
2019-12-03 04:49:50 +00:00
l.volumesLock.RLock()
defer l.volumesLock.RUnlock()
v, ok := l.volumes[vid]
return v, ok
}
func (l *DiskLocation) VolumesLen() int {
2019-12-03 04:49:50 +00:00
l.volumesLock.RLock()
defer l.volumesLock.RUnlock()
return len(l.volumes)
}
func (l *DiskLocation) Close() {
2019-12-03 04:49:50 +00:00
l.volumesLock.Lock()
for _, v := range l.volumes {
v.Close()
}
2019-12-03 04:49:50 +00:00
l.volumesLock.Unlock()
2019-05-28 04:40:51 +00:00
l.ecVolumesLock.Lock()
2019-06-06 06:20:26 +00:00
for _, ecVolume := range l.ecVolumes {
ecVolume.Close()
}
2019-05-28 04:40:51 +00:00
l.ecVolumesLock.Unlock()
return
}
func (l *DiskLocation) LocateVolume(vid needle.VolumeId) (os.FileInfo, bool) {
if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
for _, fileInfo := range fileInfos {
2020-11-27 01:21:55 +00:00
volId, _, err := volumeIdFromFileName(fileInfo.Name())
if vid == volId && err == nil {
return fileInfo, true
}
}
}
return nil, false
}
func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64) {
l.volumesLock.RLock()
defer l.volumesLock.RUnlock()
for _, vol := range l.volumes {
if vol.IsReadOnly() {
continue
}
datSize, idxSize, _ := vol.FileStat()
unUsedSpace += volumeSizeLimit - (datSize + idxSize)
}
return
}
func (l *DiskLocation) CheckDiskSpace() {
2020-07-03 23:34:31 +00:00
for {
if dir, e := filepath.Abs(l.Directory); e == nil {
s := stats.NewDiskStatus(dir)
stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "all").Set(float64(s.All))
stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "used").Set(float64(s.Used))
stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "free").Set(float64(s.Free))
2020-07-03 23:34:31 +00:00
if (s.PercentFree < l.MinFreeSpacePercent) != l.isDiskSpaceLow {
l.isDiskSpaceLow = !l.isDiskSpaceLow
}
if l.isDiskSpaceLow {
glog.V(0).Infof("dir %s freePercent %.2f%% < min %.2f%%, isLowDiskSpace: %v", dir, s.PercentFree, l.MinFreeSpacePercent, l.isDiskSpaceLow)
} else {
glog.V(4).Infof("dir %s freePercent %.2f%% < min %.2f%%, isLowDiskSpace: %v", dir, s.PercentFree, l.MinFreeSpacePercent, l.isDiskSpaceLow)
}
}
2020-07-03 23:34:31 +00:00
time.Sleep(time.Minute)
}
2020-06-05 22:27:10 +00:00
}