seaweedfs/weed/storage/disk_location.go

204 lines
4.4 KiB
Go
Raw Normal View History

2016-04-27 03:10:26 +00:00
package storage
import (
"io/ioutil"
2016-11-11 03:53:22 +00:00
"os"
2016-04-27 03:10:26 +00:00
"strings"
"sync"
2016-04-27 03:10:26 +00:00
"fmt"
2017-01-20 18:18:43 +00:00
"github.com/chrislusf/seaweedfs/weed/glog"
2019-04-19 04:43:36 +00:00
"github.com/chrislusf/seaweedfs/weed/storage/needle"
2016-04-27 03:10:26 +00:00
)
type DiskLocation struct {
Directory string
MaxVolumeCount int
2019-04-19 04:43:36 +00:00
volumes map[needle.VolumeId]*Volume
sync.RWMutex
2016-04-27 03:10:26 +00:00
}
2016-04-27 03:45:35 +00:00
func NewDiskLocation(dir string, maxVolumeCount int) *DiskLocation {
location := &DiskLocation{Directory: dir, MaxVolumeCount: maxVolumeCount}
2019-04-19 04:43:36 +00:00
location.volumes = make(map[needle.VolumeId]*Volume)
2016-04-27 03:45:35 +00:00
return location
2016-04-27 03:10:26 +00:00
}
2019-04-19 04:43:36 +00:00
func (l *DiskLocation) volumeIdFromPath(dir os.FileInfo) (needle.VolumeId, string, error) {
2016-11-13 05:24:52 +00:00
name := dir.Name()
if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
collection := ""
2017-01-20 18:18:43 +00:00
base := name[:len(name)-len(".dat")]
2016-11-13 05:24:52 +00:00
i := strings.LastIndex(base, "_")
if i > 0 {
2017-01-20 18:18:43 +00:00
collection, base = base[0:i], base[i+1:]
2016-11-13 05:24:52 +00:00
}
2019-04-19 04:43:36 +00:00
vol, err := needle.NewVolumeId(base)
return vol, collection, err
}
return 0, "", fmt.Errorf("Path is not a volume: %s", name)
}
func (l *DiskLocation) loadExistingVolume(dir os.FileInfo, needleMapKind NeedleMapType, mutex *sync.RWMutex) {
name := dir.Name()
if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
vid, collection, err := l.volumeIdFromPath(dir)
if err == nil {
2016-11-13 05:24:52 +00:00
mutex.RLock()
_, found := l.volumes[vid]
mutex.RUnlock()
if !found {
2017-01-08 19:01:46 +00:00
if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0); e == nil {
2016-11-13 05:24:52 +00:00
mutex.Lock()
l.volumes[vid] = v
mutex.Unlock()
2019-04-19 07:39:34 +00:00
size, _, _ := v.FileStat()
2017-01-06 18:22:20 +00:00
glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s",
2019-04-19 07:39:34 +00:00
l.Directory+"/"+name, v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
// println("volume", vid, "last append at", v.lastAppendAtNs)
2016-11-13 05:24:52 +00:00
} else {
glog.V(0).Infof("new volume %s error %s", name, e)
}
2016-11-13 05:24:52 +00:00
}
}
}
}
2016-04-27 03:45:35 +00:00
func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, concurrency int) {
2016-11-13 05:24:52 +00:00
task_queue := make(chan os.FileInfo, 10*concurrency)
2016-11-11 03:53:22 +00:00
go func() {
if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
for _, dir := range dirs {
task_queue <- dir
}
}
close(task_queue)
}()
var wg sync.WaitGroup
var mutex sync.RWMutex
for workerNum := 0; workerNum < concurrency; workerNum++ {
wg.Add(1)
go func() {
defer wg.Done()
for dir := range task_queue {
2016-11-13 05:24:52 +00:00
l.loadExistingVolume(dir, needleMapKind, &mutex)
2016-04-27 03:10:26 +00:00
}
2016-11-11 03:53:22 +00:00
}()
2016-04-27 03:10:26 +00:00
}
2016-11-11 03:53:22 +00:00
wg.Wait()
2016-11-13 05:24:52 +00:00
}
func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
l.Lock()
defer l.Unlock()
l.concurrentLoadingVolumes(needleMapKind, 10)
2016-11-13 05:24:52 +00:00
2016-04-27 03:10:26 +00:00
glog.V(0).Infoln("Store started on dir:", l.Directory, "with", len(l.volumes), "volumes", "max", l.MaxVolumeCount)
}
2016-04-27 03:45:35 +00:00
func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) {
l.Lock()
defer l.Unlock()
2016-04-27 03:45:35 +00:00
for k, v := range l.volumes {
if v.Collection == collection {
e = l.deleteVolumeById(k)
if e != nil {
return
}
}
}
return
}
2019-04-19 04:43:36 +00:00
func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (e error) {
2016-04-27 03:45:35 +00:00
v, ok := l.volumes[vid]
if !ok {
return
}
e = v.Destroy()
if e != nil {
return
}
delete(l.volumes, vid)
return
}
2019-04-19 04:43:36 +00:00
func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapType) bool {
if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
for _, dir := range dirs {
volId, _, err := l.volumeIdFromPath(dir)
if vid == volId && err == nil {
var mutex sync.RWMutex
l.loadExistingVolume(dir, needleMapKind, &mutex)
return true
}
}
}
return false
}
2019-04-19 04:43:36 +00:00
func (l *DiskLocation) DeleteVolume(vid needle.VolumeId) error {
2017-01-20 18:18:43 +00:00
l.Lock()
defer l.Unlock()
_, ok := l.volumes[vid]
if !ok {
return fmt.Errorf("Volume not found, VolumeId: %d", vid)
}
return l.deleteVolumeById(vid)
}
2019-04-19 04:43:36 +00:00
func (l *DiskLocation) UnloadVolume(vid needle.VolumeId) error {
2017-01-20 18:18:43 +00:00
l.Lock()
defer l.Unlock()
v, ok := l.volumes[vid]
if !ok {
return fmt.Errorf("Volume not loaded, VolumeId: %d", vid)
}
v.Close()
delete(l.volumes, vid)
return nil
}
2019-04-19 04:43:36 +00:00
func (l *DiskLocation) SetVolume(vid needle.VolumeId, volume *Volume) {
l.Lock()
defer l.Unlock()
l.volumes[vid] = volume
}
2019-04-19 04:43:36 +00:00
func (l *DiskLocation) FindVolume(vid needle.VolumeId) (*Volume, bool) {
l.RLock()
defer l.RUnlock()
v, ok := l.volumes[vid]
return v, ok
}
func (l *DiskLocation) VolumesLen() int {
l.RLock()
defer l.RUnlock()
return len(l.volumes)
}
func (l *DiskLocation) Close() {
l.Lock()
defer l.Unlock()
for _, v := range l.volumes {
v.Close()
}
return
}