2012-09-10 07:18:07 +00:00
|
|
|
package topology
|
|
|
|
|
|
|
|
import (
|
2012-09-11 00:08:52 +00:00
|
|
|
"errors"
|
2015-01-08 07:54:50 +00:00
|
|
|
"fmt"
|
2012-09-10 07:18:07 +00:00
|
|
|
"math/rand"
|
2013-07-16 04:34:43 +00:00
|
|
|
"sync"
|
2018-11-23 08:24:51 +00:00
|
|
|
"time"
|
2014-10-26 18:34:55 +00:00
|
|
|
|
2016-06-03 01:09:14 +00:00
|
|
|
"github.com/chrislusf/seaweedfs/weed/glog"
|
|
|
|
"github.com/chrislusf/seaweedfs/weed/storage"
|
2012-09-10 07:18:07 +00:00
|
|
|
)
|
|
|
|
|
2014-03-10 18:43:54 +00:00
|
|
|
// mapping from volume to its locations, inverted from server to volume
|
2012-09-10 07:18:07 +00:00
|
|
|
type VolumeLayout struct {
|
2016-06-27 22:28:23 +00:00
|
|
|
rp *storage.ReplicaPlacement
|
|
|
|
ttl *storage.TTL
|
|
|
|
vid2location map[storage.VolumeId]*VolumeLocationList
|
|
|
|
writables []storage.VolumeId // transient array of writable volume id
|
2017-05-23 00:05:27 +00:00
|
|
|
readonlyVolumes map[storage.VolumeId]bool // transient set of readonly volumes
|
2016-06-27 22:28:23 +00:00
|
|
|
oversizedVolumes map[storage.VolumeId]bool // set of oversized volumes
|
|
|
|
volumeSizeLimit uint64
|
|
|
|
accessLock sync.RWMutex
|
2012-09-10 07:18:07 +00:00
|
|
|
}
|
|
|
|
|
2018-11-23 08:24:51 +00:00
|
|
|
type VolumeLayoutStats struct {
|
|
|
|
TotalSize uint64
|
|
|
|
UsedSize uint64
|
|
|
|
FileCount uint64
|
|
|
|
}
|
|
|
|
|
2014-09-20 19:38:59 +00:00
|
|
|
func NewVolumeLayout(rp *storage.ReplicaPlacement, ttl *storage.TTL, volumeSizeLimit uint64) *VolumeLayout {
|
2012-09-10 07:18:07 +00:00
|
|
|
return &VolumeLayout{
|
2016-06-27 22:28:23 +00:00
|
|
|
rp: rp,
|
|
|
|
ttl: ttl,
|
|
|
|
vid2location: make(map[storage.VolumeId]*VolumeLocationList),
|
|
|
|
writables: *new([]storage.VolumeId),
|
2017-05-23 00:05:27 +00:00
|
|
|
readonlyVolumes: make(map[storage.VolumeId]bool),
|
2016-06-27 22:28:23 +00:00
|
|
|
oversizedVolumes: make(map[storage.VolumeId]bool),
|
|
|
|
volumeSizeLimit: volumeSizeLimit,
|
2012-09-10 07:18:07 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-01-08 07:54:50 +00:00
|
|
|
func (vl *VolumeLayout) String() string {
|
|
|
|
return fmt.Sprintf("rp:%v, ttl:%v, vid2location:%v, writables:%v, volumeSizeLimit:%v", vl.rp, vl.ttl, vl.vid2location, vl.writables, vl.volumeSizeLimit)
|
|
|
|
}
|
|
|
|
|
2012-09-10 07:18:07 +00:00
|
|
|
func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
|
2013-07-16 04:34:43 +00:00
|
|
|
vl.accessLock.Lock()
|
|
|
|
defer vl.accessLock.Unlock()
|
|
|
|
|
2012-09-10 07:18:07 +00:00
|
|
|
if _, ok := vl.vid2location[v.Id]; !ok {
|
2012-09-19 23:48:04 +00:00
|
|
|
vl.vid2location[v.Id] = NewVolumeLocationList()
|
2012-09-10 07:18:07 +00:00
|
|
|
}
|
2014-03-19 11:48:13 +00:00
|
|
|
vl.vid2location[v.Id].Set(dn)
|
2018-11-22 22:31:10 +00:00
|
|
|
glog.V(4).Infof("volume %d added to %s len %s copy %d", v.Id, dn.Id(), vl.vid2location[v.Id].Length(), v.ReplicaPlacement.GetCopyCount())
|
2016-08-09 12:12:39 +00:00
|
|
|
for _, dn := range vl.vid2location[v.Id].list {
|
|
|
|
if v_info, err := dn.GetVolumesById(v.Id); err == nil {
|
|
|
|
if v_info.ReadOnly {
|
|
|
|
glog.V(3).Infof("vid %d removed from writable", v.Id)
|
|
|
|
vl.removeFromWritable(v.Id)
|
2017-05-23 00:05:27 +00:00
|
|
|
vl.readonlyVolumes[v.Id] = true
|
2016-08-09 12:12:39 +00:00
|
|
|
return
|
2017-05-23 00:05:27 +00:00
|
|
|
} else {
|
|
|
|
delete(vl.readonlyVolumes, v.Id)
|
2016-08-09 12:12:39 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
glog.V(3).Infof("vid %d removed from writable", v.Id)
|
|
|
|
vl.removeFromWritable(v.Id)
|
2017-05-23 00:05:27 +00:00
|
|
|
delete(vl.readonlyVolumes, v.Id)
|
2016-08-09 12:12:39 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
2014-03-19 11:48:13 +00:00
|
|
|
if vl.vid2location[v.Id].Length() == vl.rp.GetCopyCount() && vl.isWritable(v) {
|
2016-06-27 22:28:23 +00:00
|
|
|
if _, ok := vl.oversizedVolumes[v.Id]; !ok {
|
|
|
|
vl.addToWritable(v.Id)
|
|
|
|
}
|
2014-03-19 11:48:13 +00:00
|
|
|
} else {
|
2016-06-27 22:28:23 +00:00
|
|
|
vl.rememberOversizedVolumne(v)
|
2014-03-19 11:48:13 +00:00
|
|
|
vl.removeFromWritable(v.Id)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-27 22:28:23 +00:00
|
|
|
func (vl *VolumeLayout) rememberOversizedVolumne(v *storage.VolumeInfo) {
|
|
|
|
if vl.isOversized(v) {
|
|
|
|
vl.oversizedVolumes[v.Id] = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-09-20 19:38:59 +00:00
|
|
|
func (vl *VolumeLayout) UnRegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
|
|
|
|
vl.accessLock.Lock()
|
|
|
|
defer vl.accessLock.Unlock()
|
|
|
|
|
|
|
|
vl.removeFromWritable(v.Id)
|
|
|
|
delete(vl.vid2location, v.Id)
|
|
|
|
}
|
|
|
|
|
2016-04-14 08:28:40 +00:00
|
|
|
func (vl *VolumeLayout) addToWritable(vid storage.VolumeId) {
|
2014-03-19 11:48:13 +00:00
|
|
|
for _, id := range vl.writables {
|
|
|
|
if vid == id {
|
|
|
|
return
|
2012-09-11 00:08:52 +00:00
|
|
|
}
|
2012-09-10 07:18:07 +00:00
|
|
|
}
|
2014-03-19 11:48:13 +00:00
|
|
|
vl.writables = append(vl.writables, vid)
|
2012-09-10 07:18:07 +00:00
|
|
|
}
|
|
|
|
|
2016-06-27 22:28:23 +00:00
|
|
|
func (vl *VolumeLayout) isOversized(v *storage.VolumeInfo) bool {
|
2016-06-29 08:05:00 +00:00
|
|
|
return uint64(v.Size) >= vl.volumeSizeLimit
|
2016-06-27 22:28:23 +00:00
|
|
|
}
|
|
|
|
|
2013-01-20 11:40:04 +00:00
|
|
|
func (vl *VolumeLayout) isWritable(v *storage.VolumeInfo) bool {
|
2016-06-29 08:05:00 +00:00
|
|
|
return !vl.isOversized(v) &&
|
2013-04-15 02:30:26 +00:00
|
|
|
v.Version == storage.CurrentVersion &&
|
|
|
|
!v.ReadOnly
|
2012-12-18 00:48:54 +00:00
|
|
|
}
|
|
|
|
|
2018-07-11 09:01:33 +00:00
|
|
|
func (vl *VolumeLayout) isEmpty() bool {
|
|
|
|
vl.accessLock.RLock()
|
|
|
|
defer vl.accessLock.RUnlock()
|
|
|
|
|
|
|
|
return len(vl.vid2location) == 0
|
|
|
|
}
|
|
|
|
|
2013-01-04 09:02:52 +00:00
|
|
|
func (vl *VolumeLayout) Lookup(vid storage.VolumeId) []*DataNode {
|
2016-04-14 08:28:40 +00:00
|
|
|
vl.accessLock.RLock()
|
|
|
|
defer vl.accessLock.RUnlock()
|
|
|
|
|
2013-04-15 02:30:26 +00:00
|
|
|
if location := vl.vid2location[vid]; location != nil {
|
|
|
|
return location.list
|
|
|
|
}
|
2013-03-20 08:23:15 +00:00
|
|
|
return nil
|
2012-09-23 22:45:26 +00:00
|
|
|
}
|
|
|
|
|
2014-03-10 18:43:54 +00:00
|
|
|
func (vl *VolumeLayout) ListVolumeServers() (nodes []*DataNode) {
|
2016-04-14 08:28:40 +00:00
|
|
|
vl.accessLock.RLock()
|
|
|
|
defer vl.accessLock.RUnlock()
|
|
|
|
|
2014-03-10 18:43:54 +00:00
|
|
|
for _, location := range vl.vid2location {
|
|
|
|
nodes = append(nodes, location.list...)
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2015-04-06 21:17:36 +00:00
|
|
|
func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (*storage.VolumeId, uint64, *VolumeLocationList, error) {
|
2016-04-14 08:28:40 +00:00
|
|
|
vl.accessLock.RLock()
|
|
|
|
defer vl.accessLock.RUnlock()
|
|
|
|
|
2012-09-11 00:08:52 +00:00
|
|
|
len_writers := len(vl.writables)
|
|
|
|
if len_writers <= 0 {
|
2013-08-09 06:57:22 +00:00
|
|
|
glog.V(0).Infoln("No more writable volumes!")
|
2012-09-17 00:31:15 +00:00
|
|
|
return nil, 0, nil, errors.New("No more writable volumes!")
|
2012-09-11 00:08:52 +00:00
|
|
|
}
|
2014-04-13 08:29:52 +00:00
|
|
|
if option.DataCenter == "" {
|
2013-06-20 01:10:38 +00:00
|
|
|
vid := vl.writables[rand.Intn(len_writers)]
|
|
|
|
locationList := vl.vid2location[vid]
|
|
|
|
if locationList != nil {
|
|
|
|
return &vid, count, locationList, nil
|
|
|
|
}
|
|
|
|
return nil, 0, nil, errors.New("Strangely vid " + vid.String() + " is on no machine!")
|
2015-03-10 07:20:31 +00:00
|
|
|
}
|
|
|
|
var vid storage.VolumeId
|
|
|
|
var locationList *VolumeLocationList
|
|
|
|
counter := 0
|
|
|
|
for _, v := range vl.writables {
|
|
|
|
volumeLocationList := vl.vid2location[v]
|
|
|
|
for _, dn := range volumeLocationList.list {
|
|
|
|
if dn.GetDataCenter().Id() == NodeId(option.DataCenter) {
|
|
|
|
if option.Rack != "" && dn.GetRack().Id() != NodeId(option.Rack) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if option.DataNode != "" && dn.Id() != NodeId(option.DataNode) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
counter++
|
|
|
|
if rand.Intn(counter) < 1 {
|
|
|
|
vid, locationList = v, volumeLocationList
|
2013-06-20 01:10:38 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2012-09-11 00:08:52 +00:00
|
|
|
}
|
2015-03-10 07:20:31 +00:00
|
|
|
return &vid, count, locationList, nil
|
2012-09-17 00:31:15 +00:00
|
|
|
}
|
|
|
|
|
2014-04-13 08:29:52 +00:00
|
|
|
func (vl *VolumeLayout) GetActiveVolumeCount(option *VolumeGrowOption) int {
|
2016-04-14 08:28:40 +00:00
|
|
|
vl.accessLock.RLock()
|
|
|
|
defer vl.accessLock.RUnlock()
|
|
|
|
|
2014-04-13 08:29:52 +00:00
|
|
|
if option.DataCenter == "" {
|
2013-06-20 01:10:38 +00:00
|
|
|
return len(vl.writables)
|
|
|
|
}
|
|
|
|
counter := 0
|
|
|
|
for _, v := range vl.writables {
|
|
|
|
for _, dn := range vl.vid2location[v].list {
|
2014-04-13 08:29:52 +00:00
|
|
|
if dn.GetDataCenter().Id() == NodeId(option.DataCenter) {
|
|
|
|
if option.Rack != "" && dn.GetRack().Id() != NodeId(option.Rack) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if option.DataNode != "" && dn.Id() != NodeId(option.DataNode) {
|
|
|
|
continue
|
|
|
|
}
|
2013-06-20 01:10:38 +00:00
|
|
|
counter++
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return counter
|
2012-09-17 00:31:15 +00:00
|
|
|
}
|
|
|
|
|
2012-09-19 23:48:04 +00:00
|
|
|
func (vl *VolumeLayout) removeFromWritable(vid storage.VolumeId) bool {
|
2013-08-12 23:39:49 +00:00
|
|
|
toDeleteIndex := -1
|
|
|
|
for k, id := range vl.writables {
|
|
|
|
if id == vid {
|
|
|
|
toDeleteIndex = k
|
|
|
|
break
|
2012-09-19 23:48:04 +00:00
|
|
|
}
|
|
|
|
}
|
2013-08-12 23:39:49 +00:00
|
|
|
if toDeleteIndex >= 0 {
|
|
|
|
glog.V(0).Infoln("Volume", vid, "becomes unwritable")
|
|
|
|
vl.writables = append(vl.writables[0:toDeleteIndex], vl.writables[toDeleteIndex+1:]...)
|
|
|
|
return true
|
|
|
|
}
|
2012-09-19 23:48:04 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
func (vl *VolumeLayout) setVolumeWritable(vid storage.VolumeId) bool {
|
|
|
|
for _, v := range vl.writables {
|
|
|
|
if v == vid {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
2013-08-09 06:57:22 +00:00
|
|
|
glog.V(0).Infoln("Volume", vid, "becomes writable")
|
2012-09-19 23:48:04 +00:00
|
|
|
vl.writables = append(vl.writables, vid)
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
func (vl *VolumeLayout) SetVolumeUnavailable(dn *DataNode, vid storage.VolumeId) bool {
|
2013-07-16 04:34:43 +00:00
|
|
|
vl.accessLock.Lock()
|
|
|
|
defer vl.accessLock.Unlock()
|
|
|
|
|
2014-03-10 18:43:54 +00:00
|
|
|
if location, ok := vl.vid2location[vid]; ok {
|
|
|
|
if location.Remove(dn) {
|
|
|
|
if location.Length() < vl.rp.GetCopyCount() {
|
|
|
|
glog.V(0).Infoln("Volume", vid, "has", location.Length(), "replica, less than required", vl.rp.GetCopyCount())
|
|
|
|
return vl.removeFromWritable(vid)
|
|
|
|
}
|
2012-09-20 09:11:08 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
2012-09-19 08:45:30 +00:00
|
|
|
}
|
2012-09-19 23:48:04 +00:00
|
|
|
func (vl *VolumeLayout) SetVolumeAvailable(dn *DataNode, vid storage.VolumeId) bool {
|
2013-07-16 04:34:43 +00:00
|
|
|
vl.accessLock.Lock()
|
|
|
|
defer vl.accessLock.Unlock()
|
|
|
|
|
2014-03-19 11:48:13 +00:00
|
|
|
vl.vid2location[vid].Set(dn)
|
|
|
|
if vl.vid2location[vid].Length() >= vl.rp.GetCopyCount() {
|
|
|
|
return vl.setVolumeWritable(vid)
|
2012-09-19 23:48:04 +00:00
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
2012-09-19 08:45:30 +00:00
|
|
|
|
2012-09-19 23:48:04 +00:00
|
|
|
func (vl *VolumeLayout) SetVolumeCapacityFull(vid storage.VolumeId) bool {
|
2013-07-16 04:34:43 +00:00
|
|
|
vl.accessLock.Lock()
|
|
|
|
defer vl.accessLock.Unlock()
|
|
|
|
|
2013-08-09 06:57:22 +00:00
|
|
|
// glog.V(0).Infoln("Volume", vid, "reaches full capacity.")
|
2012-09-19 23:48:04 +00:00
|
|
|
return vl.removeFromWritable(vid)
|
2012-09-19 08:45:30 +00:00
|
|
|
}
|
|
|
|
|
2013-11-12 10:19:06 +00:00
|
|
|
func (vl *VolumeLayout) ToMap() map[string]interface{} {
|
2012-09-17 00:31:15 +00:00
|
|
|
m := make(map[string]interface{})
|
2014-03-03 06:16:54 +00:00
|
|
|
m["replication"] = vl.rp.String()
|
2014-09-20 19:38:59 +00:00
|
|
|
m["ttl"] = vl.ttl.String()
|
2012-09-17 00:31:15 +00:00
|
|
|
m["writables"] = vl.writables
|
|
|
|
//m["locations"] = vl.vid2location
|
|
|
|
return m
|
2012-09-10 07:18:07 +00:00
|
|
|
}
|
2018-11-23 08:24:51 +00:00
|
|
|
|
|
|
|
func (vl *VolumeLayout) Stats() *VolumeLayoutStats {
|
|
|
|
vl.accessLock.RLock()
|
|
|
|
defer vl.accessLock.RUnlock()
|
|
|
|
|
|
|
|
ret := &VolumeLayoutStats{}
|
|
|
|
|
|
|
|
freshThreshold := time.Now().Unix() - 60
|
|
|
|
|
|
|
|
for vid, vll := range vl.vid2location {
|
|
|
|
size, fileCount := vll.Stats(vid, freshThreshold)
|
|
|
|
ret.FileCount += uint64(fileCount)
|
|
|
|
ret.UsedSize += size
|
|
|
|
if vl.readonlyVolumes[vid] {
|
|
|
|
ret.TotalSize += size
|
|
|
|
} else {
|
|
|
|
ret.TotalSize += vl.volumeSizeLimit
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret
|
|
|
|
}
|