seaweedfs/weed/topology/topology.go
2022-09-10 11:26:19 -07:00

341 lines
9.5 KiB
Go

package topology
import (
"encoding/json"
"errors"
"fmt"
"math/rand"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
hashicorpRaft "github.com/hashicorp/raft"
"github.com/seaweedfs/raft"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/sequence"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type Topology struct {
vacuumLockCounter int64
NodeImpl
collectionMap *util.ConcurrentReadMap
ecShardMap map[needle.VolumeId]*EcShardLocations
ecShardMapLock sync.RWMutex
pulse int64
volumeSizeLimit uint64
replicationAsMin bool
Sequence sequence.Sequencer
chanFullVolumes chan storage.VolumeInfo
chanCrowdedVolumes chan storage.VolumeInfo
Configuration *Configuration
RaftServer raft.Server
RaftServerAccessLock sync.RWMutex
HashicorpRaft *hashicorpRaft.Raft
UuidAccessLock sync.RWMutex
UuidMap map[string][]string
}
func NewTopology(id string, seq sequence.Sequencer, volumeSizeLimit uint64, pulse int, replicationAsMin bool) *Topology {
t := &Topology{}
t.id = NodeId(id)
t.nodeType = "Topology"
t.NodeImpl.value = t
t.diskUsages = newDiskUsages()
t.children = make(map[NodeId]Node)
t.collectionMap = util.NewConcurrentReadMap()
t.ecShardMap = make(map[needle.VolumeId]*EcShardLocations)
t.pulse = int64(pulse)
t.volumeSizeLimit = volumeSizeLimit
t.replicationAsMin = replicationAsMin
t.Sequence = seq
t.chanFullVolumes = make(chan storage.VolumeInfo)
t.chanCrowdedVolumes = make(chan storage.VolumeInfo)
t.Configuration = &Configuration{}
return t
}
func (t *Topology) IsLeader() bool {
t.RaftServerAccessLock.RLock()
defer t.RaftServerAccessLock.RUnlock()
if t.RaftServer != nil {
if t.RaftServer.State() == raft.Leader {
return true
}
if leader, err := t.Leader(); err == nil {
if pb.ServerAddress(t.RaftServer.Name()) == leader {
return true
}
}
} else if t.HashicorpRaft != nil {
if t.HashicorpRaft.State() == hashicorpRaft.Leader {
return true
}
}
return false
}
func (t *Topology) Leader() (l pb.ServerAddress, err error) {
for count := 0; count < 3; count++ {
l, err = t.MaybeLeader()
if err != nil {
return
}
if l != "" {
break
}
time.Sleep(time.Duration(5+count) * time.Second)
}
return
}
func (t *Topology) MaybeLeader() (l pb.ServerAddress, err error) {
t.RaftServerAccessLock.RLock()
defer t.RaftServerAccessLock.RUnlock()
if t.RaftServer != nil {
l = pb.ServerAddress(t.RaftServer.Leader())
} else if t.HashicorpRaft != nil {
l = pb.ServerAddress(t.HashicorpRaft.Leader())
} else {
err = errors.New("Raft Server not ready yet!")
}
return
}
func (t *Topology) Lookup(collection string, vid needle.VolumeId) (dataNodes []*DataNode) {
// maybe an issue if lots of collections?
if collection == "" {
for _, c := range t.collectionMap.Items() {
if list := c.(*Collection).Lookup(vid); list != nil {
return list
}
}
} else {
if c, ok := t.collectionMap.Find(collection); ok {
return c.(*Collection).Lookup(vid)
}
}
if locations, found := t.LookupEcShards(vid); found {
for _, loc := range locations.Locations {
dataNodes = append(dataNodes, loc...)
}
return dataNodes
}
return nil
}
func (t *Topology) NextVolumeId() (needle.VolumeId, error) {
vid := t.GetMaxVolumeId()
next := vid.Next()
t.RaftServerAccessLock.RLock()
defer t.RaftServerAccessLock.RUnlock()
if t.RaftServer != nil {
if _, err := t.RaftServer.Do(NewMaxVolumeIdCommand(next)); err != nil {
return 0, err
}
} else if t.HashicorpRaft != nil {
b, err := json.Marshal(NewMaxVolumeIdCommand(next))
if err != nil {
return 0, fmt.Errorf("failed marshal NewMaxVolumeIdCommand: %+v", err)
}
if future := t.HashicorpRaft.Apply(b, time.Second); future.Error() != nil {
return 0, future.Error()
}
}
return next, nil
}
// deprecated
func (t *Topology) HasWritableVolume(option *VolumeGrowOption) bool {
vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType)
active, _ := vl.GetActiveVolumeCount(option)
return active > 0
}
func (t *Topology) PickForWrite(count uint64, option *VolumeGrowOption) (string, uint64, *VolumeLocationList, error) {
vid, count, datanodes, err := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl, option.DiskType).PickForWrite(count, option)
if err != nil {
return "", 0, nil, fmt.Errorf("failed to find writable volumes for collection:%s replication:%s ttl:%s error: %v", option.Collection, option.ReplicaPlacement.String(), option.Ttl.String(), err)
}
if datanodes.Length() == 0 {
return "", 0, nil, fmt.Errorf("no writable volumes available for collection:%s replication:%s ttl:%s", option.Collection, option.ReplicaPlacement.String(), option.Ttl.String())
}
fileId := t.Sequence.NextFileId(count)
return needle.NewFileId(*vid, fileId, rand.Uint32()).String(), count, datanodes, nil
}
func (t *Topology) GetVolumeLayout(collectionName string, rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) *VolumeLayout {
return t.collectionMap.Get(collectionName, func() interface{} {
return NewCollection(collectionName, t.volumeSizeLimit, t.replicationAsMin)
}).(*Collection).GetOrCreateVolumeLayout(rp, ttl, diskType)
}
func (t *Topology) ListCollections(includeNormalVolumes, includeEcVolumes bool) (ret []string) {
mapOfCollections := make(map[string]bool)
for _, c := range t.collectionMap.Items() {
mapOfCollections[c.(*Collection).Name] = true
}
if includeEcVolumes {
t.ecShardMapLock.RLock()
for _, ecVolumeLocation := range t.ecShardMap {
mapOfCollections[ecVolumeLocation.Collection] = true
}
t.ecShardMapLock.RUnlock()
}
for k := range mapOfCollections {
ret = append(ret, k)
}
return ret
}
func (t *Topology) FindCollection(collectionName string) (*Collection, bool) {
c, hasCollection := t.collectionMap.Find(collectionName)
if !hasCollection {
return nil, false
}
return c.(*Collection), hasCollection
}
func (t *Topology) DeleteCollection(collectionName string) {
t.collectionMap.Delete(collectionName)
}
func (t *Topology) DeleteLayout(collectionName string, rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) {
collection, found := t.FindCollection(collectionName)
if !found {
return
}
collection.DeleteVolumeLayout(rp, ttl, diskType)
if len(collection.storageType2VolumeLayout.Items()) == 0 {
t.DeleteCollection(collectionName)
}
}
func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
diskType := types.ToDiskType(v.DiskType)
vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
vl.RegisterVolume(&v, dn)
vl.EnsureCorrectWritables(&v)
}
func (t *Topology) UnRegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
glog.Infof("removing volume info: %+v from %v", v, dn.id)
diskType := types.ToDiskType(v.DiskType)
volumeLayout := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
volumeLayout.UnRegisterVolume(&v, dn)
if volumeLayout.isEmpty() {
t.DeleteLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
}
}
func (t *Topology) GetOrCreateDataCenter(dcName string) *DataCenter {
t.Lock()
defer t.Unlock()
for _, c := range t.children {
dc := c.(*DataCenter)
if string(dc.Id()) == dcName {
return dc
}
}
dc := NewDataCenter(dcName)
t.doLinkChildNode(dc)
return dc
}
func (t *Topology) SyncDataNodeRegistration(volumes []*master_pb.VolumeInformationMessage, dn *DataNode) (newVolumes, deletedVolumes []storage.VolumeInfo) {
// convert into in memory struct storage.VolumeInfo
var volumeInfos []storage.VolumeInfo
for _, v := range volumes {
if vi, err := storage.NewVolumeInfo(v); err == nil {
volumeInfos = append(volumeInfos, vi)
} else {
glog.V(0).Infof("Fail to convert joined volume information: %v", err)
}
}
// find out the delta volumes
var changedVolumes []storage.VolumeInfo
newVolumes, deletedVolumes, changedVolumes = dn.UpdateVolumes(volumeInfos)
for _, v := range newVolumes {
t.RegisterVolumeLayout(v, dn)
}
for _, v := range deletedVolumes {
t.UnRegisterVolumeLayout(v, dn)
}
for _, v := range changedVolumes {
diskType := types.ToDiskType(v.DiskType)
vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
vl.EnsureCorrectWritables(&v)
}
return
}
func (t *Topology) IncrementalSyncDataNodeRegistration(newVolumes, deletedVolumes []*master_pb.VolumeShortInformationMessage, dn *DataNode) {
var newVis, oldVis []storage.VolumeInfo
for _, v := range newVolumes {
vi, err := storage.NewVolumeInfoFromShort(v)
if err != nil {
glog.V(0).Infof("NewVolumeInfoFromShort %v: %v", v, err)
continue
}
newVis = append(newVis, vi)
}
for _, v := range deletedVolumes {
vi, err := storage.NewVolumeInfoFromShort(v)
if err != nil {
glog.V(0).Infof("NewVolumeInfoFromShort %v: %v", v, err)
continue
}
oldVis = append(oldVis, vi)
}
dn.DeltaUpdateVolumes(newVis, oldVis)
for _, vi := range newVis {
t.RegisterVolumeLayout(vi, dn)
}
for _, vi := range oldVis {
t.UnRegisterVolumeLayout(vi, dn)
}
return
}
func (t *Topology) DataNodeRegistration(dcName, rackName string, dn *DataNode) {
if dn.Parent() != nil {
return
}
// registration to topo
dc := t.GetOrCreateDataCenter(dcName)
rack := dc.GetOrCreateRack(rackName)
rack.LinkChildNode(dn)
glog.Infof("[%s] reLink To topo ", dn.Id())
}