seaweedfs/weed/cluster/cluster.go

316 lines
7.8 KiB
Go
Raw Normal View History

2021-11-08 07:52:40 +00:00
package cluster
2021-11-04 07:54:38 +00:00
import (
"github.com/chrislusf/seaweedfs/weed/pb"
2021-11-06 11:07:38 +00:00
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
2021-11-04 07:54:38 +00:00
"math"
"sync"
"time"
)
2021-11-08 08:09:11 +00:00
const (
2022-04-01 23:55:26 +00:00
MasterType = "master"
VolumeServerType = "volumeServer"
FilerType = "filer"
BrokerType = "broker"
2021-11-08 08:09:11 +00:00
)
2022-05-02 04:59:16 +00:00
type FilerGroup string
type Filers struct {
filers map[pb.ServerAddress]*ClusterNode
leaders *Leaders
}
type Leaders struct {
leaders [3]pb.ServerAddress
}
2021-11-04 07:54:38 +00:00
type ClusterNode struct {
Address pb.ServerAddress
Version string
counter int
2022-05-30 23:23:52 +00:00
CreatedTs time.Time
2021-11-04 07:54:38 +00:00
}
type Cluster struct {
2022-05-02 04:59:16 +00:00
filerGroup2filers map[FilerGroup]*Filers
filersLock sync.RWMutex
brokers map[pb.ServerAddress]*ClusterNode
brokersLock sync.RWMutex
2021-11-04 07:54:38 +00:00
}
func NewCluster() *Cluster {
return &Cluster{
2022-05-02 04:59:16 +00:00
filerGroup2filers: make(map[FilerGroup]*Filers),
brokers: make(map[pb.ServerAddress]*ClusterNode),
2021-11-04 07:54:38 +00:00
}
}
2022-05-02 04:59:16 +00:00
func (cluster *Cluster) getFilers(filerGroup FilerGroup, createIfNotFound bool) *Filers {
cluster.filersLock.Lock()
defer cluster.filersLock.Unlock()
filers, found := cluster.filerGroup2filers[filerGroup]
if !found && createIfNotFound {
filers = &Filers{
filers: make(map[pb.ServerAddress]*ClusterNode),
leaders: &Leaders{},
}
cluster.filerGroup2filers[filerGroup] = filers
}
return filers
}
func (cluster *Cluster) AddClusterNode(ns, nodeType string, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
filerGroup := FilerGroup(ns)
2021-11-04 07:54:38 +00:00
switch nodeType {
2021-11-08 08:09:11 +00:00
case FilerType:
2022-05-02 04:59:16 +00:00
filers := cluster.getFilers(filerGroup, true)
if existingNode, found := filers.filers[address]; found {
2021-11-04 07:54:38 +00:00
existingNode.counter++
2021-11-06 11:07:38 +00:00
return nil
2021-11-04 07:54:38 +00:00
}
2022-05-02 04:59:16 +00:00
filers.filers[address] = &ClusterNode{
2021-11-04 07:54:38 +00:00
Address: address,
Version: version,
counter: 1,
2022-05-30 23:23:52 +00:00
CreatedTs: time.Now(),
2021-11-04 07:54:38 +00:00
}
2022-05-02 04:59:16 +00:00
return cluster.ensureFilerLeaders(filers, true, filerGroup, nodeType, address)
2021-11-09 16:50:55 +00:00
case BrokerType:
cluster.brokersLock.Lock()
defer cluster.brokersLock.Unlock()
if existingNode, found := cluster.brokers[address]; found {
existingNode.counter++
return nil
}
cluster.brokers[address] = &ClusterNode{
Address: address,
Version: version,
counter: 1,
2022-05-30 23:23:52 +00:00
CreatedTs: time.Now(),
2021-11-09 16:50:55 +00:00
}
return []*master_pb.KeepConnectedResponse{
{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(address),
IsAdd: true,
},
},
}
2021-11-08 08:09:11 +00:00
case MasterType:
2022-04-06 16:17:04 +00:00
return []*master_pb.KeepConnectedResponse{
{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(address),
IsAdd: true,
},
},
}
2021-11-04 07:54:38 +00:00
}
2021-11-06 11:07:38 +00:00
return nil
2021-11-04 07:54:38 +00:00
}
2022-05-02 04:59:16 +00:00
func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
filerGroup := FilerGroup(ns)
2021-11-04 07:54:38 +00:00
switch nodeType {
2021-11-08 08:09:11 +00:00
case FilerType:
2022-05-02 04:59:16 +00:00
filers := cluster.getFilers(filerGroup, false)
if filers == nil {
return nil
}
if existingNode, found := filers.filers[address]; !found {
2021-11-06 11:07:38 +00:00
return nil
2021-11-04 07:54:38 +00:00
} else {
existingNode.counter--
if existingNode.counter <= 0 {
2022-05-02 04:59:16 +00:00
delete(filers.filers, address)
return cluster.ensureFilerLeaders(filers, false, filerGroup, nodeType, address)
2021-11-04 07:54:38 +00:00
}
}
2021-11-09 16:50:55 +00:00
case BrokerType:
cluster.brokersLock.Lock()
defer cluster.brokersLock.Unlock()
if existingNode, found := cluster.brokers[address]; !found {
return nil
} else {
existingNode.counter--
if existingNode.counter <= 0 {
delete(cluster.brokers, address)
return []*master_pb.KeepConnectedResponse{
{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(address),
IsAdd: false,
},
},
}
}
}
2021-11-08 08:09:11 +00:00
case MasterType:
2022-04-06 16:17:04 +00:00
return []*master_pb.KeepConnectedResponse{
{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(address),
IsAdd: false,
},
},
}
2021-11-04 07:54:38 +00:00
}
2021-11-06 11:07:38 +00:00
return nil
2021-11-04 07:54:38 +00:00
}
2022-05-02 04:59:16 +00:00
func (cluster *Cluster) ListClusterNode(filerGroup FilerGroup, nodeType string) (nodes []*ClusterNode) {
2021-11-04 07:54:38 +00:00
switch nodeType {
2021-11-08 08:09:11 +00:00
case FilerType:
2022-05-02 04:59:16 +00:00
filers := cluster.getFilers(filerGroup, false)
if filers == nil {
return
}
2021-11-08 08:13:40 +00:00
cluster.filersLock.RLock()
defer cluster.filersLock.RUnlock()
2022-05-02 04:59:16 +00:00
for _, node := range filers.filers {
2021-11-04 07:54:38 +00:00
nodes = append(nodes, node)
}
2021-11-09 16:50:55 +00:00
case BrokerType:
cluster.brokersLock.RLock()
defer cluster.brokersLock.RUnlock()
for _, node := range cluster.brokers {
nodes = append(nodes, node)
}
2021-11-08 08:09:11 +00:00
case MasterType:
2021-11-04 07:54:38 +00:00
}
return
}
2021-11-06 21:23:35 +00:00
2022-05-02 04:59:16 +00:00
func (cluster *Cluster) IsOneLeader(filerGroup FilerGroup, address pb.ServerAddress) bool {
filers := cluster.getFilers(filerGroup, false)
if filers == nil {
return false
}
return filers.leaders.isOneLeader(address)
2021-11-06 21:23:35 +00:00
}
2021-11-04 07:54:38 +00:00
2022-05-02 04:59:16 +00:00
func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGroup FilerGroup, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
2021-11-04 07:54:38 +00:00
if isAdd {
2022-05-02 04:59:16 +00:00
if filers.leaders.addLeaderIfVacant(address) {
2021-11-04 07:54:38 +00:00
// has added the address as one leader
2021-11-06 11:07:38 +00:00
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
2022-05-02 04:59:16 +00:00
FilerGroup: string(filerGroup),
NodeType: nodeType,
Address: string(address),
IsLeader: true,
IsAdd: true,
2021-11-06 11:07:38 +00:00
},
})
} else {
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
2022-05-02 04:59:16 +00:00
FilerGroup: string(filerGroup),
NodeType: nodeType,
Address: string(address),
IsLeader: false,
IsAdd: true,
2021-11-06 11:07:38 +00:00
},
})
2021-11-04 07:54:38 +00:00
}
} else {
2022-05-02 04:59:16 +00:00
if filers.leaders.removeLeaderIfExists(address) {
2021-11-06 11:07:38 +00:00
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
2022-05-02 04:59:16 +00:00
FilerGroup: string(filerGroup),
NodeType: nodeType,
Address: string(address),
IsLeader: true,
IsAdd: false,
2021-11-06 11:07:38 +00:00
},
})
2021-11-04 07:54:38 +00:00
// pick the freshest one, since it is less likely to go away
var shortestDuration int64 = math.MaxInt64
now := time.Now()
var candidateAddress pb.ServerAddress
2022-05-02 04:59:16 +00:00
for _, node := range filers.filers {
if filers.leaders.isOneLeader(node.Address) {
2021-11-04 07:54:38 +00:00
continue
}
2022-05-30 23:23:52 +00:00
duration := now.Sub(node.CreatedTs).Nanoseconds()
2021-11-04 07:54:38 +00:00
if duration < shortestDuration {
shortestDuration = duration
candidateAddress = node.Address
}
}
if candidateAddress != "" {
2022-05-02 04:59:16 +00:00
filers.leaders.addLeaderIfVacant(candidateAddress)
2021-11-06 11:07:38 +00:00
// added a new leader
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(candidateAddress),
IsLeader: true,
IsAdd: true,
},
})
2021-11-04 07:54:38 +00:00
}
2021-11-06 11:07:38 +00:00
} else {
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
2022-05-02 04:59:16 +00:00
FilerGroup: string(filerGroup),
NodeType: nodeType,
Address: string(address),
IsLeader: false,
IsAdd: false,
2021-11-06 11:07:38 +00:00
},
})
2021-11-04 07:54:38 +00:00
}
}
2021-11-06 11:07:38 +00:00
return
2021-11-04 07:54:38 +00:00
}
func (leaders *Leaders) addLeaderIfVacant(address pb.ServerAddress) (hasChanged bool) {
if leaders.isOneLeader(address) {
return
}
for i := 0; i < len(leaders.leaders); i++ {
if leaders.leaders[i] == "" {
leaders.leaders[i] = address
hasChanged = true
return
}
}
return
}
func (leaders *Leaders) removeLeaderIfExists(address pb.ServerAddress) (hasChanged bool) {
if !leaders.isOneLeader(address) {
return
}
for i := 0; i < len(leaders.leaders); i++ {
if leaders.leaders[i] == address {
leaders.leaders[i] = ""
hasChanged = true
return
}
}
return
}
func (leaders *Leaders) isOneLeader(address pb.ServerAddress) bool {
for i := 0; i < len(leaders.leaders); i++ {
if leaders.leaders[i] == address {
return true
}
}
return false
}
func (leaders *Leaders) GetLeaders() (addresses []pb.ServerAddress) {
for i := 0; i < len(leaders.leaders); i++ {
if leaders.leaders[i] != "" {
addresses = append(addresses, leaders.leaders[i])
}
}
return
}