seaweedfs/weed/server/master_server.go

420 lines
13 KiB
Go
Raw Normal View History

package weed_server
import (
2022-04-06 16:17:04 +00:00
"context"
"fmt"
2022-01-24 14:09:43 +00:00
"github.com/chrislusf/seaweedfs/weed/stats"
"net/http"
"net/http/httputil"
"net/url"
"os"
"regexp"
"strings"
"sync"
"time"
2021-11-29 08:49:49 +00:00
"github.com/chrislusf/seaweedfs/weed/cluster"
"github.com/chrislusf/seaweedfs/weed/pb"
2015-05-03 19:37:49 +00:00
"github.com/chrislusf/raft"
"github.com/gorilla/mux"
2022-04-06 16:17:04 +00:00
hashicorpRaft "github.com/hashicorp/raft"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog"
2018-07-29 04:02:56 +00:00
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/sequence"
"github.com/chrislusf/seaweedfs/weed/shell"
"github.com/chrislusf/seaweedfs/weed/topology"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/wdclient"
)
2019-11-11 01:15:17 +00:00
const (
2022-04-06 16:17:04 +00:00
SequencerType = "master.sequencer.type"
SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
2022-04-07 10:31:37 +00:00
RaftServerRemovalTime = 72 * time.Minute
2019-11-11 01:15:17 +00:00
)
2019-06-23 10:08:27 +00:00
type MasterOption struct {
Master pb.ServerAddress
2020-07-14 14:34:16 +00:00
MetaFolder string
2021-08-13 00:54:34 +00:00
VolumeSizeLimitMB uint32
2020-07-14 14:34:16 +00:00
VolumePreallocate bool
// PulseSeconds int
2019-06-23 10:08:27 +00:00
DefaultReplicaPlacement string
GarbageThreshold float64
WhiteList []string
DisableHttp bool
MetricsAddress string
MetricsIntervalSec int
IsFollower bool
2019-06-23 10:08:27 +00:00
}
type MasterServer struct {
master_pb.UnimplementedSeaweedServer
2019-06-23 10:08:27 +00:00
option *MasterOption
guard *security.Guard
preallocateSize int64
2021-05-06 10:46:14 +00:00
Topo *topology.Topology
vg *topology.VolumeGrowth
vgCh chan *topology.VolumeGrowRequest
2014-02-05 09:54:52 +00:00
2020-12-13 05:19:40 +00:00
boundedLeaderChan chan int
2022-04-07 10:31:37 +00:00
2022-07-19 06:50:52 +00:00
onPeerUpdateDoneCn chan string
onPeerUpdateDoneCnExist bool
// notifying clients
clientChansLock sync.RWMutex
2021-11-06 11:07:38 +00:00
clientChans map[string]chan *master_pb.KeepConnectedResponse
2019-02-18 20:11:52 +00:00
grpcDialOption grpc.DialOption
MasterClient *wdclient.MasterClient
2020-07-14 14:34:16 +00:00
adminLocks *AdminLocks
2021-11-08 07:52:40 +00:00
Cluster *cluster.Cluster
}
func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.ServerAddress) *MasterServer {
2017-01-08 19:01:46 +00:00
v := util.GetViper()
signingKey := v.GetString("jwt.signing.key")
2019-05-04 15:42:25 +00:00
v.SetDefault("jwt.signing.expires_after_seconds", 10)
expiresAfterSec := v.GetInt("jwt.signing.expires_after_seconds")
2019-06-06 07:29:02 +00:00
readSigningKey := v.GetString("jwt.signing.read.key")
v.SetDefault("jwt.signing.read.expires_after_seconds", 60)
readExpiresAfterSec := v.GetInt("jwt.signing.read.expires_after_seconds")
v.SetDefault("master.replication.treat_replication_as_minimums", false)
replicationAsMin := v.GetBool("master.replication.treat_replication_as_minimums")
2021-05-06 10:46:14 +00:00
v.SetDefault("master.volume_growth.copy_1", 7)
v.SetDefault("master.volume_growth.copy_2", 6)
v.SetDefault("master.volume_growth.copy_3", 3)
v.SetDefault("master.volume_growth.copy_other", 1)
v.SetDefault("master.volume_growth.threshold", 0.9)
2017-01-08 19:01:46 +00:00
var preallocateSize int64
2019-06-23 10:08:27 +00:00
if option.VolumePreallocate {
preallocateSize = int64(option.VolumeSizeLimitMB) * (1 << 20)
2017-01-08 19:01:46 +00:00
}
grpcDialOption := security.LoadClientTLS(v, "grpc.master")
ms := &MasterServer{
2019-06-23 10:08:27 +00:00
option: option,
preallocateSize: preallocateSize,
2021-07-01 08:21:14 +00:00
vgCh: make(chan *topology.VolumeGrowRequest, 1<<6),
2021-11-06 11:07:38 +00:00
clientChans: make(map[string]chan *master_pb.KeepConnectedResponse),
grpcDialOption: grpcDialOption,
2022-05-02 04:59:16 +00:00
MasterClient: wdclient.NewMasterClient(grpcDialOption, "", cluster.MasterType, option.Master, "", peers),
2020-05-26 01:39:53 +00:00
adminLocks: NewAdminLocks(),
2021-11-08 07:52:40 +00:00
Cluster: cluster.NewCluster(),
}
2020-12-13 05:19:40 +00:00
ms.boundedLeaderChan = make(chan int, 16)
2022-07-19 06:50:52 +00:00
ms.onPeerUpdateDoneCn = make(chan string)
2022-04-07 10:31:37 +00:00
2022-04-06 16:17:04 +00:00
ms.MasterClient.OnPeerUpdate = ms.OnPeerUpdate
seq := ms.createSequencer(option)
if nil == seq {
glog.Fatalf("create sequencer failed.")
}
ms.Topo = topology.NewTopology("topo", seq, uint64(ms.option.VolumeSizeLimitMB)*1024*1024, 5, replicationAsMin)
ms.vg = topology.NewDefaultVolumeGrowth()
2019-06-23 10:08:27 +00:00
glog.V(0).Infoln("Volume Size Limit is", ms.option.VolumeSizeLimitMB, "MB")
2019-06-23 10:08:27 +00:00
ms.guard = security.NewGuard(ms.option.WhiteList, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec)
2015-01-05 22:20:04 +00:00
2020-12-14 08:24:41 +00:00
handleStaticResources2(r)
r.HandleFunc("/", ms.proxyToLeader(ms.uiStatusHandler))
r.HandleFunc("/ui/index.html", ms.uiStatusHandler)
2019-06-23 10:08:27 +00:00
if !ms.option.DisableHttp {
r.HandleFunc("/dir/assign", ms.proxyToLeader(ms.guard.WhiteList(ms.dirAssignHandler)))
r.HandleFunc("/dir/lookup", ms.guard.WhiteList(ms.dirLookupHandler))
r.HandleFunc("/dir/status", ms.proxyToLeader(ms.guard.WhiteList(ms.dirStatusHandler)))
r.HandleFunc("/col/delete", ms.proxyToLeader(ms.guard.WhiteList(ms.collectionDeleteHandler)))
r.HandleFunc("/vol/grow", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeGrowHandler)))
r.HandleFunc("/vol/status", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeStatusHandler)))
r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeVacuumHandler)))
r.HandleFunc("/submit", ms.guard.WhiteList(ms.submitFromMasterServerHandler))
2020-02-22 05:45:03 +00:00
/*
r.HandleFunc("/stats/health", ms.guard.WhiteList(statsHealthHandler))
r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler))
r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler))
*/
r.HandleFunc("/{fileId}", ms.redirectHandler)
}
2021-05-06 10:46:14 +00:00
ms.Topo.StartRefreshWritableVolumes(
ms.grpcDialOption,
ms.option.GarbageThreshold,
v.GetFloat64("master.volume_growth.threshold"),
ms.preallocateSize,
)
ms.ProcessGrowRequest()
if !option.IsFollower {
ms.startAdminScripts()
}
return ms
}
2014-02-05 09:54:52 +00:00
func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
2022-04-04 12:51:51 +00:00
var raftServerName string
if raftServer.raftServer != nil {
ms.Topo.RaftServer = raftServer.raftServer
ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) {
glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc()
if ms.Topo.RaftServer.Leader() != "" {
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
}
})
raftServerName = ms.Topo.RaftServer.Name()
} else if raftServer.RaftHashicorp != nil {
ms.Topo.HashicorpRaft = raftServer.RaftHashicorp
leaderCh := raftServer.RaftHashicorp.LeaderCh()
prevLeader := ms.Topo.HashicorpRaft.Leader()
go func() {
for {
select {
case isLeader := <-leaderCh:
leader := ms.Topo.HashicorpRaft.Leader()
glog.V(0).Infof("is leader %+v change event: %+v => %+v", isLeader, prevLeader, leader)
stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", leader)).Inc()
prevLeader = leader
}
}
}()
raftServerName = ms.Topo.HashicorpRaft.String()
}
if ms.Topo.IsLeader() {
2022-04-04 12:51:51 +00:00
glog.V(0).Infoln("[", raftServerName, "]", "I am the leader!")
2014-02-10 07:37:29 +00:00
} else {
2022-04-04 12:51:51 +00:00
if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.")
2022-04-04 12:51:51 +00:00
} else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
glog.V(0).Infoln("[", ms.Topo.HashicorpRaft.String(), "]", ms.Topo.HashicorpRaft.Leader(), "is the leader.")
}
2014-02-10 07:37:29 +00:00
}
2014-02-05 09:54:52 +00:00
}
func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
2014-02-05 09:54:52 +00:00
return func(w http.ResponseWriter, r *http.Request) {
if ms.Topo.IsLeader() {
2014-02-05 09:54:52 +00:00
f(w, r)
2022-04-07 14:37:40 +00:00
return
}
var raftServerLeader string
if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
raftServerLeader = ms.Topo.RaftServer.Leader()
} else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
raftServerLeader = string(ms.Topo.HashicorpRaft.Leader())
}
if raftServerLeader == "" {
f(w, r)
2022-04-07 14:37:40 +00:00
return
}
ms.boundedLeaderChan <- 1
defer func() { <-ms.boundedLeaderChan }()
targetUrl, err := url.Parse("http://" + raftServerLeader)
if err != nil {
writeJsonError(w, r, http.StatusInternalServerError,
fmt.Errorf("Leader URL http://%s Parse Error: %v", raftServerLeader, err))
return
}
glog.V(4).Infoln("proxying to leader", raftServerLeader)
proxy := httputil.NewSingleHostReverseProxy(targetUrl)
director := proxy.Director
proxy.Director = func(req *http.Request) {
actualHost, err := security.GetActualRemoteHost(req)
if err == nil {
req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost)
}
director(req)
2014-02-05 09:54:52 +00:00
}
2022-04-07 14:37:40 +00:00
proxy.Transport = util.Transport
proxy.ServeHTTP(w, r)
2014-02-05 09:54:52 +00:00
}
}
func (ms *MasterServer) startAdminScripts() {
v := util.GetViper()
adminScripts := v.GetString("master.maintenance.scripts")
if adminScripts == "" {
return
}
2021-11-29 08:49:49 +00:00
glog.V(0).Infof("adminScripts: %v", adminScripts)
v.SetDefault("master.maintenance.sleep_minutes", 17)
sleepMinutes := v.GetInt("master.maintenance.sleep_minutes")
scriptLines := strings.Split(adminScripts, "\n")
if !strings.Contains(adminScripts, "lock") {
scriptLines = append(append([]string{}, "lock"), scriptLines...)
scriptLines = append(scriptLines, "unlock")
}
masterAddress := string(ms.option.Master)
var shellOptions shell.ShellOptions
shellOptions.GrpcDialOption = security.LoadClientTLS(v, "grpc.master")
shellOptions.Masters = &masterAddress
shellOptions.Directory = "/"
2022-05-03 14:12:33 +00:00
emptyFilerGroup := ""
shellOptions.FilerGroup = &emptyFilerGroup
commandEnv := shell.NewCommandEnv(&shellOptions)
reg, _ := regexp.Compile(`'.*?'|".*?"|\S+`)
go commandEnv.MasterClient.KeepConnectedToMaster()
go func() {
commandEnv.MasterClient.WaitUntilConnected()
for {
time.Sleep(time.Duration(sleepMinutes) * time.Minute)
if ms.Topo.IsLeader() {
2022-05-02 04:59:16 +00:00
shellOptions.FilerAddress = ms.GetOneFiler(cluster.FilerGroup(*shellOptions.FilerGroup))
if shellOptions.FilerAddress == "" {
continue
}
for _, line := range scriptLines {
for _, c := range strings.Split(line, ";") {
processEachCmd(reg, c, commandEnv)
}
}
}
}
}()
}
func processEachCmd(reg *regexp.Regexp, line string, commandEnv *shell.CommandEnv) {
cmds := reg.FindAllString(line, -1)
if len(cmds) == 0 {
return
}
args := make([]string, len(cmds[1:]))
for i := range args {
args[i] = strings.Trim(string(cmds[1+i]), "\"'")
}
cmd := strings.ToLower(cmds[0])
for _, c := range shell.Commands {
if c.Name() == cmd {
glog.V(0).Infof("executing: %s %v", cmd, args)
if err := c.Do(args, commandEnv, os.Stdout); err != nil {
glog.V(0).Infof("error: %v", err)
}
}
}
}
func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer {
var seq sequence.Sequencer
v := util.GetViper()
seqType := strings.ToLower(v.GetString(SequencerType))
glog.V(1).Infof("[%s] : [%s]", SequencerType, seqType)
2019-11-11 01:15:17 +00:00
switch strings.ToLower(seqType) {
case "snowflake":
var err error
snowflakeId := v.GetInt(SequencerSnowflakeId)
seq, err = sequence.NewSnowflakeSequencer(string(option.Master), snowflakeId)
if err != nil {
glog.Error(err)
seq = nil
}
default:
seq = sequence.NewMemorySequencer()
}
return seq
}
2022-04-06 16:17:04 +00:00
func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startFrom time.Time) {
2022-04-06 16:17:04 +00:00
if update.NodeType != cluster.MasterType || ms.Topo.HashicorpRaft == nil {
return
}
2022-04-07 13:52:13 +00:00
glog.V(4).Infof("OnPeerUpdate: %+v", update)
2022-04-06 16:17:04 +00:00
peerAddress := pb.ServerAddress(update.Address)
2022-04-07 10:31:37 +00:00
peerName := string(peerAddress)
2022-04-07 11:23:22 +00:00
isLeader := ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader
2022-04-06 16:17:04 +00:00
if update.IsAdd {
2022-04-07 11:23:22 +00:00
if isLeader {
2022-04-07 10:31:37 +00:00
raftServerFound := false
for _, server := range ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers {
if string(server.ID) == peerName {
raftServerFound = true
}
}
if !raftServerFound {
2022-04-07 11:23:22 +00:00
glog.V(0).Infof("adding new raft server: %s", peerName)
2022-04-07 10:31:37 +00:00
ms.Topo.HashicorpRaft.AddVoter(
hashicorpRaft.ServerID(peerName),
hashicorpRaft.ServerAddress(peerAddress.ToGrpcAddress()), 0, 0)
}
}
2022-07-19 06:50:52 +00:00
if ms.onPeerUpdateDoneCnExist {
ms.onPeerUpdateDoneCn <- peerName
2022-04-06 16:17:04 +00:00
}
2022-04-07 11:23:22 +00:00
} else if isLeader {
2022-04-06 16:17:04 +00:00
go func(peerName string) {
2022-07-19 06:50:52 +00:00
raftServerRemovalTimeAfter := time.After(RaftServerRemovalTime)
raftServerPingTicker := time.NewTicker(5 * time.Minute)
defer func() {
ms.onPeerUpdateDoneCnExist = false
}()
2022-04-06 16:17:04 +00:00
for {
select {
case <-raftServerPingTicker.C:
err := ms.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
_, err := client.Ping(context.Background(), &master_pb.PingRequest{
Target: peerName,
TargetType: cluster.MasterType,
})
return err
})
if err != nil {
glog.Warningf("raft server %s ping failed %+v", peerName, err)
} else {
glog.V(0).Infof("raft server %s remove canceled on ping success", peerName)
return
}
2022-07-19 06:50:52 +00:00
case <-raftServerRemovalTimeAfter:
2022-04-07 10:31:37 +00:00
err := ms.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
_, err := client.RaftRemoveServer(context.Background(), &master_pb.RaftRemoveServerRequest{
Id: peerName,
Force: false,
})
return err
})
if err != nil {
2022-04-07 11:23:22 +00:00
glog.Warningf("failed to removing old raft server %s: %v", peerName, err)
return
2022-04-06 16:17:04 +00:00
}
2022-07-19 06:50:52 +00:00
glog.V(0).Infof("old raft server %s removed", peerName)
2022-04-06 16:17:04 +00:00
return
2022-07-19 06:50:52 +00:00
case peerDone := <-ms.onPeerUpdateDoneCn:
2022-04-07 10:31:37 +00:00
if peerName == peerDone {
glog.V(0).Infof("raft server %s remove canceled on onPeerUpdate", peerName)
2022-04-07 10:31:37 +00:00
return
}
2022-04-06 16:17:04 +00:00
}
}
2022-04-07 11:23:22 +00:00
}(peerName)
2022-07-19 06:50:52 +00:00
ms.onPeerUpdateDoneCnExist = true
2022-04-06 16:17:04 +00:00
}
}