seaweedfs/weed/command/master.go

315 lines
11 KiB
Go
Raw Normal View History

package command
import (
"fmt"
2022-10-06 16:30:30 +00:00
hashicorpRaft "github.com/hashicorp/raft"
"net/http"
"os"
"path"
"strings"
"time"
2022-07-30 17:26:56 +00:00
"golang.org/x/exp/slices"
"github.com/gorilla/mux"
"github.com/seaweedfs/raft/protobuf"
"github.com/spf13/viper"
"google.golang.org/grpc/reflection"
2022-07-30 17:26:56 +00:00
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
"github.com/seaweedfs/seaweedfs/weed/util"
)
2019-06-23 10:08:27 +00:00
var (
m MasterOptions
)
type MasterOptions struct {
2020-07-14 14:34:16 +00:00
port *int
2021-09-12 09:19:10 +00:00
portGrpc *int
2020-07-14 14:34:16 +00:00
ip *string
ipBind *string
metaFolder *string
peers *string
volumeSizeLimitMB *uint
volumePreallocate *bool
// pulseSeconds *int
2019-06-23 10:08:27 +00:00
defaultReplication *string
garbageThreshold *float64
whiteList *string
disableHttp *bool
metricsAddress *string
metricsIntervalSec *int
2020-10-03 09:03:41 +00:00
raftResumeState *bool
2022-01-19 16:43:22 +00:00
metricsHttpPort *int
heartbeatInterval *time.Duration
electionTimeout *time.Duration
2022-04-04 08:50:56 +00:00
raftHashicorp *bool
raftBootstrap *bool
2019-06-23 10:08:27 +00:00
}
func init() {
2012-09-04 02:18:02 +00:00
cmdMaster.Run = runMaster // break init cycle
2019-06-23 10:08:27 +00:00
m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
2021-09-20 21:05:59 +00:00
m.portGrpc = cmdMaster.Flag.Int("port.grpc", 0, "grpc listen port")
m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address, also used as identifier")
m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
2019-06-23 10:08:27 +00:00
m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
2020-04-13 19:58:45 +00:00
m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
2019-06-23 10:08:27 +00:00
m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
// m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
m.defaultReplication = cmdMaster.Flag.String("defaultReplication", "", "Default replication type if not specified.")
2019-06-23 10:08:27 +00:00
m.garbageThreshold = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
m.whiteList = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
m.disableHttp = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
m.metricsAddress = cmdMaster.Flag.String("metrics.address", "", "Prometheus gateway address <host>:<port>")
2019-06-23 10:08:27 +00:00
m.metricsIntervalSec = cmdMaster.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
2022-01-19 16:43:22 +00:00
m.metricsHttpPort = cmdMaster.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
2020-10-03 17:16:47 +00:00
m.raftResumeState = cmdMaster.Flag.Bool("resumeState", false, "resume previous state on start master server")
m.heartbeatInterval = cmdMaster.Flag.Duration("heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
m.electionTimeout = cmdMaster.Flag.Duration("electionTimeout", 10*time.Second, "election timeout of master servers")
2022-04-04 08:50:56 +00:00
m.raftHashicorp = cmdMaster.Flag.Bool("raftHashicorp", false, "use hashicorp raft")
m.raftBootstrap = cmdMaster.Flag.Bool("raftBootstrap", false, "Whether to bootstrap the Raft cluster")
}
var cmdMaster = &Command{
2012-09-04 02:18:02 +00:00
UsageLine: "master -port=9333",
Short: "start a master server",
2019-02-10 05:07:12 +00:00
Long: `start a master server to provide volume=>location mapping service and sequence number of file ids
The configuration file "security.toml" is read from ".", "$HOME/.seaweedfs/", "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order.
2019-02-10 05:07:12 +00:00
The example security.toml configuration file can be generated by "weed scaffold -config=security"
`,
}
var (
2019-06-23 10:08:27 +00:00
masterCpuProfile = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
masterMemProfile = cmdMaster.Flag.String("memprofile", "", "memory profile output file")
)
func runMaster(cmd *Command, args []string) bool {
2019-02-18 20:11:52 +00:00
util.LoadConfiguration("security", false)
util.LoadConfiguration("master", false)
2019-02-18 20:11:52 +00:00
grace.SetupProfiling(*masterCpuProfile, *masterMemProfile)
2017-06-22 08:33:58 +00:00
parent, _ := util.FullPath(*m.metaFolder).DirAndName()
2020-10-25 03:12:04 +00:00
if util.FileExists(string(parent)) && !util.FileExists(*m.metaFolder) {
os.MkdirAll(*m.metaFolder, 0755)
}
if err := util.TestFolderWritable(util.ResolvePath(*m.metaFolder)); err != nil {
2019-06-23 10:08:27 +00:00
glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *m.metaFolder, err)
}
2019-07-28 08:55:05 +00:00
2022-08-07 08:34:32 +00:00
masterWhiteList := util.StringSplit(*m.whiteList, ",")
2019-06-23 10:08:27 +00:00
if *m.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
}
go stats_collect.StartMetricsServer(*m.ipBind, *m.metricsHttpPort)
2019-07-28 08:55:05 +00:00
startMaster(m, masterWhiteList)
2019-07-28 08:55:05 +00:00
return true
}
2019-07-28 08:55:05 +00:00
func startMaster(masterOption MasterOptions, masterWhiteList []string) {
backend.LoadConfiguration(util.GetViper())
2021-09-20 21:05:59 +00:00
if *masterOption.portGrpc == 0 {
*masterOption.portGrpc = 10000 + *masterOption.port
}
if *masterOption.ipBind == "" {
*masterOption.ipBind = *masterOption.ip
}
2021-09-20 21:05:59 +00:00
myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.portGrpc, *masterOption.peers)
masterPeers := make(map[string]pb.ServerAddress)
for _, peer := range peers {
masterPeers[string(peer)] = peer
}
2019-07-28 08:55:05 +00:00
r := mux.NewRouter()
ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers)
2021-09-08 02:29:42 +00:00
listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
2020-06-02 07:10:35 +00:00
glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
masterListener, masterLocalListener, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0)
if e != nil {
2015-01-14 01:04:41 +00:00
glog.Fatalf("Master startup error: %v", e)
2012-09-28 17:21:06 +00:00
}
2019-07-28 08:55:05 +00:00
// start raftServer
metaDir := path.Join(*masterOption.metaFolder, fmt.Sprintf("m%d", *masterOption.port))
raftServerOption := &weed_server.RaftServerOption{
GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.master"),
2022-03-26 20:13:19 +00:00
Peers: masterPeers,
ServerAddr: myMasterAddress,
DataDir: util.ResolvePath(metaDir),
Topo: ms.Topo,
RaftResumeState: *masterOption.raftResumeState,
HeartbeatInterval: *masterOption.heartbeatInterval,
ElectionTimeout: *masterOption.electionTimeout,
2022-04-04 08:50:56 +00:00
RaftBootstrap: *m.raftBootstrap,
}
2022-04-04 08:50:56 +00:00
var raftServer *weed_server.RaftServer
var err error
if *m.raftHashicorp {
2022-04-04 12:51:51 +00:00
if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
glog.Fatalf("NewHashicorpRaftServer: %s", err)
}
2022-04-04 08:50:56 +00:00
} else {
raftServer, err = weed_server.NewRaftServer(raftServerOption)
2022-04-04 12:51:51 +00:00
if raftServer == nil {
glog.Fatalf("please verify %s is writable, see https://github.com/seaweedfs/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
2022-04-04 12:51:51 +00:00
}
2019-07-28 08:55:05 +00:00
}
ms.SetRaftServer(raftServer)
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
2022-07-30 17:26:56 +00:00
r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD")
2022-04-04 08:50:56 +00:00
if *m.raftHashicorp {
2022-04-04 14:16:06 +00:00
r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
2022-04-04 08:50:56 +00:00
}
2019-07-28 08:55:05 +00:00
// starting grpc server
2021-09-12 09:19:10 +00:00
grpcPort := *masterOption.portGrpc
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*masterOption.ipBind, grpcPort, 0)
2019-07-28 08:55:05 +00:00
if err != nil {
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
}
2020-03-04 08:39:47 +00:00
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
2019-07-28 08:55:05 +00:00
master_pb.RegisterSeaweedServer(grpcS, ms)
2022-04-04 08:50:56 +00:00
if *m.raftHashicorp {
raftServer.TransportManager.Register(grpcS)
} else {
protobuf.RegisterRaftServer(grpcS, raftServer)
}
2019-07-28 08:55:05 +00:00
reflection.Register(grpcS)
2020-06-02 07:10:35 +00:00
glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.Version(), *masterOption.ipBind, grpcPort)
if grpcLocalL != nil {
go grpcS.Serve(grpcLocalL)
}
2019-07-28 08:55:05 +00:00
go grpcS.Serve(grpcL)
2017-01-10 09:01:12 +00:00
2022-04-04 12:51:51 +00:00
timeSleep := 1500 * time.Millisecond
if !*m.raftHashicorp {
go func() {
time.Sleep(timeSleep)
ms.Topo.RaftServerAccessLock.RLock()
isEmptyMaster := ms.Topo.RaftServer.Leader() == "" && ms.Topo.RaftServer.IsLogEmpty()
if isEmptyMaster && isTheFirstOne(myMasterAddress, peers) && ms.MasterClient.FindLeaderFromOtherPeers(myMasterAddress) == "" {
raftServer.DoJoinCommand()
}
ms.Topo.RaftServerAccessLock.RUnlock()
2022-04-04 12:51:51 +00:00
}()
}
go ms.MasterClient.KeepConnectedToMaster()
// start http server
var (
clientCertFile,
certFile,
keyFile string
)
useTLS := false
useMTLS := false
if viper.GetString("https.master.key") != "" {
useTLS = true
certFile = viper.GetString("https.master.cert")
keyFile = viper.GetString("https.master.key")
}
if viper.GetString("https.master.ca") != "" {
useMTLS = true
clientCertFile = viper.GetString("https.master.ca")
}
2017-01-10 09:01:12 +00:00
httpS := &http.Server{Handler: r}
if masterLocalListener != nil {
go httpS.Serve(masterLocalListener)
}
if useMTLS {
httpS.TLSConfig = security.LoadClientTLSHTTP(clientCertFile)
}
if useTLS {
go httpS.ServeTLS(masterListener, certFile, keyFile)
} else {
go httpS.Serve(masterListener)
}
2017-01-10 09:01:12 +00:00
2022-10-06 16:30:30 +00:00
grace.OnInterrupt(ms.Shutdown)
grace.OnInterrupt(grpcS.Stop)
2022-10-06 16:30:30 +00:00
grace.OnReload(func() {
if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader {
ms.Topo.HashicorpRaft.LeadershipTransfer()
}
})
2019-07-28 08:55:05 +00:00
select {}
}
2018-08-12 21:25:31 +00:00
func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers string) (masterAddress pb.ServerAddress, cleanedPeers []pb.ServerAddress) {
2020-04-13 19:58:45 +00:00
glog.V(0).Infof("current: %s:%d peers:%s", masterIp, masterPort, peers)
masterAddress = pb.NewServerAddress(masterIp, masterPort, masterGrpcPort)
cleanedPeers = pb.ServerAddresses(peers).ToAddresses()
2018-08-12 21:25:31 +00:00
hasSelf := false
for _, peer := range cleanedPeers {
if peer.ToHttpAddress() == masterAddress.ToHttpAddress() {
2018-08-12 21:25:31 +00:00
hasSelf = true
break
}
}
if !hasSelf {
cleanedPeers = append(cleanedPeers, masterAddress)
2018-08-12 21:25:31 +00:00
}
if len(cleanedPeers)%2 == 0 {
glog.Fatalf("Only odd number of masters are supported: %+v", cleanedPeers)
2018-08-12 21:25:31 +00:00
}
return
}
2019-06-23 10:08:27 +00:00
func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
slices.SortFunc(peers, func(a, b pb.ServerAddress) bool {
return strings.Compare(string(a), string(b)) < 0
})
if len(peers) <= 0 {
return true
}
return self == peers[0]
}
2019-06-23 10:08:27 +00:00
func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOption {
masterAddress := pb.NewServerAddress(*m.ip, *m.port, *m.portGrpc)
2019-06-23 10:08:27 +00:00
return &weed_server.MasterOption{
Master: masterAddress,
2020-07-14 14:34:16 +00:00
MetaFolder: *m.metaFolder,
2021-08-13 00:54:34 +00:00
VolumeSizeLimitMB: uint32(*m.volumeSizeLimitMB),
2020-07-14 14:34:16 +00:00
VolumePreallocate: *m.volumePreallocate,
// PulseSeconds: *m.pulseSeconds,
2019-06-23 10:08:27 +00:00
DefaultReplicaPlacement: *m.defaultReplication,
GarbageThreshold: *m.garbageThreshold,
WhiteList: whiteList,
DisableHttp: *m.disableHttp,
MetricsAddress: *m.metricsAddress,
MetricsIntervalSec: *m.metricsIntervalSec,
}
2019-06-23 22:30:16 +00:00
}