2016-06-03 01:09:14 +00:00
package command
2012-08-07 08:29:22 +00:00
import (
2019-06-05 08:30:24 +00:00
"net/http"
"os"
2020-10-07 08:25:39 +00:00
"sort"
2019-06-05 08:30:24 +00:00
"strings"
2020-10-07 08:25:39 +00:00
"time"
2020-01-29 17:09:55 +00:00
2022-03-14 23:22:52 +00:00
"github.com/chrislusf/raft/protobuf"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/gorilla/mux"
"github.com/spf13/viper"
"google.golang.org/grpc/reflection"
2020-07-17 05:50:14 +00:00
"github.com/chrislusf/seaweedfs/weed/util/grace"
2019-02-19 06:38:14 +00:00
"github.com/chrislusf/seaweedfs/weed/glog"
2020-03-04 08:39:47 +00:00
"github.com/chrislusf/seaweedfs/weed/pb"
2019-02-19 06:38:14 +00:00
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
2019-02-18 20:11:52 +00:00
"github.com/chrislusf/seaweedfs/weed/security"
2022-03-14 23:22:52 +00:00
weed_server "github.com/chrislusf/seaweedfs/weed/server"
2019-11-29 02:33:18 +00:00
"github.com/chrislusf/seaweedfs/weed/storage/backend"
2019-02-19 06:38:14 +00:00
"github.com/chrislusf/seaweedfs/weed/util"
2012-08-07 08:29:22 +00:00
)
2019-06-23 10:08:27 +00:00
var (
m MasterOptions
)
type MasterOptions struct {
2020-07-14 14:34:16 +00:00
port * int
2021-09-12 09:19:10 +00:00
portGrpc * int
2020-07-14 14:34:16 +00:00
ip * string
ipBind * string
metaFolder * string
peers * string
volumeSizeLimitMB * uint
volumePreallocate * bool
2020-06-04 17:52:01 +00:00
// pulseSeconds *int
2019-06-23 10:08:27 +00:00
defaultReplication * string
garbageThreshold * float64
whiteList * string
disableHttp * bool
metricsAddress * string
metricsIntervalSec * int
2020-10-03 09:03:41 +00:00
raftResumeState * bool
2022-01-19 16:43:22 +00:00
metricsHttpPort * int
2022-02-12 10:19:49 +00:00
heartbeatInterval * time . Duration
electionTimeout * time . Duration
2019-06-23 10:08:27 +00:00
}
2012-08-07 08:29:22 +00:00
func init ( ) {
2012-09-04 02:18:02 +00:00
cmdMaster . Run = runMaster // break init cycle
2019-06-23 10:08:27 +00:00
m . port = cmdMaster . Flag . Int ( "port" , 9333 , "http listen port" )
2021-09-20 21:05:59 +00:00
m . portGrpc = cmdMaster . Flag . Int ( "port.grpc" , 0 , "grpc listen port" )
2021-03-24 00:27:57 +00:00
m . ip = cmdMaster . Flag . String ( "ip" , util . DetectedHostAddress ( ) , "master <ip>|<server> address, also used as identifier" )
2022-03-11 22:02:39 +00:00
m . ipBind = cmdMaster . Flag . String ( "ip.bind" , "" , "ip address to bind to. If empty, default to same as -ip option." )
2019-06-23 10:08:27 +00:00
m . metaFolder = cmdMaster . Flag . String ( "mdir" , os . TempDir ( ) , "data directory to store meta data" )
2020-04-13 19:58:45 +00:00
m . peers = cmdMaster . Flag . String ( "peers" , "" , "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095" )
2019-06-23 10:08:27 +00:00
m . volumeSizeLimitMB = cmdMaster . Flag . Uint ( "volumeSizeLimitMB" , 30 * 1000 , "Master stops directing writes to oversized volumes." )
m . volumePreallocate = cmdMaster . Flag . Bool ( "volumePreallocate" , false , "Preallocate disk space for volumes." )
2020-06-04 17:52:01 +00:00
// m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
2021-08-14 09:54:13 +00:00
m . defaultReplication = cmdMaster . Flag . String ( "defaultReplication" , "" , "Default replication type if not specified." )
2019-06-23 10:08:27 +00:00
m . garbageThreshold = cmdMaster . Flag . Float64 ( "garbageThreshold" , 0.3 , "threshold to vacuum and reclaim spaces" )
m . whiteList = cmdMaster . Flag . String ( "whiteList" , "" , "comma separated Ip addresses having write permission. No limit if empty." )
m . disableHttp = cmdMaster . Flag . Bool ( "disableHttp" , false , "disable http requests, only gRPC operations are allowed." )
2020-09-19 07:03:00 +00:00
m . metricsAddress = cmdMaster . Flag . String ( "metrics.address" , "" , "Prometheus gateway address <host>:<port>" )
2019-06-23 10:08:27 +00:00
m . metricsIntervalSec = cmdMaster . Flag . Int ( "metrics.intervalSeconds" , 15 , "Prometheus push interval in seconds" )
2022-01-19 16:43:22 +00:00
m . metricsHttpPort = cmdMaster . Flag . Int ( "metricsPort" , 0 , "Prometheus metrics listen port" )
2020-10-03 17:16:47 +00:00
m . raftResumeState = cmdMaster . Flag . Bool ( "resumeState" , false , "resume previous state on start master server" )
2022-02-12 10:19:49 +00:00
m . heartbeatInterval = cmdMaster . Flag . Duration ( "heartbeatInterval" , 300 * time . Millisecond , "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)" )
m . electionTimeout = cmdMaster . Flag . Duration ( "electionTimeout" , 10 * time . Second , "election timeout of master servers" )
2012-08-07 08:29:22 +00:00
}
var cmdMaster = & Command {
2012-09-04 02:18:02 +00:00
UsageLine : "master -port=9333" ,
Short : "start a master server" ,
2019-02-10 05:07:12 +00:00
Long : ` start a master server to provide volume = > location mapping service and sequence number of file ids
2020-12-08 00:46:48 +00:00
The configuration file "security.toml" is read from "." , "$HOME/.seaweedfs/" , "/usr/local/etc/seaweedfs/" , or "/etc/seaweedfs/" , in that order .
2019-02-10 05:07:12 +00:00
The example security . toml configuration file can be generated by "weed scaffold -config=security"
2012-08-07 08:29:22 +00:00
` ,
}
var (
2019-06-23 10:08:27 +00:00
masterCpuProfile = cmdMaster . Flag . String ( "cpuprofile" , "" , "cpu profile output file" )
masterMemProfile = cmdMaster . Flag . String ( "memprofile" , "" , "memory profile output file" )
2012-08-07 08:29:22 +00:00
)
func runMaster ( cmd * Command , args [ ] string ) bool {
2019-02-18 20:11:52 +00:00
2019-06-05 08:30:24 +00:00
util . LoadConfiguration ( "security" , false )
util . LoadConfiguration ( "master" , false )
2019-02-18 20:11:52 +00:00
2020-04-28 06:10:23 +00:00
grace . SetupProfiling ( * masterCpuProfile , * masterMemProfile )
2017-06-22 08:33:58 +00:00
2020-10-24 01:18:46 +00:00
parent , _ := util . FullPath ( * m . metaFolder ) . DirAndName ( )
2020-10-25 03:12:04 +00:00
if util . FileExists ( string ( parent ) ) && ! util . FileExists ( * m . metaFolder ) {
2020-10-24 01:18:46 +00:00
os . MkdirAll ( * m . metaFolder , 0755 )
}
2020-07-17 05:50:14 +00:00
if err := util . TestFolderWritable ( util . ResolvePath ( * m . metaFolder ) ) ; err != nil {
2019-06-23 10:08:27 +00:00
glog . Fatalf ( "Check Meta Folder (-mdir) Writable %s : %s" , * m . metaFolder , err )
2013-12-09 21:27:09 +00:00
}
2019-07-28 08:55:05 +00:00
var masterWhiteList [ ] string
2019-06-23 10:08:27 +00:00
if * m . whiteList != "" {
masterWhiteList = strings . Split ( * m . whiteList , "," )
2013-08-13 16:31:19 +00:00
}
2019-06-23 10:08:27 +00:00
if * m . volumeSizeLimitMB > util . VolumeSizeLimitGB * 1000 {
2017-07-17 04:40:47 +00:00
glog . Fatalf ( "volumeSizeLimitMB should be smaller than 30000" )
}
2013-08-13 16:31:19 +00:00
2022-01-19 16:43:22 +00:00
go stats_collect . StartMetricsServer ( * m . metricsHttpPort )
2019-07-28 08:55:05 +00:00
startMaster ( m , masterWhiteList )
2014-02-15 01:10:49 +00:00
2019-07-28 08:55:05 +00:00
return true
}
2014-04-26 05:09:42 +00:00
2019-07-28 08:55:05 +00:00
func startMaster ( masterOption MasterOptions , masterWhiteList [ ] string ) {
2019-07-28 10:58:13 +00:00
2020-01-29 17:09:55 +00:00
backend . LoadConfiguration ( util . GetViper ( ) )
2019-11-29 02:33:18 +00:00
2021-09-20 21:05:59 +00:00
if * masterOption . portGrpc == 0 {
* masterOption . portGrpc = 10000 + * masterOption . port
}
2022-03-11 22:02:39 +00:00
if * masterOption . ipBind == "" {
2022-03-17 21:25:25 +00:00
* masterOption . ipBind = * masterOption . ip
2022-03-11 22:02:39 +00:00
}
2021-09-20 21:05:59 +00:00
2021-09-13 05:47:52 +00:00
myMasterAddress , peers := checkPeers ( * masterOption . ip , * masterOption . port , * masterOption . portGrpc , * masterOption . peers )
2019-07-28 10:58:13 +00:00
2019-07-28 08:55:05 +00:00
r := mux . NewRouter ( )
2019-07-28 10:58:13 +00:00
ms := weed_server . NewMasterServer ( r , masterOption . toMasterOption ( masterWhiteList ) , peers )
2021-09-08 02:29:42 +00:00
listeningAddress := util . JoinHostPort ( * masterOption . ipBind , * masterOption . port )
2020-06-02 07:10:35 +00:00
glog . V ( 0 ) . Infof ( "Start Seaweed Master %s at %s" , util . Version ( ) , listeningAddress )
2022-03-16 05:28:18 +00:00
masterListener , masterLocalListner , e := util . NewIpAndLocalListeners ( * masterOption . ipBind , * masterOption . port , 0 )
2014-03-20 18:07:15 +00:00
if e != nil {
2015-01-14 01:04:41 +00:00
glog . Fatalf ( "Master startup error: %v" , e )
2012-09-28 17:21:06 +00:00
}
2022-03-14 23:22:52 +00:00
2019-07-28 08:55:05 +00:00
// start raftServer
2022-02-12 10:19:49 +00:00
raftServerOption := & weed_server . RaftServerOption {
GrpcDialOption : security . LoadClientTLS ( util . GetViper ( ) , "grpc.master" ) ,
Peers : peers ,
ServerAddr : myMasterAddress ,
DataDir : util . ResolvePath ( * masterOption . metaFolder ) ,
Topo : ms . Topo ,
RaftResumeState : * masterOption . raftResumeState ,
HeartbeatInterval : * masterOption . heartbeatInterval ,
ElectionTimeout : * masterOption . electionTimeout ,
}
raftServer , err := weed_server . NewRaftServer ( raftServerOption )
2019-07-28 08:55:05 +00:00
if raftServer == nil {
2020-10-02 18:01:20 +00:00
glog . Fatalf ( "please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s" , * masterOption . metaFolder , err )
2019-07-28 08:55:05 +00:00
}
ms . SetRaftServer ( raftServer )
r . HandleFunc ( "/cluster/status" , raftServer . StatusHandler ) . Methods ( "GET" )
// starting grpc server
2021-09-12 09:19:10 +00:00
grpcPort := * masterOption . portGrpc
2022-03-16 05:28:18 +00:00
grpcL , grpcLocalL , err := util . NewIpAndLocalListeners ( * masterOption . ipBind , grpcPort , 0 )
2019-07-28 08:55:05 +00:00
if err != nil {
glog . Fatalf ( "master failed to listen on grpc port %d: %v" , grpcPort , err )
}
2020-03-04 08:39:47 +00:00
grpcS := pb . NewGrpcServer ( security . LoadServerTLS ( util . GetViper ( ) , "grpc.master" ) )
2019-07-28 08:55:05 +00:00
master_pb . RegisterSeaweedServer ( grpcS , ms )
protobuf . RegisterRaftServer ( grpcS , raftServer )
reflection . Register ( grpcS )
2020-06-02 07:10:35 +00:00
glog . V ( 0 ) . Infof ( "Start Seaweed Master %s grpc server at %s:%d" , util . Version ( ) , * masterOption . ipBind , grpcPort )
2022-03-16 05:28:18 +00:00
if grpcLocalL != nil {
go grpcS . Serve ( grpcLocalL )
}
2019-07-28 08:55:05 +00:00
go grpcS . Serve ( grpcL )
2017-01-10 09:01:12 +00:00
2020-10-07 08:25:39 +00:00
go func ( ) {
time . Sleep ( 1500 * time . Millisecond )
if ms . Topo . RaftServer . Leader ( ) == "" && ms . Topo . RaftServer . IsLogEmpty ( ) && isTheFirstOne ( myMasterAddress , peers ) {
2020-10-16 18:02:44 +00:00
if ms . MasterClient . FindLeaderFromOtherPeers ( myMasterAddress ) == "" {
2020-10-07 08:25:39 +00:00
raftServer . DoJoinCommand ( )
}
}
} ( )
2021-11-06 00:52:15 +00:00
go ms . MasterClient . KeepConnectedToMaster ( )
2019-07-28 10:58:13 +00:00
2019-01-18 22:14:47 +00:00
// start http server
2022-03-14 23:22:52 +00:00
var (
clientCertFile ,
certFile ,
keyFile string
)
useTLS := false
useMTLS := false
if viper . GetString ( "https.master.key" ) != "" {
useTLS = true
certFile = viper . GetString ( "https.master.cert" )
keyFile = viper . GetString ( "https.master.key" )
}
if viper . GetString ( "https.master.ca" ) != "" {
useMTLS = true
clientCertFile = viper . GetString ( "https.master.ca" )
}
2017-01-10 09:01:12 +00:00
httpS := & http . Server { Handler : r }
2022-03-16 05:28:18 +00:00
if masterLocalListner != nil {
go httpS . Serve ( masterLocalListner )
}
2022-03-14 23:22:52 +00:00
if useMTLS {
httpS . TLSConfig = security . LoadClientTLSHTTP ( clientCertFile )
}
if useTLS {
go httpS . ServeTLS ( masterListener , certFile , keyFile )
} else {
go httpS . Serve ( masterListener )
}
2017-01-10 09:01:12 +00:00
2019-07-28 08:55:05 +00:00
select { }
2012-08-07 08:29:22 +00:00
}
2018-08-12 21:25:31 +00:00
2021-09-13 05:47:52 +00:00
func checkPeers ( masterIp string , masterPort int , masterGrpcPort int , peers string ) ( masterAddress pb . ServerAddress , cleanedPeers [ ] pb . ServerAddress ) {
2020-04-13 19:58:45 +00:00
glog . V ( 0 ) . Infof ( "current: %s:%d peers:%s" , masterIp , masterPort , peers )
2021-09-13 05:47:52 +00:00
masterAddress = pb . NewServerAddress ( masterIp , masterPort , masterGrpcPort )
cleanedPeers = pb . ServerAddresses ( peers ) . ToAddresses ( )
2018-08-12 21:25:31 +00:00
hasSelf := false
for _ , peer := range cleanedPeers {
2021-09-13 05:47:52 +00:00
if peer . ToHttpAddress ( ) == masterAddress . ToHttpAddress ( ) {
2018-08-12 21:25:31 +00:00
hasSelf = true
break
}
}
if ! hasSelf {
2019-07-28 10:58:13 +00:00
cleanedPeers = append ( cleanedPeers , masterAddress )
2018-08-12 21:25:31 +00:00
}
2019-07-28 10:58:13 +00:00
if len ( cleanedPeers ) % 2 == 0 {
2021-09-13 05:47:52 +00:00
glog . Fatalf ( "Only odd number of masters are supported: %+v" , cleanedPeers )
2018-08-12 21:25:31 +00:00
}
return
}
2019-06-23 10:08:27 +00:00
2021-09-13 05:47:52 +00:00
func isTheFirstOne ( self pb . ServerAddress , peers [ ] pb . ServerAddress ) bool {
sort . Slice ( peers , func ( i , j int ) bool {
return strings . Compare ( string ( peers [ i ] ) , string ( peers [ j ] ) ) < 0
} )
2020-10-07 08:25:39 +00:00
if len ( peers ) <= 0 {
return true
}
return self == peers [ 0 ]
}
2019-06-23 10:08:27 +00:00
func ( m * MasterOptions ) toMasterOption ( whiteList [ ] string ) * weed_server . MasterOption {
2021-09-13 05:47:52 +00:00
masterAddress := pb . NewServerAddress ( * m . ip , * m . port , * m . portGrpc )
2019-06-23 10:08:27 +00:00
return & weed_server . MasterOption {
2021-09-13 05:47:52 +00:00
Master : masterAddress ,
2020-07-14 14:34:16 +00:00
MetaFolder : * m . metaFolder ,
2021-08-13 00:54:34 +00:00
VolumeSizeLimitMB : uint32 ( * m . volumeSizeLimitMB ) ,
2020-07-14 14:34:16 +00:00
VolumePreallocate : * m . volumePreallocate ,
2020-06-04 17:52:01 +00:00
// PulseSeconds: *m.pulseSeconds,
2019-06-23 10:08:27 +00:00
DefaultReplicaPlacement : * m . defaultReplication ,
GarbageThreshold : * m . garbageThreshold ,
WhiteList : whiteList ,
DisableHttp : * m . disableHttp ,
MetricsAddress : * m . metricsAddress ,
MetricsIntervalSec : * m . metricsIntervalSec ,
}
2019-06-23 22:30:16 +00:00
}