2013-12-04 07:22:26 +00:00
package main
import (
"code.google.com/p/weed-fs/go/glog"
2013-12-09 21:27:09 +00:00
"code.google.com/p/weed-fs/go/util"
2013-12-04 07:22:26 +00:00
"code.google.com/p/weed-fs/go/weed/weed_server"
"github.com/gorilla/mux"
"net/http"
"os"
"runtime"
2014-05-07 17:17:06 +00:00
"runtime/pprof"
2013-12-04 07:22:26 +00:00
"strconv"
"strings"
2014-03-16 06:03:49 +00:00
"sync"
2013-12-04 07:22:26 +00:00
"time"
)
2014-05-07 17:17:06 +00:00
type ServerOptions struct {
cpuprofile * string
}
2014-03-31 03:57:25 +00:00
var (
2014-05-07 17:17:06 +00:00
serverOptions ServerOptions
2014-05-13 07:03:10 +00:00
filerOptions FilerOptions
2014-03-31 03:57:25 +00:00
)
2013-12-04 07:22:26 +00:00
func init ( ) {
cmdServer . Run = runServer // break init cycle
}
var cmdServer = & Command {
UsageLine : "server -port=8080 -dir=/tmp -max=5 -ip=server_name" ,
Short : "start a server, including volume server, and automatically elect a master server" ,
Long : ` start both a volume server to provide storage spaces
and a master server to provide volume = > location mapping service and sequence number of file ids
This is provided as a convenient way to start both volume server and master server .
2014-04-26 05:09:42 +00:00
The servers are exactly the same as starting them separately .
2013-12-04 07:22:26 +00:00
So other volume servers can use this embedded master server also .
2014-04-26 05:09:42 +00:00
Optionally , one filer server can be started . Logically , filer servers should not be in a cluster .
They run with meta data on disk , not shared . So each filer server is different .
2013-12-04 07:22:26 +00:00
` ,
}
var (
2014-04-26 05:09:42 +00:00
serverIp = cmdServer . Flag . String ( "ip" , "" , "ip or server name" )
serverPublicIp = cmdServer . Flag . String ( "publicIp" , "" , "ip or server name" )
2014-03-03 06:16:54 +00:00
serverMaxCpu = cmdServer . Flag . Int ( "maxCpu" , 0 , "maximum number of CPUs. 0 means all available CPUs" )
2014-03-20 18:07:15 +00:00
serverTimeout = cmdServer . Flag . Int ( "idleTimeout" , 10 , "connection idle seconds" )
2014-03-03 06:16:54 +00:00
serverDataCenter = cmdServer . Flag . String ( "dataCenter" , "" , "current volume server's data center name" )
serverRack = cmdServer . Flag . String ( "rack" , "" , "current volume server's rack name" )
serverWhiteListOption = cmdServer . Flag . String ( "whiteList" , "" , "comma separated Ip addresses having write permission. No limit if empty." )
2014-04-26 05:09:42 +00:00
serverPeers = cmdServer . Flag . String ( "master.peers" , "" , "other master nodes in comma separated ip:masterPort list" )
masterPort = cmdServer . Flag . Int ( "master.port" , 9333 , "master server http listen port" )
masterMetaFolder = cmdServer . Flag . String ( "master.dir" , "" , "data directory to store meta data, default to same as -dir specified" )
masterVolumeSizeLimitMB = cmdServer . Flag . Uint ( "master.volumeSizeLimitMB" , 30 * 1000 , "Master stops directing writes to oversized volumes." )
masterConfFile = cmdServer . Flag . String ( "master.conf" , "/etc/weedfs/weedfs.conf" , "xml configuration file" )
masterDefaultReplicaPlacement = cmdServer . Flag . String ( "master.defaultReplicaPlacement" , "000" , "Default replication type if not specified." )
volumePort = cmdServer . Flag . Int ( "volume.port" , 8080 , "volume server http listen port" )
2014-03-03 06:16:54 +00:00
volumeDataFolders = cmdServer . Flag . String ( "dir" , os . TempDir ( ) , "directories to store data files. dir[,dir]..." )
2014-04-26 05:09:42 +00:00
volumeMaxDataVolumeCounts = cmdServer . Flag . String ( "volume.max" , "7" , "maximum numbers of volumes, count[,count]..." )
2014-03-16 06:03:49 +00:00
volumePulse = cmdServer . Flag . Int ( "pulseSeconds" , 5 , "number of seconds between heartbeats" )
2014-03-31 03:57:25 +00:00
isStartingFiler = cmdServer . Flag . Bool ( "filer" , false , "whether to start filer" )
2013-12-04 07:22:26 +00:00
serverWhiteList [ ] string
)
2014-03-31 03:57:25 +00:00
func init ( ) {
2014-05-07 17:17:06 +00:00
serverOptions . cpuprofile = cmdServer . Flag . String ( "cpuprofile" , "" , "write cpu profile to file" )
2014-05-13 07:03:10 +00:00
filerOptions . master = cmdServer . Flag . String ( "filer.master" , "" , "default to current master server" )
filerOptions . collection = cmdServer . Flag . String ( "filer.collection" , "" , "all data will be stored in this collection" )
filerOptions . port = cmdServer . Flag . Int ( "filer.port" , 8888 , "filer server http listen port" )
filerOptions . dir = cmdServer . Flag . String ( "filer.dir" , "" , "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified" )
filerOptions . defaultReplicaPlacement = cmdServer . Flag . String ( "filer.defaultReplicaPlacement" , "" , "Default replication type if not specified during runtime." )
2014-03-31 03:57:25 +00:00
}
2013-12-04 07:22:26 +00:00
func runServer ( cmd * Command , args [ ] string ) bool {
2014-05-07 17:17:06 +00:00
if * serverOptions . cpuprofile != "" {
f , err := os . Create ( * serverOptions . cpuprofile )
if err != nil {
glog . Fatal ( err )
}
pprof . StartCPUProfile ( f )
defer pprof . StopCPUProfile ( )
}
2014-03-31 03:57:25 +00:00
2014-04-26 05:09:42 +00:00
if * serverPublicIp == "" {
if * serverIp == "" {
* serverPublicIp = "localhost"
} else {
* serverPublicIp = * serverIp
}
}
2014-05-13 07:03:10 +00:00
* filerOptions . master = * serverPublicIp + ":" + strconv . Itoa ( * masterPort )
2014-03-31 03:57:25 +00:00
2014-05-13 07:03:10 +00:00
if * filerOptions . defaultReplicaPlacement == "" {
* filerOptions . defaultReplicaPlacement = * masterDefaultReplicaPlacement
2014-03-31 03:57:25 +00:00
}
2013-12-04 07:22:26 +00:00
if * serverMaxCpu < 1 {
* serverMaxCpu = runtime . NumCPU ( )
}
runtime . GOMAXPROCS ( * serverMaxCpu )
folders := strings . Split ( * volumeDataFolders , "," )
maxCountStrings := strings . Split ( * volumeMaxDataVolumeCounts , "," )
maxCounts := make ( [ ] int , 0 )
for _ , maxString := range maxCountStrings {
if max , e := strconv . Atoi ( maxString ) ; e == nil {
maxCounts = append ( maxCounts , max )
} else {
2014-04-17 07:16:44 +00:00
glog . Fatalf ( "The max specified in -max not a valid number %s" , maxString )
2013-12-04 07:22:26 +00:00
}
}
if len ( folders ) != len ( maxCounts ) {
glog . Fatalf ( "%d directories by -dir, but only %d max is set by -max" , len ( folders ) , len ( maxCounts ) )
}
for _ , folder := range folders {
2013-12-09 21:27:09 +00:00
if err := util . TestFolderWritable ( folder ) ; err != nil {
glog . Fatalf ( "Check Data Folder(-dir) Writable %s : %s" , folder , err )
2013-12-04 07:22:26 +00:00
}
}
2014-03-31 03:57:25 +00:00
if * masterMetaFolder == "" {
* masterMetaFolder = folders [ 0 ]
}
2014-05-13 07:03:10 +00:00
if * filerOptions . dir == "" {
* filerOptions . dir = * masterMetaFolder + "/filer"
os . MkdirAll ( * filerOptions . dir , 0700 )
2014-03-31 03:57:25 +00:00
}
if err := util . TestFolderWritable ( * masterMetaFolder ) ; err != nil {
glog . Fatalf ( "Check Meta Folder (-mdir=\"%s\") Writable: %s" , * masterMetaFolder , err )
}
2014-05-13 07:03:10 +00:00
if err := util . TestFolderWritable ( * filerOptions . dir ) ; err != nil {
glog . Fatalf ( "Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s" , * filerOptions . dir , err )
2014-03-31 03:57:25 +00:00
}
2013-12-04 07:22:26 +00:00
if * serverWhiteListOption != "" {
serverWhiteList = strings . Split ( * serverWhiteListOption , "," )
}
2014-03-31 03:57:25 +00:00
if * isStartingFiler {
go func ( ) {
r := http . NewServeMux ( )
2014-05-13 07:03:10 +00:00
_ , nfs_err := weed_server . NewFilerServer ( r , * filerOptions . port , * filerOptions . master , * filerOptions . dir , * filerOptions . collection )
2014-03-31 03:57:25 +00:00
if nfs_err != nil {
glog . Fatalf ( nfs_err . Error ( ) )
}
2014-05-13 07:03:10 +00:00
glog . V ( 0 ) . Infoln ( "Start Weed Filer" , util . VERSION , "at port" , strconv . Itoa ( * filerOptions . port ) )
2014-03-31 03:57:25 +00:00
filerListener , e := util . NewListener (
2014-05-13 07:03:10 +00:00
":" + strconv . Itoa ( * filerOptions . port ) ,
2014-03-31 03:57:25 +00:00
time . Duration ( 10 ) * time . Second ,
)
if e != nil {
glog . Fatalf ( e . Error ( ) )
}
if e := http . Serve ( filerListener , r ) ; e != nil {
glog . Fatalf ( "Filer Fail to serve:%s" , e . Error ( ) )
}
} ( )
}
2014-03-30 18:28:04 +00:00
2014-03-16 06:03:49 +00:00
var raftWaitForMaster sync . WaitGroup
var volumeWait sync . WaitGroup
raftWaitForMaster . Add ( 1 )
volumeWait . Add ( 1 )
2013-12-04 07:22:26 +00:00
go func ( ) {
r := mux . NewRouter ( )
2014-03-25 20:46:59 +00:00
ms := weed_server . NewMasterServer ( r , * masterPort , * masterMetaFolder ,
2014-03-03 06:16:54 +00:00
* masterVolumeSizeLimitMB , * volumePulse , * masterConfFile , * masterDefaultReplicaPlacement , * garbageThreshold , serverWhiteList ,
2013-12-04 07:22:26 +00:00
)
2014-04-26 05:09:42 +00:00
glog . V ( 0 ) . Infoln ( "Start Weed Master" , util . VERSION , "at" , * serverIp + ":" + strconv . Itoa ( * masterPort ) )
masterListener , e := util . NewListener ( * serverIp + ":" + strconv . Itoa ( * masterPort ) , time . Duration ( * serverTimeout ) * time . Second )
2014-03-20 18:07:15 +00:00
if e != nil {
glog . Fatalf ( e . Error ( ) )
2013-12-04 07:22:26 +00:00
}
2013-12-09 21:27:09 +00:00
go func ( ) {
2014-03-16 06:03:49 +00:00
raftWaitForMaster . Wait ( )
2013-12-09 21:27:09 +00:00
time . Sleep ( 100 * time . Millisecond )
2014-04-26 05:09:42 +00:00
myAddress := * serverPublicIp + ":" + strconv . Itoa ( * masterPort )
2013-12-09 21:34:05 +00:00
var peers [ ] string
if * serverPeers != "" {
peers = strings . Split ( * serverPeers , "," )
2013-12-09 21:27:09 +00:00
}
2014-04-17 06:43:27 +00:00
raftServer := weed_server . NewRaftServer ( r , peers , myAddress , * masterMetaFolder , ms . Topo , * volumePulse )
2014-02-05 09:54:52 +00:00
ms . SetRaftServer ( raftServer )
2014-03-16 06:03:49 +00:00
volumeWait . Done ( )
2013-12-09 21:27:09 +00:00
} ( )
2014-03-16 06:03:49 +00:00
raftWaitForMaster . Done ( )
2014-03-20 18:07:15 +00:00
if e := http . Serve ( masterListener , r ) ; e != nil {
glog . Fatalf ( "Master Fail to serve:%s" , e . Error ( ) )
2013-12-04 07:22:26 +00:00
}
} ( )
2014-03-16 06:03:49 +00:00
volumeWait . Wait ( )
2013-12-09 21:27:09 +00:00
time . Sleep ( 100 * time . Millisecond )
r := http . NewServeMux ( )
2014-05-13 07:03:10 +00:00
volumeServer := weed_server . NewVolumeServer ( r , * serverIp , * volumePort , * serverPublicIp , folders , maxCounts ,
2013-12-04 07:22:26 +00:00
* serverIp + ":" + strconv . Itoa ( * masterPort ) , * volumePulse , * serverDataCenter , * serverRack , serverWhiteList ,
)
2014-04-26 05:09:42 +00:00
glog . V ( 0 ) . Infoln ( "Start Weed volume server" , util . VERSION , "at" , * serverIp + ":" + strconv . Itoa ( * volumePort ) )
2014-03-20 18:07:15 +00:00
volumeListener , e := util . NewListener (
* serverIp + ":" + strconv . Itoa ( * volumePort ) ,
time . Duration ( * serverTimeout ) * time . Second ,
)
2013-12-04 07:22:26 +00:00
if e != nil {
2014-03-20 18:07:15 +00:00
glog . Fatalf ( e . Error ( ) )
}
2014-05-13 07:03:10 +00:00
2014-05-13 22:04:04 +00:00
OnInterrupt ( func ( ) {
volumeServer . Shutdown ( )
pprof . StopCPUProfile ( )
} )
2014-05-13 07:03:10 +00:00
2014-03-20 18:07:15 +00:00
if e := http . Serve ( volumeListener , r ) ; e != nil {
glog . Fatalf ( "Fail to serve:%s" , e . Error ( ) )
2013-12-04 07:22:26 +00:00
}
return true
}