Refactoring volume server options.

go/weed/volume.go

@@ -13,8 +13,42 @@ import (
 	"github.com/chrislusf/weed-fs/go/weed/weed_server"
 )
 
+var (
+	v VolumeServerOptions
+)
+
+type VolumeServerOptions struct {
+	port                  *int
+	adminPort             *int
+	folders               []string
+	folderMaxLimits       []int
+	ip                    *string
+	publicIp              *string
+	bindIp                *string
+	master                *string
+	pulseSeconds          *int
+	idleConnectionTimeout *int
+	maxCpu                *int
+	dataCenter            *string
+	rack                  *string
+	whiteList             []string
+	fixJpgOrientation     *bool
+}
+
 func init() {
 	cmdVolume.Run = runVolume // break init cycle
+	v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
+	v.adminPort = cmdVolume.Flag.Int("port.admin", 8443, "https admin port, active when SSL certs are specified. Not ready yet.")
+	v.ip = cmdVolume.Flag.String("ip", "", "ip or server name")
+	v.publicIp = cmdVolume.Flag.String("publicIp", "", "Publicly accessible <ip|server_name>")
+	v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
+	v.master = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
+	v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
+	v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds")
+	v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
+	v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
+	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
+	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", true, "Adjust jpg orientation when uploading.")
 }
 
 var cmdVolume = &Command{
@@ -26,74 +60,60 @@ var cmdVolume = &Command{
 }
 
 var (
-	vport                 = cmdVolume.Flag.Int("port", 8080, "http listen port")
-	volumeSecurePort      = cmdVolume.Flag.Int("port.secure", 8443, "https listen port, active when SSL certs are specified. Not ready yet.")
 	volumeFolders         = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
 	maxVolumeCounts       = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...")
-	ip                    = cmdVolume.Flag.String("ip", "", "ip or server name")
-	publicIp              = cmdVolume.Flag.String("publicIp", "", "Publicly accessible <ip|server_name>")
-	volumeBindIp          = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
-	masterNode            = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
-	vpulse                = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
-	vTimeout              = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds")
-	vMaxCpu               = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
-	dataCenter            = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
-	rack                  = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
 	volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
-	fixJpgOrientation     = cmdVolume.Flag.Bool("images.fix.orientation", true, "Adjust jpg orientation when uploading.")
-
-	volumeWhiteList []string
 )
 
 func runVolume(cmd *Command, args []string) bool {
-	if *vMaxCpu < 1 {
+	if *v.maxCpu < 1 {
-		*vMaxCpu = runtime.NumCPU()
+		*v.maxCpu = runtime.NumCPU()
 	}
-	runtime.GOMAXPROCS(*vMaxCpu)
+	runtime.GOMAXPROCS(*v.maxCpu)
-	folders := strings.Split(*volumeFolders, ",")
+
+	v.folders = strings.Split(*volumeFolders, ",")
 	maxCountStrings := strings.Split(*maxVolumeCounts, ",")
-	maxCounts := make([]int, 0)
 	for _, maxString := range maxCountStrings {
 		if max, e := strconv.Atoi(maxString); e == nil {
-			maxCounts = append(maxCounts, max)
+			v.folderMaxLimits = append(v.folderMaxLimits, max)
 		} else {
 			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
 		}
 	}
-	if len(folders) != len(maxCounts) {
+	if len(v.folders) != len(v.folderMaxLimits) {
-		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
+		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
 	}
-	for _, folder := range folders {
+	for _, folder := range v.folders {
 		if err := util.TestFolderWritable(folder); err != nil {
 			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
 		}
 	}
-
-	if *publicIp == "" {
-		if *ip == "" {
-			*ip = "127.0.0.1"
-			*publicIp = "localhost"
-		} else {
-			*publicIp = *ip
-		}
-	}
 	if *volumeWhiteListOption != "" {
-		volumeWhiteList = strings.Split(*volumeWhiteListOption, ",")
+		v.whiteList = strings.Split(*volumeWhiteListOption, ",")
+	}
+
+	if *v.publicIp == "" {
+		if *v.ip == "" {
+			*v.ip = "127.0.0.1"
+			*v.publicIp = "localhost"
+		} else {
+			*v.publicIp = *v.ip
+		}
 	}
 
 	r := http.NewServeMux()
 
-	volumeServer := weed_server.NewVolumeServer(r, *ip, *vport, *publicIp, folders, maxCounts,
+	volumeServer := weed_server.NewVolumeServer(r, *v.ip, *v.port, *v.publicIp, v.folders, v.folderMaxLimits,
-		*masterNode, *vpulse, *dataCenter, *rack,
+		*v.master, *v.pulseSeconds, *v.dataCenter, *v.rack,
-		volumeWhiteList,
+		v.whiteList,
-		*fixJpgOrientation,
+		*v.fixJpgOrientation,
 	)
 
-	listeningAddress := *volumeBindIp + ":" + strconv.Itoa(*vport)
+	listeningAddress := *v.ip + ":" + strconv.Itoa(*v.port)
 
 	glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress)
 
-	listener, e := util.NewListener(listeningAddress, time.Duration(*vTimeout)*time.Second)
+	listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
 	if e != nil {
 		glog.Fatalf(e.Error())
 	}

note/security.txt

@@ -3,7 +3,7 @@ Design for Seaweed-FS security
 Design Objectives
 	Security can mean many different things. The original vision is that: if you have one machine lying around
 	somewhere with some disk space, it should be able to join your file system to contribute some disk space and
-	network bandwidth.
+	network bandwidth, securely!
 
 	To achieve this purpose, the security should be able to:
 	1. Secure the inter-server communication. Only real cluster servers can join and communicate.
@@ -14,7 +14,7 @@ Non Objective
 	User specific access control.
 
 Design Architect
-	master, and volume servers all talk securely via 2-way SSL for admin.
+	master, and volume servers all talk securely via 2-way SSL for admin operations.
 	upon joining, master gives its secret key to volume servers.
 	filer or clients talk to master to get secret key, and use the key to generate JWT to write on volume server.
 	A side benefit:
@@ -34,3 +34,18 @@ file uploading:
 	when filer/clients wants to upload, master generate a JWT
 		filer~>volume(public port)
 		master~>volume(public port)
+
+Currently, volume server has 2 ip addresses: ip and publicUrl.
+	The ip is for admin purpose, and master talk to volume server this way.
+	The publicUrl is for clients to access the server, via http GET/POST/DELETE etc.
+		The write operations are secured by JWT.
+	clients talk to master also via https? possible. Decide on this later.
+
+Dev plan:
+	1. volume server separate admin from public GET/POST/DELETE handlers
+		The step 1 may be good enough for most use cases.
+
+	If 2-way ssl are still needed
+	2. volume server add ssl support
+	3. https connections to operate on volume servers
+