starting a shell

weed-fs/bin/.gitignore

@@ -0,0 +1 @@
+/weed

weed-fs/note/replication.txt

@@ -0,0 +1,59 @@
+1. each file can choose the replication factor
+2. replication granularity is in volume level
+3. if not enough spaces, we can automatically decrease some volume's the replication factor, especially for cold data
+4. support migrating data to cheaper storage
+5. manual volume placement, access-based volume placement, auction based volume placement
+
+When a new volume server is started, it reports 
+  1. how many volumes it can hold
+  2. current list of existing volumes
+Each volume server remembers:
+  1. current volume ids, replica locations
+
+The master assign volume ids based on
+  1. replication factor
+     data center, rack
+  2. concurrent write support
+On master, stores the replication configuration
+{
+  replication:{
+    {factor:1, min_volume_count:3, weight:10},
+    {factor:2, min_volume_count:2, weight:20},
+    {factor:3, min_volume_count:3, weight:30}
+  },
+  port:9333,
+  
+}
+Or manually via command line
+  1. add volume with specified replication factor
+  2. add volume with specified volume id
+
+
+If duplicated volume ids are reported from different volume servers, 
+the master determines the replication factor of the volume,
+if less than the replication factor, the volume is in readonly mode
+if more than the replication factor, the volume will purge the smallest/oldest volume
+if equal, the volume will function as usual
+
+maybe use gossip to send the volumeServer~volumes information
+
+
+Use cases:
+  on volume server
+    1. weed volume -mserver="xx.xx.xx.xx:9333" -publicUrl="good.com:8080" -dir="/tmp" -volumes=50
+  on weed master
+    1. weed master -port=9333
+      generate a default json configuration file if doesn't exist
+      
+Bootstrap
+  1. at the very beginning, the system has no volumes at all.
+  2. if maxReplicationFactor==1, always initialize volumes right away
+  3. if nServersHasFreeSpaces >= maxReplicationFactor, auto initialize
+  4. if maxReplicationFactor>1
+     weed shell
+     > disable_auto_initialize
+     > enable_auto_initialize
+     > assign_free_volume vid "server1:port","server2:port","server3:port"
+     > status
+  5. 
+  

weed-fs/src/cmd/weed/shell.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+  "bufio"
+  "os"
+  "fmt"
+)
+
+func init() {
+  cmdShell.Run = runShell // break init cycle
+}
+
+var cmdShell = &Command{
+  UsageLine: "shell",
+  Short:     "run interactive commands, now just echo",
+  Long: `run interactive commands.
+
+  `,
+}
+
+var (
+)
+
+func runShell(command *Command, args []string) bool {
+  r := bufio.NewReader(os.Stdin)
+  o := bufio.NewWriter(os.Stdout)
+  e := bufio.NewWriter(os.Stderr)
+  prompt := func () { 
+    o.WriteString("> ")
+    o.Flush()
+  };
+  readLine := func () string { 
+    ret, err := r.ReadString('\n')
+    if err != nil { 
+       fmt.Fprint(e,err); 
+       os.Exit(1)
+    }
+    return ret
+  }
+  execCmd := func (cmd string) int { 
+    if cmd != "" { 
+      o.WriteString(cmd)
+    }
+    return 0
+  }
+
+  cmd := ""
+  for { 
+    prompt()
+    cmd = readLine()
+    execCmd(cmd)
+  } 
+  return true
+}

weed-fs/src/cmd/weed/weed.go

@@ -1,12 +1,12 @@
 package main
 
 import (
-  "encoding/json"
+	"encoding/json"
 	"flag"
 	"fmt"
-  "net/http"
 	"io"
 	"log"
+	"net/http"
 	"os"
 	"strings"
 	"sync"
@@ -22,7 +22,8 @@ var port *int
 var commands = []*Command{
 	cmdFix,
 	cmdMaster,
-  cmdUpload,
+	cmdUpload,
+	cmdShell,
 	cmdVersion,
 	cmdVolume,
 }
@@ -50,6 +51,12 @@ func main() {
 
 	if args[0] == "help" {
 		help(args[1:])
+		for _, cmd := range commands {
+			if cmd.Name() == args[1] && cmd.Run != nil {
+				fmt.Fprintf(os.Stderr, "Default Parameters:\n")
+				cmd.Flag.PrintDefaults()
+			}
+		}
 		return
 	}
 
@@ -59,10 +66,10 @@ func main() {
 			cmd.Flag.Parse(args[1:])
 			args = cmd.Flag.Args()
 			if !cmd.Run(cmd, args) {
+				fmt.Fprintf(os.Stderr, "\n")
+				cmd.Flag.Usage()
 				fmt.Fprintf(os.Stderr, "Default Parameters:\n")
 				cmd.Flag.PrintDefaults()
-        fmt.Fprintf(os.Stderr, "\n")
-				cmd.Flag.Usage()
 			}
 			exit()
 			return
@@ -173,15 +180,15 @@ func exitIfErrors() {
 	}
 }
 func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) {
-  w.Header().Set("Content-Type", "application/javascript")
+	w.Header().Set("Content-Type", "application/javascript")
-  bytes, _ := json.Marshal(obj)
+	bytes, _ := json.Marshal(obj)
-  callback := r.FormValue("callback")
+	callback := r.FormValue("callback")
-  if callback == "" {
+	if callback == "" {
-    w.Write(bytes)
+		w.Write(bytes)
-  } else {
+	} else {
-    w.Write([]uint8(callback))
+		w.Write([]uint8(callback))
-    w.Write([]uint8("("))
+		w.Write([]uint8("("))
-    fmt.Fprint(w, string(bytes))
+		fmt.Fprint(w, string(bytes))
-    w.Write([]uint8(")"))
+		w.Write([]uint8(")"))
-  }
+	}
 }

weed-fs/src/pkg/topology/data_center.go

@@ -0,0 +1,6 @@
+package topology
+
+import (
+
+)
+

weed-fs/src/pkg/topology/node.go

@@ -0,0 +1,16 @@
+package topology
+
+import (
+
+)
+
+type VolumeInfo struct {
+  Id   uint32
+  Size int64
+}
+type Node struct {
+  volumes     map[uint64]VolumeInfo
+  volumeLimit int
+  Port        int
+  PublicUrl   string
+}

weed-fs/src/pkg/topology/rack.go

@@ -0,0 +1,10 @@
+package topology
+
+import (
+
+)
+
+type Rack struct {
+  nodes     map[uint64]*Node
+  IpRanges   []string
+}

weed-fs/src/pkg/util/config.go

@@ -0,0 +1,128 @@
+// Copyright 2011 Numerotron Inc.
+// Use of this source code is governed by an MIT-style license
+// that can be found in the LICENSE file.
+//
+// Developed at www.stathat.com by Patrick Crosby
+// Contact us on twitter with any questions:  twitter.com/stat_hat
+
+// The jconfig package provides a simple, basic configuration file parser using JSON.
+package util
+
+import (
+	"bytes"
+	"encoding/json"
+	"log"
+	"os"
+)
+
+type Config struct {
+	data     map[string]interface{}
+	filename string
+}
+
+func newConfig() *Config {
+	result := new(Config)
+	result.data = make(map[string]interface{})
+	return result
+}
+
+// Loads config information from a JSON file
+func LoadConfig(filename string) *Config {
+	result := newConfig()
+	result.filename = filename
+	err := result.parse()
+	if err != nil {
+		log.Fatalf("error loading config file %s: %s", filename, err)
+	}
+	return result
+}
+
+// Loads config information from a JSON string
+func LoadConfigString(s string) *Config {
+	result := newConfig()
+	err := json.Unmarshal([]byte(s), &result.data)
+	if err != nil {
+		log.Fatalf("error parsing config string %s: %s", s, err)
+	}
+	return result
+}
+
+func (c *Config) StringMerge(s string) {
+	next := LoadConfigString(s)
+	c.merge(next.data)
+}
+
+func (c *Config) LoadMerge(filename string) {
+	next := LoadConfig(filename)
+	c.merge(next.data)
+}
+
+func (c *Config) merge(ndata map[string]interface{}) {
+	for k, v := range ndata {
+		c.data[k] = v
+	}
+}
+
+func (c *Config) parse() error {
+	f, err := os.Open(c.filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	b := new(bytes.Buffer)
+	_, err = b.ReadFrom(f)
+	if err != nil {
+		return err
+	}
+	err = json.Unmarshal(b.Bytes(), &c.data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Returns a string for the config variable key
+func (c *Config) GetString(key string) string {
+	result, present := c.data[key]
+	if !present {
+		return ""
+	}
+	return result.(string)
+}
+
+// Returns an int for the config variable key
+func (c *Config) GetInt(key string) int {
+	x, ok := c.data[key]
+	if !ok {
+		return -1
+	}
+	return int(x.(float64))
+}
+
+// Returns a float for the config variable key
+func (c *Config) GetFloat(key string) float64 {
+	x, ok := c.data[key]
+	if !ok {
+		return -1
+	}
+	return x.(float64)
+}
+
+// Returns a bool for the config variable key
+func (c *Config) GetBool(key string) bool {
+	x, ok := c.data[key]
+	if !ok {
+		return false
+	}
+	return x.(bool)
+}
+
+// Returns an array for the config variable key
+func (c *Config) GetArray(key string) []interface{} {
+	result, present := c.data[key]
+	if !present {
+		return []interface{}(nil)
+	}
+	return result.([]interface{})
+}