mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
starting a shell
This commit is contained in:
parent
0109cc0e30
commit
0121f35c12
1
weed-fs/bin/.gitignore
vendored
Normal file
1
weed-fs/bin/.gitignore
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
/weed
|
59
weed-fs/note/replication.txt
Normal file
59
weed-fs/note/replication.txt
Normal file
|
@ -0,0 +1,59 @@
|
|||
1. each file can choose the replication factor
|
||||
2. replication granularity is in volume level
|
||||
3. if not enough spaces, we can automatically decrease some volume's the replication factor, especially for cold data
|
||||
4. support migrating data to cheaper storage
|
||||
5. manual volume placement, access-based volume placement, auction based volume placement
|
||||
|
||||
When a new volume server is started, it reports
|
||||
1. how many volumes it can hold
|
||||
2. current list of existing volumes
|
||||
Each volume server remembers:
|
||||
1. current volume ids, replica locations
|
||||
|
||||
The master assign volume ids based on
|
||||
1. replication factor
|
||||
data center, rack
|
||||
2. concurrent write support
|
||||
On master, stores the replication configuration
|
||||
{
|
||||
replication:{
|
||||
{factor:1, min_volume_count:3, weight:10},
|
||||
{factor:2, min_volume_count:2, weight:20},
|
||||
{factor:3, min_volume_count:3, weight:30}
|
||||
},
|
||||
port:9333,
|
||||
|
||||
}
|
||||
Or manually via command line
|
||||
1. add volume with specified replication factor
|
||||
2. add volume with specified volume id
|
||||
|
||||
|
||||
If duplicated volume ids are reported from different volume servers,
|
||||
the master determines the replication factor of the volume,
|
||||
if less than the replication factor, the volume is in readonly mode
|
||||
if more than the replication factor, the volume will purge the smallest/oldest volume
|
||||
if equal, the volume will function as usual
|
||||
|
||||
maybe use gossip to send the volumeServer~volumes information
|
||||
|
||||
|
||||
Use cases:
|
||||
on volume server
|
||||
1. weed volume -mserver="xx.xx.xx.xx:9333" -publicUrl="good.com:8080" -dir="/tmp" -volumes=50
|
||||
on weed master
|
||||
1. weed master -port=9333
|
||||
generate a default json configuration file if doesn't exist
|
||||
|
||||
Bootstrap
|
||||
1. at the very beginning, the system has no volumes at all.
|
||||
2. if maxReplicationFactor==1, always initialize volumes right away
|
||||
3. if nServersHasFreeSpaces >= maxReplicationFactor, auto initialize
|
||||
4. if maxReplicationFactor>1
|
||||
weed shell
|
||||
> disable_auto_initialize
|
||||
> enable_auto_initialize
|
||||
> assign_free_volume vid "server1:port","server2:port","server3:port"
|
||||
> status
|
||||
5.
|
||||
|
54
weed-fs/src/cmd/weed/shell.go
Normal file
54
weed-fs/src/cmd/weed/shell.go
Normal file
|
@ -0,0 +1,54 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmdShell.Run = runShell // break init cycle
|
||||
}
|
||||
|
||||
var cmdShell = &Command{
|
||||
UsageLine: "shell",
|
||||
Short: "run interactive commands, now just echo",
|
||||
Long: `run interactive commands.
|
||||
|
||||
`,
|
||||
}
|
||||
|
||||
var (
|
||||
)
|
||||
|
||||
func runShell(command *Command, args []string) bool {
|
||||
r := bufio.NewReader(os.Stdin)
|
||||
o := bufio.NewWriter(os.Stdout)
|
||||
e := bufio.NewWriter(os.Stderr)
|
||||
prompt := func () {
|
||||
o.WriteString("> ")
|
||||
o.Flush()
|
||||
};
|
||||
readLine := func () string {
|
||||
ret, err := r.ReadString('\n')
|
||||
if err != nil {
|
||||
fmt.Fprint(e,err);
|
||||
os.Exit(1)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
execCmd := func (cmd string) int {
|
||||
if cmd != "" {
|
||||
o.WriteString(cmd)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
cmd := ""
|
||||
for {
|
||||
prompt()
|
||||
cmd = readLine()
|
||||
execCmd(cmd)
|
||||
}
|
||||
return true
|
||||
}
|
|
@ -4,9 +4,9 @@ import (
|
|||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
@ -23,6 +23,7 @@ var commands = []*Command{
|
|||
cmdFix,
|
||||
cmdMaster,
|
||||
cmdUpload,
|
||||
cmdShell,
|
||||
cmdVersion,
|
||||
cmdVolume,
|
||||
}
|
||||
|
@ -50,6 +51,12 @@ func main() {
|
|||
|
||||
if args[0] == "help" {
|
||||
help(args[1:])
|
||||
for _, cmd := range commands {
|
||||
if cmd.Name() == args[1] && cmd.Run != nil {
|
||||
fmt.Fprintf(os.Stderr, "Default Parameters:\n")
|
||||
cmd.Flag.PrintDefaults()
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -59,10 +66,10 @@ func main() {
|
|||
cmd.Flag.Parse(args[1:])
|
||||
args = cmd.Flag.Args()
|
||||
if !cmd.Run(cmd, args) {
|
||||
fmt.Fprintf(os.Stderr, "Default Parameters:\n")
|
||||
cmd.Flag.PrintDefaults()
|
||||
fmt.Fprintf(os.Stderr, "\n")
|
||||
cmd.Flag.Usage()
|
||||
fmt.Fprintf(os.Stderr, "Default Parameters:\n")
|
||||
cmd.Flag.PrintDefaults()
|
||||
}
|
||||
exit()
|
||||
return
|
||||
|
|
6
weed-fs/src/pkg/topology/data_center.go
Normal file
6
weed-fs/src/pkg/topology/data_center.go
Normal file
|
@ -0,0 +1,6 @@
|
|||
package topology
|
||||
|
||||
import (
|
||||
|
||||
)
|
||||
|
16
weed-fs/src/pkg/topology/node.go
Normal file
16
weed-fs/src/pkg/topology/node.go
Normal file
|
@ -0,0 +1,16 @@
|
|||
package topology
|
||||
|
||||
import (
|
||||
|
||||
)
|
||||
|
||||
type VolumeInfo struct {
|
||||
Id uint32
|
||||
Size int64
|
||||
}
|
||||
type Node struct {
|
||||
volumes map[uint64]VolumeInfo
|
||||
volumeLimit int
|
||||
Port int
|
||||
PublicUrl string
|
||||
}
|
10
weed-fs/src/pkg/topology/rack.go
Normal file
10
weed-fs/src/pkg/topology/rack.go
Normal file
|
@ -0,0 +1,10 @@
|
|||
package topology
|
||||
|
||||
import (
|
||||
|
||||
)
|
||||
|
||||
type Rack struct {
|
||||
nodes map[uint64]*Node
|
||||
IpRanges []string
|
||||
}
|
128
weed-fs/src/pkg/util/config.go
Normal file
128
weed-fs/src/pkg/util/config.go
Normal file
|
@ -0,0 +1,128 @@
|
|||
// Copyright 2011 Numerotron Inc.
|
||||
// Use of this source code is governed by an MIT-style license
|
||||
// that can be found in the LICENSE file.
|
||||
//
|
||||
// Developed at www.stathat.com by Patrick Crosby
|
||||
// Contact us on twitter with any questions: twitter.com/stat_hat
|
||||
|
||||
// The jconfig package provides a simple, basic configuration file parser using JSON.
|
||||
package util
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
data map[string]interface{}
|
||||
filename string
|
||||
}
|
||||
|
||||
func newConfig() *Config {
|
||||
result := new(Config)
|
||||
result.data = make(map[string]interface{})
|
||||
return result
|
||||
}
|
||||
|
||||
// Loads config information from a JSON file
|
||||
func LoadConfig(filename string) *Config {
|
||||
result := newConfig()
|
||||
result.filename = filename
|
||||
err := result.parse()
|
||||
if err != nil {
|
||||
log.Fatalf("error loading config file %s: %s", filename, err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Loads config information from a JSON string
|
||||
func LoadConfigString(s string) *Config {
|
||||
result := newConfig()
|
||||
err := json.Unmarshal([]byte(s), &result.data)
|
||||
if err != nil {
|
||||
log.Fatalf("error parsing config string %s: %s", s, err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (c *Config) StringMerge(s string) {
|
||||
next := LoadConfigString(s)
|
||||
c.merge(next.data)
|
||||
}
|
||||
|
||||
func (c *Config) LoadMerge(filename string) {
|
||||
next := LoadConfig(filename)
|
||||
c.merge(next.data)
|
||||
}
|
||||
|
||||
func (c *Config) merge(ndata map[string]interface{}) {
|
||||
for k, v := range ndata {
|
||||
c.data[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) parse() error {
|
||||
f, err := os.Open(c.filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
b := new(bytes.Buffer)
|
||||
_, err = b.ReadFrom(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = json.Unmarshal(b.Bytes(), &c.data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Returns a string for the config variable key
|
||||
func (c *Config) GetString(key string) string {
|
||||
result, present := c.data[key]
|
||||
if !present {
|
||||
return ""
|
||||
}
|
||||
return result.(string)
|
||||
}
|
||||
|
||||
// Returns an int for the config variable key
|
||||
func (c *Config) GetInt(key string) int {
|
||||
x, ok := c.data[key]
|
||||
if !ok {
|
||||
return -1
|
||||
}
|
||||
return int(x.(float64))
|
||||
}
|
||||
|
||||
// Returns a float for the config variable key
|
||||
func (c *Config) GetFloat(key string) float64 {
|
||||
x, ok := c.data[key]
|
||||
if !ok {
|
||||
return -1
|
||||
}
|
||||
return x.(float64)
|
||||
}
|
||||
|
||||
// Returns a bool for the config variable key
|
||||
func (c *Config) GetBool(key string) bool {
|
||||
x, ok := c.data[key]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return x.(bool)
|
||||
}
|
||||
|
||||
// Returns an array for the config variable key
|
||||
func (c *Config) GetArray(key string) []interface{} {
|
||||
result, present := c.data[key]
|
||||
if !present {
|
||||
return []interface{}(nil)
|
||||
}
|
||||
return result.([]interface{})
|
||||
}
|
Loading…
Reference in a new issue