mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
go fmt for all source codes
This commit is contained in:
parent
ca9056d673
commit
b0c7df0c3b
|
@ -24,7 +24,6 @@ type Command struct {
|
||||||
|
|
||||||
// Flag is a set of flags specific to this command.
|
// Flag is a set of flags specific to this command.
|
||||||
Flag flag.FlagSet
|
Flag flag.FlagSet
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Name returns the command's name: the first word in the usage line.
|
// Name returns the command's name: the first word in the usage line.
|
||||||
|
|
|
@ -2,8 +2,8 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"os"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
@ -18,26 +18,25 @@ var cmdShell = &Command{
|
||||||
`,
|
`,
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var ()
|
||||||
)
|
|
||||||
|
|
||||||
func runShell(command *Command, args []string) bool {
|
func runShell(command *Command, args []string) bool {
|
||||||
r := bufio.NewReader(os.Stdin)
|
r := bufio.NewReader(os.Stdin)
|
||||||
o := bufio.NewWriter(os.Stdout)
|
o := bufio.NewWriter(os.Stdout)
|
||||||
e := bufio.NewWriter(os.Stderr)
|
e := bufio.NewWriter(os.Stderr)
|
||||||
prompt := func () {
|
prompt := func() {
|
||||||
o.WriteString("> ")
|
o.WriteString("> ")
|
||||||
o.Flush()
|
o.Flush()
|
||||||
};
|
}
|
||||||
readLine := func () string {
|
readLine := func() string {
|
||||||
ret, err := r.ReadString('\n')
|
ret, err := r.ReadString('\n')
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprint(e,err);
|
fmt.Fprint(e, err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
return ret
|
return ret
|
||||||
}
|
}
|
||||||
execCmd := func (cmd string) int {
|
execCmd := func(cmd string) int {
|
||||||
if cmd != "" {
|
if cmd != "" {
|
||||||
o.WriteString(cmd)
|
o.WriteString(cmd)
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,7 +42,7 @@ var (
|
||||||
store *storage.Store
|
store *storage.Store
|
||||||
)
|
)
|
||||||
|
|
||||||
var fileNameEscaper = strings.NewReplacer("\\","\\\\","\"","\\\"")
|
var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
|
||||||
|
|
||||||
func statusHandler(w http.ResponseWriter, r *http.Request) {
|
func statusHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
m := make(map[string]interface{})
|
m := make(map[string]interface{})
|
||||||
|
@ -156,7 +156,9 @@ func GetHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
|
if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
|
||||||
w.Header().Set("Content-Encoding", "gzip")
|
w.Header().Set("Content-Encoding", "gzip")
|
||||||
} else {
|
} else {
|
||||||
n.Data = storage.UnGzipData(n.Data)
|
if n.Data, err = storage.UnGzipData(n.Data); err != nil {
|
||||||
|
debug("lookup error:", err, r.URL.Path)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,8 +3,8 @@ package directory
|
||||||
import (
|
import (
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"pkg/storage"
|
"pkg/storage"
|
||||||
"strings"
|
|
||||||
"pkg/util"
|
"pkg/util"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
type FileId struct {
|
type FileId struct {
|
||||||
|
@ -16,7 +16,7 @@ type FileId struct {
|
||||||
func NewFileId(VolumeId storage.VolumeId, Key uint64, Hashcode uint32) *FileId {
|
func NewFileId(VolumeId storage.VolumeId, Key uint64, Hashcode uint32) *FileId {
|
||||||
return &FileId{VolumeId: VolumeId, Key: Key, Hashcode: Hashcode}
|
return &FileId{VolumeId: VolumeId, Key: Key, Hashcode: Hashcode}
|
||||||
}
|
}
|
||||||
func ParseFileId(fid string) *FileId{
|
func ParseFileId(fid string) *FileId {
|
||||||
a := strings.Split(fid, ",")
|
a := strings.Split(fid, ",")
|
||||||
if len(a) != 2 {
|
if len(a) != 2 {
|
||||||
println("Invalid fid", fid, ", split length", len(a))
|
println("Invalid fid", fid, ", split length", len(a))
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
package operation
|
package operation
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
|
||||||
"log"
|
"log"
|
||||||
|
"net/http"
|
||||||
)
|
)
|
||||||
|
|
||||||
func Delete(url string) error {
|
func Delete(url string) error {
|
||||||
|
|
|
@ -2,11 +2,11 @@ package operation
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
_ "fmt"
|
||||||
"net/url"
|
"net/url"
|
||||||
"pkg/storage"
|
"pkg/storage"
|
||||||
"pkg/util"
|
"pkg/util"
|
||||||
_ "fmt"
|
|
||||||
"errors"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Location struct {
|
type Location struct {
|
||||||
|
@ -31,7 +31,7 @@ func Lookup(server string, vid storage.VolumeId) (*LookupResult, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if ret.Error != ""{
|
if ret.Error != "" {
|
||||||
return nil, errors.New(ret.Error)
|
return nil, errors.New(ret.Error)
|
||||||
}
|
}
|
||||||
return &ret, nil
|
return &ret, nil
|
||||||
|
|
|
@ -3,13 +3,13 @@ package operation
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
_ "fmt"
|
_ "fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"log"
|
"log"
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
"net/http"
|
"net/http"
|
||||||
"errors"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type UploadResult struct {
|
type UploadResult struct {
|
||||||
|
@ -40,7 +40,7 @@ func Upload(uploadUrl string, filename string, reader io.Reader) (*UploadResult,
|
||||||
log.Println("failing to read upload resonse", uploadUrl, resp_body)
|
log.Println("failing to read upload resonse", uploadUrl, resp_body)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if ret.Error != ""{
|
if ret.Error != "" {
|
||||||
return nil, errors.New(ret.Error)
|
return nil, errors.New(ret.Error)
|
||||||
}
|
}
|
||||||
return &ret, nil
|
return &ret, nil
|
||||||
|
|
|
@ -182,7 +182,7 @@ func (vg *VolumeGrowth) GrowByCountAndType(count int, repType storage.Replicatio
|
||||||
func (vg *VolumeGrowth) grow(topo *topology.Topology, vid storage.VolumeId, repType storage.ReplicationType, servers ...*topology.DataNode) error {
|
func (vg *VolumeGrowth) grow(topo *topology.Topology, vid storage.VolumeId, repType storage.ReplicationType, servers ...*topology.DataNode) error {
|
||||||
for _, server := range servers {
|
for _, server := range servers {
|
||||||
if err := operation.AllocateVolume(server, vid, repType); err == nil {
|
if err := operation.AllocateVolume(server, vid, repType); err == nil {
|
||||||
vi := storage.VolumeInfo{Id: vid, Size: 0, RepType: repType, Version:storage.CurrentVersion}
|
vi := storage.VolumeInfo{Id: vid, Size: 0, RepType: repType, Version: storage.CurrentVersion}
|
||||||
server.AddOrUpdateVolume(vi)
|
server.AddOrUpdateVolume(vi)
|
||||||
topo.RegisterVolumeLayout(&vi, server)
|
topo.RegisterVolumeLayout(&vi, server)
|
||||||
fmt.Println("Created Volume", vid, "on", server)
|
fmt.Println("Created Volume", vid, "on", server)
|
||||||
|
|
|
@ -80,7 +80,7 @@ func setup(topologyLayout string) *topology.Topology {
|
||||||
fmt.Println("data:", data)
|
fmt.Println("data:", data)
|
||||||
|
|
||||||
//need to connect all nodes first before server adding volumes
|
//need to connect all nodes first before server adding volumes
|
||||||
topo := topology.NewTopology("mynetwork","/etc/weedfs/weedfs.conf","/tmp","testing",32*1024, 5)
|
topo := topology.NewTopology("mynetwork", "/etc/weedfs/weedfs.conf", "/tmp", "testing", 32*1024, 5)
|
||||||
mTopology := data.(map[string]interface{})
|
mTopology := data.(map[string]interface{})
|
||||||
for dcKey, dcValue := range mTopology {
|
for dcKey, dcValue := range mTopology {
|
||||||
dc := topology.NewDataCenter(dcKey)
|
dc := topology.NewDataCenter(dcKey)
|
||||||
|
@ -96,7 +96,7 @@ func setup(topologyLayout string) *topology.Topology {
|
||||||
rack.LinkChildNode(server)
|
rack.LinkChildNode(server)
|
||||||
for _, v := range serverMap["volumes"].([]interface{}) {
|
for _, v := range serverMap["volumes"].([]interface{}) {
|
||||||
m := v.(map[string]interface{})
|
m := v.(map[string]interface{})
|
||||||
vi := storage.VolumeInfo{Id: storage.VolumeId(int64(m["id"].(float64))), Size: int64(m["size"].(float64)), Version:storage.CurrentVersion}
|
vi := storage.VolumeInfo{Id: storage.VolumeId(int64(m["id"].(float64))), Size: int64(m["size"].(float64)), Version: storage.CurrentVersion}
|
||||||
server.AddOrUpdateVolume(vi)
|
server.AddOrUpdateVolume(vi)
|
||||||
}
|
}
|
||||||
server.UpAdjustMaxVolumeCountDelta(int(serverMap["limit"].(float64)))
|
server.UpAdjustMaxVolumeCountDelta(int(serverMap["limit"].(float64)))
|
||||||
|
@ -122,9 +122,8 @@ func TestRemoveDataCenter(t *testing.T) {
|
||||||
func TestReserveOneVolume(t *testing.T) {
|
func TestReserveOneVolume(t *testing.T) {
|
||||||
topo := setup(topologyLayout)
|
topo := setup(topologyLayout)
|
||||||
rand.Seed(time.Now().UnixNano())
|
rand.Seed(time.Now().UnixNano())
|
||||||
vg:=&VolumeGrowth{copy1factor:3,copy2factor:2,copy3factor:1,copyAll:4}
|
vg := &VolumeGrowth{copy1factor: 3, copy2factor: 2, copy3factor: 1, copyAll: 4}
|
||||||
if c, e := vg.GrowByCountAndType(1,storage.Copy000,topo);e==nil{
|
if c, e := vg.GrowByCountAndType(1, storage.Copy000, topo); e == nil {
|
||||||
t.Log("reserved", c)
|
t.Log("reserved", c)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2,10 +2,10 @@ package sequence
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/gob"
|
"encoding/gob"
|
||||||
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"sync"
|
"sync"
|
||||||
"log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
|
@ -1,10 +1,10 @@
|
||||||
package storage
|
package storage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"pkg/util"
|
"pkg/util"
|
||||||
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMemoryUsage(t *testing.T) {
|
func TestMemoryUsage(t *testing.T) {
|
||||||
|
|
|
@ -18,11 +18,11 @@ func TestXYZ(t *testing.T) {
|
||||||
m.Set(Key(i), i+11, i+5)
|
m.Set(Key(i), i+11, i+5)
|
||||||
}
|
}
|
||||||
|
|
||||||
// for i := uint32(0); i < 100; i++ {
|
// for i := uint32(0); i < 100; i++ {
|
||||||
// if v := m.Get(Key(i)); v != nil {
|
// if v := m.Get(Key(i)); v != nil {
|
||||||
// println(i, "=", v.Key, v.Offset, v.Size)
|
// println(i, "=", v.Key, v.Offset, v.Size)
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
|
|
||||||
for i := uint32(0); i < 10*batch; i++ {
|
for i := uint32(0); i < 10*batch; i++ {
|
||||||
v, ok := m.Get(Key(i))
|
v, ok := m.Get(Key(i))
|
||||||
|
@ -51,7 +51,7 @@ func TestXYZ(t *testing.T) {
|
||||||
t.Fatal("key", i, "should have been deleted needle value", v)
|
t.Fatal("key", i, "should have been deleted needle value", v)
|
||||||
}
|
}
|
||||||
} else if i%2 == 0 {
|
} else if i%2 == 0 {
|
||||||
if v==nil{
|
if v == nil {
|
||||||
t.Fatal("key", i, "missing")
|
t.Fatal("key", i, "missing")
|
||||||
}
|
}
|
||||||
if v.Size != i {
|
if v.Size != i {
|
||||||
|
|
|
@ -12,28 +12,13 @@ import (
|
||||||
* Default more not to gzip since gzip can be done on client side.
|
* Default more not to gzip since gzip can be done on client side.
|
||||||
*/
|
*/
|
||||||
func IsGzippable(ext, mtype string) bool {
|
func IsGzippable(ext, mtype string) bool {
|
||||||
if strings.HasPrefix(mtype, "text/"){
|
if strings.HasPrefix(mtype, "text/") {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
if ext == ".zip" {
|
switch ext {
|
||||||
|
case ".zip", ".rar", ".gz", ".bz2", ".xz":
|
||||||
return false
|
return false
|
||||||
}
|
case ".pdf", ".txt", ".html", ".css", ".js", ".json":
|
||||||
if ext == ".rar" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if ext == ".gz" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if ext == ".pdf" {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if ext == ".css" {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if ext == ".js" {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if ext == ".json" {
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(mtype, "application/") {
|
if strings.HasPrefix(mtype, "application/") {
|
||||||
|
@ -46,18 +31,21 @@ func IsGzippable(ext, mtype string) bool {
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
func GzipData(input []byte) []byte {
|
|
||||||
|
func GzipData(input []byte) ([]byte, error) {
|
||||||
buf := new(bytes.Buffer)
|
buf := new(bytes.Buffer)
|
||||||
w, _ := gzip.NewWriterLevel(buf, flate.BestCompression)
|
w, _ := gzip.NewWriterLevel(buf, flate.BestCompression)
|
||||||
if _, err := w.Write(input); err != nil {
|
if _, err := w.Write(input); err != nil {
|
||||||
println("error compressing data:", err)
|
println("error compressing data:", err)
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := w.Close(); err != nil {
|
if err := w.Close(); err != nil {
|
||||||
println("error closing compressed data:", err)
|
println("error closing compressed data:", err)
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
return buf.Bytes()
|
return buf.Bytes(), nil
|
||||||
}
|
}
|
||||||
func UnGzipData(input []byte) []byte {
|
func UnGzipData(input []byte) ([]byte, error) {
|
||||||
buf := bytes.NewBuffer(input)
|
buf := bytes.NewBuffer(input)
|
||||||
r, _ := gzip.NewReader(buf)
|
r, _ := gzip.NewReader(buf)
|
||||||
defer r.Close()
|
defer r.Close()
|
||||||
|
@ -65,5 +53,5 @@ func UnGzipData(input []byte) []byte {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
println("error uncompressing data:", err)
|
println("error uncompressing data:", err)
|
||||||
}
|
}
|
||||||
return output
|
return output, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -64,7 +64,9 @@ func NewNeedle(r *http.Request) (n *Needle, fname string, e error) {
|
||||||
mtype = contentType
|
mtype = contentType
|
||||||
}
|
}
|
||||||
if IsGzippable(ext, mtype) {
|
if IsGzippable(ext, mtype) {
|
||||||
data = GzipData(data)
|
if data, e = GzipData(data); e != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
n.SetGzipped()
|
n.SetGzipped()
|
||||||
}
|
}
|
||||||
if ext == ".gz" {
|
if ext == ".gz" {
|
||||||
|
|
|
@ -32,7 +32,7 @@ func NewReplicationTypeFromString(t string) (ReplicationType, error) {
|
||||||
case "200":
|
case "200":
|
||||||
return Copy200, nil
|
return Copy200, nil
|
||||||
}
|
}
|
||||||
return Copy000, errors.New("Unknown Replication Type:"+t)
|
return Copy000, errors.New("Unknown Replication Type:" + t)
|
||||||
}
|
}
|
||||||
func NewReplicationTypeFromByte(b byte) (ReplicationType, error) {
|
func NewReplicationTypeFromByte(b byte) (ReplicationType, error) {
|
||||||
switch b {
|
switch b {
|
||||||
|
@ -49,7 +49,7 @@ func NewReplicationTypeFromByte(b byte) (ReplicationType, error) {
|
||||||
case byte(200):
|
case byte(200):
|
||||||
return Copy200, nil
|
return Copy200, nil
|
||||||
}
|
}
|
||||||
return Copy000, errors.New("Unknown Replication Type:"+string(b))
|
return Copy000, errors.New("Unknown Replication Type:" + string(b))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *ReplicationType) String() string {
|
func (r *ReplicationType) String() string {
|
||||||
|
@ -87,7 +87,7 @@ func (r *ReplicationType) Byte() byte {
|
||||||
return byte(000)
|
return byte(000)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (repType ReplicationType)GetReplicationLevelIndex() int {
|
func (repType ReplicationType) GetReplicationLevelIndex() int {
|
||||||
switch repType {
|
switch repType {
|
||||||
case Copy000:
|
case Copy000:
|
||||||
return 0
|
return 0
|
||||||
|
@ -104,7 +104,7 @@ func (repType ReplicationType)GetReplicationLevelIndex() int {
|
||||||
}
|
}
|
||||||
return -1
|
return -1
|
||||||
}
|
}
|
||||||
func (repType ReplicationType)GetCopyCount() int {
|
func (repType ReplicationType) GetCopyCount() int {
|
||||||
switch repType {
|
switch repType {
|
||||||
case Copy000:
|
case Copy000:
|
||||||
return 1
|
return 1
|
||||||
|
|
|
@ -65,13 +65,13 @@ func (s *Store) AddVolume(volumeListString string, replicationType string) error
|
||||||
}
|
}
|
||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
func (s *Store) addVolume(vid VolumeId, replicationType ReplicationType) error {
|
func (s *Store) addVolume(vid VolumeId, replicationType ReplicationType) (err error) {
|
||||||
if s.volumes[vid] != nil {
|
if s.volumes[vid] != nil {
|
||||||
return errors.New("Volume Id " + vid.String() + " already exists!")
|
return errors.New("Volume Id " + vid.String() + " already exists!")
|
||||||
}
|
}
|
||||||
log.Println("In dir", s.dir, "adds volume =", vid, ", replicationType =", replicationType)
|
log.Println("In dir", s.dir, "adds volume =", vid, ", replicationType =", replicationType)
|
||||||
s.volumes[vid] = NewVolume(s.dir, vid, replicationType)
|
s.volumes[vid], err = NewVolume(s.dir, vid, replicationType)
|
||||||
return nil
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) {
|
func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) {
|
||||||
|
@ -107,7 +107,7 @@ func (s *Store) loadExistingVolumes() {
|
||||||
base := name[:len(name)-len(".dat")]
|
base := name[:len(name)-len(".dat")]
|
||||||
if vid, err := NewVolumeId(base); err == nil {
|
if vid, err := NewVolumeId(base); err == nil {
|
||||||
if s.volumes[vid] == nil {
|
if s.volumes[vid] == nil {
|
||||||
v := NewVolume(s.dir, vid, CopyNil)
|
if v, e := NewVolume(s.dir, vid, CopyNil); e == nil {
|
||||||
s.volumes[vid] = v
|
s.volumes[vid] = v
|
||||||
log.Println("In dir", s.dir, "read volume =", vid, "replicationType =", v.replicaType, "version =", v.version, "size =", v.Size())
|
log.Println("In dir", s.dir, "read volume =", vid, "replicationType =", v.replicaType, "version =", v.version, "size =", v.Size())
|
||||||
}
|
}
|
||||||
|
@ -115,6 +115,7 @@ func (s *Store) loadExistingVolumes() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
func (s *Store) Status() []*VolumeInfo {
|
func (s *Store) Status() []*VolumeInfo {
|
||||||
var stats []*VolumeInfo
|
var stats []*VolumeInfo
|
||||||
|
|
|
@ -24,9 +24,9 @@ type Volume struct {
|
||||||
accessLock sync.Mutex
|
accessLock sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume) {
|
func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume, e error) {
|
||||||
v = &Volume{dir: dirname, Id: id, replicaType: replicationType}
|
v = &Volume{dir: dirname, Id: id, replicaType: replicationType}
|
||||||
v.load()
|
e = v.load()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
func (v *Volume) load() error {
|
func (v *Volume) load() error {
|
||||||
|
@ -79,21 +79,19 @@ func (v *Volume) maybeWriteSuperBlock() {
|
||||||
v.dataFile.Write(header)
|
v.dataFile.Write(header)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func (v *Volume) readSuperBlock() error {
|
func (v *Volume) readSuperBlock() (err error) {
|
||||||
v.dataFile.Seek(0, 0)
|
v.dataFile.Seek(0, 0)
|
||||||
header := make([]byte, SuperBlockSize)
|
header := make([]byte, SuperBlockSize)
|
||||||
if _, e := v.dataFile.Read(header); e != nil {
|
if _, e := v.dataFile.Read(header); e != nil {
|
||||||
return fmt.Errorf("cannot read superblock: %s", e)
|
return fmt.Errorf("cannot read superblock: %s", e)
|
||||||
}
|
}
|
||||||
var err error
|
|
||||||
v.version, v.replicaType, err = ParseSuperBlock(header)
|
v.version, v.replicaType, err = ParseSuperBlock(header)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
func ParseSuperBlock(header []byte) (version Version, replicaType ReplicationType, e error) {
|
func ParseSuperBlock(header []byte) (version Version, replicaType ReplicationType, err error) {
|
||||||
version = Version(header[0])
|
version = Version(header[0])
|
||||||
var err error
|
|
||||||
if replicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
|
if replicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
|
||||||
e = fmt.Errorf("cannot read replica type: %s", err)
|
err = fmt.Errorf("cannot read replica type: %s", err)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,13 +5,14 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
type VolumeId uint32
|
type VolumeId uint32
|
||||||
func NewVolumeId(vid string) (VolumeId,error) {
|
|
||||||
|
func NewVolumeId(vid string) (VolumeId, error) {
|
||||||
volumeId, err := strconv.ParseUint(vid, 10, 64)
|
volumeId, err := strconv.ParseUint(vid, 10, 64)
|
||||||
return VolumeId(volumeId), err
|
return VolumeId(volumeId), err
|
||||||
}
|
}
|
||||||
func (vid *VolumeId) String() string{
|
func (vid *VolumeId) String() string {
|
||||||
return strconv.FormatUint(uint64(*vid), 10)
|
return strconv.FormatUint(uint64(*vid), 10)
|
||||||
}
|
}
|
||||||
func (vid *VolumeId) Next() VolumeId{
|
func (vid *VolumeId) Next() VolumeId {
|
||||||
return VolumeId(uint32(*vid)+1)
|
return VolumeId(uint32(*vid) + 1)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
package storage
|
package storage
|
||||||
|
|
||||||
import (
|
import ()
|
||||||
)
|
|
||||||
|
|
||||||
type Version uint8
|
type Version uint8
|
||||||
|
|
||||||
|
|
|
@ -32,11 +32,11 @@ func TestLoadConfiguration(t *testing.T) {
|
||||||
c, err := NewConfiguration([]byte(confContent))
|
c, err := NewConfiguration([]byte(confContent))
|
||||||
|
|
||||||
fmt.Printf("%s\n", c)
|
fmt.Printf("%s\n", c)
|
||||||
if err!=nil{
|
if err != nil {
|
||||||
t.Fatalf("unmarshal error:%s",err.Error())
|
t.Fatalf("unmarshal error:%s", err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(c.Topo.DataCenters) <= 0 || c.Topo.DataCenters[0].Name != "dc1" {
|
if len(c.Topo.DataCenters) <= 0 || c.Topo.DataCenters[0].Name != "dc1" {
|
||||||
t.Fatalf("unmarshal error:%s",c)
|
t.Fatalf("unmarshal error:%s", c)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
package topology
|
package topology
|
||||||
|
|
||||||
import (
|
import ()
|
||||||
)
|
|
||||||
|
|
||||||
type DataCenter struct {
|
type DataCenter struct {
|
||||||
NodeImpl
|
NodeImpl
|
||||||
|
@ -28,7 +27,7 @@ func (dc *DataCenter) GetOrCreateRack(rackName string) *Rack {
|
||||||
return rack
|
return rack
|
||||||
}
|
}
|
||||||
|
|
||||||
func (dc *DataCenter) ToMap() interface{}{
|
func (dc *DataCenter) ToMap() interface{} {
|
||||||
m := make(map[string]interface{})
|
m := make(map[string]interface{})
|
||||||
m["Max"] = dc.GetMaxVolumeCount()
|
m["Max"] = dc.GetMaxVolumeCount()
|
||||||
m["Free"] = dc.FreeSpace()
|
m["Free"] = dc.FreeSpace()
|
||||||
|
|
|
@ -37,8 +37,8 @@ func (nl *NodeList) RandomlyPickN(n int, min int) ([]Node, bool) {
|
||||||
list = append(list, n)
|
list = append(list, n)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if n > len(list){
|
if n > len(list) {
|
||||||
return nil,false
|
return nil, false
|
||||||
}
|
}
|
||||||
for i := n; i > 0; i-- {
|
for i := n; i > 0; i-- {
|
||||||
r := rand.Intn(i)
|
r := rand.Intn(i)
|
||||||
|
|
|
@ -1,38 +1,38 @@
|
||||||
package topology
|
package topology
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
_ "fmt"
|
||||||
"strconv"
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
_ "fmt"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestXYZ(t *testing.T) {
|
func TestXYZ(t *testing.T) {
|
||||||
topo := NewTopology("topo","/etc/weed.conf", "/tmp","test",234,5)
|
topo := NewTopology("topo", "/etc/weed.conf", "/tmp", "test", 234, 5)
|
||||||
for i := 0; i < 5; i++ {
|
for i := 0; i < 5; i++ {
|
||||||
dc := NewDataCenter("dc" + strconv.Itoa(i))
|
dc := NewDataCenter("dc" + strconv.Itoa(i))
|
||||||
dc.activeVolumeCount = i
|
dc.activeVolumeCount = i
|
||||||
dc.maxVolumeCount = 5
|
dc.maxVolumeCount = 5
|
||||||
topo.LinkChildNode(dc)
|
topo.LinkChildNode(dc)
|
||||||
}
|
}
|
||||||
nl := NewNodeList(topo.Children(),nil)
|
nl := NewNodeList(topo.Children(), nil)
|
||||||
|
|
||||||
picked, ret := nl.RandomlyPickN(1)
|
picked, ret := nl.RandomlyPickN(1)
|
||||||
if !ret || len(picked)!=1 {
|
if !ret || len(picked) != 1 {
|
||||||
t.Errorf("need to randomly pick 1 node")
|
t.Errorf("need to randomly pick 1 node")
|
||||||
}
|
}
|
||||||
|
|
||||||
picked, ret = nl.RandomlyPickN(4)
|
picked, ret = nl.RandomlyPickN(4)
|
||||||
if !ret || len(picked)!=4 {
|
if !ret || len(picked) != 4 {
|
||||||
t.Errorf("need to randomly pick 4 nodes")
|
t.Errorf("need to randomly pick 4 nodes")
|
||||||
}
|
}
|
||||||
|
|
||||||
picked, ret = nl.RandomlyPickN(5)
|
picked, ret = nl.RandomlyPickN(5)
|
||||||
if !ret || len(picked)!=5 {
|
if !ret || len(picked) != 5 {
|
||||||
t.Errorf("need to randomly pick 5 nodes")
|
t.Errorf("need to randomly pick 5 nodes")
|
||||||
}
|
}
|
||||||
|
|
||||||
picked, ret = nl.RandomlyPickN(6)
|
picked, ret = nl.RandomlyPickN(6)
|
||||||
if ret || len(picked)!=0 {
|
if ret || len(picked) != 0 {
|
||||||
t.Errorf("can not randomly pick 6 nodes:", ret, picked)
|
t.Errorf("can not randomly pick 6 nodes:", ret, picked)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -78,7 +78,7 @@ func setup(topologyLayout string) *Topology {
|
||||||
}
|
}
|
||||||
|
|
||||||
//need to connect all nodes first before server adding volumes
|
//need to connect all nodes first before server adding volumes
|
||||||
topo := NewTopology("mynetwork","/etc/weed.conf","/tmp","test",234,5)
|
topo := NewTopology("mynetwork", "/etc/weed.conf", "/tmp", "test", 234, 5)
|
||||||
mTopology := data.(map[string]interface{})
|
mTopology := data.(map[string]interface{})
|
||||||
for dcKey, dcValue := range mTopology {
|
for dcKey, dcValue := range mTopology {
|
||||||
dc := NewDataCenter(dcKey)
|
dc := NewDataCenter(dcKey)
|
||||||
|
@ -94,7 +94,7 @@ func setup(topologyLayout string) *Topology {
|
||||||
rack.LinkChildNode(server)
|
rack.LinkChildNode(server)
|
||||||
for _, v := range serverMap["volumes"].([]interface{}) {
|
for _, v := range serverMap["volumes"].([]interface{}) {
|
||||||
m := v.(map[string]interface{})
|
m := v.(map[string]interface{})
|
||||||
vi := storage.VolumeInfo{Id: storage.VolumeId(int64(m["id"].(float64))), Size: int64(m["size"].(float64)), Version:storage.CurrentVersion}
|
vi := storage.VolumeInfo{Id: storage.VolumeId(int64(m["id"].(float64))), Size: int64(m["size"].(float64)), Version: storage.CurrentVersion}
|
||||||
server.AddOrUpdateVolume(vi)
|
server.AddOrUpdateVolume(vi)
|
||||||
}
|
}
|
||||||
server.UpAdjustMaxVolumeCountDelta(int(serverMap["limit"].(float64)))
|
server.UpAdjustMaxVolumeCountDelta(int(serverMap["limit"].(float64)))
|
||||||
|
@ -122,6 +122,6 @@ func TestReserveOneVolume(t *testing.T) {
|
||||||
rand.Seed(time.Now().UnixNano())
|
rand.Seed(time.Now().UnixNano())
|
||||||
rand.Seed(1)
|
rand.Seed(1)
|
||||||
ret, node, vid := topo.RandomlyReserveOneVolume()
|
ret, node, vid := topo.RandomlyReserveOneVolume()
|
||||||
fmt.Println("assigned :", ret, ", node :", node,", volume id:", vid)
|
fmt.Println("assigned :", ret, ", node :", node, ", volume id:", vid)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -104,7 +104,7 @@ func vacuumVolume_Check(urlLocation string, vid storage.VolumeId, garbageThresho
|
||||||
values.Add("garbageThreshold", garbageThreshold)
|
values.Add("garbageThreshold", garbageThreshold)
|
||||||
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_check", values)
|
jsonBlob, err := util.Post("http://"+urlLocation+"/admin/vacuum_volume_check", values)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println("parameters:",values)
|
fmt.Println("parameters:", values)
|
||||||
return err, false
|
return err, false
|
||||||
}
|
}
|
||||||
var ret VacuumVolumeResult
|
var ret VacuumVolumeResult
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
package topology
|
package topology
|
||||||
|
|
||||||
import (
|
import ()
|
||||||
)
|
|
||||||
|
|
||||||
func (t *Topology) ToMap() interface{} {
|
func (t *Topology) ToMap() interface{} {
|
||||||
m := make(map[string]interface{})
|
m := make(map[string]interface{})
|
||||||
|
|
|
@ -31,7 +31,7 @@ func (dnll *VolumeLocationList) Add(loc *DataNode) bool {
|
||||||
func (dnll *VolumeLocationList) Remove(loc *DataNode) bool {
|
func (dnll *VolumeLocationList) Remove(loc *DataNode) bool {
|
||||||
for i, dnl := range dnll.list {
|
for i, dnl := range dnll.list {
|
||||||
if loc.Ip == dnl.Ip && loc.Port == dnl.Port {
|
if loc.Ip == dnl.Ip && loc.Port == dnl.Port {
|
||||||
dnll.list = append(dnll.list[:i],dnll.list[i+1:]...)
|
dnll.list = append(dnll.list[:i], dnll.list[i+1:]...)
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,34 +1,33 @@
|
||||||
package util
|
package util
|
||||||
|
|
||||||
func BytesToUint64(b []byte)(v uint64){
|
func BytesToUint64(b []byte) (v uint64) {
|
||||||
length := uint(len(b))
|
length := uint(len(b))
|
||||||
for i :=uint(0);i<length-1;i++ {
|
for i := uint(0); i < length-1; i++ {
|
||||||
v += uint64(b[i])
|
v += uint64(b[i])
|
||||||
v <<= 8
|
v <<= 8
|
||||||
}
|
}
|
||||||
v+=uint64(b[length-1])
|
v += uint64(b[length-1])
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
func BytesToUint32(b []byte)(v uint32){
|
func BytesToUint32(b []byte) (v uint32) {
|
||||||
length := uint(len(b))
|
length := uint(len(b))
|
||||||
for i :=uint(0);i<length-1;i++ {
|
for i := uint(0); i < length-1; i++ {
|
||||||
v += uint32(b[i])
|
v += uint32(b[i])
|
||||||
v <<= 8
|
v <<= 8
|
||||||
}
|
}
|
||||||
v+=uint32(b[length-1])
|
v += uint32(b[length-1])
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
func Uint64toBytes(b []byte, v uint64){
|
func Uint64toBytes(b []byte, v uint64) {
|
||||||
for i :=uint(0);i<8;i++ {
|
for i := uint(0); i < 8; i++ {
|
||||||
b[7-i] = byte(v>>(i*8))
|
b[7-i] = byte(v >> (i * 8))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func Uint32toBytes(b []byte, v uint32){
|
func Uint32toBytes(b []byte, v uint32) {
|
||||||
for i :=uint(0);i<4;i++ {
|
for i := uint(0); i < 4; i++ {
|
||||||
b[3-i] = byte(v>>(i*8))
|
b[3-i] = byte(v >> (i * 8))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func Uint8toBytes(b []byte, v uint8){
|
func Uint8toBytes(b []byte, v uint8) {
|
||||||
b[0] = byte(v)
|
b[0] = byte(v)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -4,10 +4,10 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
)
|
)
|
||||||
|
|
||||||
func ParseInt(text string, defaultValue int) int{
|
func ParseInt(text string, defaultValue int) int {
|
||||||
count, parseError := strconv.ParseUint(text,10,64)
|
count, parseError := strconv.ParseUint(text, 10, 64)
|
||||||
if parseError!=nil {
|
if parseError != nil {
|
||||||
if len(text)>0{
|
if len(text) > 0 {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
return defaultValue
|
return defaultValue
|
||||||
|
|
Loading…
Reference in a new issue