gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

issue 43 "go fmt" chagnes from "Ryan S. Brown" <sb@ryansb.com>

Browse source 
some basic changes to parse upload url
This commit is contained in:
Chris Lu 2013-09-01 23:58:21 -07:00
parent 2e70cc8be7
commit 82b74c7940
28 changed files with 102 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go/glog/convenient_api.go
View file

@ -12,5 +12,5 @@ func ToStderr() {
logging.toStderr = true
}
func ToStderrAndLog() {
logging.alsoToStderr = true
logging.alsoToStderr = true
}

10
go/operation/delete_content.go
View file

@ -6,19 +6,19 @@ import (
"net/http"
)
func DeleteFile(server string, fileId string) (error) {
func DeleteFile(server string, fileId string) error {
fid, parseErr := storage.ParseFileId(fileId)
if parseErr != nil {
return parseErr
}
lookup, lookupError := Lookup(server,fid.VolumeId)
lookup, lookupError := Lookup(server, fid.VolumeId)
if lookupError != nil {
return lookupError
return lookupError
}
if len(lookup.Locations) == 0 {
return nil
return nil
}
return Delete("http://"+lookup.Locations[0].PublicUrl+"/"+fileId)
return Delete("http://" + lookup.Locations[0].PublicUrl + "/" + fileId)
}
func Delete(url string) error {
req, err := http.NewRequest("DELETE", url, nil)

4
go/operation/submit.go
View file

@ -95,8 +95,8 @@ func (fi FilePart) Upload(server string, fid string) (int, error) {
if fi.ModTime != 0 {
fileUrl += "?ts=" + strconv.Itoa(int(fi.ModTime))
}
if closer, ok := fi.Reader.(io.Closer); ok{
defer closer.Close()
if closer, ok := fi.Reader.(io.Closer); ok {
defer closer.Close()
}
ret, e := Upload(fileUrl, fi.FileName, fi.Reader, fi.IsGzipped, fi.MimeType)
if e != nil {

4
go/replication/store_replicate.go
View file

@ -2,9 +2,9 @@ package replication
import (
"bytes"
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/operation"
"code.google.com/p/weed-fs/go/storage"
"code.google.com/p/weed-fs/go/glog"
"net/http"
"strconv"
)
@ -25,7 +25,7 @@ func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.Volum
if needToReplicate { //send to other replica locations
if r.FormValue("type") != "replicate" {
if !distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool {
_, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified,10), string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime))
_, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified, 10), string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime))
return err == nil
}) {
ret = 0

8
go/replication/volume_growth.go
View file

@ -1,12 +1,12 @@
package replication
import (
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/operation"
"code.google.com/p/weed-fs/go/storage"
"code.google.com/p/weed-fs/go/topology"
"errors"
"fmt"
"code.google.com/p/weed-fs/go/glog"
"math/rand"
"sync"
)
@ -39,13 +39,13 @@ func (vg *VolumeGrowth) AutomaticGrowByType(repType storage.ReplicationType, dat
factor = 1
count, err = vg.GrowByCountAndType(vg.copy1factor, repType, dataCenter, topo)
case storage.Copy001:
factor = 2
factor = 2
count, err = vg.GrowByCountAndType(vg.copy2factor, repType, dataCenter, topo)
case storage.Copy010:
factor = 2
factor = 2
count, err = vg.GrowByCountAndType(vg.copy2factor, repType, dataCenter, topo)
case storage.Copy100:
factor = 2
factor = 2
count, err = vg.GrowByCountAndType(vg.copy2factor, repType, dataCenter, topo)
case storage.Copy110:
factor = 3

2
go/sequence/sequence.go
View file

@ -1,8 +1,8 @@
package sequence
import (
"encoding/gob"
"code.google.com/p/weed-fs/go/glog"
"encoding/gob"
"os"
"path"
"sync"

3
go/storage/cdb_map.go
View file

@ -76,6 +76,9 @@ func (m cdbMap) FileCount() int {
func (m *cdbMap) DeletedCount() int {
return m.DeletionCounter
}
func (m *cdbMap) NextFileKey(count int) (uint64) {
return 0
}
func getMetric(c *cdb.Cdb, m *mapMetric) error {
data, err := c.Data([]byte{'M'})

2
go/storage/compact_map_perf_test.go
View file

@ -1,8 +1,8 @@
package storage
import (
"code.google.com/p/weed-fs/go/util"
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/util"
"os"
"testing"
)

2
go/storage/compress.go
View file

@ -2,9 +2,9 @@ package storage
import (
"bytes"
"code.google.com/p/weed-fs/go/glog"
"compress/flate"
"compress/gzip"
"code.google.com/p/weed-fs/go/glog"
"io/ioutil"
"strings"
)

21
go/storage/needle.go
View file

@ -1,10 +1,10 @@
package storage
import (
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/util"
"encoding/hex"
"io/ioutil"
"code.google.com/p/weed-fs/go/glog"
"mime"
"net/http"
"path"
@ -86,19 +86,19 @@ func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string
fileName = fileName[:len(fileName)-3]
}
modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
return
return
}
func NewNeedle(r *http.Request) (n *Needle, e error) {
fname, mimeType, isGzipped := "", "", false
fname, mimeType, isGzipped := "", "", false
n = new(Needle)
fname, n.Data, mimeType, isGzipped, n.LastModified, e = ParseUpload(r)
if e != nil {
return
return
}
if len(fname) < 256 {
n.Name = []byte(fname)
n.SetHasName()
}
if len(fname) < 256 {
n.Name = []byte(fname)
n.SetHasName()
}
if len(mimeType) < 256 {
n.Mime = []byte(mimeType)
n.SetHasMime()
@ -108,7 +108,7 @@ func NewNeedle(r *http.Request) (n *Needle, e error) {
}
if n.LastModified == 0 {
n.LastModified = uint64(time.Now().Unix())
n.SetHasLastModifiedDate()
n.SetHasLastModifiedDate()
}
n.Checksum = NewCRC(n.Data)
@ -127,9 +127,6 @@ func NewNeedle(r *http.Request) (n *Needle, e error) {
func (n *Needle) ParsePath(fid string) {
length := len(fid)
if length <= 8 {
if length > 0 {
glog.V(0).Infoln("Invalid fid", fid, "length", length)
}
return
}
delta := ""

22
go/storage/needle_map.go
View file

@ -2,6 +2,7 @@ package storage
import (
"bufio"
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/util"
"fmt"
"io"
@ -18,6 +19,7 @@ type NeedleMapper interface {
FileCount() int
DeletedCount() int
Visit(visit func(NeedleValue) error) (err error)
NextFileKey(count int) uint64
}
type mapMetric struct {
@ -25,6 +27,7 @@ type mapMetric struct {
FileCounter int `json:"FileCounter"`
DeletionByteCounter uint64 `json:"DeletionByteCounter"`
FileByteCounter uint64 `json:"FileByteCounter"`
MaximumFileKey uint64 `json:"MaxFileKey"`
}
type NeedleMap struct {
@ -53,23 +56,27 @@ const (
func LoadNeedleMap(file *os.File) (*NeedleMap, error) {
nm := NewNeedleMap(file)
e := walkIndexFile(file, func(key uint64, offset, size uint32) error {
if key > nm.MaximumFileKey {
nm.MaximumFileKey = key
}
nm.FileCounter++
nm.FileByteCounter = nm.FileByteCounter + uint64(size)
if offset > 0 {
oldSize := nm.m.Set(Key(key), offset, size)
//glog.V(0).Infoln("reading key", key, "offset", offset, "size", size, "oldSize", oldSize)
glog.V(4).Infoln("reading key", key, "offset", offset, "size", size, "oldSize", oldSize)
if oldSize > 0 {
nm.DeletionCounter++
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
}
} else {
oldSize := nm.m.Delete(Key(key))
//glog.V(0).Infoln("removing key", key, "offset", offset, "size", size, "oldSize", oldSize)
glog.V(4).Infoln("removing key", key, "offset", offset, "size", size, "oldSize", oldSize)
nm.DeletionCounter++
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
}
return nil
})
glog.V(1).Infoln("max file key:", nm.MaximumFileKey)
return nm, e
}
@ -163,3 +170,14 @@ func (nm NeedleMap) DeletedCount() int {
func (nm *NeedleMap) Visit(visit func(NeedleValue) error) (err error) {
return nm.m.Visit(visit)
}
func (nm NeedleMap) MaxFileKey() uint64 {
return nm.MaximumFileKey
}
func (nm NeedleMap) NextFileKey(count int) (ret uint64) {
if count <= 0 {
return 0
}
ret = nm.MaximumFileKey
nm.MaximumFileKey += uint64(count)
return
}

2
go/storage/needle_read_write.go
View file

@ -1,11 +1,11 @@
package storage
import (
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/util"
"errors"
"fmt"
"io"
"code.google.com/p/weed-fs/go/glog"
"os"
)

8
go/storage/store.go
View file

@ -1,11 +1,11 @@
package storage
import (
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/util"
"encoding/json"
"fmt"
"io/ioutil"
"code.google.com/p/weed-fs/go/glog"
"net/url"
"strconv"
"strings"
@ -97,10 +97,10 @@ func (s *Store) addVolume(vid VolumeId, replicationType ReplicationType) error {
if location := s.findFreeLocation(); location != nil {
glog.V(0).Infoln("In dir", location.directory, "adds volume =", vid, ", replicationType =", replicationType)
if volume, err := NewVolume(location.directory, vid, replicationType); err == nil {
location.volumes[vid] = volume
return nil
location.volumes[vid] = volume
return nil
} else {
return err
return err
}
}
return fmt.Errorf("No more free space left")

2
go/storage/volume.go
View file

@ -64,7 +64,7 @@ func (v *Volume) load(alsoLoadIndex bool) error {
v.dataFile, e = os.Open(fileName + ".dat")
v.readOnly = true
} else {
return fmt.Errorf("Unknown state about Volume Data file %s.dat", fileName)
return fmt.Errorf("Unknown state about Volume Data file %s.dat", fileName)
}
if e != nil {
if !os.IsPermission(e) {

2
go/topology/node.go
View file

@ -1,8 +1,8 @@
package topology
import (
"code.google.com/p/weed-fs/go/storage"
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/storage"
)
type NodeId string

2
go/topology/node_list.go
View file

@ -1,8 +1,8 @@
package topology
import (
"code.google.com/p/weed-fs/go/storage"
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/storage"
"math/rand"
)

28
go/topology/node_list_test.go
View file

@ -25,21 +25,21 @@ func TestXYZ(t *testing.T) {
t.Error("need to randomly pick 1 node")
}
picked, ret = nl.RandomlyPickN(1, 0, "dc1")
if !ret || len(picked) != 1 {
t.Error("need to randomly pick 1 node")
}
if picked[0].Id() != "dc1" {
t.Error("need to randomly pick 1 dc1 node")
}
picked, ret = nl.RandomlyPickN(1, 0, "dc1")
if !ret || len(picked) != 1 {
t.Error("need to randomly pick 1 node")
}
if picked[0].Id() != "dc1" {
t.Error("need to randomly pick 1 dc1 node")
}
picked, ret = nl.RandomlyPickN(2, 0, "dc1")
if !ret || len(picked) != 2 {
t.Error("need to randomly pick 1 node")
}
if picked[0].Id() != "dc1" {
t.Error("need to randomly pick 2 with one dc1 node")
}
picked, ret = nl.RandomlyPickN(2, 0, "dc1")
if !ret || len(picked) != 2 {
t.Error("need to randomly pick 1 node")
}
if picked[0].Id() != "dc1" {
t.Error("need to randomly pick 2 with one dc1 node")
}
picked, ret = nl.RandomlyPickN(4, 0, "")
if !ret || len(picked) != 4 {

2
go/topology/topology.go
View file

@ -1,11 +1,11 @@
package topology
import (
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/sequence"
"code.google.com/p/weed-fs/go/storage"
"errors"
"io/ioutil"
"code.google.com/p/weed-fs/go/glog"
"math/rand"
)

2
go/topology/topology_compact.go
View file

@ -1,11 +1,11 @@
package topology
import (
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/storage"
"code.google.com/p/weed-fs/go/util"
"encoding/json"
"errors"
"code.google.com/p/weed-fs/go/glog"
"net/url"
"time"
)

2
go/topology/topology_event_handling.go
View file

@ -1,8 +1,8 @@
package topology
import (
"code.google.com/p/weed-fs/go/storage"
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/storage"
"math/rand"
"time"
)

2
go/util/config.go
View file

@ -10,8 +10,8 @@ package util
import (
"bytes"
"encoding/json"
"code.google.com/p/weed-fs/go/glog"
"encoding/json"
"os"
)

2
go/util/post.go
View file

@ -1,8 +1,8 @@
package util
import (
"io/ioutil"
"code.google.com/p/weed-fs/go/glog"
"io/ioutil"
"net/http"
"net/url"
)

2
go/weed/export.go
View file

@ -3,9 +3,9 @@ package main
import (
"archive/tar"
"bytes"
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/storage"
"fmt"
"code.google.com/p/weed-fs/go/glog"
"os"
"path"
"strconv"

2
go/weed/fix.go
View file

@ -1,8 +1,8 @@
package main
import (
"code.google.com/p/weed-fs/go/storage"
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/storage"
"os"
"path"
"strconv"

2
go/weed/master.go
View file

@ -182,7 +182,7 @@ func volumeStatusHandler(w http.ResponseWriter, r *http.Request) {
}
func redirectHandler(w http.ResponseWriter, r *http.Request) {
vid, _, _, _ := parseURLPath(r.URL.Path)
vid, _, _, _, _ := parseURLPath(r.URL.Path)
volumeId, err := storage.NewVolumeId(vid)
if err != nil {
debug("parsing error:", err, r.URL.Path)

2
go/weed/shell.go
View file

@ -2,8 +2,8 @@ package main
import (
"bufio"
"fmt"
"code.google.com/p/weed-fs/go/glog"
"fmt"
"os"
)

12
go/weed/volume.go
View file

@ -118,7 +118,7 @@ func storeHandler(w http.ResponseWriter, r *http.Request) {
}
func GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) {
n := new(storage.Needle)
vid, fid, filename, ext := parseURLPath(r.URL.Path)
vid, fid, filename, ext, _ := parseURLPath(r.URL.Path)
volumeId, err := storage.NewVolumeId(vid)
if err != nil {
debug("parsing error:", err, r.URL.Path)
@ -207,7 +207,7 @@ func PostHandler(w http.ResponseWriter, r *http.Request) {
writeJsonError(w, r, e)
return
}
vid, _, _, _ := parseURLPath(r.URL.Path)
vid, _, _, _, _ := parseURLPath(r.URL.Path)
volumeId, ve := storage.NewVolumeId(vid)
if ve != nil {
debug("NewVolumeId error:", ve)
@ -231,7 +231,7 @@ func PostHandler(w http.ResponseWriter, r *http.Request) {
}
func DeleteHandler(w http.ResponseWriter, r *http.Request) {
n := new(storage.Needle)
vid, fid, _, _ := parseURLPath(r.URL.Path)
vid, fid, _, _, _ := parseURLPath(r.URL.Path)
volumeId, _ := storage.NewVolumeId(vid)
n.ParsePath(fid)
@ -266,7 +266,7 @@ func DeleteHandler(w http.ResponseWriter, r *http.Request) {
writeJsonQuiet(w, r, m)
}
func parseURLPath(path string) (vid, fid, filename, ext string) {
func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) {
switch strings.Count(path, "/") {
case 3:
parts := strings.Split(path, "/")
@ -284,9 +284,7 @@ func parseURLPath(path string) (vid, fid, filename, ext string) {
sepIndex := strings.LastIndex(path, "/")
commaIndex := strings.LastIndex(path[sepIndex:], ",")
if commaIndex <= 0 {
if "favicon.ico" != path[sepIndex+1:] {
glog.V(0).Infoln("unknown file id", path[sepIndex+1:])
}
vid, isVolumeIdOnly = path[sepIndex+1:], true
return
}
dotIndex := strings.LastIndex(path[sepIndex:], ".")

34
go/weed/weed.go
View file

@ -7,7 +7,7 @@ import (
"fmt"
"io"
"math/rand"
"net"
"net"
"net/http"
"os"
"strings"
@ -225,20 +225,20 @@ func debug(params ...interface{}) {
}
}
func secure(whiteList []string, f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
if len(whiteList) == 0 {
f(w, r)
return
}
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err == nil {
for _, ip := range whiteList {
if ip == host {
f(w, r)
return
}
}
}
writeJsonQuiet(w, r, map[string]interface{}{"error": "No write permisson from " + host})
}
return func(w http.ResponseWriter, r *http.Request) {
if len(whiteList) == 0 {
f(w, r)
return
}
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err == nil {
for _, ip := range whiteList {
if ip == host {
f(w, r)
return
}
}
}
writeJsonQuiet(w, r, map[string]interface{}{"error": "No write permisson from " + host})
}
}