gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

mux router cannot handle "/" correctly. switching it off for volume

Browse source 
servers
This commit is contained in:
Chris Lu 2013-12-09 13:27:09 -08:00
parent 2e2f426fe2
commit 11e91bd549
7 changed files with 70 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
go/util/file_util.go Normal file
View file

@ -0,0 +1,23 @@
package util
import (
"code.google.com/p/weed-fs/go/glog"
"errors"
"os"
)
func TestFolderWritable(folder string) (err error) {
fileInfo, err := os.Stat(folder)
if err != nil {
return err
}
if !fileInfo.IsDir() {
return errors.New("Not a valid folder!")
}
perm := fileInfo.Mode().Perm()
glog.V(0).Infoln("Folder", folder, "Permission:", perm)
if 0200&perm != 0 {
return nil
}
return errors.New("Not writable!")
}

4
go/weed/master.go
View file

@ -2,6 +2,7 @@ package main
import (
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/util"
"code.google.com/p/weed-fs/go/weed/weed_server"
"github.com/gorilla/mux"
"net/http"
@ -46,6 +47,9 @@ func runMaster(cmd *Command, args []string) bool {
*mMaxCpu = runtime.NumCPU()
}
runtime.GOMAXPROCS(*mMaxCpu)
if err := util.TestFolderWritable(*metaFolder); err != nil {
glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err)
}
if *masterWhiteListOption != "" {
masterWhiteList = strings.Split(*masterWhiteListOption, ",")
}

30
go/weed/server.go
View file

@ -2,6 +2,7 @@ package main
import (
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/util"
"code.google.com/p/weed-fs/go/weed/weed_server"
"github.com/gorilla/mux"
"net/http"
@ -40,6 +41,7 @@ var (
serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name")
serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
serverMembers = cmdServer.Flag.String("members", "", "comma separated ip:masterPort list")
masterPort = cmdServer.Flag.Int("masterPort", 9333, "master server http listen port")
masterMetaFolder = cmdServer.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
masterVolumeSizeLimitMB = cmdServer.Flag.Uint("volumeSizeLimitMB", 32*1024, "Default Volume Size in MegaBytes")
@ -60,6 +62,10 @@ func runServer(cmd *Command, args []string) bool {
}
runtime.GOMAXPROCS(*serverMaxCpu)
if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *masterMetaFolder, err)
}
folders := strings.Split(*volumeDataFolders, ",")
maxCountStrings := strings.Split(*volumeMaxDataVolumeCounts, ",")
maxCounts := make([]int, 0)
@ -74,16 +80,9 @@ func runServer(cmd *Command, args []string) bool {
glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
}
for _, folder := range folders {
fileInfo, err := os.Stat(folder)
if err != nil {
glog.Fatalf("No Existing Folder:%s", folder)
if err := util.TestFolderWritable(folder); err != nil {
glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
}
if !fileInfo.IsDir() {
glog.Fatalf("Volume Folder should not be a file:%s", folder)
}
perm := fileInfo.Mode().Perm()
glog.V(0).Infoln("Volume Folder", folder)
glog.V(0).Infoln("Permission:", perm)
}
if *volumePublicUrl == "" {
@ -105,13 +104,24 @@ func runServer(cmd *Command, args []string) bool {
Handler: r,
ReadTimeout: time.Duration(*serverReadTimeout) * time.Second,
}
go func() {
time.Sleep(100 * time.Millisecond)
var members []string
if *serverMembers != "" {
members = strings.Split(*serverMembers, ",")
}
weed_server.NewRaftServer(r, VERSION, members, *serverIp+":"+strconv.Itoa(*masterPort), *masterMetaFolder)
}()
e := masterServer.ListenAndServe()
if e != nil {
glog.Fatalf("Fail to start master:%s", e)
}
}()
r := mux.NewRouter()
time.Sleep(100 * time.Millisecond)
r := http.NewServeMux()
weed_server.NewVolumeServer(r, VERSION, *serverIp, *volumePort, *volumePublicUrl, folders, maxCounts,
*serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, serverWhiteList,
)

15
go/weed/volume.go
View file

@ -2,8 +2,8 @@ package main
import (
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/util"
"code.google.com/p/weed-fs/go/weed/weed_server"
"github.com/gorilla/mux"
"net/http"
"os"
"runtime"
@ -60,16 +60,9 @@ func runVolume(cmd *Command, args []string) bool {
glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
}
for _, folder := range folders {
fileInfo, err := os.Stat(folder)
if err != nil {
glog.Fatalf("No Existing Folder:%s", folder)
if err := util.TestFolderWritable(folder); err != nil {
glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
}
if !fileInfo.IsDir() {
glog.Fatalf("Volume Folder should not be a file:%s", folder)
}
perm := fileInfo.Mode().Perm()
glog.V(0).Infoln("Volume Folder", folder)
glog.V(0).Infoln("Permission:", perm)
}
if *publicUrl == "" {
@ -79,7 +72,7 @@ func runVolume(cmd *Command, args []string) bool {
volumeWhiteList = strings.Split(*volumeWhiteListOption, ",")
}
r := mux.NewRouter()
r := http.NewServeMux()
weed_server.NewVolumeServer(r, VERSION, *ip, *vport, *publicUrl, folders, maxCounts,
*masterNode, *vpulse, *dataCenter, *rack, volumeWhiteList,

1
go/weed/weed_server/common.go
View file

@ -14,6 +14,7 @@ import (
"strings"
)
func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) (err error) {
w.Header().Set("Content-Type", "application/javascript")
var bytes []byte

4
go/weed/weed_server/volume_server.go
View file

@ -3,8 +3,8 @@ package weed_server
import (
"code.google.com/p/weed-fs/go/glog"
"code.google.com/p/weed-fs/go/storage"
"github.com/gorilla/mux"
"math/rand"
"net/http"
"time"
)
@ -18,7 +18,7 @@ type VolumeServer struct {
version string
}
func NewVolumeServer(r *mux.Router, version string, ip string, port int, publicUrl string, folders []string, maxCounts []int,
func NewVolumeServer(r *http.ServeMux, version string, ip string, port int, publicUrl string, folders []string, maxCounts []int,
masterNode string, pulseSeconds int,
dataCenter string, rack string,
whiteList []string) *VolumeServer {

33
go/weed/weed_server/volume_server_handlers.go
View file

@ -27,7 +27,7 @@ func (vs *VolumeServer) assignVolumeHandler(w http.ResponseWriter, r *http.Reque
} else {
writeJsonQuiet(w, r, map[string]string{"error": err.Error()})
}
debug("assign volume =", r.FormValue("volume"), ", collection =", r.FormValue("collection"), ", replicationType =", r.FormValue("replicationType"), ", error =", err)
glog.V(2).Infoln("assign volume =", r.FormValue("volume"), ", collection =", r.FormValue("collection"), ", replicationType =", r.FormValue("replicationType"), ", error =", err)
}
func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) {
err, ret := vs.store.CheckCompactVolume(r.FormValue("volume"), r.FormValue("garbageThreshold"))
@ -36,7 +36,7 @@ func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.
} else {
writeJsonQuiet(w, r, map[string]interface{}{"error": err.Error(), "result": false})
}
debug("checked compacting volume =", r.FormValue("volume"), "garbageThreshold =", r.FormValue("garbageThreshold"), "vacuum =", ret)
glog.V(2).Infoln("checked compacting volume =", r.FormValue("volume"), "garbageThreshold =", r.FormValue("garbageThreshold"), "vacuum =", ret)
}
func (vs *VolumeServer) vacuumVolumeCompactHandler(w http.ResponseWriter, r *http.Request) {
err := vs.store.CompactVolume(r.FormValue("volume"))
@ -45,7 +45,7 @@ func (vs *VolumeServer) vacuumVolumeCompactHandler(w http.ResponseWriter, r *htt
} else {
writeJsonQuiet(w, r, map[string]string{"error": err.Error()})
}
debug("compacted volume =", r.FormValue("volume"), ", error =", err)
glog.V(2).Infoln("compacted volume =", r.FormValue("volume"), ", error =", err)
}
func (vs *VolumeServer) vacuumVolumeCommitHandler(w http.ResponseWriter, r *http.Request) {
err := vs.store.CommitCompactVolume(r.FormValue("volume"))
@ -54,7 +54,7 @@ func (vs *VolumeServer) vacuumVolumeCommitHandler(w http.ResponseWriter, r *http
} else {
writeJsonQuiet(w, r, map[string]string{"error": err.Error()})
}
debug("commit compact volume =", r.FormValue("volume"), ", error =", err)
glog.V(2).Infoln("commit compact volume =", r.FormValue("volume"), ", error =", err)
}
func (vs *VolumeServer) freezeVolumeHandler(w http.ResponseWriter, r *http.Request) {
//TODO: notify master that this volume will be read-only
@ -64,7 +64,7 @@ func (vs *VolumeServer) freezeVolumeHandler(w http.ResponseWriter, r *http.Reque
} else {
writeJsonQuiet(w, r, map[string]string{"error": err.Error()})
}
debug("freeze volume =", r.FormValue("volume"), ", error =", err)
glog.V(2).Infoln("freeze volume =", r.FormValue("volume"), ", error =", err)
}
func (vs *VolumeServer) submitFromVolumeServerHandler(w http.ResponseWriter, r *http.Request) {
submitForClientHandler(w, r, vs.masterNode)
@ -89,28 +89,28 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
vid, fid, filename, ext, _ := parseURLPath(r.URL.Path)
volumeId, err := storage.NewVolumeId(vid)
if err != nil {
debug("parsing error:", err, r.URL.Path)
glog.V(2).Infoln("parsing error:", err, r.URL.Path)
return
}
n.ParsePath(fid)
debug("volume", volumeId, "reading", n)
glog.V(2).Infoln("volume", volumeId, "reading", n)
if !vs.store.HasVolume(volumeId) {
lookupResult, err := operation.Lookup(vs.masterNode, volumeId)
debug("volume", volumeId, "found on", lookupResult, "error", err)
glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err)
if err == nil {
http.Redirect(w, r, "http://"+lookupResult.Locations[0].PublicUrl+r.URL.Path, http.StatusMovedPermanently)
} else {
debug("lookup error:", err, r.URL.Path)
glog.V(2).Infoln("lookup error:", err, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
}
return
}
cookie := n.Cookie
count, e := vs.store.Read(volumeId, n)
debug("read bytes", count, "error", e)
glog.V(2).Infoln("read bytes", count, "error", e)
if e != nil || count <= 0 {
debug("read error:", e, r.URL.Path)
glog.V(0).Infoln("read error:", e, r.URL.Path)
w.WriteHeader(http.StatusNotFound)
return
}
@ -156,7 +156,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
w.Header().Set("Content-Encoding", "gzip")
} else {
if n.Data, err = storage.UnGzipData(n.Data); err != nil {
debug("lookup error:", err, r.URL.Path)
glog.V(0).Infoln("lookup error:", err, r.URL.Path)
}
}
}
@ -164,21 +164,21 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
w.Header().Set("Content-Length", strconv.Itoa(len(n.Data)))
if isGetMethod {
if _, e = w.Write(n.Data); e != nil {
debug("response write error:", e)
glog.V(0).Infoln("response write error:", e)
}
}
}
func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
m := make(map[string]interface{})
if e := r.ParseForm(); e != nil {
debug("form parse error:", e)
glog.V(0).Infoln("form parse error:", e)
writeJsonError(w, r, e)
return
}
vid, _, _, _, _ := parseURLPath(r.URL.Path)
volumeId, ve := storage.NewVolumeId(vid)
if ve != nil {
debug("NewVolumeId error:", ve)
glog.V(0).Infoln("NewVolumeId error:", ve)
writeJsonError(w, r, ve)
return
}
@ -203,7 +203,7 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
volumeId, _ := storage.NewVolumeId(vid)
n.ParsePath(fid)
debug("deleting", n)
glog.V(2).Infoln("deleting", n)
cookie := n.Cookie
count, ok := vs.store.Read(volumeId, n)
@ -233,4 +233,3 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
m["size"] = uint32(count)
writeJsonQuiet(w, r, m)
}