diff --git a/weed-fs/build b/weed-fs/build new file mode 100755 index 000000000..477879ac4 --- /dev/null +++ b/weed-fs/build @@ -0,0 +1,19 @@ +# Build script generated by gb: http://go-gb.googlecode.com +# gb provides configuration-free building and distributing + +echo "Build script generated by gb: http://go-gb.googlecode.com" +if [ "$1" = "goinstall" ]; then +echo Running goinstall \ + +else +echo Building \ +&& echo "(in src/pkg/util)" gomake $1 && cd src/pkg/util && gomake $1 && cd - > /dev/null \ +&& echo "(in src/pkg/storage)" gomake $1 && cd src/pkg/storage && gomake $1 && cd - > /dev/null \ +&& echo "(in src/cmd/weedvolume/fix_volume_index)" gomake $1 && cd src/cmd/weedvolume/fix_volume_index && gomake $1 && cd - > /dev/null \ +&& echo "(in src/cmd/weedvolume)" gomake $1 && cd src/cmd/weedvolume && gomake $1 && cd - > /dev/null \ +&& echo "(in src/pkg/directory)" gomake $1 && cd src/pkg/directory && gomake $1 && cd - > /dev/null \ +&& echo "(in src/cmd/weedmaster)" gomake $1 && cd src/cmd/weedmaster && gomake $1 && cd - > /dev/null \ + +fi + +# The makefiles above are invoked in topological dependence order diff --git a/weed-fs/src/cmd/agent.go b/weed-fs/src/cmd/agent.go deleted file mode 100644 index 8949d4d02..000000000 --- a/weed-fs/src/cmd/agent.go +++ /dev/null @@ -1,47 +0,0 @@ -package main - -import ( - "http" - // "runtime" - "log" - "os" - "exec" - "fmt" -) - -func HelloServer(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "text/plain") - w.Write([]byte("hello, world!\n")) -} -func ShellHandler(w http.ResponseWriter, r *http.Request) { - name := r.FormValue("cmd") - args := r.Form["arg"] - dir := r.FormValue("dir") - w.Header().Set("Content-Type", "text/plain") - - cmd := run(w, dir, name, args) - cmd.Wait() -} - -func run(w http.ResponseWriter, dir string, name string, args []string) *exec.Cmd { - cmd := exec.Command(name, args...) - cmd.Dir = dir - cmd.Env = os.Environ() - cmd.Stdin = os.Stdin - cmd.Stdout = w - cmd.Stderr = w - - if err := cmd.Start(); err != nil { - fmt.Fprint(w, "could not execute", args, ":", cmd.Args, "\n", err,"\n") - } - return cmd -} - -func main() { - // runtime.GOMAXPROCS(1) - http.HandleFunc("/", HelloServer) - http.HandleFunc("/shell", ShellHandler) - - log.Println("Serving at http://127.0.0.1:8080/") - http.ListenAndServe(":8080", nil) -} diff --git a/weed-fs/src/cmd/weedc.go b/weed-fs/src/cmd/weedc.go deleted file mode 100644 index 0aa03f85a..000000000 --- a/weed-fs/src/cmd/weedc.go +++ /dev/null @@ -1,157 +0,0 @@ -package main - -import ( - "storage" - "flag" - "fmt" - "http" - "json" - "log" - "mime" - "rand" - "strconv" - "strings" - "time" -) - -var ( - port = flag.Int("port", 8080, "http listen port") - chunkFolder = flag.String("dir", "/tmp", "data directory to store files") - volumes = flag.String("volumes", "0,1-3,4", "comma-separated list of volume ids or range of ids") - publicUrl = flag.String("publicUrl", "localhost:8080", "public url to serve data read") - metaServer = flag.String("mserver", "localhost:9333", "master directory server to store mappings") - IsDebug = flag.Bool("debug", false, "enable debug mode") - pulse = flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") - - store *storage.Store -) - -func statusHandler(w http.ResponseWriter, r *http.Request) { - writeJson(w, r, store.Status()) -} -func addVolumeHandler(w http.ResponseWriter, r *http.Request) { - store.AddVolume(r.FormValue("volume")) - writeJson(w, r, store.Status()) -} -func storeHandler(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case "GET": - GetHandler(w, r) - case "DELETE": - DeleteHandler(w, r) - case "POST": - PostHandler(w, r) - } -} -func GetHandler(w http.ResponseWriter, r *http.Request) { - n := new(storage.Needle) - vid, fid, ext := parseURLPath(r.URL.Path) - volumeId, _ := strconv.Atoui64(vid) - n.ParsePath(fid) - - if *IsDebug { - log.Println("volume", volumeId, "reading", n) - } - cookie := n.Cookie - count, e := store.Read(volumeId, n) - if *IsDebug { - log.Println("read bytes", count, "error", e) - } - if n.Cookie != cookie { - log.Println("request with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent()) - return - } - if ext != "" { - w.Header().Set("Content-Type", mime.TypeByExtension(ext)) - } - w.Write(n.Data) -} -func PostHandler(w http.ResponseWriter, r *http.Request) { - vid, _, _ := parseURLPath(r.URL.Path) - volumeId, e := strconv.Atoui64(vid) - if e != nil { - writeJson(w, r, e) - } else { - ret := store.Write(volumeId, storage.NewNeedle(r)) - m := make(map[string]uint32) - m["size"] = ret - writeJson(w, r, m) - } -} -func DeleteHandler(w http.ResponseWriter, r *http.Request) { - n := new(storage.Needle) - vid, fid, _ := parseURLPath(r.URL.Path) - volumeId, _ := strconv.Atoui64(vid) - n.ParsePath(fid) - - cookie := n.Cookie - count, _ := store.Read(volumeId, n) - - if n.Cookie != cookie { - log.Println("delete with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent()) - return - } - - n.Size = 0 - store.Write(volumeId, n) - m := make(map[string]uint32) - m["size"] = uint32(count) - writeJson(w, r, m) -} -func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) { - w.Header().Set("Content-Type", "application/javascript") - bytes, _ := json.Marshal(obj) - callback := r.FormValue("callback") - if callback == "" { - w.Write(bytes) - } else { - w.Write([]uint8(callback)) - w.Write([]uint8("(")) - fmt.Fprint(w, string(bytes)) - w.Write([]uint8(")")) - } - //log.Println("JSON Response", string(bytes)) -} -func parseURLPath(path string) (vid, fid, ext string) { - sepIndex := strings.LastIndex(path, "/") - commaIndex := strings.LastIndex(path[sepIndex:], ",") - if commaIndex <= 0 { - log.Println("unknown file id", path[sepIndex+1:]) - return - } - dotIndex := strings.LastIndex(path[sepIndex:], ".") - vid = path[sepIndex+1 : commaIndex] - fid = path[commaIndex+1:] - ext = "" - if dotIndex > 0 { - fid = path[commaIndex+1 : dotIndex] - ext = path[dotIndex+1:] - } - return -} - -func main() { - flag.Parse() - //TODO: now default to 1G, this value should come from server? - store = storage.NewStore(*port, *publicUrl, *chunkFolder, *volumes) - defer store.Close() - http.HandleFunc("/", storeHandler) - http.HandleFunc("/status", statusHandler) - http.HandleFunc("/add_volume", addVolumeHandler) - - go func() { - for { - store.Join(*metaServer) - ns := int64(*pulse) * 1e9 - time.Sleep(ns + rand.Int63()%ns) - } - }() - log.Println("store joined at", *metaServer) - - log.Println("Start storage service at http://127.0.0.1:" + strconv.Itoa(*port), "public url", *publicUrl) - e := http.ListenAndServe(":"+strconv.Itoa(*port), nil) - if e != nil { - log.Fatalf("Fail to start:", e.String()) - } - -} diff --git a/weed-fs/src/cmd/weedmaster/Makefile b/weed-fs/src/cmd/weedmaster/Makefile new file mode 100644 index 000000000..a0c4ca7c0 --- /dev/null +++ b/weed-fs/src/cmd/weedmaster/Makefile @@ -0,0 +1,39 @@ +# Makefile generated by gb: http://go-gb.googlecode.com +# gb provides configuration-free building and distributing + +include $(GOROOT)/src/Make.inc + +TARG=weedmaster +GOFILES=\ + weedmaster.go\ + +# gb: this is the local install +GBROOT=../../.. + +# gb: compile/link against local install +GCIMPORTS+= -I $(GBROOT)/_obj +LDIMPORTS+= -L $(GBROOT)/_obj + +# gb: compile/link against GOPATH entries +GOPATHSEP=: +ifeq ($(GOHOSTOS),windows) +GOPATHSEP=; +endif +GCIMPORTS=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) +LDIMPORTS=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) + +# gb: default target is in GBROOT this way +command: + +include $(GOROOT)/src/Make.cmd + +# gb: copy to local install +$(GBROOT)/bin/$(TARG): $(TARG) + mkdir -p $(dir $@); cp -f $< $@ +command: $(GBROOT)/bin/$(TARG) + +# gb: local dependencies +$(TARG): $(GBROOT)/_obj/storage.a +<<<<<<< local +$(TARG): $(GBROOT)/_obj/directory.a +<<<<<<< local diff --git a/weed-fs/src/cmd/weeds.go b/weed-fs/src/cmd/weedmaster/weedmaster.go similarity index 100% rename from weed-fs/src/cmd/weeds.go rename to weed-fs/src/cmd/weedmaster/weedmaster.go diff --git a/weed-fs/src/cmd/weedvolume/fix_volume_index/Makefile b/weed-fs/src/cmd/weedvolume/fix_volume_index/Makefile new file mode 100644 index 000000000..54e0d7d74 --- /dev/null +++ b/weed-fs/src/cmd/weedvolume/fix_volume_index/Makefile @@ -0,0 +1,37 @@ +# Makefile generated by gb: http://go-gb.googlecode.com +# gb provides configuration-free building and distributing + +include $(GOROOT)/src/Make.inc + +TARG=fix_volume_index +GOFILES=\ + fix_volume_index.go\ + +# gb: this is the local install +GBROOT=../../../.. + +# gb: compile/link against local install +GCIMPORTS+= -I $(GBROOT)/_obj +LDIMPORTS+= -L $(GBROOT)/_obj + +# gb: compile/link against GOPATH entries +GOPATHSEP=: +ifeq ($(GOHOSTOS),windows) +GOPATHSEP=; +endif +GCIMPORTS=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) +LDIMPORTS=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) + +# gb: default target is in GBROOT this way +command: + +include $(GOROOT)/src/Make.cmd + +# gb: copy to local install +$(GBROOT)/bin/$(TARG): $(TARG) + mkdir -p $(dir $@); cp -f $< $@ +command: $(GBROOT)/bin/$(TARG) + +# gb: local dependencies +$(TARG): $(GBROOT)/_obj/storage.a +<<<<<<< local diff --git a/weed-fs/src/cmd/fix_volume_index.go b/weed-fs/src/cmd/weedvolume/fix_volume_index/fix_volume_index.go similarity index 100% rename from weed-fs/src/cmd/fix_volume_index.go rename to weed-fs/src/cmd/weedvolume/fix_volume_index/fix_volume_index.go diff --git a/weed-fs/src/pkg/directory/Makefile b/weed-fs/src/pkg/directory/Makefile new file mode 100644 index 000000000..55cc4ce50 --- /dev/null +++ b/weed-fs/src/pkg/directory/Makefile @@ -0,0 +1,35 @@ +# Makefile generated by gb: http://go-gb.googlecode.com +# gb provides configuration-free building and distributing + +include $(GOROOT)/src/Make.inc + +TARG=directory +GOFILES=\ + file_id.go\ + volume_mapping.go\ + +# gb: this is the local install +GBROOT=../../.. + +# gb: compile/link against local install +GCIMPORTS+= -I $(GBROOT)/_obj +LDIMPORTS+= -L $(GBROOT)/_obj +# gb: compile/link against GOPATH entries +GOPATHSEP=: +ifeq ($(GOHOSTOS),windows) +GOPATHSEP=; +endif +GCIMPORTS=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) +LDIMPORTS=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) + +# gb: copy to local install +$(GBROOT)/_obj/$(TARG).a: _obj/$(TARG).a + mkdir -p $(dir $@); cp -f $< $@ + +package: $(GBROOT)/_obj/$(TARG).a + +include $(GOROOT)/src/Make.pkg + +# gb: local dependencies +_obj/$(TARG).a: $(GBROOT)/_obj/storage.a +_obj/$(TARG).a: $(GBROOT)/_obj/util.a diff --git a/weed-fs/src/pkg/storage/Makefile b/weed-fs/src/pkg/storage/Makefile new file mode 100644 index 000000000..ab50c9e8a --- /dev/null +++ b/weed-fs/src/pkg/storage/Makefile @@ -0,0 +1,36 @@ +# Makefile generated by gb: http://go-gb.googlecode.com +# gb provides configuration-free building and distributing + +include $(GOROOT)/src/Make.inc + +TARG=storage +GOFILES=\ + needle.go\ + needle_map.go\ + store.go\ + volume.go\ + +# gb: this is the local install +GBROOT=../../.. + +# gb: compile/link against local install +GCIMPORTS+= -I $(GBROOT)/_obj +LDIMPORTS+= -L $(GBROOT)/_obj +# gb: compile/link against GOPATH entries +GOPATHSEP=: +ifeq ($(GOHOSTOS),windows) +GOPATHSEP=; +endif +GCIMPORTS=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) +LDIMPORTS=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) + +# gb: copy to local install +$(GBROOT)/_obj/$(TARG).a: _obj/$(TARG).a + mkdir -p $(dir $@); cp -f $< $@ + +package: $(GBROOT)/_obj/$(TARG).a + +include $(GOROOT)/src/Make.pkg + +# gb: local dependencies +_obj/$(TARG).a: $(GBROOT)/_obj/util.a diff --git a/weed-fs/src/pkg/util/Makefile b/weed-fs/src/pkg/util/Makefile new file mode 100644 index 000000000..bf3a57f05 --- /dev/null +++ b/weed-fs/src/pkg/util/Makefile @@ -0,0 +1,31 @@ +# Makefile generated by gb: http://go-gb.googlecode.com +# gb provides configuration-free building and distributing + +include $(GOROOT)/src/Make.inc + +TARG=util +GOFILES=\ + bytes.go\ + post.go\ + +# gb: this is the local install +GBROOT=../../.. + +# gb: compile/link against local install +GCIMPORTS+= -I $(GBROOT)/_obj +LDIMPORTS+= -L $(GBROOT)/_obj +# gb: compile/link against GOPATH entries +GOPATHSEP=: +ifeq ($(GOHOSTOS),windows) +GOPATHSEP=; +endif +GCIMPORTS=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) +LDIMPORTS=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH) + +# gb: copy to local install +$(GBROOT)/_obj/$(TARG).a: _obj/$(TARG).a + mkdir -p $(dir $@); cp -f $< $@ + +package: $(GBROOT)/_obj/$(TARG).a + +include $(GOROOT)/src/Make.pkg