From 79b0a581327b0153acb869570eae20fa1a1e8faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Thu, 23 Jun 2016 11:19:09 +0800 Subject: [PATCH 1/2] add options params for the Assign func --- weed/operation/assign_file_id.go | 33 +++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go index acc2d3034..a893e836b 100644 --- a/weed/operation/assign_file_id.go +++ b/weed/operation/assign_file_id.go @@ -19,7 +19,13 @@ type AssignResult struct { Error string `json:"error,omitempty"` } -func Assign(server string, count uint64, replication string, collection string, ttl string) (*AssignResult, error) { +/* +options params meaning: +options[0] main data Center +options[1] main rack +options[2] main data node +*/ +func Assign(server string, count uint64, replication string, collection string, ttl string, options ...string) (*AssignResult, error) { values := make(url.Values) values.Add("count", strconv.FormatUint(count, 10)) if replication != "" { @@ -31,6 +37,31 @@ func Assign(server string, count uint64, replication string, collection string, if ttl != "" { values.Add("ttl", ttl) } + + var dataCenter, rack, dataNode string + switch len(options) { + case 1: + dataCenter = options[0] + case 2: + dataCenter = options[0] + rack = options[1] + case 3: + dataCenter = options[0] + rack = options[1] + dataNode = options[2] + default: + } + + if dataCenter != "" { + values.Add("dataCenter", dataCenter) + } + if rack != "" { + values.Add("rack", rack) + } + if dataNode != "" { + values.Add("dataNode", dataNode) + } + jsonBlob, err := util.Post("http://"+server+"/dir/assign", values) glog.V(2).Info("assign result :", string(jsonBlob)) if err != nil { From 1f630945422953429eff852a6b6a03b9c004e9ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Sun, 26 Jun 2016 10:50:18 +0800 Subject: [PATCH 2/2] refactor Volume Assign function --- weed/command/benchmark.go | 6 ++- weed/operation/assign_file_id.go | 59 +++++++++------------- weed/operation/submit.go | 16 +++++- weed/server/common.go | 8 ++- weed/server/filer_server_handlers_write.go | 8 ++- 5 files changed, 57 insertions(+), 40 deletions(-) diff --git a/weed/command/benchmark.go b/weed/command/benchmark.go index 7e0802e30..7bbb82319 100644 --- a/weed/command/benchmark.go +++ b/weed/command/benchmark.go @@ -205,7 +205,11 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { start := time.Now() fileSize := int64(*b.fileSize + rand.Intn(64)) fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize} - if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil { + ar := &operation.VolumeAssignRequest{ + Count: 1, + Collection: *b.collection, + } + if assignResult, err := operation.Assign(*b.server, ar); err == nil { fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection if _, err := fp.Upload(0, *b.server, secret); err == nil { if rand.Intn(100) < *b.deletePercentage { diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go index a893e836b..f2365890c 100644 --- a/weed/operation/assign_file_id.go +++ b/weed/operation/assign_file_id.go @@ -11,6 +11,16 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" ) +type VolumeAssignRequest struct { + Count uint64 + Replication string + Collection string + Ttl string + DataCenter string + Rack string + DataNode string +} + type AssignResult struct { Fid string `json:"fid,omitempty"` Url string `json:"url,omitempty"` @@ -19,47 +29,26 @@ type AssignResult struct { Error string `json:"error,omitempty"` } -/* -options params meaning: -options[0] main data Center -options[1] main rack -options[2] main data node -*/ -func Assign(server string, count uint64, replication string, collection string, ttl string, options ...string) (*AssignResult, error) { +func Assign(server string, r *VolumeAssignRequest) (*AssignResult, error) { values := make(url.Values) - values.Add("count", strconv.FormatUint(count, 10)) - if replication != "" { - values.Add("replication", replication) + values.Add("count", strconv.FormatUint(r.Count, 10)) + if r.Replication != "" { + values.Add("replication", r.Replication) } - if collection != "" { - values.Add("collection", collection) + if r.Collection != "" { + values.Add("collection", r.Collection) } - if ttl != "" { - values.Add("ttl", ttl) + if r.Ttl != "" { + values.Add("ttl", r.Ttl) } - - var dataCenter, rack, dataNode string - switch len(options) { - case 1: - dataCenter = options[0] - case 2: - dataCenter = options[0] - rack = options[1] - case 3: - dataCenter = options[0] - rack = options[1] - dataNode = options[2] - default: + if r.DataCenter != "" { + values.Add("dataCenter", r.DataCenter) } - - if dataCenter != "" { - values.Add("dataCenter", dataCenter) + if r.Rack != "" { + values.Add("rack", r.Rack) } - if rack != "" { - values.Add("rack", rack) - } - if dataNode != "" { - values.Add("dataNode", dataNode) + if r.DataNode != "" { + values.Add("dataNode", r.DataNode) } jsonBlob, err := util.Post("http://"+server+"/dir/assign", values) diff --git a/weed/operation/submit.go b/weed/operation/submit.go index 19bbd7a70..54b6e164e 100644 --- a/weed/operation/submit.go +++ b/weed/operation/submit.go @@ -44,7 +44,13 @@ func SubmitFiles(master string, files []FilePart, for index, file := range files { results[index].FileName = file.FileName } - ret, err := Assign(master, uint64(len(files)), replication, collection, ttl) + ar := &VolumeAssignRequest{ + Count: uint64(len(files)), + Replication: replication, + Collection: collection, + Ttl: ttl, + } + ret, err := Assign(master, ar) if err != nil { for index, _ := range files { results[index].Error = err.Error() @@ -164,7 +170,13 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret func upload_one_chunk(filename string, reader io.Reader, master, replication string, collection string, ttl string, jwt security.EncodedJwt, ) (fid string, size uint32, e error) { - ret, err := Assign(master, 1, replication, collection, ttl) + ar := &VolumeAssignRequest{ + Count: 1, + Replication: replication, + Collection: collection, + Ttl: ttl, + } + ret, err := Assign(master, ar) if err != nil { return "", 0, err } diff --git a/weed/server/common.go b/weed/server/common.go index 312bcea14..dcd31f823 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -94,7 +94,13 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st debug("assigning file id for", fname) r.ParseForm() - assignResult, ae := operation.Assign(masterUrl, 1, r.FormValue("replication"), r.FormValue("collection"), r.FormValue("ttl")) + ar := &operation.VolumeAssignRequest{ + Count: 1, + Replication: r.FormValue("replication"), + Collection: r.FormValue("collection"), + Ttl: r.FormValue("ttl"), + } + assignResult, ae := operation.Assign(masterUrl, ar) if ae != nil { writeJsonError(w, r, http.StatusInternalServerError, ae) return diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 597004f7d..e2d40f532 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -87,7 +87,13 @@ func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Reques } func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, replication, collection string) (fileId, urlLocation string, err error) { - assignResult, ae := operation.Assign(fs.getMasterNode(), 1, replication, collection, r.URL.Query().Get("ttl")) + ar := &operation.VolumeAssignRequest{ + Count: 1, + Replication: replication, + Collection: collection, + Ttl: r.URL.Query().Get("ttl"), + } + assignResult, ae := operation.Assign(fs.getMasterNode(), ar) if ae != nil { glog.V(0).Infoln("failing to assign a file id", ae.Error()) writeJsonError(w, r, http.StatusInternalServerError, ae)