From da9b672d1bce089c7a74e6b4bb68bb68cc4097f2 Mon Sep 17 00:00:00 2001 From: sparklxb Date: Sun, 8 Jan 2017 22:34:26 +0800 Subject: [PATCH] support additional header name-value pairs --- weed/operation/upload_content.go | 13 +++------ weed/server/common.go | 4 +-- weed/server/filer_server_handlers_write.go | 3 +-- weed/storage/needle.go | 31 +++++++++++++--------- weed/topology/store_replicate.go | 15 ++++++++--- 5 files changed, 36 insertions(+), 30 deletions(-) diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index b5784322a..30c7f1ea3 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -36,13 +36,13 @@ func init() { var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") -func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairs []byte, jwt security.EncodedJwt) (*UploadResult, error) { +func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { return upload_content(uploadUrl, func(w io.Writer) (err error) { _, err = io.Copy(w, reader) return - }, filename, isGzipped, mtype, pairs, jwt) + }, filename, isGzipped, mtype, pairMap, jwt) } -func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairs []byte, jwt security.EncodedJwt) (*UploadResult, error) { +func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { body_buf := bytes.NewBufferString("") body_writer := multipart.NewWriter(body_buf) h := make(textproto.MIMEHeader) @@ -59,13 +59,6 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error if jwt != "" { h.Set("Authorization", "BEARER "+string(jwt)) } - pairMap := make(map[string]string) - if len(pairs) != 0 { - err := json.Unmarshal(pairs, &pairMap) - if err != nil { - glog.V(0).Infoln("Unmarshal pairs error:", err) - } - } file_writer, cp_err := body_writer.CreatePart(h) if cp_err != nil { diff --git a/weed/server/common.go b/weed/server/common.go index 3c9e3014f..c5956143f 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -86,7 +86,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st } debug("parsing upload file...") - fname, data, mimeType, pairs, isGzipped, lastModified, _, _, pe := storage.ParseUpload(r) + fname, data, mimeType, pairMap, isGzipped, lastModified, _, _, pe := storage.ParseUpload(r) if pe != nil { writeJsonError(w, r, http.StatusBadRequest, pe) return @@ -112,7 +112,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st } debug("upload file to store", url) - uploadResult, err := operation.Upload(url, fname, bytes.NewReader(data), isGzipped, mimeType, pairs, jwt) + uploadResult, err := operation.Upload(url, fname, bytes.NewReader(data), isGzipped, mimeType, pairMap, jwt) if err != nil { writeJsonError(w, r, http.StatusInternalServerError, err) return diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index aed393dcd..80f222ce6 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -13,10 +13,9 @@ import ( "net/http" "net/textproto" "net/url" - "strings" - "path" "strconv" + "strings" "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" diff --git a/weed/storage/needle.go b/weed/storage/needle.go index ea013e290..5bafe2f62 100644 --- a/weed/storage/needle.go +++ b/weed/storage/needle.go @@ -1,6 +1,7 @@ package storage import ( + "encoding/json" "fmt" "io/ioutil" "math" @@ -11,8 +12,6 @@ import ( "strings" "time" - "encoding/json" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/images" "github.com/chrislusf/seaweedfs/weed/operation" @@ -60,17 +59,15 @@ func (n *Needle) String() (str string) { } func ParseUpload(r *http.Request) ( - fileName string, data []byte, mimeType string, pairs []byte, isGzipped bool, + fileName string, data []byte, mimeType string, pairMap map[string]string, isGzipped bool, modifiedTime uint64, ttl *TTL, isChunkedFile bool, e error) { - pairMap := make(map[string]string) + pairMap = make(map[string]string) for k, v := range r.Header { if len(v) > 0 && strings.HasPrefix(k, PairNamePrefix) { pairMap[k] = v[0] } } - if len(pairMap) != 0 { - pairs, _ = json.Marshal(pairMap) - } + form, fe := r.MultipartReader() if fe != nil { glog.V(0).Infoln("MultipartReader [ERROR]", fe) @@ -158,10 +155,10 @@ func ParseUpload(r *http.Request) ( return } func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) { - var pair []byte + var pairMap map[string]string fname, mimeType, isGzipped, isChunkedFile := "", "", false, false n = new(Needle) - fname, n.Data, mimeType, pair, isGzipped, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r) + fname, n.Data, mimeType, pairMap, isGzipped, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r) if e != nil { return } @@ -173,10 +170,18 @@ func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) { n.Mime = []byte(mimeType) n.SetHasMime() } - if len(pair) < 65536 { - n.Pairs = pair - n.PairsSize = uint16(len(pair)) - n.SetHasPairs() + if len(pairMap) != 0 { + trimmedPairMap := make(map[string]string) + for k, v := range pairMap { + trimmedPairMap[k[len(PairNamePrefix):]] = v + } + + pairs, _ := json.Marshal(trimmedPairMap) + if len(pairs) < 65536 { + n.Pairs = pairs + n.PairsSize = uint16(len(pairs)) + n.SetHasPairs() + } } if isGzipped { n.SetGzipped() diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index e76771140..aa312ac03 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -2,14 +2,14 @@ package topology import ( "bytes" + "encoding/json" "errors" "fmt" "net/http" + "net/url" "strconv" "strings" - "net/url" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/security" @@ -55,9 +55,18 @@ func ReplicatedWrite(masterNode string, s *storage.Store, q.Set("cm", "true") } u.RawQuery = q.Encode() + + pairMap := make(map[string]string) + if needle.HasPairs() { + err := json.Unmarshal(needle.Pairs, &pairMap) + if err != nil { + glog.V(0).Infoln("Unmarshal pairs error:", err) + } + } + _, err := operation.Upload(u.String(), string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime), - needle.Pairs, jwt) + pairMap, jwt) return err }); err != nil { ret = 0