support additional header name-value pairs

weed/operation/upload_content.go

@@ -36,13 +36,13 @@ func init() {
 
 var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
-func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairs []byte, jwt security.EncodedJwt) (*UploadResult, error) {
+func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
 	return upload_content(uploadUrl, func(w io.Writer) (err error) {
 		_, err = io.Copy(w, reader)
 		return
-	}, filename, isGzipped, mtype, pairs, jwt)
+	}, filename, isGzipped, mtype, pairMap, jwt)
 }
-func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairs []byte, jwt security.EncodedJwt) (*UploadResult, error) {
+func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
 	body_buf := bytes.NewBufferString("")
 	body_writer := multipart.NewWriter(body_buf)
 	h := make(textproto.MIMEHeader)
@@ -59,13 +59,6 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 	if jwt != "" {
 		h.Set("Authorization", "BEARER "+string(jwt))
 	}
-	pairMap := make(map[string]string)
-	if len(pairs) != 0 {
-		err := json.Unmarshal(pairs, &pairMap)
-		if err != nil {
-			glog.V(0).Infoln("Unmarshal pairs error:", err)
-		}
-	}
 
 	file_writer, cp_err := body_writer.CreatePart(h)
 	if cp_err != nil {

weed/server/common.go

@@ -86,7 +86,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st
 	}
 
 	debug("parsing upload file...")
-	fname, data, mimeType, pairs, isGzipped, lastModified, _, _, pe := storage.ParseUpload(r)
+	fname, data, mimeType, pairMap, isGzipped, lastModified, _, _, pe := storage.ParseUpload(r)
 	if pe != nil {
 		writeJsonError(w, r, http.StatusBadRequest, pe)
 		return
@@ -112,7 +112,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st
 	}
 
 	debug("upload file to store", url)
-	uploadResult, err := operation.Upload(url, fname, bytes.NewReader(data), isGzipped, mimeType, pairs, jwt)
+	uploadResult, err := operation.Upload(url, fname, bytes.NewReader(data), isGzipped, mimeType, pairMap, jwt)
 	if err != nil {
 		writeJsonError(w, r, http.StatusInternalServerError, err)
 		return

weed/server/filer_server_handlers_write.go

@@ -13,10 +13,9 @@ import (
 	"net/http"
 	"net/textproto"
 	"net/url"
-	"strings"
-
 	"path"
 	"strconv"
+	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"

weed/storage/needle.go

@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"math"
@@ -11,8 +12,6 @@ import (
 	"strings"
 	"time"
 
-	"encoding/json"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/images"
 	"github.com/chrislusf/seaweedfs/weed/operation"
@@ -60,17 +59,15 @@ func (n *Needle) String() (str string) {
 }
 
 func ParseUpload(r *http.Request) (
-	fileName string, data []byte, mimeType string, pairs []byte, isGzipped bool,
+	fileName string, data []byte, mimeType string, pairMap map[string]string, isGzipped bool,
 	modifiedTime uint64, ttl *TTL, isChunkedFile bool, e error) {
-	pairMap := make(map[string]string)
+	pairMap = make(map[string]string)
 	for k, v := range r.Header {
 		if len(v) > 0 && strings.HasPrefix(k, PairNamePrefix) {
 			pairMap[k] = v[0]
 		}
 	}
-	if len(pairMap) != 0 {
+
-		pairs, _ = json.Marshal(pairMap)
-	}
 	form, fe := r.MultipartReader()
 	if fe != nil {
 		glog.V(0).Infoln("MultipartReader [ERROR]", fe)
@@ -158,10 +155,10 @@ func ParseUpload(r *http.Request) (
 	return
 }
 func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
-	var pair []byte
+	var pairMap map[string]string
 	fname, mimeType, isGzipped, isChunkedFile := "", "", false, false
 	n = new(Needle)
-	fname, n.Data, mimeType, pair, isGzipped, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r)
+	fname, n.Data, mimeType, pairMap, isGzipped, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r)
 	if e != nil {
 		return
 	}
@@ -173,10 +170,18 @@ func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
 		n.Mime = []byte(mimeType)
 		n.SetHasMime()
 	}
-	if len(pair) < 65536 {
+	if len(pairMap) != 0 {
-		n.Pairs = pair
+		trimmedPairMap := make(map[string]string)
-		n.PairsSize = uint16(len(pair))
+		for k, v := range pairMap {
-		n.SetHasPairs()
+			trimmedPairMap[k[len(PairNamePrefix):]] = v
+		}
+
+		pairs, _ := json.Marshal(trimmedPairMap)
+		if len(pairs) < 65536 {
+			n.Pairs = pairs
+			n.PairsSize = uint16(len(pairs))
+			n.SetHasPairs()
+		}
 	}
 	if isGzipped {
 		n.SetGzipped()

weed/topology/store_replicate.go

@@ -2,14 +2,14 @@ package topology
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 
-	"net/url"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/security"
@@ -55,9 +55,18 @@ func ReplicatedWrite(masterNode string, s *storage.Store,
 					q.Set("cm", "true")
 				}
 				u.RawQuery = q.Encode()
+
+				pairMap := make(map[string]string)
+				if needle.HasPairs() {
+					err := json.Unmarshal(needle.Pairs, &pairMap)
+					if err != nil {
+						glog.V(0).Infoln("Unmarshal pairs error:", err)
+					}
+				}
+
 				_, err := operation.Upload(u.String(),
 					string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped(), string(needle.Mime),
-					needle.Pairs, jwt)
+					pairMap, jwt)
 				return err
 			}); err != nil {
 				ret = 0