gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

support gzip file upload, fix problem during replication of gzipped data

Browse source
This commit is contained in:
Chris Lu 2013-07-15 11:04:43 -07:00
parent c6bd4e656e
commit 70fe7e6b5d
4 changed files with 11 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
go/operation/upload_content.go
View file

@ -21,12 +21,16 @@ type UploadResult struct {
Error string
}
func Upload(uploadUrl string, filename string, reader io.Reader) (*UploadResult, error) {
func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool) (*UploadResult, error) {
body_buf := bytes.NewBufferString("")
body_writer := multipart.NewWriter(body_buf)
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, filename))
h.Set("Content-Type", mime.TypeByExtension(strings.ToLower(filepath.Ext(filename))))
println("content is gzipped", isGzipped)
if isGzipped {
h.Set("Content-Encoding", "gzip")
}
file_writer, err := body_writer.CreatePart(h)
if err != nil {
log.Println("error creating form file", err)

2
go/replication/store_replicate.go
View file

@ -25,7 +25,7 @@ func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.Volum
if needToReplicate { //send to other replica locations
if r.FormValue("type") != "replicate" {
if !distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool {
_, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified,10), string(needle.Name), bytes.NewReader(needle.Data))
_, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified,10), string(needle.Name), bytes.NewReader(needle.Data), needle.IsGzipped())
return err == nil
}) {
ret = 0

6
go/storage/needle.go
View file

@ -4,8 +4,8 @@ import (
"code.google.com/p/weed-fs/go/util"
"encoding/hex"
"errors"
"log"
"io/ioutil"
"log"
"mime"
"net/http"
"path"
@ -73,7 +73,9 @@ func NewNeedle(r *http.Request) (n *Needle, e error) {
n.SetHasMime()
mtype = contentType
}
if IsGzippable(ext, mtype) {
if part.Header.Get("Content-Encoding") == "gzip" {
n.SetGzipped()
} else if IsGzippable(ext, mtype) {
if data, e = GzipData(data); e != nil {
return
}

2
go/weed/upload.go
View file

@ -78,7 +78,7 @@ func upload(filename string, server string, fid string) (int, error) {
debug("Failed to stat file:", filename)
return 0, fiErr
}
ret, e := operation.Upload("http://"+server+"/"+fid+"?ts="+strconv.Itoa(int(fi.ModTime().Unix())), path.Base(filename), fh)
ret, e := operation.Upload("http://"+server+"/"+fid+"?ts="+strconv.Itoa(int(fi.ModTime().Unix())), path.Base(filename), fh, false)
if e != nil {
return 0, e
}