From 4c200acd7dd663bed5f566c5538114ef66e5cbe3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 10 Jul 2013 00:25:14 -0700 Subject: [PATCH] 1. ensure replicated file has the same timestamp 2. upload can specify modified time by &ts=... 3. correctly return code 304 --- go/operation/upload_content.go | 5 ++++- go/replication/store_replicate.go | 10 +++++----- go/storage/needle.go | 5 ++++- go/weed/upload.go | 7 ++++++- go/weed/volume.go | 2 +- 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/go/operation/upload_content.go b/go/operation/upload_content.go index cae657b2c..27ebb0f5a 100644 --- a/go/operation/upload_content.go +++ b/go/operation/upload_content.go @@ -4,10 +4,12 @@ import ( "bytes" "encoding/json" "errors" + "path/filepath" _ "fmt" "io" "io/ioutil" "log" + "mime" "mime/multipart" "net/http" ) @@ -29,7 +31,8 @@ func Upload(uploadUrl string, filename string, reader io.Reader) (*UploadResult, log.Println("error copying data", err) return nil, err } - content_type := body_writer.FormDataContentType() + content_type := mime.TypeByExtension(filepath.Ext(filename)) + content_type := body_writer.FormDataContentType() if err = body_writer.Close(); err != nil { log.Println("error closing body", err) return nil, err diff --git a/go/replication/store_replicate.go b/go/replication/store_replicate.go index 8306a31b4..f6e84e2ef 100644 --- a/go/replication/store_replicate.go +++ b/go/replication/store_replicate.go @@ -23,9 +23,9 @@ func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.Volum needToReplicate = s.GetVolume(volumeId).NeedToReplicate() } if needToReplicate { //send to other replica locations - if r.FormValue("type") != "standard" { + if r.FormValue("type") != "replicate" { if !distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool { - _, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=standard", string(needle.Name), bytes.NewReader(needle.Data)) + _, err := operation.Upload("http://"+location.Url+r.URL.Path+"?type=replicate&ts="+strconv.FormatUint(needle.LastModified,10), string(needle.Name), bytes.NewReader(needle.Data)) return err == nil }) { ret = 0 @@ -39,7 +39,7 @@ func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.Volum volumeId.String() + ": " + err.Error() } else { distributedOperation(masterNode, s, volumeId, func(location operation.Location) bool { - return nil == operation.Delete("http://"+location.Url+r.URL.Path+"?type=standard") + return nil == operation.Delete("http://"+location.Url+r.URL.Path+"?type=replicate") }) } } @@ -59,9 +59,9 @@ func ReplicatedDelete(masterNode string, store *storage.Store, volumeId storage. needToReplicate = store.GetVolume(volumeId).NeedToReplicate() } if needToReplicate { //send to other replica locations - if r.FormValue("type") != "standard" { + if r.FormValue("type") != "replicate" { if !distributedOperation(masterNode, store, volumeId, func(location operation.Location) bool { - return nil == operation.Delete("http://"+location.Url+r.URL.Path+"?type=standard") + return nil == operation.Delete("http://"+location.Url+r.URL.Path+"?type=replicate") }) { ret = 0 } diff --git a/go/storage/needle.go b/go/storage/needle.go index 9c0acc6b9..9822e5d01 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -90,7 +90,10 @@ func NewNeedle(r *http.Request) (n *Needle, e error) { } n.SetHasName() } - n.LastModified = uint64(time.Now().Unix()) + var parseError error + if n.LastModified, parseError = strconv.ParseUint(r.FormValue("ts"), 10, 64); parseError != nil { + n.LastModified = uint64(time.Now().Unix()) + } n.SetHasLastModifiedDate() n.Data = data diff --git a/go/weed/upload.go b/go/weed/upload.go index a47551ddf..96b326a39 100644 --- a/go/weed/upload.go +++ b/go/weed/upload.go @@ -66,7 +66,12 @@ func upload(filename string, server string, fid string) (int, error) { debug("Failed to open file:", filename) return 0, err } - ret, e := operation.Upload("http://"+server+"/"+fid, path.Base(filename), fh) + fi, fiErr := fh.Stat() + if fiErr!=nil { + debug("Failed to stat file:", filename) + return 0, fiErr + } + ret, e := operation.Upload("http://"+server+"/"+fid+"?ts="+strconv.Itoa(int(fi.ModTime().Unix()) ), path.Base(filename), fh) if e != nil { return 0, e } diff --git a/go/weed/volume.go b/go/weed/volume.go index 039212793..d6c0af123 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -149,7 +149,7 @@ func GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat)) if r.Header.Get("If-Modified-Since") != "" { if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil { - if t.Unix() <= int64(n.LastModified) { + if t.Unix() >= int64(n.LastModified) { w.WriteHeader(http.StatusNotModified) return }