complete submit chunked file

go/operation/chunked_file.go

@@ -85,10 +85,9 @@ func (cm *ChunkManifest) DeleteChunks(master string) error {
 	return nil
 }
 
-func (cm *ChunkManifest) StoredHelper() error {
-	//TODO
-	return nil
-}
+//func (cm *ChunkManifest) StoredHelper() error {
+//	return nil
+//}
 
 func httpRangeDownload(fileUrl string, w io.Writer, offset int64) (written int64, e error) {
 	req, err := http.NewRequest("GET", fileUrl, nil)

go/operation/submit.go

@@ -9,6 +9,8 @@ import (
 	"strconv"
 	"strings"
 
+	"net/url"
+
 	"github.com/chrislusf/seaweedfs/go/glog"
 	"github.com/chrislusf/seaweedfs/go/security"
 )
@@ -117,7 +119,13 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 	if maxMB > 0 && fi.FileSize > int64(maxMB*1024*1024) {
 		chunkSize := int64(maxMB * 1024 * 1024)
 		chunks := fi.FileSize/chunkSize + 1
-		var fids []string
+		cm := ChunkManifest{
+			Name:   fi.FileName,
+			Size:   fi.FileSize,
+			Mime:   fi.MimeType,
+			Chunks: make([]*ChunkInfo, 0, chunks),
+		}
+
 		for i := int64(0); i < chunks; i++ {
 			id, count, e := upload_one_chunk(
 				fi.FileName+"-"+strconv.FormatInt(i+1, 10),
@@ -125,12 +133,24 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 				master, fi.Replication, fi.Collection, fi.Ttl,
 				jwt)
 			if e != nil {
+				// delete all uploaded chunks
+				cm.DeleteChunks(master)
 				return 0, e
 			}
-			fids = append(fids, id)
+			cm.Chunks = append(cm.Chunks,
+				&ChunkInfo{
+					Offset: i * chunkSize,
+					Size:   int64(count),
+					Fid:    id,
+				},
+			)
 			retSize += count
 		}
-		err = upload_file_id_list(fileUrl, fi.FileName+"-list", fids, jwt)
+		err = upload_chunked_file_manifest(fileUrl, &cm, jwt)
+		if err != nil {
+			// delete all uploaded chunks
+			cm.DeleteChunks(master)
+		}
 	} else {
 		ret, e := Upload(fileUrl, fi.FileName, fi.Reader, fi.IsGzipped, fi.MimeType, jwt)
 		if e != nil {
@@ -158,10 +178,17 @@ func upload_one_chunk(filename string, reader io.Reader, master,
 	return fid, uploadResult.Size, nil
 }
 
-func upload_file_id_list(fileUrl, filename string, fids []string, jwt security.EncodedJwt) error {
-	var buf bytes.Buffer
-	buf.WriteString(strings.Join(fids, "\n"))
-	glog.V(4).Info("Uploading final list ", filename, " to ", fileUrl, "...")
-	_, e := Upload(fileUrl, filename, &buf, false, "text/plain", jwt)
+func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt security.EncodedJwt) error {
+	buf, e := manifest.GetData()
+	if e != nil {
+		return e
+	}
+	bufReader := bytes.NewReader(buf)
+	glog.V(4).Info("Uploading chunks manifest ", manifest.Name, " to ", fileUrl, "...")
+	u, _ := url.Parse(fileUrl)
+	q := u.Query()
+	q.Set("cm", "1")
+	u.RawQuery = q.Encode()
+	_, e = Upload(u.String(), manifest.Name, bufReader, false, "text/plain", jwt)
 	return e
 }

go/storage/needle.go

@@ -132,7 +132,7 @@ func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string
 	}
 	modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
 	ttl, _ = ReadTTL(r.FormValue("ttl"))
-	isChunkedFile, _ = strconv.ParseBool(r.FormValue("cf"))
+	isChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
 	return
 }
 func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {

go/weed/weed_server/volume_server_handlers_read.go

@@ -270,7 +270,7 @@ func (vs *VolumeServer) tryHandleChunkedFile(n *storage.Needle, fileName string,
 		return true
 	}
 
-	chunkedFileReader := operation.ChunkedFileReader{
+	chunkedFileReader := &operation.ChunkedFileReader{
 		Manifest: chunkManifest,
 		Master:   vs.GetMasterNode(),
 	}