seaweedfs/go/operation/submit.go

154 lines
4.2 KiB
Go
Raw Normal View History

package operation
import (
"bytes"
"code.google.com/p/weed-fs/go/glog"
"io"
"mime"
"os"
"path"
"strconv"
"strings"
)
2013-08-14 17:07:42 +00:00
type FilePart struct {
Reader io.Reader
FileName string
FileSize int64
IsGzipped bool
MimeType string
ModTime int64 //in seconds
Replication string
Collection string
Server string //this comes from assign result
Fid string //this comes from assign result, but customizable
2013-08-14 17:07:42 +00:00
}
type SubmitResult struct {
2014-04-15 17:01:13 +00:00
FileName string `json:"fileName,omitempty"`
FileUrl string `json:"fileUrl,omitempty"`
Fid string `json:"fid,omitempty"`
Size uint32 `json:"size,omitempty"`
Error string `json:"error,omitempty"`
}
func SubmitFiles(master string, files []FilePart, replication string, collection string, maxMB int) ([]SubmitResult, error) {
results := make([]SubmitResult, len(files))
for index, file := range files {
2013-08-14 17:07:42 +00:00
results[index].FileName = file.FileName
}
ret, err := Assign(master, len(files), replication, collection)
if err != nil {
for index, _ := range files {
results[index].Error = err.Error()
}
return results, err
}
for index, file := range files {
2014-03-10 01:50:45 +00:00
file.Fid = ret.Fid
if index > 0 {
2014-03-10 01:50:45 +00:00
file.Fid = file.Fid + "_" + strconv.Itoa(index)
}
2014-03-10 01:50:45 +00:00
file.Server = ret.PublicUrl
file.Replication = replication
file.Collection = collection
results[index].Size, err = file.Upload(maxMB, master)
if err != nil {
results[index].Error = err.Error()
}
2014-03-10 01:50:45 +00:00
results[index].Fid = file.Fid
results[index].FileUrl = file.Server + "/" + file.Fid
}
return results, nil
}
2013-08-14 17:07:42 +00:00
func NewFileParts(fullPathFilenames []string) (ret []FilePart, err error) {
ret = make([]FilePart, len(fullPathFilenames))
for index, file := range fullPathFilenames {
if ret[index], err = newFilePart(file); err != nil {
2013-08-14 17:07:42 +00:00
return
}
}
2013-08-14 17:07:42 +00:00
return
}
func newFilePart(fullPathFilename string) (ret FilePart, err error) {
2013-08-14 17:07:42 +00:00
fh, openErr := os.Open(fullPathFilename)
if openErr != nil {
glog.V(0).Info("Failed to open file: ", fullPathFilename)
return ret, openErr
}
ret.Reader = fh
2013-08-14 17:07:42 +00:00
if fi, fiErr := fh.Stat(); fiErr != nil {
glog.V(0).Info("Failed to stat file:", fullPathFilename)
return ret, fiErr
} else {
ret.ModTime = fi.ModTime().UTC().Unix()
ret.FileSize = fi.Size()
2013-08-14 17:07:42 +00:00
}
ext := strings.ToLower(path.Ext(fullPathFilename))
ret.IsGzipped = ext == ".gz"
if ret.IsGzipped {
ret.FileName = fullPathFilename[0 : len(fullPathFilename)-3]
}
2013-08-14 17:07:42 +00:00
ret.FileName = fullPathFilename
if ext != "" {
ret.MimeType = mime.TypeByExtension(ext)
}
return ret, nil
}
2014-04-15 17:01:13 +00:00
func (fi FilePart) Upload(maxMB int, master string) (retSize uint32, err error) {
2014-03-10 01:50:45 +00:00
fileUrl := "http://" + fi.Server + "/" + fi.Fid
2013-08-14 17:07:42 +00:00
if fi.ModTime != 0 {
fileUrl += "?ts=" + strconv.Itoa(int(fi.ModTime))
}
if closer, ok := fi.Reader.(io.Closer); ok {
defer closer.Close()
}
if maxMB > 0 && fi.FileSize > int64(maxMB*1024*1024) {
chunkSize := int64(maxMB * 1024 * 1024)
chunks := fi.FileSize/chunkSize + 1
fids := make([]string, 0)
for i := int64(0); i < chunks; i++ {
id, count, e := upload_one_chunk(fi.FileName+"-"+strconv.FormatInt(i+1, 10), io.LimitReader(fi.Reader, chunkSize), master, fi.Replication, fi.Collection)
if e != nil {
return 0, e
}
fids = append(fids, id)
retSize += count
}
err = upload_file_id_list(fileUrl, fi.FileName+"-list", fids)
} else {
ret, e := Upload(fileUrl, fi.FileName, fi.Reader, fi.IsGzipped, fi.MimeType)
if e != nil {
return 0, e
}
return ret.Size, e
}
return
}
2014-04-15 17:01:13 +00:00
func upload_one_chunk(filename string, reader io.Reader, master, replication string, collection string) (fid string, size uint32, e error) {
ret, err := Assign(master, 1, replication, collection)
if err != nil {
return "", 0, err
}
fileUrl, fid := "http://"+ret.PublicUrl+"/"+ret.Fid, ret.Fid
glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
uploadResult, uploadError := Upload(fileUrl, filename, reader, false, "application/octet-stream")
2013-11-19 10:12:56 +00:00
if uploadError != nil {
return fid, 0, uploadError
}
return fid, uploadResult.Size, nil
}
func upload_file_id_list(fileUrl, filename string, fids []string) error {
var buf bytes.Buffer
buf.WriteString(strings.Join(fids, "\n"))
glog.V(4).Info("Uploading final list ", filename, " to ", fileUrl, "...")
_, e := Upload(fileUrl, filename, &buf, false, "text/plain")
return e
}