refactoring

This commit is contained in:
Chris Lu 2021-09-06 16:20:49 -07:00
parent 64f6532fbe
commit 6923af7280
15 changed files with 228 additions and 49 deletions

View file

@ -65,7 +65,16 @@ func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, st
targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid) targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
_, err = operation.UploadData(targetUrl, fmt.Sprintf("test%d", i), false, data, false, "bench/test", nil, assignResult.Auth) uploadOption := &operation.UploadOption{
UploadUrl: targetUrl,
Filename: fmt.Sprintf("test%d", i),
Cipher: false,
IsInputCompressed: true,
MimeType: "bench/test",
PairMap: nil,
Jwt: assignResult.Auth,
}
_, err = operation.UploadData(data, uploadOption)
if err != nil { if err != nil {
log.Fatalf("upload: %v", err) log.Fatalf("upload: %v", err)
} }

View file

@ -392,8 +392,16 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
} }
targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId
uploadOption := &operation.UploadOption{
uploadResult, err := operation.UploadData(targetUrl, fileName, worker.options.cipher, data, false, mimeType, nil, security.EncodedJwt(assignResult.Auth)) UploadUrl: targetUrl,
Filename: fileName,
Cipher: worker.options.cipher,
IsInputCompressed: false,
MimeType: mimeType,
PairMap: nil,
Jwt: security.EncodedJwt(assignResult.Auth),
}
uploadResult, err := operation.UploadData(data, uploadOption)
if err != nil { if err != nil {
return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
} }
@ -498,7 +506,16 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
replication = assignResult.Replication replication = assignResult.Replication
} }
uploadResult, err, _ := operation.Upload(targetUrl, fileName+"-"+strconv.FormatInt(i+1, 10), worker.options.cipher, io.NewSectionReader(f, i*chunkSize, chunkSize), false, "", nil, security.EncodedJwt(assignResult.Auth)) uploadOption := &operation.UploadOption{
UploadUrl: targetUrl,
Filename: fileName+"-"+strconv.FormatInt(i+1, 10),
Cipher: worker.options.cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: security.EncodedJwt(assignResult.Auth),
}
uploadResult, err, _ := operation.Upload(io.NewSectionReader(f, i*chunkSize, chunkSize), uploadOption)
if err != nil { if err != nil {
uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
return return
@ -630,8 +647,16 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off
return nil, collection, replication, fmt.Errorf("filerGrpcAddress assign volume: %v", flushErr) return nil, collection, replication, fmt.Errorf("filerGrpcAddress assign volume: %v", flushErr)
} }
fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) uploadOption := &operation.UploadOption{
uploadResult, flushErr, _ := operation.Upload(fileUrl, name, worker.options.cipher, reader, false, "", nil, auth) UploadUrl: fmt.Sprintf("http://%s/%s", host, fileId),
Filename: name,
Cipher: worker.options.cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: auth,
}
uploadResult, flushErr, _ := operation.Upload(reader, uploadOption)
if flushErr != nil { if flushErr != nil {
return nil, collection, replication, fmt.Errorf("upload data: %v", flushErr) return nil, collection, replication, fmt.Errorf("upload data: %v", flushErr)
} }

View file

@ -66,7 +66,16 @@ func (f *Filer) assignAndUpload(targetFile string, data []byte) (*operation.Assi
// upload data // upload data
targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
uploadResult, err := operation.UploadData(targetUrl, "", f.Cipher, data, false, "", nil, assignResult.Auth) uploadOption := &operation.UploadOption{
UploadUrl: targetUrl,
Filename: "",
Cipher: f.Cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: assignResult.Auth,
}
uploadResult, err := operation.UploadData(data, uploadOption)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err) return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
} }

View file

@ -58,7 +58,16 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath, writeOnly bool) filer.Sa
if wfs.option.VolumeServerAccess == "filerProxy" { if wfs.option.VolumeServerAccess == "filerProxy" {
fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.getCurrentFiler(), fileId) fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.getCurrentFiler(), fileId)
} }
uploadResult, err, data := operation.Upload(fileUrl, filename, wfs.option.Cipher, reader, false, "", nil, auth) uploadOption := &operation.UploadOption{
UploadUrl: fileUrl,
Filename: filename,
Cipher: wfs.option.Cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: auth,
}
uploadResult, err, data := operation.Upload(reader, uploadOption)
if err != nil { if err != nil {
glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err) glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err)
return nil, "", "", fmt.Errorf("upload data: %v", err) return nil, "", "", fmt.Errorf("upload data: %v", err)

View file

@ -88,7 +88,16 @@ func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConf
// upload data // upload data
targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid) targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
uploadResult, err := operation.UploadData(targetUrl, "", broker.option.Cipher, data, false, "", nil, assignResult.Auth) uploadOption := &operation.UploadOption{
UploadUrl: targetUrl,
Filename: "",
Cipher: broker.option.Cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: assignResult.Auth,
}
uploadResult, err := operation.UploadData(data, uploadOption)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err) return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
} }

View file

@ -53,7 +53,16 @@ func TestCreateNeedleFromRequest(t *testing.T) {
assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip") assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip")
fmt.Printf("needle: %v, originalSize: %d\n", n, originalSize) fmt.Printf("needle: %v, originalSize: %d\n", n, originalSize)
} }
uploadResult, err, data := Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader([]byte(textContent)), false, "", nil, "") uploadOption := &UploadOption{
UploadUrl: "http://localhost:8080/389,0f084d17353afda0",
Filename: "t.txt",
Cipher: false,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: "",
}
uploadResult, err, data := Upload(bytes.NewReader([]byte(textContent)), uploadOption)
if len(data) != len(textContent) { if len(data) != len(textContent) {
t.Errorf("data actual %d expected %d", len(data), len(textContent)) t.Errorf("data actual %d expected %d", len(data), len(textContent))
} }
@ -72,7 +81,16 @@ func TestCreateNeedleFromRequest(t *testing.T) {
fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize) fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize)
} }
gzippedData, _ := util.GzipData([]byte(textContent)) gzippedData, _ := util.GzipData([]byte(textContent))
Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(gzippedData), true, "text/plain", nil, "") uploadOption := &UploadOption{
UploadUrl: "http://localhost:8080/389,0f084d17353afda0",
Filename: "t.txt",
Cipher: false,
IsInputCompressed: true,
MimeType: "text/plain",
PairMap: nil,
Jwt: "",
}
Upload(bytes.NewReader(gzippedData), uploadOption)
} }
/* /*

View file

@ -206,7 +206,16 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption) cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
} }
} else { } else {
ret, e, _ := Upload(fileUrl, baseName, false, fi.Reader, false, fi.MimeType, nil, jwt) uploadOption := &UploadOption{
UploadUrl: fileUrl,
Filename: baseName,
Cipher: false,
IsInputCompressed: false,
MimeType: fi.MimeType,
PairMap: nil,
Jwt: jwt,
}
ret, e, _ := Upload(fi.Reader, uploadOption)
if e != nil { if e != nil {
return 0, e return 0, e
} }
@ -219,7 +228,16 @@ func upload_one_chunk(filename string, reader io.Reader, masterFn GetMasterFn,
fileUrl string, jwt security.EncodedJwt, fileUrl string, jwt security.EncodedJwt,
) (size uint32, e error) { ) (size uint32, e error) {
glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...") glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
uploadResult, uploadError, _ := Upload(fileUrl, filename, false, reader, false, "", nil, jwt) uploadOption := &UploadOption{
UploadUrl: fileUrl,
Filename: filename,
Cipher: false,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: jwt,
}
uploadResult, uploadError, _ := Upload(reader, uploadOption)
if uploadError != nil { if uploadError != nil {
return 0, uploadError return 0, uploadError
} }
@ -236,6 +254,15 @@ func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt s
q := u.Query() q := u.Query()
q.Set("cm", "true") q.Set("cm", "true")
u.RawQuery = q.Encode() u.RawQuery = q.Encode()
_, e = UploadData(u.String(), manifest.Name, false, buf, false, "application/json", nil, jwt) uploadOption := &UploadOption{
UploadUrl: u.String(),
Filename: manifest.Name,
Cipher: false,
IsInputCompressed: false,
MimeType: "application/json",
PairMap: nil,
Jwt: jwt,
}
_, e = UploadData(buf, uploadOption)
return e return e
} }

View file

@ -20,6 +20,16 @@ import (
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
) )
type UploadOption struct {
UploadUrl string
Filename string
Cipher bool
IsInputCompressed bool
MimeType string
PairMap map[string]string
Jwt security.EncodedJwt
}
type UploadResult struct { type UploadResult struct {
Name string `json:"name,omitempty"` Name string `json:"name,omitempty"`
Size uint32 `json:"size,omitempty"` Size uint32 `json:"size,omitempty"`
@ -65,18 +75,18 @@ func init() {
var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`) var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
// Upload sends a POST request to a volume server to upload the content with adjustable compression level // Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { func UploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
uploadResult, err = retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt) uploadResult, err = retriedUploadData(data, option)
return return
} }
// Upload sends a POST request to a volume server to upload the content with fast compression // Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) { func Upload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) {
uploadResult, err, data = doUpload(uploadUrl, filename, cipher, reader, isInputCompressed, mtype, pairMap, jwt) uploadResult, err, data = doUpload(reader, option)
return return
} }
func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) { func doUpload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) {
bytesReader, ok := reader.(*util.BytesReader) bytesReader, ok := reader.(*util.BytesReader)
if ok { if ok {
data = bytesReader.Bytes data = bytesReader.Bytes
@ -87,38 +97,38 @@ func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader,
return return
} }
} }
uploadResult, uploadErr := retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt) uploadResult, uploadErr := retriedUploadData(data, option)
return uploadResult, uploadErr, data return uploadResult, uploadErr, data
} }
func retriedUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { func retriedUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
for i := 0; i < 3; i++ { for i := 0; i < 3; i++ {
uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt) uploadResult, err = doUploadData(data, option)
if err == nil { if err == nil {
uploadResult.RetryCount = i uploadResult.RetryCount = i
return return
} else { } else {
glog.Warningf("uploading to %s: %v", uploadUrl, err) glog.Warningf("uploading to %s: %v", option.UploadUrl, err)
} }
time.Sleep(time.Millisecond * time.Duration(237*(i+1))) time.Sleep(time.Millisecond * time.Duration(237*(i+1)))
} }
return return
} }
func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { func doUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
contentIsGzipped := isInputCompressed contentIsGzipped := option.IsInputCompressed
shouldGzipNow := false shouldGzipNow := false
if !isInputCompressed { if !option.IsInputCompressed {
if mtype == "" { if option.MimeType == "" {
mtype = http.DetectContentType(data) option.MimeType = http.DetectContentType(data)
// println("detect1 mimetype to", mtype) // println("detect1 mimetype to", MimeType)
if mtype == "application/octet-stream" { if option.MimeType == "application/octet-stream" {
mtype = "" option.MimeType = ""
} }
} }
if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed { if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
shouldGzipNow = true shouldGzipNow = true
} else if !iAmSure && mtype == "" && len(data) > 16*1024 { } else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 {
var compressed []byte var compressed []byte
compressed, err = util.GzipData(data[0:128]) compressed, err = util.GzipData(data[0:128])
shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90% shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
@ -131,14 +141,14 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
// this could be double copying // this could be double copying
clearDataLen = len(data) clearDataLen = len(data)
clearData := data clearData := data
if shouldGzipNow && !cipher { if shouldGzipNow && !option.Cipher {
compressed, compressErr := util.GzipData(data) compressed, compressErr := util.GzipData(data)
// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed)) // fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
if compressErr == nil { if compressErr == nil {
data = compressed data = compressed
contentIsGzipped = true contentIsGzipped = true
} }
} else if isInputCompressed { } else if option.IsInputCompressed {
// just to get the clear data length // just to get the clear data length
clearData, err = util.DecompressData(data) clearData, err = util.DecompressData(data)
if err == nil { if err == nil {
@ -146,7 +156,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
} }
} }
if cipher { if option.Cipher {
// encrypt(gzip(data)) // encrypt(gzip(data))
// encrypt // encrypt
@ -158,23 +168,23 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
} }
// upload data // upload data
uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) { uploadResult, err = upload_content(option.UploadUrl, func(w io.Writer) (err error) {
_, err = w.Write(encryptedData) _, err = w.Write(encryptedData)
return return
}, "", false, len(encryptedData), "", nil, jwt) }, "", false, len(encryptedData), "", nil, option.Jwt)
if uploadResult == nil { if uploadResult == nil {
return return
} }
uploadResult.Name = filename uploadResult.Name = option.Filename
uploadResult.Mime = mtype uploadResult.Mime = option.MimeType
uploadResult.CipherKey = cipherKey uploadResult.CipherKey = cipherKey
uploadResult.Size = uint32(clearDataLen) uploadResult.Size = uint32(clearDataLen)
} else { } else {
// upload data // upload data
uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) { uploadResult, err = upload_content(option.UploadUrl, func(w io.Writer) (err error) {
_, err = w.Write(data) _, err = w.Write(data)
return return
}, filename, contentIsGzipped, len(data), mtype, pairMap, jwt) }, option.Filename, contentIsGzipped, len(data), option.MimeType, option.PairMap, option.Jwt)
if uploadResult == nil { if uploadResult == nil {
return return
} }

View file

@ -107,7 +107,16 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string)
glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header) glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header)
// fetch data as is, regardless whether it is encrypted or not // fetch data as is, regardless whether it is encrypted or not
uploadResult, err, _ := operation.Upload(fileUrl, filename, false, resp.Body, "gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, auth) uploadOption := &operation.UploadOption{
UploadUrl: fileUrl,
Filename: filename,
Cipher: false,
IsInputCompressed: "gzip" == header.Get("Content-Encoding"),
MimeType: header.Get("Content-Type"),
PairMap: nil,
Jwt: auth,
}
uploadResult, err, _ := operation.Upload(resp.Body, uploadOption)
if err != nil { if err != nil {
glog.V(0).Infof("upload source data %v to %s: %v", sourceChunk.GetFileIdString(), fileUrl, err) glog.V(0).Infof("upload source data %v to %s: %v", sourceChunk.GetFileIdString(), fileUrl, err)
return "", fmt.Errorf("upload data: %v", err) return "", fmt.Errorf("upload data: %v", err)

View file

@ -161,7 +161,16 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn ope
} }
debug("upload file to store", url) debug("upload file to store", url)
uploadResult, err := operation.UploadData(url, pu.FileName, false, pu.Data, pu.IsGzipped, pu.MimeType, pu.PairMap, assignResult.Auth) uploadOption := &operation.UploadOption{
UploadUrl: url,
Filename: pu.FileName,
Cipher: false,
IsInputCompressed: pu.IsGzipped,
MimeType: pu.MimeType,
PairMap: pu.PairMap,
Jwt: assignResult.Auth,
}
uploadResult, err := operation.UploadData(pu.Data, uploadOption)
if err != nil { if err != nil {
writeJsonError(w, r, http.StatusInternalServerError, err) writeJsonError(w, r, http.StatusInternalServerError, err)
return return

View file

@ -241,7 +241,16 @@ func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAs
} }
// upload the chunk to the volume server // upload the chunk to the volume server
uploadResult, uploadErr, _ := operation.Upload(urlLocation, name, fs.option.Cipher, reader, false, "", nil, auth) uploadOption := &operation.UploadOption{
UploadUrl: urlLocation,
Filename: name,
Cipher: fs.option.Cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: auth,
}
uploadResult, uploadErr, _ := operation.Upload(reader, uploadOption)
if uploadErr != nil { if uploadErr != nil {
return nil, "", "", uploadErr return nil, "", "", uploadErr
} }

View file

@ -44,7 +44,16 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht
// println("detect2 mimetype to", pu.MimeType) // println("detect2 mimetype to", pu.MimeType)
} }
uploadResult, uploadError := operation.UploadData(urlLocation, pu.FileName, true, uncompressedData, false, pu.MimeType, pu.PairMap, auth) uploadOption := &operation.UploadOption{
UploadUrl: urlLocation,
Filename: pu.FileName,
Cipher: true,
IsInputCompressed: false,
MimeType: pu.MimeType,
PairMap: pu.PairMap,
Jwt: auth,
}
uploadResult, uploadError := operation.UploadData(uncompressedData, uploadOption)
if uploadError != nil { if uploadError != nil {
return nil, fmt.Errorf("upload to volume server: %v", uploadError) return nil, fmt.Errorf("upload to volume server: %v", uploadError)
} }

View file

@ -127,7 +127,16 @@ func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fil
stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds()) stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds())
}() }()
uploadResult, err, data := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth) uploadOption := &operation.UploadOption{
UploadUrl: urlLocation,
Filename: fileName,
Cipher: fs.option.Cipher,
IsInputCompressed: false,
MimeType: contentType,
PairMap: pairMap,
Jwt: auth,
}
uploadResult, err, data := operation.Upload(limitedReader, uploadOption)
if uploadResult != nil && uploadResult.RetryCount > 0 { if uploadResult != nil && uploadResult.RetryCount > 0 {
stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount)) stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount))
} }

View file

@ -413,7 +413,16 @@ func (f *WebDavFile) saveDataAsChunk(reader io.Reader, name string, offset int64
} }
fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
uploadResult, flushErr, _ := operation.Upload(fileUrl, f.name, f.fs.option.Cipher, reader, false, "", nil, auth) uploadOption := &operation.UploadOption{
UploadUrl: fileUrl,
Filename: f.name,
Cipher: f.fs.option.Cipher,
IsInputCompressed: false,
MimeType: "",
PairMap: nil,
Jwt: auth,
}
uploadResult, flushErr, _ := operation.Upload(reader, uploadOption)
if flushErr != nil { if flushErr != nil {
glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, flushErr) glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, flushErr)
return nil, f.collection, f.replication, fmt.Errorf("upload data: %v", flushErr) return nil, f.collection, f.replication, fmt.Errorf("upload data: %v", flushErr)

View file

@ -83,7 +83,16 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt
// volume server do not know about encryption // volume server do not know about encryption
// TODO optimize here to compress data only once // TODO optimize here to compress data only once
_, err := operation.UploadData(u.String(), string(n.Name), false, n.Data, n.IsCompressed(), string(n.Mime), pairMap, jwt) uploadOption := &operation.UploadOption{
UploadUrl: u.String(),
Filename: string(n.Name),
Cipher: false,
IsInputCompressed: n.IsCompressed(),
MimeType: string(n.Mime),
PairMap: pairMap,
Jwt: jwt,
}
_, err := operation.UploadData(n.Data, uploadOption)
return err return err
}); err != nil { }); err != nil {
err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err) err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err)