mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
refactoring
This commit is contained in:
parent
64f6532fbe
commit
6923af7280
|
@ -65,7 +65,16 @@ func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, st
|
||||||
|
|
||||||
targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
|
targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
|
||||||
|
|
||||||
_, err = operation.UploadData(targetUrl, fmt.Sprintf("test%d", i), false, data, false, "bench/test", nil, assignResult.Auth)
|
uploadOption := &operation.UploadOption{
|
||||||
|
UploadUrl: targetUrl,
|
||||||
|
Filename: fmt.Sprintf("test%d", i),
|
||||||
|
Cipher: false,
|
||||||
|
IsInputCompressed: true,
|
||||||
|
MimeType: "bench/test",
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: assignResult.Auth,
|
||||||
|
}
|
||||||
|
_, err = operation.UploadData(data, uploadOption)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("upload: %v", err)
|
log.Fatalf("upload: %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -392,8 +392,16 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
|
||||||
}
|
}
|
||||||
|
|
||||||
targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId
|
targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId
|
||||||
|
uploadOption := &operation.UploadOption{
|
||||||
uploadResult, err := operation.UploadData(targetUrl, fileName, worker.options.cipher, data, false, mimeType, nil, security.EncodedJwt(assignResult.Auth))
|
UploadUrl: targetUrl,
|
||||||
|
Filename: fileName,
|
||||||
|
Cipher: worker.options.cipher,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: mimeType,
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: security.EncodedJwt(assignResult.Auth),
|
||||||
|
}
|
||||||
|
uploadResult, err := operation.UploadData(data, uploadOption)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
|
return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
|
||||||
}
|
}
|
||||||
|
@ -498,7 +506,16 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
|
||||||
replication = assignResult.Replication
|
replication = assignResult.Replication
|
||||||
}
|
}
|
||||||
|
|
||||||
uploadResult, err, _ := operation.Upload(targetUrl, fileName+"-"+strconv.FormatInt(i+1, 10), worker.options.cipher, io.NewSectionReader(f, i*chunkSize, chunkSize), false, "", nil, security.EncodedJwt(assignResult.Auth))
|
uploadOption := &operation.UploadOption{
|
||||||
|
UploadUrl: targetUrl,
|
||||||
|
Filename: fileName+"-"+strconv.FormatInt(i+1, 10),
|
||||||
|
Cipher: worker.options.cipher,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: "",
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: security.EncodedJwt(assignResult.Auth),
|
||||||
|
}
|
||||||
|
uploadResult, err, _ := operation.Upload(io.NewSectionReader(f, i*chunkSize, chunkSize), uploadOption)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
|
uploadError = fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
|
||||||
return
|
return
|
||||||
|
@ -630,8 +647,16 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off
|
||||||
return nil, collection, replication, fmt.Errorf("filerGrpcAddress assign volume: %v", flushErr)
|
return nil, collection, replication, fmt.Errorf("filerGrpcAddress assign volume: %v", flushErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
|
uploadOption := &operation.UploadOption{
|
||||||
uploadResult, flushErr, _ := operation.Upload(fileUrl, name, worker.options.cipher, reader, false, "", nil, auth)
|
UploadUrl: fmt.Sprintf("http://%s/%s", host, fileId),
|
||||||
|
Filename: name,
|
||||||
|
Cipher: worker.options.cipher,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: "",
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: auth,
|
||||||
|
}
|
||||||
|
uploadResult, flushErr, _ := operation.Upload(reader, uploadOption)
|
||||||
if flushErr != nil {
|
if flushErr != nil {
|
||||||
return nil, collection, replication, fmt.Errorf("upload data: %v", flushErr)
|
return nil, collection, replication, fmt.Errorf("upload data: %v", flushErr)
|
||||||
}
|
}
|
||||||
|
|
|
@ -66,7 +66,16 @@ func (f *Filer) assignAndUpload(targetFile string, data []byte) (*operation.Assi
|
||||||
|
|
||||||
// upload data
|
// upload data
|
||||||
targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
|
targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
|
||||||
uploadResult, err := operation.UploadData(targetUrl, "", f.Cipher, data, false, "", nil, assignResult.Auth)
|
uploadOption := &operation.UploadOption{
|
||||||
|
UploadUrl: targetUrl,
|
||||||
|
Filename: "",
|
||||||
|
Cipher: f.Cipher,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: "",
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: assignResult.Auth,
|
||||||
|
}
|
||||||
|
uploadResult, err := operation.UploadData(data, uploadOption)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
|
return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -58,7 +58,16 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath, writeOnly bool) filer.Sa
|
||||||
if wfs.option.VolumeServerAccess == "filerProxy" {
|
if wfs.option.VolumeServerAccess == "filerProxy" {
|
||||||
fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.getCurrentFiler(), fileId)
|
fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.getCurrentFiler(), fileId)
|
||||||
}
|
}
|
||||||
uploadResult, err, data := operation.Upload(fileUrl, filename, wfs.option.Cipher, reader, false, "", nil, auth)
|
uploadOption := &operation.UploadOption{
|
||||||
|
UploadUrl: fileUrl,
|
||||||
|
Filename: filename,
|
||||||
|
Cipher: wfs.option.Cipher,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: "",
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: auth,
|
||||||
|
}
|
||||||
|
uploadResult, err, data := operation.Upload(reader, uploadOption)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err)
|
glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err)
|
||||||
return nil, "", "", fmt.Errorf("upload data: %v", err)
|
return nil, "", "", fmt.Errorf("upload data: %v", err)
|
||||||
|
|
|
@ -88,7 +88,16 @@ func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConf
|
||||||
|
|
||||||
// upload data
|
// upload data
|
||||||
targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
|
targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
|
||||||
uploadResult, err := operation.UploadData(targetUrl, "", broker.option.Cipher, data, false, "", nil, assignResult.Auth)
|
uploadOption := &operation.UploadOption{
|
||||||
|
UploadUrl: targetUrl,
|
||||||
|
Filename: "",
|
||||||
|
Cipher: broker.option.Cipher,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: "",
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: assignResult.Auth,
|
||||||
|
}
|
||||||
|
uploadResult, err := operation.UploadData(data, uploadOption)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
|
return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -53,7 +53,16 @@ func TestCreateNeedleFromRequest(t *testing.T) {
|
||||||
assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip")
|
assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip")
|
||||||
fmt.Printf("needle: %v, originalSize: %d\n", n, originalSize)
|
fmt.Printf("needle: %v, originalSize: %d\n", n, originalSize)
|
||||||
}
|
}
|
||||||
uploadResult, err, data := Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader([]byte(textContent)), false, "", nil, "")
|
uploadOption := &UploadOption{
|
||||||
|
UploadUrl: "http://localhost:8080/389,0f084d17353afda0",
|
||||||
|
Filename: "t.txt",
|
||||||
|
Cipher: false,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: "",
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: "",
|
||||||
|
}
|
||||||
|
uploadResult, err, data := Upload(bytes.NewReader([]byte(textContent)), uploadOption)
|
||||||
if len(data) != len(textContent) {
|
if len(data) != len(textContent) {
|
||||||
t.Errorf("data actual %d expected %d", len(data), len(textContent))
|
t.Errorf("data actual %d expected %d", len(data), len(textContent))
|
||||||
}
|
}
|
||||||
|
@ -72,7 +81,16 @@ func TestCreateNeedleFromRequest(t *testing.T) {
|
||||||
fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize)
|
fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize)
|
||||||
}
|
}
|
||||||
gzippedData, _ := util.GzipData([]byte(textContent))
|
gzippedData, _ := util.GzipData([]byte(textContent))
|
||||||
Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(gzippedData), true, "text/plain", nil, "")
|
uploadOption := &UploadOption{
|
||||||
|
UploadUrl: "http://localhost:8080/389,0f084d17353afda0",
|
||||||
|
Filename: "t.txt",
|
||||||
|
Cipher: false,
|
||||||
|
IsInputCompressed: true,
|
||||||
|
MimeType: "text/plain",
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: "",
|
||||||
|
}
|
||||||
|
Upload(bytes.NewReader(gzippedData), uploadOption)
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -206,7 +206,16 @@ func (fi FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jw
|
||||||
cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
|
cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
ret, e, _ := Upload(fileUrl, baseName, false, fi.Reader, false, fi.MimeType, nil, jwt)
|
uploadOption := &UploadOption{
|
||||||
|
UploadUrl: fileUrl,
|
||||||
|
Filename: baseName,
|
||||||
|
Cipher: false,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: fi.MimeType,
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: jwt,
|
||||||
|
}
|
||||||
|
ret, e, _ := Upload(fi.Reader, uploadOption)
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return 0, e
|
return 0, e
|
||||||
}
|
}
|
||||||
|
@ -219,7 +228,16 @@ func upload_one_chunk(filename string, reader io.Reader, masterFn GetMasterFn,
|
||||||
fileUrl string, jwt security.EncodedJwt,
|
fileUrl string, jwt security.EncodedJwt,
|
||||||
) (size uint32, e error) {
|
) (size uint32, e error) {
|
||||||
glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
|
glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
|
||||||
uploadResult, uploadError, _ := Upload(fileUrl, filename, false, reader, false, "", nil, jwt)
|
uploadOption := &UploadOption{
|
||||||
|
UploadUrl: fileUrl,
|
||||||
|
Filename: filename,
|
||||||
|
Cipher: false,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: "",
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: jwt,
|
||||||
|
}
|
||||||
|
uploadResult, uploadError, _ := Upload(reader, uploadOption)
|
||||||
if uploadError != nil {
|
if uploadError != nil {
|
||||||
return 0, uploadError
|
return 0, uploadError
|
||||||
}
|
}
|
||||||
|
@ -236,6 +254,15 @@ func upload_chunked_file_manifest(fileUrl string, manifest *ChunkManifest, jwt s
|
||||||
q := u.Query()
|
q := u.Query()
|
||||||
q.Set("cm", "true")
|
q.Set("cm", "true")
|
||||||
u.RawQuery = q.Encode()
|
u.RawQuery = q.Encode()
|
||||||
_, e = UploadData(u.String(), manifest.Name, false, buf, false, "application/json", nil, jwt)
|
uploadOption := &UploadOption{
|
||||||
|
UploadUrl: u.String(),
|
||||||
|
Filename: manifest.Name,
|
||||||
|
Cipher: false,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: "application/json",
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: jwt,
|
||||||
|
}
|
||||||
|
_, e = UploadData(buf, uploadOption)
|
||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
|
|
|
@ -20,6 +20,16 @@ import (
|
||||||
"github.com/chrislusf/seaweedfs/weed/util"
|
"github.com/chrislusf/seaweedfs/weed/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type UploadOption struct {
|
||||||
|
UploadUrl string
|
||||||
|
Filename string
|
||||||
|
Cipher bool
|
||||||
|
IsInputCompressed bool
|
||||||
|
MimeType string
|
||||||
|
PairMap map[string]string
|
||||||
|
Jwt security.EncodedJwt
|
||||||
|
}
|
||||||
|
|
||||||
type UploadResult struct {
|
type UploadResult struct {
|
||||||
Name string `json:"name,omitempty"`
|
Name string `json:"name,omitempty"`
|
||||||
Size uint32 `json:"size,omitempty"`
|
Size uint32 `json:"size,omitempty"`
|
||||||
|
@ -65,18 +75,18 @@ func init() {
|
||||||
var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
|
var fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
|
||||||
|
|
||||||
// Upload sends a POST request to a volume server to upload the content with adjustable compression level
|
// Upload sends a POST request to a volume server to upload the content with adjustable compression level
|
||||||
func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
|
func UploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
|
||||||
uploadResult, err = retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
|
uploadResult, err = retriedUploadData(data, option)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Upload sends a POST request to a volume server to upload the content with fast compression
|
// Upload sends a POST request to a volume server to upload the content with fast compression
|
||||||
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
|
func Upload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) {
|
||||||
uploadResult, err, data = doUpload(uploadUrl, filename, cipher, reader, isInputCompressed, mtype, pairMap, jwt)
|
uploadResult, err, data = doUpload(reader, option)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error, data []byte) {
|
func doUpload(reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) {
|
||||||
bytesReader, ok := reader.(*util.BytesReader)
|
bytesReader, ok := reader.(*util.BytesReader)
|
||||||
if ok {
|
if ok {
|
||||||
data = bytesReader.Bytes
|
data = bytesReader.Bytes
|
||||||
|
@ -87,38 +97,38 @@ func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader,
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
uploadResult, uploadErr := retriedUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
|
uploadResult, uploadErr := retriedUploadData(data, option)
|
||||||
return uploadResult, uploadErr, data
|
return uploadResult, uploadErr, data
|
||||||
}
|
}
|
||||||
|
|
||||||
func retriedUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
|
func retriedUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
|
||||||
for i := 0; i < 3; i++ {
|
for i := 0; i < 3; i++ {
|
||||||
uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputCompressed, mtype, pairMap, jwt)
|
uploadResult, err = doUploadData(data, option)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
uploadResult.RetryCount = i
|
uploadResult.RetryCount = i
|
||||||
return
|
return
|
||||||
} else {
|
} else {
|
||||||
glog.Warningf("uploading to %s: %v", uploadUrl, err)
|
glog.Warningf("uploading to %s: %v", option.UploadUrl, err)
|
||||||
}
|
}
|
||||||
time.Sleep(time.Millisecond * time.Duration(237*(i+1)))
|
time.Sleep(time.Millisecond * time.Duration(237*(i+1)))
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputCompressed bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
|
func doUploadData(data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
|
||||||
contentIsGzipped := isInputCompressed
|
contentIsGzipped := option.IsInputCompressed
|
||||||
shouldGzipNow := false
|
shouldGzipNow := false
|
||||||
if !isInputCompressed {
|
if !option.IsInputCompressed {
|
||||||
if mtype == "" {
|
if option.MimeType == "" {
|
||||||
mtype = http.DetectContentType(data)
|
option.MimeType = http.DetectContentType(data)
|
||||||
// println("detect1 mimetype to", mtype)
|
// println("detect1 mimetype to", MimeType)
|
||||||
if mtype == "application/octet-stream" {
|
if option.MimeType == "application/octet-stream" {
|
||||||
mtype = ""
|
option.MimeType = ""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed {
|
if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
|
||||||
shouldGzipNow = true
|
shouldGzipNow = true
|
||||||
} else if !iAmSure && mtype == "" && len(data) > 16*1024 {
|
} else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 {
|
||||||
var compressed []byte
|
var compressed []byte
|
||||||
compressed, err = util.GzipData(data[0:128])
|
compressed, err = util.GzipData(data[0:128])
|
||||||
shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
|
shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
|
||||||
|
@ -131,14 +141,14 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
|
||||||
// this could be double copying
|
// this could be double copying
|
||||||
clearDataLen = len(data)
|
clearDataLen = len(data)
|
||||||
clearData := data
|
clearData := data
|
||||||
if shouldGzipNow && !cipher {
|
if shouldGzipNow && !option.Cipher {
|
||||||
compressed, compressErr := util.GzipData(data)
|
compressed, compressErr := util.GzipData(data)
|
||||||
// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
|
// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
|
||||||
if compressErr == nil {
|
if compressErr == nil {
|
||||||
data = compressed
|
data = compressed
|
||||||
contentIsGzipped = true
|
contentIsGzipped = true
|
||||||
}
|
}
|
||||||
} else if isInputCompressed {
|
} else if option.IsInputCompressed {
|
||||||
// just to get the clear data length
|
// just to get the clear data length
|
||||||
clearData, err = util.DecompressData(data)
|
clearData, err = util.DecompressData(data)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
@ -146,7 +156,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if cipher {
|
if option.Cipher {
|
||||||
// encrypt(gzip(data))
|
// encrypt(gzip(data))
|
||||||
|
|
||||||
// encrypt
|
// encrypt
|
||||||
|
@ -158,23 +168,23 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
|
||||||
}
|
}
|
||||||
|
|
||||||
// upload data
|
// upload data
|
||||||
uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
|
uploadResult, err = upload_content(option.UploadUrl, func(w io.Writer) (err error) {
|
||||||
_, err = w.Write(encryptedData)
|
_, err = w.Write(encryptedData)
|
||||||
return
|
return
|
||||||
}, "", false, len(encryptedData), "", nil, jwt)
|
}, "", false, len(encryptedData), "", nil, option.Jwt)
|
||||||
if uploadResult == nil {
|
if uploadResult == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
uploadResult.Name = filename
|
uploadResult.Name = option.Filename
|
||||||
uploadResult.Mime = mtype
|
uploadResult.Mime = option.MimeType
|
||||||
uploadResult.CipherKey = cipherKey
|
uploadResult.CipherKey = cipherKey
|
||||||
uploadResult.Size = uint32(clearDataLen)
|
uploadResult.Size = uint32(clearDataLen)
|
||||||
} else {
|
} else {
|
||||||
// upload data
|
// upload data
|
||||||
uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
|
uploadResult, err = upload_content(option.UploadUrl, func(w io.Writer) (err error) {
|
||||||
_, err = w.Write(data)
|
_, err = w.Write(data)
|
||||||
return
|
return
|
||||||
}, filename, contentIsGzipped, len(data), mtype, pairMap, jwt)
|
}, option.Filename, contentIsGzipped, len(data), option.MimeType, option.PairMap, option.Jwt)
|
||||||
if uploadResult == nil {
|
if uploadResult == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -107,7 +107,16 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string)
|
||||||
glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header)
|
glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header)
|
||||||
|
|
||||||
// fetch data as is, regardless whether it is encrypted or not
|
// fetch data as is, regardless whether it is encrypted or not
|
||||||
uploadResult, err, _ := operation.Upload(fileUrl, filename, false, resp.Body, "gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, auth)
|
uploadOption := &operation.UploadOption{
|
||||||
|
UploadUrl: fileUrl,
|
||||||
|
Filename: filename,
|
||||||
|
Cipher: false,
|
||||||
|
IsInputCompressed: "gzip" == header.Get("Content-Encoding"),
|
||||||
|
MimeType: header.Get("Content-Type"),
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: auth,
|
||||||
|
}
|
||||||
|
uploadResult, err, _ := operation.Upload(resp.Body, uploadOption)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.V(0).Infof("upload source data %v to %s: %v", sourceChunk.GetFileIdString(), fileUrl, err)
|
glog.V(0).Infof("upload source data %v to %s: %v", sourceChunk.GetFileIdString(), fileUrl, err)
|
||||||
return "", fmt.Errorf("upload data: %v", err)
|
return "", fmt.Errorf("upload data: %v", err)
|
||||||
|
|
|
@ -161,7 +161,16 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn ope
|
||||||
}
|
}
|
||||||
|
|
||||||
debug("upload file to store", url)
|
debug("upload file to store", url)
|
||||||
uploadResult, err := operation.UploadData(url, pu.FileName, false, pu.Data, pu.IsGzipped, pu.MimeType, pu.PairMap, assignResult.Auth)
|
uploadOption := &operation.UploadOption{
|
||||||
|
UploadUrl: url,
|
||||||
|
Filename: pu.FileName,
|
||||||
|
Cipher: false,
|
||||||
|
IsInputCompressed: pu.IsGzipped,
|
||||||
|
MimeType: pu.MimeType,
|
||||||
|
PairMap: pu.PairMap,
|
||||||
|
Jwt: assignResult.Auth,
|
||||||
|
}
|
||||||
|
uploadResult, err := operation.UploadData(pu.Data, uploadOption)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||||
return
|
return
|
||||||
|
|
|
@ -241,7 +241,16 @@ func (fs *FilerServer) saveAsChunk(so *operation.StorageOption) filer.SaveDataAs
|
||||||
}
|
}
|
||||||
|
|
||||||
// upload the chunk to the volume server
|
// upload the chunk to the volume server
|
||||||
uploadResult, uploadErr, _ := operation.Upload(urlLocation, name, fs.option.Cipher, reader, false, "", nil, auth)
|
uploadOption := &operation.UploadOption{
|
||||||
|
UploadUrl: urlLocation,
|
||||||
|
Filename: name,
|
||||||
|
Cipher: fs.option.Cipher,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: "",
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: auth,
|
||||||
|
}
|
||||||
|
uploadResult, uploadErr, _ := operation.Upload(reader, uploadOption)
|
||||||
if uploadErr != nil {
|
if uploadErr != nil {
|
||||||
return nil, "", "", uploadErr
|
return nil, "", "", uploadErr
|
||||||
}
|
}
|
||||||
|
|
|
@ -44,7 +44,16 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht
|
||||||
// println("detect2 mimetype to", pu.MimeType)
|
// println("detect2 mimetype to", pu.MimeType)
|
||||||
}
|
}
|
||||||
|
|
||||||
uploadResult, uploadError := operation.UploadData(urlLocation, pu.FileName, true, uncompressedData, false, pu.MimeType, pu.PairMap, auth)
|
uploadOption := &operation.UploadOption{
|
||||||
|
UploadUrl: urlLocation,
|
||||||
|
Filename: pu.FileName,
|
||||||
|
Cipher: true,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: pu.MimeType,
|
||||||
|
PairMap: pu.PairMap,
|
||||||
|
Jwt: auth,
|
||||||
|
}
|
||||||
|
uploadResult, uploadError := operation.UploadData(uncompressedData, uploadOption)
|
||||||
if uploadError != nil {
|
if uploadError != nil {
|
||||||
return nil, fmt.Errorf("upload to volume server: %v", uploadError)
|
return nil, fmt.Errorf("upload to volume server: %v", uploadError)
|
||||||
}
|
}
|
||||||
|
|
|
@ -127,7 +127,16 @@ func (fs *FilerServer) doUpload(urlLocation string, limitedReader io.Reader, fil
|
||||||
stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds())
|
stats.FilerRequestHistogram.WithLabelValues("chunkUpload").Observe(time.Since(start).Seconds())
|
||||||
}()
|
}()
|
||||||
|
|
||||||
uploadResult, err, data := operation.Upload(urlLocation, fileName, fs.option.Cipher, limitedReader, false, contentType, pairMap, auth)
|
uploadOption := &operation.UploadOption{
|
||||||
|
UploadUrl: urlLocation,
|
||||||
|
Filename: fileName,
|
||||||
|
Cipher: fs.option.Cipher,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: contentType,
|
||||||
|
PairMap: pairMap,
|
||||||
|
Jwt: auth,
|
||||||
|
}
|
||||||
|
uploadResult, err, data := operation.Upload(limitedReader, uploadOption)
|
||||||
if uploadResult != nil && uploadResult.RetryCount > 0 {
|
if uploadResult != nil && uploadResult.RetryCount > 0 {
|
||||||
stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount))
|
stats.FilerRequestCounter.WithLabelValues("chunkUploadRetry").Add(float64(uploadResult.RetryCount))
|
||||||
}
|
}
|
||||||
|
|
|
@ -413,7 +413,16 @@ func (f *WebDavFile) saveDataAsChunk(reader io.Reader, name string, offset int64
|
||||||
}
|
}
|
||||||
|
|
||||||
fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
|
fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
|
||||||
uploadResult, flushErr, _ := operation.Upload(fileUrl, f.name, f.fs.option.Cipher, reader, false, "", nil, auth)
|
uploadOption := &operation.UploadOption{
|
||||||
|
UploadUrl: fileUrl,
|
||||||
|
Filename: f.name,
|
||||||
|
Cipher: f.fs.option.Cipher,
|
||||||
|
IsInputCompressed: false,
|
||||||
|
MimeType: "",
|
||||||
|
PairMap: nil,
|
||||||
|
Jwt: auth,
|
||||||
|
}
|
||||||
|
uploadResult, flushErr, _ := operation.Upload(reader, uploadOption)
|
||||||
if flushErr != nil {
|
if flushErr != nil {
|
||||||
glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, flushErr)
|
glog.V(0).Infof("upload data %v to %s: %v", f.name, fileUrl, flushErr)
|
||||||
return nil, f.collection, f.replication, fmt.Errorf("upload data: %v", flushErr)
|
return nil, f.collection, f.replication, fmt.Errorf("upload data: %v", flushErr)
|
||||||
|
|
|
@ -83,7 +83,16 @@ func ReplicatedWrite(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOpt
|
||||||
|
|
||||||
// volume server do not know about encryption
|
// volume server do not know about encryption
|
||||||
// TODO optimize here to compress data only once
|
// TODO optimize here to compress data only once
|
||||||
_, err := operation.UploadData(u.String(), string(n.Name), false, n.Data, n.IsCompressed(), string(n.Mime), pairMap, jwt)
|
uploadOption := &operation.UploadOption{
|
||||||
|
UploadUrl: u.String(),
|
||||||
|
Filename: string(n.Name),
|
||||||
|
Cipher: false,
|
||||||
|
IsInputCompressed: n.IsCompressed(),
|
||||||
|
MimeType: string(n.Mime),
|
||||||
|
PairMap: pairMap,
|
||||||
|
Jwt: jwt,
|
||||||
|
}
|
||||||
|
_, err := operation.UploadData(n.Data, uploadOption)
|
||||||
return err
|
return err
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err)
|
err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err)
|
||||||
|
|
Loading…
Reference in a new issue