gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

refactoring

Browse source
This commit is contained in:
Chris Lu 2020-03-14 14:08:00 -07:00
parent 18cda6adbb
commit ee635dcc36
1 changed files with 10 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

84
weed/operation/upload_content.go
View file

@ -2,7 +2,6 @@ package operation
import ( import (
"bytes" "bytes"
"compress/flate"
"crypto/md5" "crypto/md5"
"encoding/json" "encoding/json"
"errors" "errors"
@ -59,13 +58,22 @@ func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isI
func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { func Upload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
hash := md5.New() hash := md5.New()
reader = io.TeeReader(reader, hash) reader = io.TeeReader(reader, hash)
uploadResult, err = doUpload(uploadUrl, filename, cipher, reader, isInputGzipped, mtype, pairMap, flate.BestSpeed, jwt) uploadResult, err = doUpload(uploadUrl, filename, cipher, reader, isInputGzipped, mtype, pairMap, jwt)
if uploadResult != nil { if uploadResult != nil {
uploadResult.Md5 = fmt.Sprintf("%x", hash.Sum(nil)) uploadResult.Md5 = fmt.Sprintf("%x", hash.Sum(nil))
} }
return return
} }
func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
data, readErr := ioutil.ReadAll(reader)
if readErr != nil {
err = fmt.Errorf("read input: %v", readErr)
return
}
return doUploadData(uploadUrl, filename, cipher, data, isInputGzipped, mtype, pairMap, jwt)
}
func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) { func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
contentIsGzipped := isInputGzipped contentIsGzipped := isInputGzipped
shouldGzipNow := false shouldGzipNow := false
@ -132,78 +140,6 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
return uploadResult, err return uploadResult, err
} }
func doUpload(uploadUrl string, filename string, cipher bool, reader io.Reader, isInputGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
contentIsGzipped := isInputGzipped
shouldGzipNow := false
if !isInputGzipped {
if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); mtype == "" || iAmSure && shouldBeZipped {
shouldGzipNow = true
contentIsGzipped = true
}
}
var clearDataLen int
// gzip if possible
// this could be double copying
data, readErr := ioutil.ReadAll(reader)
if readErr != nil {
err = fmt.Errorf("read input: %v", readErr)
return
}
clearDataLen = len(data)
if shouldGzipNow {
data, err = util.GzipData(data)
} else if isInputGzipped {
// just to get the clear data length
clearData, err := util.UnGzipData(data)
if err == nil {
clearDataLen = len(clearData)
}
}
if cipher {
// encrypt(gzip(data))
// encrypt
cipherKey := util.GenCipherKey()
encryptedData, encryptionErr := util.Encrypt(data, cipherKey)
if encryptionErr != nil {
err = fmt.Errorf("encrypt input: %v", encryptionErr)
return
}
// upload data
uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
_, err = w.Write(encryptedData)
return
}, "", false, "", nil, jwt)
if uploadResult != nil {
uploadResult.Name = filename
uploadResult.Mime = mtype
uploadResult.CipherKey = cipherKey
uploadResult.Size = uint32(clearDataLen)
}
} else {
// upload data
uploadResult, err = upload_content(uploadUrl, func(w io.Writer) (err error) {
_, err = w.Write(data)
return
}, filename, contentIsGzipped, mtype, pairMap, jwt)
}
if uploadResult == nil {
return
}
uploadResult.Size = uint32(clearDataLen)
if contentIsGzipped {
uploadResult.Gzip = 1
}
return uploadResult, err
}
func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
body_buf := bytes.NewBufferString("") body_buf := bytes.NewBufferString("")
body_writer := multipart.NewWriter(body_buf) body_writer := multipart.NewWriter(body_buf)