more efficient client side gzip compression

This commit is contained in:
Chris Lu 2019-04-06 14:14:28 -07:00
parent 02191f6156
commit 174bf1e8b2
2 changed files with 64 additions and 36 deletions

View file

@ -16,40 +16,9 @@ import (
*/
func IsGzippable(ext, mtype string, data []byte) bool {
// text
if strings.HasPrefix(mtype, "text/") {
return true
}
// images
switch ext {
case ".svg", ".bmp":
return true
}
if strings.HasPrefix(mtype, "image/") {
return false
}
// by file name extension
switch ext {
case ".zip", ".rar", ".gz", ".bz2", ".xz":
return false
case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
return true
case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
return true
case ".png", ".jpg", ".jpeg":
return false
}
// by mime type
if strings.HasPrefix(mtype, "application/") {
if strings.HasSuffix(mtype, "xml") {
return true
}
if strings.HasSuffix(mtype, "script") {
return true
}
shouldBeZipped, iAmSure := IsGzippableFileType(ext, mtype)
if iAmSure {
return shouldBeZipped
}
isMostlyText := util.IsText(data)
@ -57,6 +26,50 @@ func IsGzippable(ext, mtype string, data []byte) bool {
return isMostlyText
}
/*
* Default more not to gzip since gzip can be done on client side.
*/
func IsGzippableFileType(ext, mtype string) (shouldBeZipped, iAmSure bool) {
// text
if strings.HasPrefix(mtype, "text/") {
return true, true
}
// images
switch ext {
case ".svg", ".bmp":
return true, true
}
if strings.HasPrefix(mtype, "image/") {
return false, true
}
// by file name extension
switch ext {
case ".zip", ".rar", ".gz", ".bz2", ".xz":
return false, true
case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
return true, true
case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
return true, true
case ".png", ".jpg", ".jpeg":
return false, true
}
// by mime type
if strings.HasPrefix(mtype, "application/") {
if strings.HasSuffix(mtype, "xml") {
return true, true
}
if strings.HasSuffix(mtype, "script") {
return true, true
}
}
return false, false
}
func GzipData(input []byte) ([]byte, error) {
buf := new(bytes.Buffer)
w, _ := gzip.NewWriterLevel(buf, flate.BestCompression)

View file

@ -2,6 +2,7 @@ package operation
import (
"bytes"
"compress/gzip"
"encoding/json"
"errors"
"fmt"
@ -39,10 +40,24 @@ var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
// Upload sends a POST request to a volume server to upload the content
func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
contentIsGzipped := isGzipped
shouldGzipNow := false
if !isGzipped {
if shouldBeZipped, iAmSure := IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped {
shouldGzipNow = true
contentIsGzipped = true
}
}
return upload_content(uploadUrl, func(w io.Writer) (err error) {
_, err = io.Copy(w, reader)
if shouldGzipNow {
gzWriter := gzip.NewWriter(w)
_, err = io.Copy(gzWriter, reader)
gzWriter.Close()
} else {
_, err = io.Copy(w, reader)
}
return
}, filename, isGzipped, mtype, pairMap, jwt)
}, filename, contentIsGzipped, mtype, pairMap, jwt)
}
func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
body_buf := bytes.NewBufferString("")