seaweedfs/weed/util/compression.go

154 lines
3 KiB
Go
Raw Normal View History

2019-04-19 04:43:36 +00:00
package util
import (
"bytes"
2020-06-20 15:01:00 +00:00
"fmt"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
// "github.com/klauspost/compress/zstd"
)
2020-09-12 11:08:03 +00:00
var (
2020-09-03 18:00:20 +00:00
UnsupportedCompression = fmt.Errorf("unsupported compression")
)
2020-09-12 11:08:03 +00:00
func MaybeGzipData(input []byte) []byte {
2020-09-03 18:00:20 +00:00
if IsGzippedContent(input) {
return input
}
gzipped, err := GzipData(input)
if err != nil {
return input
}
2020-09-12 11:08:03 +00:00
if len(gzipped)*10 > len(input)*9 {
2020-09-03 18:00:20 +00:00
return input
}
return gzipped
}
2020-09-12 11:08:03 +00:00
func MaybeDecompressData(input []byte) []byte {
2020-09-03 18:00:20 +00:00
uncompressed, err := DecompressData(input)
if err != nil {
if err != UnsupportedCompression {
glog.Errorf("decompressed data: %v", err)
}
return input
}
return uncompressed
}
2019-04-19 04:43:36 +00:00
func GzipData(input []byte) ([]byte, error) {
2021-08-20 10:36:51 +00:00
w := new(bytes.Buffer)
_, err := GzipStream(w, bytes.NewReader(input))
if err != nil {
2019-04-19 04:43:36 +00:00
return nil, err
}
2021-08-20 10:36:51 +00:00
return w.Bytes(), nil
}
func ungzipData(input []byte) ([]byte, error) {
w := new(bytes.Buffer)
_, err := GunzipStream(w, bytes.NewReader(input))
if err != nil {
2019-04-19 04:43:36 +00:00
return nil, err
}
2021-08-20 10:36:51 +00:00
return w.Bytes(), nil
2019-04-19 04:43:36 +00:00
}
2020-06-24 18:39:09 +00:00
func DecompressData(input []byte) ([]byte, error) {
2020-06-20 05:45:27 +00:00
if IsGzippedContent(input) {
return ungzipData(input)
}
/*
2020-11-26 19:22:30 +00:00
if IsZstdContent(input) {
return unzstdData(input)
}
*/
2020-09-03 18:00:20 +00:00
return input, UnsupportedCompression
2020-06-20 05:45:27 +00:00
}
func IsGzippedContent(data []byte) bool {
if len(data) < 2 {
return false
}
return data[0] == 31 && data[1] == 139
}
/*
var zstdEncoder, _ = zstd.NewWriter(nil)
func ZstdData(input []byte) ([]byte, error) {
return zstdEncoder.EncodeAll(input, nil), nil
}
var decoder, _ = zstd.NewReader(nil)
func unzstdData(input []byte) ([]byte, error) {
return decoder.DecodeAll(input, nil)
}
2020-06-23 16:12:02 +00:00
func IsZstdContent(data []byte) bool {
if len(data) < 4 {
return false
}
2020-06-24 18:39:09 +00:00
return data[3] == 0xFD && data[2] == 0x2F && data[1] == 0xB5 && data[0] == 0x28
2020-06-23 16:12:02 +00:00
}
*/
2020-06-23 16:12:02 +00:00
/*
2020-06-23 16:12:02 +00:00
* Default not to compressed since compression can be done on client side.
*/func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool) {
2018-12-22 23:05:31 +00:00
// text
2013-01-17 08:56:56 +00:00
if strings.HasPrefix(mtype, "text/") {
return true, true
}
2018-12-22 23:05:31 +00:00
// images
switch ext {
2020-01-30 04:51:58 +00:00
case ".svg", ".bmp", ".wav":
return true, true
2018-12-22 23:05:31 +00:00
}
if strings.HasPrefix(mtype, "image/") {
return false, true
2018-12-22 23:05:31 +00:00
}
2019-02-06 13:59:15 +00:00
// by file name extension
2013-01-17 08:56:56 +00:00
switch ext {
2021-04-28 08:51:49 +00:00
case ".zip", ".rar", ".gz", ".bz2", ".xz", ".zst", ".br":
return false, true
2014-07-08 16:32:55 +00:00
case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
return true, true
2018-12-22 23:05:31 +00:00
case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
return true, true
case ".png", ".jpg", ".jpeg":
return false, true
}
2018-12-22 23:05:31 +00:00
// by mime type
if strings.HasPrefix(mtype, "application/") {
2020-06-23 16:12:02 +00:00
if strings.HasSuffix(mtype, "zstd") {
return false, true
}
if strings.HasSuffix(mtype, "xml") {
return true, true
}
if strings.HasSuffix(mtype, "script") {
return true, true
}
2021-04-28 20:36:53 +00:00
if strings.HasSuffix(mtype, "vnd.rar") {
return false, true
}
2020-01-30 04:51:58 +00:00
}
if strings.HasPrefix(mtype, "audio/") {
switch strings.TrimPrefix(mtype, "audio/") {
case "wave", "wav", "x-wav", "x-pn-wav":
return true, true
}
}
2018-12-22 23:05:31 +00:00
return false, false
}