gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

filer.copy: add adjustable compression level

Browse source
This commit is contained in:
Chris Lu 2019-04-18 20:21:28 -07:00
parent 072644969e
commit a2d34d4802
2 changed files with 31 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
weed/command/filer_copy.go
View file

@ -29,16 +29,17 @@ var (
) )
type CopyOptions struct { type CopyOptions struct {
filerGrpcPort *int filerGrpcPort *int
master *string master *string
include *string include *string
replication *string replication *string
collection *string collection *string
ttl *string ttl *string
maxMB *int maxMB *int
grpcDialOption grpc.DialOption grpcDialOption grpc.DialOption
masterClient *wdclient.MasterClient masterClient *wdclient.MasterClient
concurrency *int concurrency *int
compressionLevel *int
} }
func init() { func init() {
@ -52,6 +53,7 @@ func init() {
copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit") copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
copy.filerGrpcPort = cmdCopy.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to filer port + 10000") copy.filerGrpcPort = cmdCopy.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to filer port + 10000")
copy.concurrency = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines") copy.concurrency = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
copy.compressionLevel = cmdCopy.Flag.Int("compressionLevel", 9, "local file compression level 1 ~ 9")
} }
var cmdCopy = &Command{ var cmdCopy = &Command{
@ -265,7 +267,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(ctx context.Context, task FileCopy
targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth) uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth, *worker.options.compressionLevel)
if err != nil { if err != nil {
return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err) return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
} }

20
weed/operation/upload_content.go
View file

@ -39,8 +39,23 @@ func init() {
var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
// Upload sends a POST request to a volume server to upload the content // Upload sends a POST request to a volume server to upload the content with adjustable compression level
func UploadWithLocalCompressionLevel(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt, compressionLevel int) (*UploadResult, error) {
if compressionLevel < 1 {
compressionLevel = 1
}
if compressionLevel > 9 {
compressionLevel = 9
}
return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, compressionLevel, jwt)
}
// Upload sends a POST request to a volume server to upload the content with fast compression
func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, flate.BestSpeed, jwt)
}
func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (*UploadResult, error) {
contentIsGzipped := isGzipped contentIsGzipped := isGzipped
shouldGzipNow := false shouldGzipNow := false
if !isGzipped { if !isGzipped {
@ -51,7 +66,7 @@ func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool,
} }
return upload_content(uploadUrl, func(w io.Writer) (err error) { return upload_content(uploadUrl, func(w io.Writer) (err error) {
if shouldGzipNow { if shouldGzipNow {
gzWriter, _ := gzip.NewWriterLevel(w, flate.BestSpeed) gzWriter, _ := gzip.NewWriterLevel(w, compression)
_, err = io.Copy(gzWriter, reader) _, err = io.Copy(gzWriter, reader)
gzWriter.Close() gzWriter.Close()
} else { } else {
@ -60,6 +75,7 @@ func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool,
return return
}, filename, contentIsGzipped, mtype, pairMap, jwt) }, filename, contentIsGzipped, mtype, pairMap, jwt)
} }
func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) { func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
body_buf := bytes.NewBufferString("") body_buf := bytes.NewBufferString("")
body_writer := multipart.NewWriter(body_buf) body_writer := multipart.NewWriter(body_buf)