gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

adjust compression threshold

Browse source
This commit is contained in:
Chris Lu 2020-10-23 23:05:55 -07:00
parent 12eafd634d
commit ef9c32ea0d
2 changed files with 5 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
weed/filesys/dirty_page.go
View file

@ -9,10 +9,12 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
) )
var ( var (
concurrentWriterLimit = runtime.NumCPU() concurrentWriterLimit = runtime.NumCPU()
concurrentWriters = util.NewLimitedConcurrentExecutor(4 * concurrentWriterLimit)
) )
type ContinuousDirtyPages struct { type ContinuousDirtyPages struct {
@ -107,7 +109,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64,
mtime := time.Now().UnixNano() mtime := time.Now().UnixNano()
pages.writeWaitGroup.Add(1) pages.writeWaitGroup.Add(1)
go func() { concurrentWriters.Execute(func() {
defer pages.writeWaitGroup.Done() defer pages.writeWaitGroup.Done()
dir, _ := pages.f.fullpath().DirAndName() dir, _ := pages.f.fullpath().DirAndName()
@ -123,7 +125,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64,
pages.collection, pages.replication = collection, replication pages.collection, pages.replication = collection, replication
pages.f.addChunks([]*filer_pb.FileChunk{chunk}) pages.f.addChunks([]*filer_pb.FileChunk{chunk})
glog.V(3).Infof("%s saveToStorage [%d,%d)", pages.f.fullpath(), offset, offset+size) glog.V(3).Infof("%s saveToStorage [%d,%d)", pages.f.fullpath(), offset, offset+size)
}() })
} }
func max(x, y int64) int64 { func max(x, y int64) int64 {

2
weed/operation/upload_content.go
View file

@ -119,7 +119,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
} }
if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed { if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed {
shouldGzipNow = true shouldGzipNow = true
} else if !iAmSure && mtype == "" && len(data) > 128 { } else if !iAmSure && mtype == "" && len(data) > 16*1024 {
var compressed []byte var compressed []byte
compressed, err = util.GzipData(data[0:128]) compressed, err = util.GzipData(data[0:128])
shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90% shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%