diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index 6fda134aa..a7e3decdc 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -9,10 +9,12 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" ) var ( concurrentWriterLimit = runtime.NumCPU() + concurrentWriters = util.NewLimitedConcurrentExecutor(4 * concurrentWriterLimit) ) type ContinuousDirtyPages struct { @@ -107,7 +109,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, mtime := time.Now().UnixNano() pages.writeWaitGroup.Add(1) - go func() { + concurrentWriters.Execute(func() { defer pages.writeWaitGroup.Done() dir, _ := pages.f.fullpath().DirAndName() @@ -123,7 +125,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, pages.collection, pages.replication = collection, replication pages.f.addChunks([]*filer_pb.FileChunk{chunk}) glog.V(3).Infof("%s saveToStorage [%d,%d)", pages.f.fullpath(), offset, offset+size) - }() + }) } func max(x, y int64) int64 { diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 5ba9e37a0..dc92125a8 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -119,7 +119,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i } if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeCompressed { shouldGzipNow = true - } else if !iAmSure && mtype == "" && len(data) > 128 { + } else if !iAmSure && mtype == "" && len(data) > 16*1024 { var compressed []byte compressed, err = util.GzipData(data[0:128]) shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%