mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
filer.sync: limit concurrency when fetching file chunks
fix https://github.com/seaweedfs/seaweedfs/issues/3787
This commit is contained in:
parent
ec46a34f33
commit
0452ae6a6c
|
@ -24,7 +24,8 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk, path st
|
|||
var wg sync.WaitGroup
|
||||
for chunkIndex, sourceChunk := range sourceChunks {
|
||||
wg.Add(1)
|
||||
go func(chunk *filer_pb.FileChunk, index int) {
|
||||
fs.executor.Execute(func() {
|
||||
func(chunk *filer_pb.FileChunk, index int) {
|
||||
defer wg.Done()
|
||||
replicatedChunk, e := fs.replicateOneChunk(chunk, path)
|
||||
if e != nil {
|
||||
|
@ -33,6 +34,7 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk, path st
|
|||
}
|
||||
replicatedChunks[index] = replicatedChunk
|
||||
}(sourceChunk, chunkIndex)
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
|
|
|
@ -32,6 +32,7 @@ type FilerSink struct {
|
|||
address string
|
||||
writeChunkByFiler bool
|
||||
isIncremental bool
|
||||
executor *util.LimitedConcurrentExecutor
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
@ -53,6 +54,7 @@ func (fs *FilerSink) IsIncremental() bool {
|
|||
func (fs *FilerSink) Initialize(configuration util.Configuration, prefix string) error {
|
||||
fs.isIncremental = configuration.GetBool(prefix + "is_incremental")
|
||||
fs.dataCenter = configuration.GetString(prefix + "dataCenter")
|
||||
fs.executor = util.NewLimitedConcurrentExecutor(32)
|
||||
return fs.DoInitialize(
|
||||
"",
|
||||
configuration.GetString(prefix+"grpcAddress"),
|
||||
|
|
Loading…
Reference in a new issue