gitlab-com-seaweedfs/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2024-01-19 02:48:24 +00:00
Code Issues Projects Releases Wiki Activity

async wait

Browse source
This commit is contained in:
chrislu 2022-09-25 17:57:26 -07:00
parent 3da8d7309a
commit 24317a9763
2 changed files with 35 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
weed/mq/messages/message_pipeline.go
View file

@ -25,7 +25,7 @@ type MessagePipeline struct {
movedBuffersChan chan MessageBufferReference movedBuffersChan chan MessageBufferReference
onMessageFn OnMessageFunc onMessageFn OnMessageFunc
mover MessageBufferMover mover MessageBufferMover
moverPool *util.LimitedConcurrentExecutor moverPool *util.LimitedAsyncExecutor
// control pipeline // control pipeline
doneChan chan struct{} doneChan chan struct{}
@ -47,15 +47,15 @@ func NewMessagePipeline(producerId int32, workerCount int, batchSize int, timeou
doneChan: make(chan struct{}), doneChan: make(chan struct{}),
batchSize: batchSize, batchSize: batchSize,
timeout: timeout, timeout: timeout,
moverPool: util.NewLimitedConcurrentExecutor(workerCount), moverPool: util.NewLimitedAsyncExecutor(workerCount),
mover: mover, mover: mover,
} }
go t.doLoopUpload() go t.doLoopUpload()
return t return t
} }
func (mp *MessagePipeline) OutputChan() chan MessageBufferReference { func (mp *MessagePipeline) NextMessageBufferReference() MessageBufferReference {
return mp.movedBuffersChan return mp.moverPool.NextFuture().Await().(MessageBufferReference)
} }
func (mp *MessagePipeline) AddMessage(message *Message) { func (mp *MessagePipeline) AddMessage(message *Message) {
@ -105,18 +105,20 @@ func (mp *MessagePipeline) doLoopUpload() {
select { select {
case messageBuffer := <-mp.sealedBuffersChan: case messageBuffer := <-mp.sealedBuffersChan:
ticker.Reset(mp.timeout) ticker.Reset(mp.timeout)
mp.moverPool.Execute(func() { mp.moverPool.Execute(func() any {
var output MessageBufferReference
util.RetryForever("message mover", func() error { util.RetryForever("message mover", func() error {
if messageReference, flushErr := mp.mover.MoveBuffer(messageBuffer); flushErr != nil { if messageReference, flushErr := mp.mover.MoveBuffer(messageBuffer); flushErr != nil {
return flushErr return flushErr
} else { } else {
mp.movedBuffersChan <- messageReference output = messageReference
} }
return nil return nil
}, func(err error) (shouldContinue bool) { }, func(err error) (shouldContinue bool) {
log.Printf("failed: %v", err) log.Printf("failed: %v", err)
return true return true
}) })
return output
}) })
case <-ticker.C: case <-ticker.C:
if atomic.LoadInt64(&mp.atomicPipelineStatus) == -2 { if atomic.LoadInt64(&mp.atomicPipelineStatus) == -2 {

30
weed/mq/messages/message_pipeline_test.go
View file

@ -5,11 +5,11 @@ import (
"time" "time"
) )
func TestAddMessage(t *testing.T) { func TestAddMessage1(t *testing.T) {
mp := NewMessagePipeline(0, 3, 10, time.Second, &EmptyMover{}) mp := NewMessagePipeline(0, 3, 10, time.Second, &EmptyMover{})
go func() { go func() {
outChan := mp.OutputChan() for i := 0; i < 100; i++ {
for mr := range outChan { mr := mp.NextMessageBufferReference()
println(mr.sequence, mr.fileId) println(mr.sequence, mr.fileId)
} }
}() }()
@ -26,4 +26,28 @@ func TestAddMessage(t *testing.T) {
mp.ShutdownStart() mp.ShutdownStart()
mp.ShutdownWait() mp.ShutdownWait()
}
func TestAddMessage2(t *testing.T) {
mp := NewMessagePipeline(0, 3, 10, time.Second, &EmptyMover{})
for i := 0; i < 100; i++ {
message := &Message{
Key: []byte("key"),
Content: []byte("data"),
Properties: nil,
Ts: time.Now(),
}
mp.AddMessage(message)
}
mp.ShutdownStart()
mp.ShutdownWait()
for i := 0; i < 100; i++ {
mr := mp.NextMessageBufferReference()
println(mr.sequence, mr.fileId)
}
} }