mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2024-01-19 02:48:24 +00:00
Use exponential backoff to query leader. (#4313)
`topology.Leader()` was using a backoff that typically resulted in at least a 5s delay when initially starting a master and raft server. This changes the backoff algorithm to use exponential backoff starting with 100ms and waiting up to 20s for leader selection. Related to #4307
This commit is contained in:
parent
dd71f54c6b
commit
57ab1f8516
2
go.mod
2
go.mod
|
@ -10,6 +10,7 @@ require (
|
||||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||||
github.com/Shopify/sarama v1.38.1
|
github.com/Shopify/sarama v1.38.1
|
||||||
github.com/aws/aws-sdk-go v1.44.219
|
github.com/aws/aws-sdk-go v1.44.219
|
||||||
|
github.com/cenkalti/backoff/v4 v4.2.0
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
github.com/bwmarrin/snowflake v0.3.0
|
github.com/bwmarrin/snowflake v0.3.0
|
||||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||||
|
@ -176,6 +177,7 @@ require (
|
||||||
github.com/aws/smithy-go v1.13.5 // indirect
|
github.com/aws/smithy-go v1.13.5 // indirect
|
||||||
github.com/benbjohnson/clock v1.3.0 // indirect
|
github.com/benbjohnson/clock v1.3.0 // indirect
|
||||||
github.com/boltdb/bolt v1.3.1 // indirect
|
github.com/boltdb/bolt v1.3.1 // indirect
|
||||||
|
github.com/cenkalti/backoff/v4 v4.2.0
|
||||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
|
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
|
||||||
|
|
2
go.sum
2
go.sum
|
@ -691,10 +691,12 @@ github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgIS
|
||||||
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
|
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
|
||||||
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
|
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
|
||||||
github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg=
|
github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg=
|
||||||
|
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
|
||||||
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
|
||||||
github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
|
github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
|
||||||
github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
|
github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
|
||||||
github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
|
github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
|
||||||
|
github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4=
|
||||||
github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||||
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||||
|
|
|
@ -11,6 +11,8 @@ import (
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||||
|
|
||||||
|
backoff "github.com/cenkalti/backoff/v4"
|
||||||
|
|
||||||
hashicorpRaft "github.com/hashicorp/raft"
|
hashicorpRaft "github.com/hashicorp/raft"
|
||||||
"github.com/seaweedfs/raft"
|
"github.com/seaweedfs/raft"
|
||||||
|
|
||||||
|
@ -96,19 +98,10 @@ func (t *Topology) IsLeader() bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *Topology) Leader() (l pb.ServerAddress, err error) {
|
func (t *Topology) Leader() (l pb.ServerAddress, err error) {
|
||||||
for count := 0; count < 3; count++ {
|
exponentialBackoff := backoff.NewExponentialBackOff()
|
||||||
l, err = t.MaybeLeader()
|
exponentialBackoff.InitialInterval = 100 * time.Millisecond
|
||||||
if err != nil {
|
exponentialBackoff.MaxElapsedTime = 20 * time.Second
|
||||||
return
|
return backoff.RetryWithData(t.MaybeLeader, exponentialBackoff)
|
||||||
}
|
|
||||||
if l != "" {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(time.Duration(5+count) * time.Second)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *Topology) MaybeLeader() (l pb.ServerAddress, err error) {
|
func (t *Topology) MaybeLeader() (l pb.ServerAddress, err error) {
|
||||||
|
|
Loading…
Reference in a new issue