From 57ab1f851656276f62124c3dc5e3fc0d15eed646 Mon Sep 17 00:00:00 2001 From: Stewart Miles Date: Wed, 15 Mar 2023 17:49:46 -0700 Subject: [PATCH] Use exponential backoff to query leader. (#4313) `topology.Leader()` was using a backoff that typically resulted in at least a 5s delay when initially starting a master and raft server. This changes the backoff algorithm to use exponential backoff starting with 100ms and waiting up to 20s for leader selection. Related to #4307 --- go.mod | 2 ++ go.sum | 2 ++ weed/topology/topology.go | 19 ++++++------------- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 13719b9bf..f2183db64 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Shopify/sarama v1.38.1 github.com/aws/aws-sdk-go v1.44.219 + github.com/cenkalti/backoff/v4 v4.2.0 github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -176,6 +177,7 @@ require ( github.com/aws/smithy-go v1.13.5 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/boltdb/bolt v1.3.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.0 github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect github.com/d4l3k/messagediff v1.2.1 // indirect github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect diff --git a/go.sum b/go.sum index 5431363f9..deccae7ba 100644 --- a/go.sum +++ b/go.sum @@ -691,10 +691,12 @@ github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgIS github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/weed/topology/topology.go b/weed/topology/topology.go index c7b492267..825c02961 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -11,6 +11,8 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/storage/types" + backoff "github.com/cenkalti/backoff/v4" + hashicorpRaft "github.com/hashicorp/raft" "github.com/seaweedfs/raft" @@ -96,19 +98,10 @@ func (t *Topology) IsLeader() bool { } func (t *Topology) Leader() (l pb.ServerAddress, err error) { - for count := 0; count < 3; count++ { - l, err = t.MaybeLeader() - if err != nil { - return - } - if l != "" { - break - } - - time.Sleep(time.Duration(5+count) * time.Second) - } - - return + exponentialBackoff := backoff.NewExponentialBackOff() + exponentialBackoff.InitialInterval = 100 * time.Millisecond + exponentialBackoff.MaxElapsedTime = 20 * time.Second + return backoff.RetryWithData(t.MaybeLeader, exponentialBackoff) } func (t *Topology) MaybeLeader() (l pb.ServerAddress, err error) {