From ccf1795e6f02de0b774ff542555f68468e25c556 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sat, 23 Nov 2024 19:58:45 -0800 Subject: [PATCH] wait a bit before getting the next volume id if the leader is recently elected --- weed/server/master_server.go | 3 +++ weed/topology/topology.go | 2 ++ weed/topology/volume_growth.go | 5 +++++ 3 files changed, 10 insertions(+) diff --git a/weed/server/master_server.go b/weed/server/master_server.go index 9b7150d83..e3b5deea6 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -184,12 +184,15 @@ func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) { stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc() if ms.Topo.RaftServer.Leader() != "" { glog.V(0).Infof("[%s] %s becomes leader.", ms.Topo.RaftServer.Name(), ms.Topo.RaftServer.Leader()) + ms.Topo.LastLeaderChangeTime = time.Now() } }) raftServerName = fmt.Sprintf("[%s]", ms.Topo.RaftServer.Name()) } else if raftServer.RaftHashicorp != nil { ms.Topo.HashicorpRaft = raftServer.RaftHashicorp raftServerName = ms.Topo.HashicorpRaft.String() + // TODO fix this for hashicorp raft + ms.Topo.LastLeaderChangeTime = time.Now() } ms.Topo.RaftServerAccessLock.Unlock() diff --git a/weed/topology/topology.go b/weed/topology/topology.go index ba3be97c4..e436b453a 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -55,6 +55,8 @@ type Topology struct { UuidAccessLock sync.RWMutex UuidMap map[string][]string + + LastLeaderChangeTime time.Time } func NewTopology(id string, seq sequence.Sequencer, volumeSizeLimit uint64, pulse int, replicationAsMin bool) *Topology { diff --git a/weed/topology/volume_growth.go b/weed/topology/volume_growth.go index d856bbc22..23e1d5fd6 100644 --- a/weed/topology/volume_growth.go +++ b/weed/topology/volume_growth.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/server/constants" "math/rand/v2" "reflect" "sync" @@ -126,6 +127,10 @@ func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topolo if e != nil { return nil, e } + for !topo.LastLeaderChangeTime.Add(constants.VolumePulseSeconds * 2).Before(time.Now()) { + glog.V(0).Infof("wait for volume servers to join back") + time.Sleep(constants.VolumePulseSeconds / 2) + } vid, raftErr := topo.NextVolumeId() if raftErr != nil { return nil, raftErr