Browse Source

Fix master leader election startup issue (#8340)

* Fix master leader election startup issue

Fixes #error-log-leader-not-selected-yet

* Fix master leader election startup issue

This change improves server address comparison using the 'Equals' method and handles recursion in topology leader lookup, resolving the 'leader not selected yet' error during master startup.

* Merge user improvements: use MaybeLeader for non-blocking checks

* not useful test

* Address code review: optimize Equals, fix deadlock in IsLeader, safe access in Leader
pull/8342/head
Chris Lu 6 days ago
committed by GitHub
parent
commit
b08bb8237c
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 7
      weed/pb/server_address.go
  2. 6
      weed/server/master_grpc_server.go
  3. 16
      weed/topology/topology.go

7
weed/pb/server_address.go

@ -62,6 +62,13 @@ func (sa ServerAddress) ToHttpAddress() string {
return string(sa)
}
func (sa ServerAddress) Equals(other ServerAddress) bool {
if sa == other {
return true
}
return sa.ToHttpAddress() == other.ToHttpAddress()
}
func (sa ServerAddress) ToGrpcAddress() string {
portsSepIndex := strings.LastIndex(string(sa), ":")
if portsSepIndex < 0 {

6
weed/server/master_grpc_server.go

@ -110,10 +110,10 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
if !ms.Topo.IsLeader() {
// tell the volume servers about the leader
newLeader, err := ms.Topo.Leader()
if err != nil {
newLeader, err := ms.Topo.MaybeLeader()
if err != nil || newLeader == "" {
glog.Warningf("SendHeartbeat find leader: %v", err)
return err
return raft.NotLeaderError
}
if err := stream.Send(&master_pb.HeartbeatResponse{
Leader: string(newLeader),

16
weed/topology/topology.go

@ -117,8 +117,10 @@ func (t *Topology) IsLeader() bool {
if t.RaftServer.State() == raft.Leader {
return true
}
if leader, err := t.Leader(); err == nil {
if pb.ServerAddress(t.RaftServer.Name()) == leader {
// Directly check leader to avoid re-acquiring lock via MaybeLeader()
leader := pb.ServerAddress(t.RaftServer.Leader())
if leader != "" {
if pb.ServerAddress(t.RaftServer.Name()).Equals(leader) {
return true
}
}
@ -175,6 +177,16 @@ func (t *Topology) Leader() (l pb.ServerAddress, err error) {
func() (l pb.ServerAddress, err error) {
l, err = t.MaybeLeader()
if err == nil && l == "" {
// Thread-safe check if we are the leader
t.RaftServerAccessLock.RLock()
if t.RaftServer != nil && t.RaftServer.State() == raft.Leader {
l = pb.ServerAddress(t.RaftServer.Name())
}
t.RaftServerAccessLock.RUnlock()
if l != "" {
return l, nil
}
err = leaderNotSelected
}
return l, err

Loading…
Cancel
Save