Browse Source
Fix master leader election startup issue (#8340 )
* Fix master leader election startup issue
Fixes #error-log-leader-not-selected-yet
* Fix master leader election startup issue
This change improves server address comparison using the 'Equals' method and handles recursion in topology leader lookup, resolving the 'leader not selected yet' error during master startup.
* Merge user improvements: use MaybeLeader for non-blocking checks
* not useful test
* Address code review: optimize Equals, fix deadlock in IsLeader, safe access in Leader
pull/8342/head
Chris Lu
6 days ago
committed by
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with
24 additions and
5 deletions
weed/pb/server_address.go
weed/server/master_grpc_server.go
weed/topology/topology.go
@ -62,6 +62,13 @@ func (sa ServerAddress) ToHttpAddress() string {
return string ( sa )
return string ( sa )
}
}
func ( sa ServerAddress ) Equals ( other ServerAddress ) bool {
if sa == other {
return true
}
return sa . ToHttpAddress ( ) == other . ToHttpAddress ( )
}
func ( sa ServerAddress ) ToGrpcAddress ( ) string {
func ( sa ServerAddress ) ToGrpcAddress ( ) string {
portsSepIndex := strings . LastIndex ( string ( sa ) , ":" )
portsSepIndex := strings . LastIndex ( string ( sa ) , ":" )
if portsSepIndex < 0 {
if portsSepIndex < 0 {
@ -110,10 +110,10 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
if ! ms . Topo . IsLeader ( ) {
if ! ms . Topo . IsLeader ( ) {
// tell the volume servers about the leader
// tell the volume servers about the leader
newLeader , err := ms . Topo . Leader ( )
if err != nil {
newLeader , err := ms . Topo . Maybe Leader( )
if err != nil || newLeader == "" {
glog . Warningf ( "SendHeartbeat find leader: %v" , err )
glog . Warningf ( "SendHeartbeat find leader: %v" , err )
return err
return raft . NotLead erErro r
}
}
if err := stream . Send ( & master_pb . HeartbeatResponse {
if err := stream . Send ( & master_pb . HeartbeatResponse {
Leader : string ( newLeader ) ,
Leader : string ( newLeader ) ,
@ -117,8 +117,10 @@ func (t *Topology) IsLeader() bool {
if t . RaftServer . State ( ) == raft . Leader {
if t . RaftServer . State ( ) == raft . Leader {
return true
return true
}
}
if leader , err := t . Leader ( ) ; err == nil {
if pb . ServerAddress ( t . RaftServer . Name ( ) ) == leader {
// Directly check leader to avoid re-acquiring lock via MaybeLeader()
leader := pb . ServerAddress ( t . RaftServer . Leader ( ) )
if leader != "" {
if pb . ServerAddress ( t . RaftServer . Name ( ) ) . Equals ( leader ) {
return true
return true
}
}
}
}
@ -175,6 +177,16 @@ func (t *Topology) Leader() (l pb.ServerAddress, err error) {
func ( ) ( l pb . ServerAddress , err error ) {
func ( ) ( l pb . ServerAddress , err error ) {
l , err = t . MaybeLeader ( )
l , err = t . MaybeLeader ( )
if err == nil && l == "" {
if err == nil && l == "" {
// Thread-safe check if we are the leader
t . RaftServerAccessLock . RLock ( )
if t . RaftServer != nil && t . RaftServer . State ( ) == raft . Leader {
l = pb . ServerAddress ( t . RaftServer . Name ( ) )
}
t . RaftServerAccessLock . RUnlock ( )
if l != "" {
return l , nil
}
err = leaderNotSelected
err = leaderNotSelected
}
}
return l , err
return l , err