rollback over onPeerupdate implementation of automatic clean-up of failed servers in favor of synchronous ping

weed/filer/meta_aggregator.go

@@ -102,7 +102,7 @@ func (ma *MetaAggregator) loopSubscribeToOneFiler(f *Filer, self pb.ServerAddres
 		if err != nil {
 			errLvl := glog.Level(0)
 			if strings.Contains(err.Error(), "duplicated local subscription detected") {
-				errLvl = glog.Level(1)
+				errLvl = glog.Level(4)
 			}
 			glog.V(errLvl).Infof("subscribing remote %s meta change: %v", peer, err)
 		}

weed/server/master_server.go

@@ -1,7 +1,6 @@
 package weed_server
 
 import (
-	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/stats"
 	"net/http"
@@ -34,8 +33,6 @@ import (
 const (
 	SequencerType        = "master.sequencer.type"
 	SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
-	RaftServerRemovalTime         = 72 * time.Minute
-	ResetRaftServerRemovalTimeMsg = "ResetRaftServerRemovalTime"
 )
 
 type MasterOption struct {
@@ -66,9 +63,6 @@ type MasterServer struct {
 
 	boundedLeaderChan chan int
 
-	onPeerUpdateDoneCns map[string]*chan string
-	onPeerUpdateLock    sync.RWMutex
-
 	// notifying clients
 	clientChansLock sync.RWMutex
 	clientChans     map[string]chan *master_pb.KeepConnectedResponse
@@ -121,7 +115,6 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
 	ms.boundedLeaderChan = make(chan int, 16)
 
 	ms.MasterClient.OnPeerUpdate = ms.OnPeerUpdate
-	ms.onPeerUpdateDoneCns = make(map[string]*chan string)
 
 	seq := ms.createSequencer(option)
 	if nil == seq {
@@ -352,8 +345,7 @@ func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startF
 	peerAddress := pb.ServerAddress(update.Address)
 	peerName := string(peerAddress)
 	isLeader := ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader
-	if update.IsAdd {
-		if isLeader {
+	if update.IsAdd && isLeader {
 		raftServerFound := false
 		for _, server := range ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers {
 			if string(server.ID) == peerName {
@@ -367,82 +359,4 @@ func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startF
 				hashicorpRaft.ServerAddress(peerAddress.ToGrpcAddress()), 0, 0)
 		}
 	}
-		ms.onPeerUpdateLock.RLock()
-		isGtZero := len(ms.onPeerUpdateDoneCns) > 0
-		ms.onPeerUpdateLock.RUnlock()
-		if isGtZero {
-			var chanPtrs []*chan string
-			ms.onPeerUpdateLock.RLock()
-			for _, cn := range ms.onPeerUpdateDoneCns {
-				chanPtrs = append(chanPtrs, cn)
-			}
-			ms.onPeerUpdateLock.RUnlock()
-			for _, onPeerUpdateDoneCn := range chanPtrs {
-				*onPeerUpdateDoneCn <- peerName
-			}
-		}
-	} else if isLeader {
-		if onPeerUpdateDoneCnPrev, ok := ms.onPeerUpdateDoneCns[peerName]; ok {
-			*onPeerUpdateDoneCnPrev <- ResetRaftServerRemovalTimeMsg
-			return
-		}
-		onPeerUpdateDoneCn := make(chan string)
-		ms.onPeerUpdateLock.Lock()
-		ms.onPeerUpdateDoneCns[peerName] = &onPeerUpdateDoneCn
-		ms.onPeerUpdateLock.Unlock()
-
-		go func(peerName string) {
-			raftServerRemovalTimeAfter := time.After(RaftServerRemovalTime)
-			raftServerPingTicker := time.NewTicker(5 * time.Minute)
-			defer func() {
-				ms.onPeerUpdateLock.Lock()
-				delete(ms.onPeerUpdateDoneCns, peerName)
-				ms.onPeerUpdateLock.Unlock()
-				close(onPeerUpdateDoneCn)
-			}()
-			for {
-				select {
-				case <-raftServerPingTicker.C:
-					err := ms.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
-						_, err := client.Ping(context.Background(), &master_pb.PingRequest{
-							Target:     peerName,
-							TargetType: cluster.MasterType,
-						})
-						return err
-					})
-					if err != nil {
-						glog.Warningf("raft server %s ping failed %+v", peerName, err)
-					} else {
-						glog.V(0).Infof("raft server %s remove canceled on ping success", peerName)
-						return
-					}
-				case <-raftServerRemovalTimeAfter:
-					err := ms.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
-						_, err := client.RaftRemoveServer(context.Background(), &master_pb.RaftRemoveServerRequest{
-							Id:    peerName,
-							Force: false,
-						})
-						return err
-					})
-					if err != nil {
-						glog.Warningf("failed to removing old raft server %s: %v", peerName, err)
-						return
-					}
-					glog.V(0).Infof("old raft server %s removed", peerName)
-					return
-				case peerDone := <-onPeerUpdateDoneCn:
-					if peerName == peerDone {
-						glog.V(0).Infof("raft server %s remove canceled on onPeerUpdate", peerName)
-						return
-					}
-					if peerDone == ResetRaftServerRemovalTimeMsg {
-						raftServerRemovalTimeAfter = time.After(RaftServerRemovalTime)
-						glog.V(0).Infof("rest wait %v for raft server %s activity, otherwise delete",
-							RaftServerRemovalTime, peerName)
-					}
-				}
-			}
-		}(peerName)
-		glog.V(0).Infof("wait %v for raft server %s activity, otherwise delete", RaftServerRemovalTime, peerName)
-	}
 }