diff --git a/k8s/charts/seaweedfs/Chart.yaml b/k8s/charts/seaweedfs/Chart.yaml index 4fb338484..643a021c0 100644 --- a/k8s/charts/seaweedfs/Chart.yaml +++ b/k8s/charts/seaweedfs/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "4.10" +appVersion: "4.11" # Dev note: Trigger a helm chart release by `git tag -a helm-` -version: 4.0.410 +version: 4.0.411 diff --git a/weed/server/master_server.go b/weed/server/master_server.go index ee6acd90d..d80605de8 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -418,7 +418,7 @@ func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startF glog.V(4).Infof("OnPeerUpdate: %+v", update) peerAddress := pb.ServerAddress(update.Address) - peerName := string(peerAddress) + peerName := raftServerID(peerAddress) if ms.Topo.HashicorpRaft.State() != hashicorpRaft.Leader { return } diff --git a/weed/server/raft_hashicorp.go b/weed/server/raft_hashicorp.go index 33cf25412..1ff213c19 100644 --- a/weed/server/raft_hashicorp.go +++ b/weed/server/raft_hashicorp.go @@ -11,7 +11,6 @@ import ( "path" "path/filepath" "sort" - "strings" "time" transport "github.com/Jille/raft-grpc-transport" @@ -34,26 +33,34 @@ const ( ) func getPeerIdx(self pb.ServerAddress, mapPeers map[string]pb.ServerAddress) int { - peers := make([]pb.ServerAddress, 0, len(mapPeers)) + peerIDs := make([]string, 0, len(mapPeers)) + seen := make(map[string]struct{}, len(mapPeers)) for _, peer := range mapPeers { - peers = append(peers, peer) - } - sort.Slice(peers, func(i, j int) bool { - return strings.Compare(string(peers[i]), string(peers[j])) < 0 - }) - for i, peer := range peers { - if string(peer) == string(self) { - return i + id := raftServerID(peer) + if _, ok := seen[id]; ok { + continue } + seen[id] = struct{}{} + peerIDs = append(peerIDs, id) + } + sort.Strings(peerIDs) + selfID := raftServerID(self) + idx := sort.SearchStrings(peerIDs, selfID) + if idx < len(peerIDs) && peerIDs[idx] == selfID { + return idx } return -1 } +func raftServerID(server pb.ServerAddress) string { + return server.ToHttpAddress() +} + func (s *RaftServer) AddPeersConfiguration() (cfg raft.Configuration) { for _, peer := range s.peers { cfg.Servers = append(cfg.Servers, raft.Server{ Suffrage: raft.Voter, - ID: raft.ServerID(peer), + ID: raft.ServerID(raftServerID(peer)), Address: raft.ServerAddress(peer.ToGrpcAddress()), }) } @@ -98,7 +105,12 @@ func (s *RaftServer) monitorLeaderLoop(updatePeers bool) { } func (s *RaftServer) updatePeers() { - peerLeader := string(s.serverAddr) + peerLeader := raftServerID(s.serverAddr) + desiredPeers := make(map[string]pb.ServerAddress, len(s.peers)) + for _, peer := range s.peers { + desiredPeers[raftServerID(peer)] = peer + } + existsPeerName := make(map[string]bool) for _, server := range s.RaftHashicorp.GetConfiguration().Configuration().Servers { if string(server.ID) == peerLeader { @@ -106,8 +118,7 @@ func (s *RaftServer) updatePeers() { } existsPeerName[string(server.ID)] = true } - for _, peer := range s.peers { - peerName := string(peer) + for peerName, peer := range desiredPeers { if peerName == peerLeader || existsPeerName[peerName] { continue } @@ -116,12 +127,12 @@ func (s *RaftServer) updatePeers() { raft.ServerID(peerName), raft.ServerAddress(peer.ToGrpcAddress()), 0, 0) } for peer := range existsPeerName { - if _, found := s.peers[peer]; !found { + if _, found := desiredPeers[peer]; !found { glog.V(0).Infof("removing old peer: %s", peer) s.RaftHashicorp.RemoveServer(raft.ServerID(peer), 0, 0) } } - if _, found := s.peers[peerLeader]; !found { + if _, found := desiredPeers[peerLeader]; !found { glog.V(0).Infof("removing old leader peer: %s", peerLeader) s.RaftHashicorp.RemoveServer(raft.ServerID(peerLeader), 0, 0) } @@ -136,7 +147,7 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) { } c := raft.DefaultConfig() - c.LocalID = raft.ServerID(s.serverAddr) // TODO maybee the IP:port address will change + c.LocalID = raft.ServerID(raftServerID(s.serverAddr)) c.HeartbeatTimeout = time.Duration(float64(option.HeartbeatInterval) * (rand.Float64()*0.25 + 1)) c.ElectionTimeout = option.ElectionTimeout if c.LeaderLeaseTimeout > c.HeartbeatTimeout { diff --git a/weed/server/raft_hashicorp_test.go b/weed/server/raft_hashicorp_test.go new file mode 100644 index 000000000..e9261e3ac --- /dev/null +++ b/weed/server/raft_hashicorp_test.go @@ -0,0 +1,87 @@ +package weed_server + +import ( + "sort" + "testing" + + "github.com/hashicorp/raft" + "github.com/seaweedfs/seaweedfs/weed/pb" +) + +func TestRaftServerID(t *testing.T) { + tests := []struct { + name string + addr pb.ServerAddress + want string + }{ + { + name: "without grpc suffix", + addr: pb.ServerAddress("master-0:9333"), + want: "master-0:9333", + }, + { + name: "with grpc suffix", + addr: pb.NewServerAddress("master-0", 9333, 19333), + want: "master-0:9333", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := raftServerID(tt.addr); got != tt.want { + t.Fatalf("raftServerID(%q) = %q, want %q", tt.addr, got, tt.want) + } + }) + } +} + +func TestGetPeerIdxUsesCanonicalID(t *testing.T) { + peers := map[string]pb.ServerAddress{ + "master-0:9333": pb.ServerAddress("master-0:9333"), + "master-1:9333": pb.ServerAddress("master-1:9333"), + "master-2:9333": pb.ServerAddress("master-2:9333"), + } + self := pb.NewServerAddress("master-2", 9333, 19333) + + if got := getPeerIdx(self, peers); got != 2 { + t.Fatalf("getPeerIdx(%q) = %d, want 2", self, got) + } +} + +func TestAddPeersConfigurationUsesCanonicalIDs(t *testing.T) { + rs := &RaftServer{ + peers: map[string]pb.ServerAddress{ + "master-0:9333": pb.ServerAddress("master-0:9333"), + "master-1:9333": pb.ServerAddress("master-1:9333"), + "master-2:9333": pb.ServerAddress("master-2:9333"), + }, + } + + cfg := rs.AddPeersConfiguration() + if len(cfg.Servers) != 3 { + t.Fatalf("len(cfg.Servers) = %d, want 3", len(cfg.Servers)) + } + + var ids []string + var addrs []string + for _, s := range cfg.Servers { + if s.Suffrage != raft.Voter { + t.Fatalf("server %q has suffrage %q, want %q", s.ID, s.Suffrage, raft.Voter) + } + ids = append(ids, string(s.ID)) + addrs = append(addrs, string(s.Address)) + } + sort.Strings(ids) + sort.Strings(addrs) + + wantIDs := []string{"master-0:9333", "master-1:9333", "master-2:9333"} + wantAddrs := []string{"master-0:19333", "master-1:19333", "master-2:19333"} + for i := range wantIDs { + if ids[i] != wantIDs[i] { + t.Fatalf("ids[%d] = %q, want %q", i, ids[i], wantIDs[i]) + } + if addrs[i] != wantAddrs[i] { + t.Fatalf("addrs[%d] = %q, want %q", i, addrs[i], wantAddrs[i]) + } + } +} diff --git a/weed/util/version/constants.go b/weed/util/version/constants.go index 87c704dd1..0bb04e437 100644 --- a/weed/util/version/constants.go +++ b/weed/util/version/constants.go @@ -9,7 +9,7 @@ import ( var ( MAJOR_VERSION = int32(4) - MINOR_VERSION = int32(10) + MINOR_VERSION = int32(11) VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION) VERSION = util.SizeLimit + " " + VERSION_NUMBER COMMIT = ""