You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

334 lines
8.8 KiB

3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
3 years ago
  1. package cluster
  2. import (
  3. "github.com/chrislusf/seaweedfs/weed/pb"
  4. "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
  5. "math"
  6. "sync"
  7. "time"
  8. )
  9. const (
  10. MasterType = "master"
  11. VolumeServerType = "volumeServer"
  12. FilerType = "filer"
  13. BrokerType = "broker"
  14. )
  15. type FilerGroupName string
  16. type DataCenter string
  17. type Rack string
  18. type Leaders struct {
  19. leaders [3]pb.ServerAddress
  20. }
  21. type ClusterNode struct {
  22. Address pb.ServerAddress
  23. Version string
  24. counter int
  25. CreatedTs time.Time
  26. DataCenter DataCenter
  27. Rack Rack
  28. }
  29. type GroupMembers struct {
  30. members map[pb.ServerAddress]*ClusterNode
  31. leaders *Leaders
  32. }
  33. type ClusterNodeGroups struct {
  34. groupMembers map[FilerGroupName]*GroupMembers
  35. sync.RWMutex
  36. }
  37. type Cluster struct {
  38. filerGroups *ClusterNodeGroups
  39. brokerGroups *ClusterNodeGroups
  40. }
  41. func newClusterNodeGroups() *ClusterNodeGroups {
  42. return &ClusterNodeGroups{
  43. groupMembers: map[FilerGroupName]*GroupMembers{},
  44. }
  45. }
  46. func (g *ClusterNodeGroups) getGroupMembers(filerGroup FilerGroupName, createIfNotFound bool) *GroupMembers {
  47. filers, found := g.groupMembers[filerGroup]
  48. if !found && createIfNotFound {
  49. filers = &GroupMembers{
  50. members: make(map[pb.ServerAddress]*ClusterNode),
  51. leaders: &Leaders{},
  52. }
  53. g.groupMembers[filerGroup] = filers
  54. }
  55. return filers
  56. }
  57. func (m *GroupMembers) addMember(dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) *ClusterNode {
  58. if existingNode, found := m.members[address]; found {
  59. existingNode.counter++
  60. return nil
  61. }
  62. t := &ClusterNode{
  63. Address: address,
  64. Version: version,
  65. counter: 1,
  66. CreatedTs: time.Now(),
  67. DataCenter: dataCenter,
  68. Rack: rack,
  69. }
  70. m.members[address] = t
  71. return t
  72. }
  73. func (m *GroupMembers) removeMember(address pb.ServerAddress) bool {
  74. if existingNode, found := m.members[address]; !found {
  75. return false
  76. } else {
  77. existingNode.counter--
  78. if existingNode.counter <= 0 {
  79. delete(m.members, address)
  80. return true
  81. }
  82. }
  83. return false
  84. }
  85. func (g *ClusterNodeGroups) AddClusterNode(filerGroup FilerGroupName, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
  86. g.Lock()
  87. defer g.Unlock()
  88. m := g.getGroupMembers(filerGroup, true)
  89. if t := m.addMember(dataCenter, rack, address, version); t != nil {
  90. return ensureGroupLeaders(m, true, filerGroup, nodeType, address)
  91. }
  92. return nil
  93. }
  94. func (g *ClusterNodeGroups) RemoveClusterNode(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
  95. g.Lock()
  96. defer g.Unlock()
  97. m := g.getGroupMembers(filerGroup, false)
  98. if m == nil {
  99. return nil
  100. }
  101. if m.removeMember(address) {
  102. return ensureGroupLeaders(m, false, filerGroup, nodeType, address)
  103. }
  104. return nil
  105. }
  106. func (g *ClusterNodeGroups) ListClusterNode(filerGroup FilerGroupName) (nodes []*ClusterNode) {
  107. g.Lock()
  108. defer g.Unlock()
  109. m := g.getGroupMembers(filerGroup, false)
  110. if m == nil {
  111. return nil
  112. }
  113. for _, node := range m.members {
  114. nodes = append(nodes, node)
  115. }
  116. return
  117. }
  118. func (g *ClusterNodeGroups) IsOneLeader(filerGroup FilerGroupName, address pb.ServerAddress) bool {
  119. g.Lock()
  120. defer g.Unlock()
  121. m := g.getGroupMembers(filerGroup, false)
  122. if m == nil {
  123. return false
  124. }
  125. return m.leaders.isOneLeader(address)
  126. }
  127. func NewCluster() *Cluster {
  128. return &Cluster{
  129. filerGroups: newClusterNodeGroups(),
  130. brokerGroups: newClusterNodeGroups(),
  131. }
  132. }
  133. func (cluster *Cluster) getGroupMembers(filerGroup FilerGroupName, nodeType string, createIfNotFound bool) *GroupMembers {
  134. switch nodeType {
  135. case FilerType:
  136. return cluster.filerGroups.getGroupMembers(filerGroup, createIfNotFound)
  137. case BrokerType:
  138. return cluster.brokerGroups.getGroupMembers(filerGroup, createIfNotFound)
  139. }
  140. return nil
  141. }
  142. func (cluster *Cluster) AddClusterNode(ns, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
  143. filerGroup := FilerGroupName(ns)
  144. switch nodeType {
  145. case FilerType:
  146. return cluster.filerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
  147. case BrokerType:
  148. return cluster.brokerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
  149. case MasterType:
  150. return []*master_pb.KeepConnectedResponse{
  151. {
  152. ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
  153. NodeType: nodeType,
  154. Address: string(address),
  155. IsAdd: true,
  156. },
  157. },
  158. }
  159. }
  160. return nil
  161. }
  162. func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
  163. filerGroup := FilerGroupName(ns)
  164. switch nodeType {
  165. case FilerType:
  166. return cluster.filerGroups.RemoveClusterNode(filerGroup, nodeType, address)
  167. case BrokerType:
  168. return cluster.brokerGroups.RemoveClusterNode(filerGroup, nodeType, address)
  169. case MasterType:
  170. return []*master_pb.KeepConnectedResponse{
  171. {
  172. ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
  173. NodeType: nodeType,
  174. Address: string(address),
  175. IsAdd: false,
  176. },
  177. },
  178. }
  179. }
  180. return nil
  181. }
  182. func (cluster *Cluster) ListClusterNode(filerGroup FilerGroupName, nodeType string) (nodes []*ClusterNode) {
  183. switch nodeType {
  184. case FilerType:
  185. return cluster.filerGroups.ListClusterNode(filerGroup)
  186. case BrokerType:
  187. return cluster.brokerGroups.ListClusterNode(filerGroup)
  188. case MasterType:
  189. }
  190. return
  191. }
  192. func (cluster *Cluster) IsOneLeader(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) bool {
  193. switch nodeType {
  194. case FilerType:
  195. return cluster.filerGroups.IsOneLeader(filerGroup, address)
  196. case BrokerType:
  197. return cluster.brokerGroups.IsOneLeader(filerGroup, address)
  198. case MasterType:
  199. }
  200. return false
  201. }
  202. func ensureGroupLeaders(m *GroupMembers, isAdd bool, filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
  203. if isAdd {
  204. if m.leaders.addLeaderIfVacant(address) {
  205. // has added the address as one leader
  206. result = append(result, &master_pb.KeepConnectedResponse{
  207. ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
  208. FilerGroup: string(filerGroup),
  209. NodeType: nodeType,
  210. Address: string(address),
  211. IsLeader: true,
  212. IsAdd: true,
  213. },
  214. })
  215. } else {
  216. result = append(result, &master_pb.KeepConnectedResponse{
  217. ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
  218. FilerGroup: string(filerGroup),
  219. NodeType: nodeType,
  220. Address: string(address),
  221. IsLeader: false,
  222. IsAdd: true,
  223. },
  224. })
  225. }
  226. } else {
  227. if m.leaders.removeLeaderIfExists(address) {
  228. result = append(result, &master_pb.KeepConnectedResponse{
  229. ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
  230. FilerGroup: string(filerGroup),
  231. NodeType: nodeType,
  232. Address: string(address),
  233. IsLeader: true,
  234. IsAdd: false,
  235. },
  236. })
  237. // pick the freshest one, since it is less likely to go away
  238. var shortestDuration int64 = math.MaxInt64
  239. now := time.Now()
  240. var candidateAddress pb.ServerAddress
  241. for _, node := range m.members {
  242. if m.leaders.isOneLeader(node.Address) {
  243. continue
  244. }
  245. duration := now.Sub(node.CreatedTs).Nanoseconds()
  246. if duration < shortestDuration {
  247. shortestDuration = duration
  248. candidateAddress = node.Address
  249. }
  250. }
  251. if candidateAddress != "" {
  252. m.leaders.addLeaderIfVacant(candidateAddress)
  253. // added a new leader
  254. result = append(result, &master_pb.KeepConnectedResponse{
  255. ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
  256. NodeType: nodeType,
  257. Address: string(candidateAddress),
  258. IsLeader: true,
  259. IsAdd: true,
  260. },
  261. })
  262. }
  263. } else {
  264. result = append(result, &master_pb.KeepConnectedResponse{
  265. ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
  266. FilerGroup: string(filerGroup),
  267. NodeType: nodeType,
  268. Address: string(address),
  269. IsLeader: false,
  270. IsAdd: false,
  271. },
  272. })
  273. }
  274. }
  275. return
  276. }
  277. func (leaders *Leaders) addLeaderIfVacant(address pb.ServerAddress) (hasChanged bool) {
  278. if leaders.isOneLeader(address) {
  279. return
  280. }
  281. for i := 0; i < len(leaders.leaders); i++ {
  282. if leaders.leaders[i] == "" {
  283. leaders.leaders[i] = address
  284. hasChanged = true
  285. return
  286. }
  287. }
  288. return
  289. }
  290. func (leaders *Leaders) removeLeaderIfExists(address pb.ServerAddress) (hasChanged bool) {
  291. if !leaders.isOneLeader(address) {
  292. return
  293. }
  294. for i := 0; i < len(leaders.leaders); i++ {
  295. if leaders.leaders[i] == address {
  296. leaders.leaders[i] = ""
  297. hasChanged = true
  298. return
  299. }
  300. }
  301. return
  302. }
  303. func (leaders *Leaders) isOneLeader(address pb.ServerAddress) bool {
  304. for i := 0; i < len(leaders.leaders); i++ {
  305. if leaders.leaders[i] == address {
  306. return true
  307. }
  308. }
  309. return false
  310. }
  311. func (leaders *Leaders) GetLeaders() (addresses []pb.ServerAddress) {
  312. for i := 0; i < len(leaders.leaders); i++ {
  313. if leaders.leaders[i] != "" {
  314. addresses = append(addresses, leaders.leaders[i])
  315. }
  316. }
  317. return
  318. }