You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

238 lines
6.1 KiB

Merge accumulated changes related to message queue (#5098) * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * tracking topic=>broker * merge * comment
1 year ago
10 months ago
1 year ago
1 year ago
11 months ago
1 year ago
10 months ago
10 months ago
10 months ago
  1. package pub_balancer
  2. import (
  3. "fmt"
  4. cmap "github.com/orcaman/concurrent-map/v2"
  5. "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
  6. "github.com/stretchr/testify/assert"
  7. "testing"
  8. )
  9. func Test_allocateOneBroker(t *testing.T) {
  10. brokers := cmap.New[*BrokerStats]()
  11. brokers.SetIfAbsent("localhost:17777", &BrokerStats{
  12. TopicPartitionCount: 0,
  13. ConsumerCount: 0,
  14. CpuUsagePercent: 0,
  15. })
  16. tests := []struct {
  17. name string
  18. args args
  19. wantAssignments []*mq_pb.BrokerPartitionAssignment
  20. }{
  21. {
  22. name: "test only one broker",
  23. args: args{
  24. brokers: brokers,
  25. partitionCount: 1,
  26. },
  27. wantAssignments: []*mq_pb.BrokerPartitionAssignment{
  28. {
  29. LeaderBroker: "localhost:17777",
  30. Partition: &mq_pb.Partition{
  31. RingSize: MaxPartitionCount,
  32. RangeStart: 0,
  33. RangeStop: MaxPartitionCount,
  34. },
  35. },
  36. },
  37. },
  38. }
  39. testThem(t, tests)
  40. }
  41. type args struct {
  42. brokers cmap.ConcurrentMap[string, *BrokerStats]
  43. partitionCount int32
  44. }
  45. func testThem(t *testing.T, tests []struct {
  46. name string
  47. args args
  48. wantAssignments []*mq_pb.BrokerPartitionAssignment
  49. }) {
  50. for _, tt := range tests {
  51. t.Run(tt.name, func(t *testing.T) {
  52. gotAssignments := AllocateTopicPartitions(tt.args.brokers, tt.args.partitionCount)
  53. assert.Equal(t, len(tt.wantAssignments), len(gotAssignments))
  54. for i, gotAssignment := range gotAssignments {
  55. assert.Equal(t, tt.wantAssignments[i].LeaderBroker, gotAssignment.LeaderBroker)
  56. assert.Equal(t, tt.wantAssignments[i].Partition.RangeStart, gotAssignment.Partition.RangeStart)
  57. assert.Equal(t, tt.wantAssignments[i].Partition.RangeStop, gotAssignment.Partition.RangeStop)
  58. assert.Equal(t, tt.wantAssignments[i].Partition.RingSize, gotAssignment.Partition.RingSize)
  59. assert.Equal(t, tt.wantAssignments[i].Partition.UnixTimeNs, gotAssignment.Partition.UnixTimeNs)
  60. }
  61. })
  62. }
  63. }
  64. func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
  65. type args struct {
  66. activeBrokers cmap.ConcurrentMap[string, *BrokerStats]
  67. followerCount int
  68. assignments []*mq_pb.BrokerPartitionAssignment
  69. }
  70. activeBrokers := cmap.New[*BrokerStats]()
  71. activeBrokers.SetIfAbsent("localhost:1", &BrokerStats{})
  72. activeBrokers.SetIfAbsent("localhost:2", &BrokerStats{})
  73. activeBrokers.SetIfAbsent("localhost:3", &BrokerStats{})
  74. activeBrokers.SetIfAbsent("localhost:4", &BrokerStats{})
  75. activeBrokers.SetIfAbsent("localhost:5", &BrokerStats{})
  76. activeBrokers.SetIfAbsent("localhost:6", &BrokerStats{})
  77. lowActiveBrokers := cmap.New[*BrokerStats]()
  78. lowActiveBrokers.SetIfAbsent("localhost:1", &BrokerStats{})
  79. lowActiveBrokers.SetIfAbsent("localhost:2", &BrokerStats{})
  80. singleActiveBroker := cmap.New[*BrokerStats]()
  81. singleActiveBroker.SetIfAbsent("localhost:1", &BrokerStats{})
  82. tests := []struct {
  83. name string
  84. args args
  85. hasChanges bool
  86. }{
  87. {
  88. name: "test empty leader",
  89. args: args{
  90. activeBrokers: activeBrokers,
  91. followerCount: 1,
  92. assignments: []*mq_pb.BrokerPartitionAssignment{
  93. {
  94. LeaderBroker: "",
  95. Partition: &mq_pb.Partition{},
  96. FollowerBrokers: []string{
  97. "localhost:2",
  98. },
  99. },
  100. },
  101. },
  102. hasChanges: true,
  103. },
  104. {
  105. name: "test empty follower",
  106. args: args{
  107. activeBrokers: activeBrokers,
  108. followerCount: 1,
  109. assignments: []*mq_pb.BrokerPartitionAssignment{
  110. {
  111. LeaderBroker: "localhost:1",
  112. Partition: &mq_pb.Partition{},
  113. FollowerBrokers: []string{
  114. "",
  115. },
  116. },
  117. },
  118. },
  119. hasChanges: true,
  120. },
  121. {
  122. name: "test dead follower",
  123. args: args{
  124. activeBrokers: activeBrokers,
  125. followerCount: 1,
  126. assignments: []*mq_pb.BrokerPartitionAssignment{
  127. {
  128. LeaderBroker: "localhost:1",
  129. Partition: &mq_pb.Partition{},
  130. FollowerBrokers: []string{
  131. "localhost:200",
  132. },
  133. },
  134. },
  135. },
  136. hasChanges: true,
  137. },
  138. {
  139. name: "test dead leader and follower",
  140. args: args{
  141. activeBrokers: activeBrokers,
  142. followerCount: 1,
  143. assignments: []*mq_pb.BrokerPartitionAssignment{
  144. {
  145. LeaderBroker: "localhost:100",
  146. Partition: &mq_pb.Partition{},
  147. FollowerBrokers: []string{
  148. "localhost:200",
  149. },
  150. },
  151. },
  152. },
  153. hasChanges: true,
  154. },
  155. {
  156. name: "test missing two followers",
  157. args: args{
  158. activeBrokers: activeBrokers,
  159. followerCount: 3,
  160. assignments: []*mq_pb.BrokerPartitionAssignment{
  161. {
  162. LeaderBroker: "localhost:1",
  163. Partition: &mq_pb.Partition{},
  164. FollowerBrokers: []string{
  165. "localhost:2",
  166. },
  167. },
  168. },
  169. },
  170. hasChanges: true,
  171. },
  172. {
  173. name: "test missing some followers",
  174. args: args{
  175. activeBrokers: activeBrokers,
  176. followerCount: 10,
  177. assignments: []*mq_pb.BrokerPartitionAssignment{
  178. {
  179. LeaderBroker: "localhost:1",
  180. Partition: &mq_pb.Partition{},
  181. FollowerBrokers: []string{
  182. "localhost:2",
  183. },
  184. },
  185. },
  186. },
  187. hasChanges: true,
  188. },
  189. {
  190. name: "test low active brokers",
  191. args: args{
  192. activeBrokers: lowActiveBrokers,
  193. followerCount: 3,
  194. assignments: []*mq_pb.BrokerPartitionAssignment{
  195. {
  196. LeaderBroker: "localhost:1",
  197. Partition: &mq_pb.Partition{},
  198. FollowerBrokers: []string{
  199. "localhost:2",
  200. },
  201. },
  202. },
  203. },
  204. hasChanges: false,
  205. },
  206. {
  207. name: "test single active broker",
  208. args: args{
  209. activeBrokers: singleActiveBroker,
  210. followerCount: 3,
  211. assignments: []*mq_pb.BrokerPartitionAssignment{
  212. {
  213. LeaderBroker: "localhost:1",
  214. Partition: &mq_pb.Partition{},
  215. FollowerBrokers: []string{
  216. "localhost:2",
  217. },
  218. },
  219. },
  220. },
  221. hasChanges: true,
  222. },
  223. }
  224. for _, tt := range tests {
  225. t.Run(tt.name, func(t *testing.T) {
  226. fmt.Printf("%v before %v\n", tt.name, tt.args.assignments)
  227. hasChanges := EnsureAssignmentsToActiveBrokers(tt.args.activeBrokers, tt.args.followerCount, tt.args.assignments)
  228. assert.Equalf(t, tt.hasChanges, hasChanges, "EnsureAssignmentsToActiveBrokers(%v, %v, %v)", tt.args.activeBrokers, tt.args.followerCount, tt.args.assignments)
  229. fmt.Printf("%v after %v\n", tt.name, tt.args.assignments)
  230. })
  231. }
  232. }