You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

87 lines
2.0 KiB

Merge accumulated changes related to message queue (#5098) * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * tracking topic=>broker * merge * comment
1 year ago
Merge accumulated changes related to message queue (#5098) * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * tracking topic=>broker * merge * comment
1 year ago
Merge accumulated changes related to message queue (#5098) * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * balance partitions on brokers * prepare topic partition first and then publish, move partition * purge unused APIs * clean up * adjust logs * add BalanceTopics() grpc API * configure topic * configure topic command * refactor * repair missing partitions * sequence of operations to ensure ordering * proto to close publishers and consumers * rename file * topic partition versioned by unixTimeNs * create local topic partition * close publishers * randomize the client name * wait until no publishers * logs * close stop publisher channel * send last ack * comments * comment * comments * support list of brokers * add cli options * Update .gitignore * logs * return io.eof directly * refactor * optionally create topic * refactoring * detect consumer disconnection * sub client wait for more messages * subscribe by time stamp * rename * rename to sub_balancer * rename * adjust comments * rename * fix compilation * rename * rename * SubscriberToSubCoordinator * sticky rebalance * go fmt * add tests * tracking topic=>broker * merge * comment
1 year ago
  1. package topic
  2. import "sync"
  3. type LocalTopic struct {
  4. Topic
  5. Partitions []*LocalPartition
  6. }
  7. func NewLocalTopic(topic Topic) *LocalTopic {
  8. return &LocalTopic{
  9. Topic: topic,
  10. Partitions: make([]*LocalPartition, 0),
  11. }
  12. }
  13. func (localTopic *LocalTopic) findPartition(partition Partition) *LocalPartition {
  14. for _, localPartition := range localTopic.Partitions {
  15. if localPartition.Partition.Equals(partition) {
  16. return localPartition
  17. }
  18. }
  19. return nil
  20. }
  21. func (localTopic *LocalTopic) removePartition(partition Partition) bool {
  22. foundPartitionIndex := -1
  23. for i, localPartition := range localTopic.Partitions {
  24. if localPartition.Partition.Equals(partition) {
  25. foundPartitionIndex = i
  26. break
  27. }
  28. }
  29. if foundPartitionIndex == -1 {
  30. return false
  31. }
  32. localTopic.Partitions = append(localTopic.Partitions[:foundPartitionIndex], localTopic.Partitions[foundPartitionIndex+1:]...)
  33. return true
  34. }
  35. func (localTopic *LocalTopic) closePartitionPublishers(unixTsNs int64) bool {
  36. var wg sync.WaitGroup
  37. for _, localPartition := range localTopic.Partitions {
  38. if localPartition.UnixTimeNs != unixTsNs {
  39. continue
  40. }
  41. wg.Add(1)
  42. go func(localPartition *LocalPartition) {
  43. defer wg.Done()
  44. localPartition.closePublishers()
  45. }(localPartition)
  46. }
  47. wg.Wait()
  48. return true
  49. }
  50. func (localTopic *LocalTopic) closePartitionSubscribers(unixTsNs int64) bool {
  51. var wg sync.WaitGroup
  52. for _, localPartition := range localTopic.Partitions {
  53. if localPartition.UnixTimeNs != unixTsNs {
  54. continue
  55. }
  56. wg.Add(1)
  57. go func(localPartition *LocalPartition) {
  58. defer wg.Done()
  59. localPartition.closeSubscribers()
  60. }(localPartition)
  61. }
  62. wg.Wait()
  63. return true
  64. }
  65. func (localTopic *LocalTopic) WaitUntilNoPublishers() {
  66. for {
  67. var wg sync.WaitGroup
  68. for _, localPartition := range localTopic.Partitions {
  69. wg.Add(1)
  70. go func(localPartition *LocalPartition) {
  71. defer wg.Done()
  72. localPartition.WaitUntilNoPublishers()
  73. }(localPartition)
  74. }
  75. wg.Wait()
  76. if len(localTopic.Partitions) == 0 {
  77. return
  78. }
  79. }
  80. }