Browse Source

use type ConsumerGroupInstanceId

pull/5890/head
chrislu 7 months ago
parent
commit
96b326a304
  1. 3
      weed/mq/sub_coordinator/consumer_group.go
  2. 9
      weed/mq/sub_coordinator/consumer_group_instance.go
  3. 8
      weed/mq/sub_coordinator/partition_consumer_mapping.go
  4. 2
      weed/mq/sub_coordinator/partition_list.go

3
weed/mq/sub_coordinator/consumer_group.go

@ -106,7 +106,7 @@ func (cg *ConsumerGroup) BalanceConsumerGroupInstances(knownPartitionSlotToBroke
cg.mapping.BalanceToConsumerInstances(partitionSlotToBrokerList, consumerInstances) cg.mapping.BalanceToConsumerInstances(partitionSlotToBrokerList, consumerInstances)
// convert cg.mapping currentMapping to map of consumer group instance id to partition slots // convert cg.mapping currentMapping to map of consumer group instance id to partition slots
consumerInstanceToPartitionSlots := make(map[string][]*PartitionSlotToConsumerInstance)
consumerInstanceToPartitionSlots := make(map[ConsumerGroupInstanceId][]*PartitionSlotToConsumerInstance)
for _, partitionSlot := range cg.mapping.currentMapping.PartitionSlots { for _, partitionSlot := range cg.mapping.currentMapping.PartitionSlots {
consumerInstanceToPartitionSlots[partitionSlot.AssignedInstanceId] = append(consumerInstanceToPartitionSlots[partitionSlot.AssignedInstanceId], partitionSlot) consumerInstanceToPartitionSlots[partitionSlot.AssignedInstanceId] = append(consumerInstanceToPartitionSlots[partitionSlot.AssignedInstanceId], partitionSlot)
} }
@ -117,7 +117,6 @@ func (cg *ConsumerGroup) BalanceConsumerGroupInstances(knownPartitionSlotToBroke
if !found { if !found {
partitionSlots = make([]*PartitionSlotToConsumerInstance, 0) partitionSlots = make([]*PartitionSlotToConsumerInstance, 0)
} }
consumerGroupInstance.Partitions = ToPartitions(partitionSlotToBrokerList.RingSize, partitionSlots)
for _, partitionSlot := range partitionSlots { for _, partitionSlot := range partitionSlots {
consumerGroupInstance.ResponseChan <- &mq_pb.SubscriberToSubCoordinatorResponse{ consumerGroupInstance.ResponseChan <- &mq_pb.SubscriberToSubCoordinatorResponse{
Message: &mq_pb.SubscriberToSubCoordinatorResponse_Assignment_{ Message: &mq_pb.SubscriberToSubCoordinatorResponse_Assignment_{

9
weed/mq/sub_coordinator/consumer_group_instance.go

@ -6,17 +6,18 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
) )
type ConsumerGroupInstanceId string
type ConsumerGroupInstance struct { type ConsumerGroupInstance struct {
InstanceId string
// the consumer group instance may not have an active partition
Partitions []*topic.Partition
InstanceId ConsumerGroupInstanceId
AssignedPartitions []topic.Partition
ResponseChan chan *mq_pb.SubscriberToSubCoordinatorResponse ResponseChan chan *mq_pb.SubscriberToSubCoordinatorResponse
MaxPartitionCount int32 MaxPartitionCount int32
} }
func NewConsumerGroupInstance(instanceId string) *ConsumerGroupInstance { func NewConsumerGroupInstance(instanceId string) *ConsumerGroupInstance {
return &ConsumerGroupInstance{ return &ConsumerGroupInstance{
InstanceId: instanceId,
InstanceId: ConsumerGroupInstanceId(instanceId),
ResponseChan: make(chan *mq_pb.SubscriberToSubCoordinatorResponse, 1), ResponseChan: make(chan *mq_pb.SubscriberToSubCoordinatorResponse, 1),
} }
} }

8
weed/mq/sub_coordinator/partition_consumer_mapping.go

@ -47,7 +47,7 @@ func (pcm *PartitionConsumerMapping) BalanceToConsumerInstances(partitionSlotToB
func doBalanceSticky(partitions []*pub_balancer.PartitionSlotToBroker, consumerInstances []*ConsumerGroupInstance, prevMapping *PartitionSlotToConsumerInstanceList) (partitionSlots []*PartitionSlotToConsumerInstance) { func doBalanceSticky(partitions []*pub_balancer.PartitionSlotToBroker, consumerInstances []*ConsumerGroupInstance, prevMapping *PartitionSlotToConsumerInstanceList) (partitionSlots []*PartitionSlotToConsumerInstance) {
// collect previous consumer instance ids // collect previous consumer instance ids
prevConsumerInstanceIds := make(map[string]struct{})
prevConsumerInstanceIds := make(map[ConsumerGroupInstanceId]struct{})
if prevMapping != nil { if prevMapping != nil {
for _, prevPartitionSlot := range prevMapping.PartitionSlots { for _, prevPartitionSlot := range prevMapping.PartitionSlots {
if prevPartitionSlot.AssignedInstanceId != "" { if prevPartitionSlot.AssignedInstanceId != "" {
@ -56,13 +56,13 @@ func doBalanceSticky(partitions []*pub_balancer.PartitionSlotToBroker, consumerI
} }
} }
// collect current consumer instance ids // collect current consumer instance ids
currConsumerInstanceIds := make(map[string]struct{})
currConsumerInstanceIds := make(map[ConsumerGroupInstanceId]struct{})
for _, consumerInstance := range consumerInstances { for _, consumerInstance := range consumerInstances {
currConsumerInstanceIds[consumerInstance.InstanceId] = struct{}{} currConsumerInstanceIds[consumerInstance.InstanceId] = struct{}{}
} }
// check deleted consumer instances // check deleted consumer instances
deletedConsumerInstanceIds := make(map[string]struct{})
deletedConsumerInstanceIds := make(map[ConsumerGroupInstanceId]struct{})
for consumerInstanceId := range prevConsumerInstanceIds { for consumerInstanceId := range prevConsumerInstanceIds {
if _, ok := currConsumerInstanceIds[consumerInstanceId]; !ok { if _, ok := currConsumerInstanceIds[consumerInstanceId]; !ok {
deletedConsumerInstanceIds[consumerInstanceId] = struct{}{} deletedConsumerInstanceIds[consumerInstanceId] = struct{}{}
@ -100,7 +100,7 @@ func doBalanceSticky(partitions []*pub_balancer.PartitionSlotToBroker, consumerI
// for all consumer instances, count the average number of partitions // for all consumer instances, count the average number of partitions
// that are assigned to them // that are assigned to them
consumerInstancePartitionCount := make(map[string]int)
consumerInstancePartitionCount := make(map[ConsumerGroupInstanceId]int)
for _, newPartitionSlot := range newPartitionSlots { for _, newPartitionSlot := range newPartitionSlots {
if newPartitionSlot.AssignedInstanceId != "" { if newPartitionSlot.AssignedInstanceId != "" {
consumerInstancePartitionCount[newPartitionSlot.AssignedInstanceId]++ consumerInstancePartitionCount[newPartitionSlot.AssignedInstanceId]++

2
weed/mq/sub_coordinator/partition_list.go

@ -7,7 +7,7 @@ type PartitionSlotToConsumerInstance struct {
RangeStop int32 RangeStop int32
UnixTimeNs int64 UnixTimeNs int64
Broker string Broker string
AssignedInstanceId string
AssignedInstanceId ConsumerGroupInstanceId
FollowerBroker string FollowerBroker string
} }

Loading…
Cancel
Save