Browse Source

Update broker_offset_manager.go

Fixed GetHighWaterMark() to use correct partition managers
Fixed GetPartitionOffsetInfo() with proper struct fields
Fixed GetOffsetMetrics() with correct types and system
pull/7231/head
chrislu 2 months ago
parent
commit
dbac0c45d8
  1. 89
      weed/mq/broker/broker_offset_manager.go

89
weed/mq/broker/broker_offset_manager.go

@ -12,9 +12,9 @@ import (
// BrokerOffsetManager manages offset assignment for all partitions in a broker
type BrokerOffsetManager struct {
mu sync.RWMutex
offsetIntegration *offset.SMQOffsetIntegration
partitionManagers map[string]*offset.PartitionOffsetManager
mu sync.RWMutex
offsetIntegration *offset.SMQOffsetIntegration
partitionManagers map[string]*offset.PartitionOffsetManager
storage offset.OffsetStorage
}
@ -34,7 +34,7 @@ func NewBrokerOffsetManagerWithStorage(storage offset.OffsetStorage) *BrokerOffs
return &BrokerOffsetManager{
offsetIntegration: offset.NewSMQOffsetIntegration(storage),
partitionManagers: make(map[string]*offset.PartitionOffsetManager),
storage: storage,
storage: storage,
}
}
@ -56,7 +56,7 @@ func NewBrokerOffsetManagerWithSQL(dbPath string) (*BrokerOffsetManager, error)
return &BrokerOffsetManager{
offsetIntegration: offset.NewSMQOffsetIntegration(sqlStorage),
partitionManagers: make(map[string]*offset.PartitionOffsetManager),
storage: sqlStorage,
storage: sqlStorage,
}, nil
}
@ -115,7 +115,18 @@ func (bom *BrokerOffsetManager) AssignBatchOffsets(t topic.Topic, p topic.Partit
// GetHighWaterMark returns the high water mark for a partition
func (bom *BrokerOffsetManager) GetHighWaterMark(t topic.Topic, p topic.Partition) (int64, error) {
partition := topicPartitionToSchemaPartition(t, p)
return bom.offsetIntegration.GetHighWaterMark(partition)
// Use the same partition manager that AssignBatchOffsets updates
bom.mu.RLock()
manager, exists := bom.partitionManagers[partitionKey(partition)]
bom.mu.RUnlock()
if !exists {
// If no manager exists, return 0 (no offsets assigned yet)
return 0, nil
}
return manager.GetHighWaterMark(), nil
}
// CreateSubscription creates an offset-based subscription
@ -145,7 +156,39 @@ func (bom *BrokerOffsetManager) CloseSubscription(subscriptionID string) error {
// GetPartitionOffsetInfo returns comprehensive offset information for a partition
func (bom *BrokerOffsetManager) GetPartitionOffsetInfo(t topic.Topic, p topic.Partition) (*offset.PartitionOffsetInfo, error) {
partition := topicPartitionToSchemaPartition(t, p)
return bom.offsetIntegration.GetPartitionOffsetInfo(partition)
// Use the same partition manager that AssignBatchOffsets updates
bom.mu.RLock()
manager, exists := bom.partitionManagers[partitionKey(partition)]
bom.mu.RUnlock()
if !exists {
// If no manager exists, return info for empty partition
return &offset.PartitionOffsetInfo{
Partition: partition,
EarliestOffset: 0,
LatestOffset: -1, // -1 indicates no records yet
HighWaterMark: 0,
RecordCount: 0,
ActiveSubscriptions: 0,
}, nil
}
// Get info from the manager
highWaterMark := manager.GetHighWaterMark()
var latestOffset int64 = -1
if highWaterMark > 0 {
latestOffset = highWaterMark - 1 // Latest assigned offset
}
return &offset.PartitionOffsetInfo{
Partition: partition,
EarliestOffset: 0, // For simplicity, assume earliest is always 0
LatestOffset: latestOffset,
HighWaterMark: highWaterMark,
RecordCount: highWaterMark,
ActiveSubscriptions: 0, // TODO: Track subscription count if needed
}, nil
}
// topicPartitionToSchemaPartition converts topic.Topic and topic.Partition to schema_pb.Partition
@ -166,13 +209,13 @@ func partitionKey(partition *schema_pb.Partition) string {
// OffsetAssignmentResult contains the result of offset assignment for logging/metrics
type OffsetAssignmentResult struct {
Topic topic.Topic
Partition topic.Partition
BaseOffset int64
LastOffset int64
Count int64
Timestamp int64
Error error
Topic topic.Topic
Partition topic.Partition
BaseOffset int64
LastOffset int64
Count int64
Timestamp int64
Error error
}
// AssignOffsetsWithResult assigns offsets and returns detailed result for logging/metrics
@ -197,7 +240,23 @@ func (bom *BrokerOffsetManager) AssignOffsetsWithResult(t topic.Topic, p topic.P
// GetOffsetMetrics returns metrics about offset usage across all partitions
func (bom *BrokerOffsetManager) GetOffsetMetrics() *offset.OffsetMetrics {
return bom.offsetIntegration.GetOffsetMetrics()
bom.mu.RLock()
defer bom.mu.RUnlock()
// Count active partitions and calculate total offsets
partitionCount := int64(len(bom.partitionManagers))
var totalOffsets int64 = 0
for _, manager := range bom.partitionManagers {
totalOffsets += manager.GetHighWaterMark()
}
return &offset.OffsetMetrics{
PartitionCount: partitionCount,
TotalOffsets: totalOffsets,
ActiveSubscriptions: 0, // TODO: Track subscription count if needed
AverageLatency: 0.0,
}
}
// Shutdown gracefully shuts down the offset manager

Loading…
Cancel
Save