chrislu
4 weeks ago
9 changed files with 144 additions and 35 deletions
-
11test/kafka/kafka-client-loadtest/go.mod
-
14test/kafka/kafka-client-loadtest/go.sum
-
86test/kafka/kafka-client-loadtest/test_offset_fetch.go
-
12weed/mq/kafka/consumer_offset/filer_storage.go
-
11weed/mq/kafka/consumer_offset/filer_storage_test.go
-
3weed/mq/kafka/protocol/find_coordinator.go
-
6weed/mq/kafka/protocol/handler.go
-
25weed/mq/kafka/protocol/offset_management.go
-
11weed/util/log_buffer/log_read_stateless.go
@ -0,0 +1,86 @@ |
|||||
|
package main |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"log" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/IBM/sarama" |
||||
|
) |
||||
|
|
||||
|
func main() { |
||||
|
log.Println("=== Testing OffsetFetch with Debug Sarama ===") |
||||
|
|
||||
|
config := sarama.NewConfig() |
||||
|
config.Version = sarama.V2_8_0_0 |
||||
|
config.Consumer.Return.Errors = true |
||||
|
config.Consumer.Offsets.Initial = sarama.OffsetOldest |
||||
|
config.Consumer.Offsets.AutoCommit.Enable = true |
||||
|
config.Consumer.Offsets.AutoCommit.Interval = 100 * time.Millisecond |
||||
|
config.Consumer.Group.Session.Timeout = 30 * time.Second |
||||
|
config.Consumer.Group.Heartbeat.Interval = 3 * time.Second |
||||
|
|
||||
|
brokers := []string{"localhost:9093"} |
||||
|
group := "test-offset-fetch-group" |
||||
|
topics := []string{"loadtest-topic-0"} |
||||
|
|
||||
|
log.Printf("Creating consumer group: group=%s brokers=%v topics=%v", group, brokers, topics) |
||||
|
|
||||
|
consumerGroup, err := sarama.NewConsumerGroup(brokers, group, config) |
||||
|
if err != nil { |
||||
|
log.Fatalf("Failed to create consumer group: %v", err) |
||||
|
} |
||||
|
defer consumerGroup.Close() |
||||
|
|
||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
||||
|
defer cancel() |
||||
|
|
||||
|
handler := &testHandler{} |
||||
|
|
||||
|
log.Println("Starting consumer group session...") |
||||
|
log.Println("Watch for 🔍 [SARAMA-DEBUG] logs to trace OffsetFetch calls") |
||||
|
|
||||
|
go func() { |
||||
|
for { |
||||
|
if err := consumerGroup.Consume(ctx, topics, handler); err != nil { |
||||
|
log.Printf("Error from consumer: %v", err) |
||||
|
} |
||||
|
if ctx.Err() != nil { |
||||
|
return |
||||
|
} |
||||
|
} |
||||
|
}() |
||||
|
|
||||
|
// Wait for context to be done
|
||||
|
<-ctx.Done() |
||||
|
log.Println("Test completed") |
||||
|
} |
||||
|
|
||||
|
type testHandler struct{} |
||||
|
|
||||
|
func (h *testHandler) Setup(session sarama.ConsumerGroupSession) error { |
||||
|
log.Printf("✓ Consumer group session setup: generation=%d memberID=%s", session.GenerationID(), session.MemberID()) |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (h *testHandler) Cleanup(session sarama.ConsumerGroupSession) error { |
||||
|
log.Println("Consumer group session cleanup") |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (h *testHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { |
||||
|
log.Printf("✓ Started consuming: topic=%s partition=%d offset=%d", claim.Topic(), claim.Partition(), claim.InitialOffset()) |
||||
|
|
||||
|
count := 0 |
||||
|
for message := range claim.Messages() { |
||||
|
count++ |
||||
|
log.Printf(" Received message #%d: offset=%d", count, message.Offset) |
||||
|
session.MarkMessage(message, "") |
||||
|
|
||||
|
if count >= 5 { |
||||
|
log.Println("Received 5 messages, stopping") |
||||
|
return nil |
||||
|
} |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue