From 133698a9c4c3f595e82343545c2e93a09a7b2d04 Mon Sep 17 00:00:00 2001 From: chrislu Date: Thu, 16 Oct 2025 23:36:48 -0700 Subject: [PATCH] tune: Adjust commit frequency to every 20 messages for optimal balance Testing showed every 50 messages too aggressive (43.6% duplicates). Every 10 messages creates too much overhead. Every 20 messages provides good middle ground: - ~600 commits per 12k messages (manageable overhead) - ~20 message loss window if consumer crashes - Balanced duplicate/missing ratio --- .../internal/consumer/consumer.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/kafka/kafka-client-loadtest/internal/consumer/consumer.go b/test/kafka/kafka-client-loadtest/internal/consumer/consumer.go index 6e11732a7..ed266091d 100644 --- a/test/kafka/kafka-client-loadtest/internal/consumer/consumer.go +++ b/test/kafka/kafka-client-loadtest/internal/consumer/consumer.go @@ -647,12 +647,12 @@ func (h *ConsumerGroupHandler) ConsumeClaim(session sarama.ConsumerGroupSession, // Mark message as processed session.MarkMessage(message, "") - // Commit offset periodically to minimize message loss - // Every 50 messages provides good balance: - // - 10s overhead reduction vs every 10 messages - // - ~50 message loss window vs 100 if consumer fails - // Auto-commit (100ms) helps in background, manual commits are failsafe - if msgCount%50 == 0 { + // Commit offset frequently to minimize both message loss and duplicates + // Every 20 messages balances: + // - ~600 commits per 12k messages (reasonable overhead) + // - ~20 message loss window if consumer fails + // - Reduces duplicate reads from rebalancing + if msgCount%20 == 0 { session.Commit() } }