Browse Source
mq(kafka): BREAKTHROUGH - Topic creation and Metadata discovery working
mq(kafka): BREAKTHROUGH - Topic creation and Metadata discovery working
- Added Server.GetHandler() method to expose protocol handler for testing
- Added Handler.AddTopicForTesting() method for direct topic registry access
- Fixed infinite Metadata loop by implementing proper topic creation
- Topic discovery now works: Metadata API returns existing topics correctly
- Auto-topic creation implemented in Produce API (for when we get there)
- Response sizes increased: 43→94 bytes (proper topic metadata included)
- Debug shows: 'Returning all existing topics: [direct-test-topic]' ✅
MAJOR PROGRESS: kafka-go now finds topics via Metadata API, but still loops
instead of proceeding to Produce API. Next: Fix Metadata v7 response format
to match kafka-go expectations so it proceeds to actual produce/consume.
This removes the CreateTopics v2 parsing complexity by bypassing that API
entirely and focusing on the core produce/consume workflow that matters most.
pull/7231/head
chrislu
2 months ago
6 changed files with 374 additions and 39 deletions
-
10test/kafka/client_integration_test.go
-
8test/kafka/debug_connection_test.go
-
187test/kafka/produce_consume_test.go
-
5weed/mq/kafka/gateway/server.go
-
140weed/mq/kafka/protocol/handler.go
-
17weed/mq/kafka/protocol/produce.go
@ -0,0 +1,187 @@ |
|||
package kafka |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"strings" |
|||
"testing" |
|||
"time" |
|||
|
|||
"github.com/seaweedfs/seaweedfs/weed/mq/kafka/gateway" |
|||
"github.com/segmentio/kafka-go" |
|||
) |
|||
|
|||
// TestKafkaGoClient_DirectProduceConsume bypasses CreateTopics and tests produce/consume directly
|
|||
func TestKafkaGoClient_DirectProduceConsume(t *testing.T) { |
|||
// Start the gateway server
|
|||
srv := gateway.NewServer(gateway.Options{ |
|||
Listen: ":0", |
|||
UseSeaweedMQ: false, |
|||
}) |
|||
|
|||
if err := srv.Start(); err != nil { |
|||
t.Fatalf("Failed to start gateway: %v", err) |
|||
} |
|||
defer srv.Close() |
|||
|
|||
brokerAddr := srv.Addr() |
|||
t.Logf("Gateway running on %s", brokerAddr) |
|||
|
|||
topicName := "direct-test-topic" |
|||
|
|||
// Pre-create the topic by making a direct call to our gateway's topic registry
|
|||
// This simulates topic already existing (like pre-created topics)
|
|||
if err := createTopicDirectly(srv, topicName); err != nil { |
|||
t.Fatalf("Failed to create topic: %v", err) |
|||
} |
|||
|
|||
// Test basic produce without kafka-go's CreateTopics API
|
|||
messages := []kafka.Message{ |
|||
{ |
|||
Key: []byte("test-key-1"), |
|||
Value: []byte("Hello from direct produce test!"), |
|||
}, |
|||
{ |
|||
Key: []byte("test-key-2"), |
|||
Value: []byte("Second test message"), |
|||
}, |
|||
} |
|||
|
|||
t.Logf("Testing direct produce to topic %s", topicName) |
|||
|
|||
// Produce messages
|
|||
if err := produceMessagesDirect(brokerAddr, topicName, messages); err != nil { |
|||
t.Fatalf("Failed to produce messages: %v", err) |
|||
} |
|||
|
|||
t.Logf("Successfully produced %d messages", len(messages)) |
|||
|
|||
// Consume messages
|
|||
t.Logf("Testing direct consume from topic %s", topicName) |
|||
|
|||
consumedMessages, err := consumeMessagesDirect(brokerAddr, topicName, len(messages)) |
|||
if err != nil { |
|||
t.Fatalf("Failed to consume messages: %v", err) |
|||
} |
|||
|
|||
// Validate consumed messages
|
|||
if len(consumedMessages) != len(messages) { |
|||
t.Errorf("Expected %d messages, got %d", len(messages), len(consumedMessages)) |
|||
} |
|||
|
|||
for i, msg := range consumedMessages { |
|||
if i < len(messages) { |
|||
expectedValue := string(messages[i].Value) |
|||
actualValue := string(msg.Value) |
|||
t.Logf("Message %d: key=%s, value=%s", i, string(msg.Key), actualValue) |
|||
|
|||
if actualValue != expectedValue { |
|||
t.Errorf("Message %d: expected value %q, got %q", i, expectedValue, actualValue) |
|||
} |
|||
} |
|||
} |
|||
|
|||
t.Logf("✅ Direct produce/consume test PASSED with %d messages", len(consumedMessages)) |
|||
} |
|||
|
|||
// createTopicDirectly creates a topic by directly adding it to the handler's registry
|
|||
func createTopicDirectly(srv interface{}, topicName string) error { |
|||
gatewayServer, ok := srv.(*gateway.Server) |
|||
if !ok { |
|||
return fmt.Errorf("invalid server type") |
|||
} |
|||
|
|||
// Get the handler and directly add the topic
|
|||
handler := gatewayServer.GetHandler() |
|||
if handler == nil { |
|||
return fmt.Errorf("handler is nil") |
|||
} |
|||
|
|||
// Add the topic with 1 partition
|
|||
handler.AddTopicForTesting(topicName, 1) |
|||
|
|||
fmt.Printf("DEBUG: Topic %s created directly in handler registry\n", topicName) |
|||
return nil |
|||
} |
|||
|
|||
func produceMessagesDirect(brokerAddr, topicName string, messages []kafka.Message) error { |
|||
// Use kafka-go Writer which should use Produce API directly
|
|||
writer := &kafka.Writer{ |
|||
Addr: kafka.TCP(brokerAddr), |
|||
Topic: topicName, |
|||
Balancer: &kafka.LeastBytes{}, |
|||
|
|||
// Reduce timeouts for faster debugging
|
|||
WriteTimeout: 10 * time.Second, |
|||
ReadTimeout: 10 * time.Second, |
|||
|
|||
// Enable detailed logging
|
|||
Logger: kafka.LoggerFunc(func(msg string, args ...interface{}) { |
|||
if strings.Contains(msg, "produce") || strings.Contains(msg, "Produce") { |
|||
fmt.Printf("PRODUCER: "+msg+"\n", args...) |
|||
} |
|||
}), |
|||
ErrorLogger: kafka.LoggerFunc(func(msg string, args ...interface{}) { |
|||
fmt.Printf("PRODUCER ERROR: "+msg+"\n", args...) |
|||
}), |
|||
} |
|||
defer writer.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) |
|||
defer cancel() |
|||
|
|||
fmt.Printf("DEBUG: Writing %d messages to topic %s\n", len(messages), topicName) |
|||
|
|||
err := writer.WriteMessages(ctx, messages...) |
|||
if err != nil { |
|||
fmt.Printf("DEBUG: WriteMessages failed: %v\n", err) |
|||
return err |
|||
} |
|||
|
|||
fmt.Printf("DEBUG: WriteMessages completed successfully\n") |
|||
return nil |
|||
} |
|||
|
|||
func consumeMessagesDirect(brokerAddr, topicName string, expectedCount int) ([]kafka.Message, error) { |
|||
// Use kafka-go Reader which should use Fetch API directly
|
|||
reader := kafka.NewReader(kafka.ReaderConfig{ |
|||
Brokers: []string{brokerAddr}, |
|||
Topic: topicName, |
|||
|
|||
// Start from the beginning
|
|||
StartOffset: kafka.FirstOffset, |
|||
|
|||
// Enable detailed logging
|
|||
Logger: kafka.LoggerFunc(func(msg string, args ...interface{}) { |
|||
if strings.Contains(msg, "fetch") || strings.Contains(msg, "Fetch") { |
|||
fmt.Printf("CONSUMER: "+msg+"\n", args...) |
|||
} |
|||
}), |
|||
ErrorLogger: kafka.LoggerFunc(func(msg string, args ...interface{}) { |
|||
fmt.Printf("CONSUMER ERROR: "+msg+"\n", args...) |
|||
}), |
|||
}) |
|||
defer reader.Close() |
|||
|
|||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) |
|||
defer cancel() |
|||
|
|||
fmt.Printf("DEBUG: Reading up to %d messages from topic %s\n", expectedCount, topicName) |
|||
|
|||
var messages []kafka.Message |
|||
for i := 0; i < expectedCount; i++ { |
|||
fmt.Printf("DEBUG: Reading message %d/%d\n", i+1, expectedCount) |
|||
|
|||
msg, err := reader.ReadMessage(ctx) |
|||
if err != nil { |
|||
fmt.Printf("DEBUG: ReadMessage %d failed: %v\n", i+1, err) |
|||
return messages, fmt.Errorf("read message %d: %w", i+1, err) |
|||
} |
|||
|
|||
fmt.Printf("DEBUG: Successfully read message %d: %d bytes\n", i+1, len(msg.Value)) |
|||
messages = append(messages, msg) |
|||
} |
|||
|
|||
fmt.Printf("DEBUG: Successfully read all %d messages\n", len(messages)) |
|||
return messages, nil |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue