You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
266 lines
7.7 KiB
266 lines
7.7 KiB
package kafka
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"fmt"
|
|
"net"
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/mq/kafka/gateway"
|
|
)
|
|
|
|
// TestRawProduceRequest tests our Produce API directly without kafka-go
|
|
func TestRawProduceRequest(t *testing.T) {
|
|
// Start the gateway server
|
|
srv := gateway.NewTestServer(gateway.Options{
|
|
Listen: ":0",
|
|
})
|
|
|
|
if err := srv.Start(); err != nil {
|
|
t.Fatalf("Failed to start gateway: %v", err)
|
|
}
|
|
defer srv.Close()
|
|
|
|
brokerAddr := srv.Addr()
|
|
t.Logf("Gateway running on %s", brokerAddr)
|
|
|
|
// Pre-create topic
|
|
topicName := "raw-test-topic"
|
|
handler := srv.GetHandler()
|
|
handler.AddTopicForTesting(topicName, 1)
|
|
|
|
// Make a raw TCP connection
|
|
conn, err := net.Dial("tcp", brokerAddr)
|
|
if err != nil {
|
|
t.Fatalf("Failed to connect: %v", err)
|
|
}
|
|
defer conn.Close()
|
|
|
|
// Test 1: ApiVersions request (should work)
|
|
t.Log("=== Testing ApiVersions API ===")
|
|
if err := sendApiVersionsRequest(conn); err != nil {
|
|
t.Fatalf("ApiVersions failed: %v", err)
|
|
}
|
|
t.Log("✅ ApiVersions API working")
|
|
|
|
// Test 2: Metadata request (should work)
|
|
t.Log("=== Testing Metadata API ===")
|
|
if err := sendMetadataRequest(conn); err != nil {
|
|
t.Fatalf("Metadata failed: %v", err)
|
|
}
|
|
t.Log("✅ Metadata API working")
|
|
|
|
// Test 3: Raw Produce request (this is what we want to test)
|
|
t.Log("=== Testing Produce API (RAW) ===")
|
|
if err := sendProduceRequest(conn, topicName); err != nil {
|
|
t.Fatalf("Produce failed: %v", err)
|
|
}
|
|
t.Log("✅ Produce API working!")
|
|
}
|
|
|
|
func sendApiVersionsRequest(conn net.Conn) error {
|
|
// Build ApiVersions request
|
|
correlationID := uint32(1)
|
|
|
|
msgBody := make([]byte, 0, 32)
|
|
msgBody = append(msgBody, 0, 18) // API key 18 (ApiVersions)
|
|
msgBody = append(msgBody, 0, 0) // API version 0
|
|
|
|
// Correlation ID
|
|
correlationBytes := make([]byte, 4)
|
|
binary.BigEndian.PutUint32(correlationBytes, correlationID)
|
|
msgBody = append(msgBody, correlationBytes...)
|
|
|
|
// Client ID (empty)
|
|
msgBody = append(msgBody, 0, 0) // empty client ID
|
|
|
|
// Send request
|
|
sizeBytes := make([]byte, 4)
|
|
binary.BigEndian.PutUint32(sizeBytes, uint32(len(msgBody)))
|
|
request := append(sizeBytes, msgBody...)
|
|
|
|
if _, err := conn.Write(request); err != nil {
|
|
return fmt.Errorf("write request: %w", err)
|
|
}
|
|
|
|
// Read response size
|
|
var responseSizeBytes [4]byte
|
|
if _, err := conn.Read(responseSizeBytes[:]); err != nil {
|
|
return fmt.Errorf("read response size: %w", err)
|
|
}
|
|
|
|
responseSize := binary.BigEndian.Uint32(responseSizeBytes[:])
|
|
|
|
// Read response body
|
|
responseBody := make([]byte, responseSize)
|
|
if _, err := conn.Read(responseBody); err != nil {
|
|
return fmt.Errorf("read response body: %w", err)
|
|
}
|
|
|
|
fmt.Printf("ApiVersions response: %d bytes\n", responseSize)
|
|
return nil
|
|
}
|
|
|
|
func sendMetadataRequest(conn net.Conn) error {
|
|
// Build Metadata request
|
|
correlationID := uint32(2)
|
|
|
|
msgBody := make([]byte, 0, 32)
|
|
msgBody = append(msgBody, 0, 3) // API key 3 (Metadata)
|
|
msgBody = append(msgBody, 0, 1) // API version 1
|
|
|
|
// Correlation ID
|
|
correlationBytes := make([]byte, 4)
|
|
binary.BigEndian.PutUint32(correlationBytes, correlationID)
|
|
msgBody = append(msgBody, correlationBytes...)
|
|
|
|
// Client ID (empty)
|
|
msgBody = append(msgBody, 0, 0) // empty client ID
|
|
|
|
// Topics array (empty = all topics)
|
|
msgBody = append(msgBody, 0xFF, 0xFF, 0xFF, 0xFF) // -1 = all topics
|
|
|
|
// Send request
|
|
sizeBytes := make([]byte, 4)
|
|
binary.BigEndian.PutUint32(sizeBytes, uint32(len(msgBody)))
|
|
request := append(sizeBytes, msgBody...)
|
|
|
|
if _, err := conn.Write(request); err != nil {
|
|
return fmt.Errorf("write request: %w", err)
|
|
}
|
|
|
|
// Read response size
|
|
var responseSizeBytes [4]byte
|
|
if _, err := conn.Read(responseSizeBytes[:]); err != nil {
|
|
return fmt.Errorf("read response size: %w", err)
|
|
}
|
|
|
|
responseSize := binary.BigEndian.Uint32(responseSizeBytes[:])
|
|
|
|
// Read response body
|
|
responseBody := make([]byte, responseSize)
|
|
if _, err := conn.Read(responseBody); err != nil {
|
|
return fmt.Errorf("read response body: %w", err)
|
|
}
|
|
|
|
fmt.Printf("Metadata response: %d bytes\n", responseSize)
|
|
return nil
|
|
}
|
|
|
|
func sendProduceRequest(conn net.Conn, topicName string) error {
|
|
// Build simple Produce request
|
|
correlationID := uint32(3)
|
|
|
|
msgBody := make([]byte, 0, 128)
|
|
msgBody = append(msgBody, 0, 0) // API key 0 (Produce)
|
|
msgBody = append(msgBody, 0, 1) // API version 1
|
|
|
|
// Correlation ID
|
|
correlationBytes := make([]byte, 4)
|
|
binary.BigEndian.PutUint32(correlationBytes, correlationID)
|
|
msgBody = append(msgBody, correlationBytes...)
|
|
|
|
// Client ID (empty)
|
|
msgBody = append(msgBody, 0, 0) // empty client ID
|
|
|
|
// Acks (-1 = all replicas)
|
|
msgBody = append(msgBody, 0xFF, 0xFF) // -1
|
|
|
|
// Timeout (5000ms)
|
|
msgBody = append(msgBody, 0, 0, 0x13, 0x88) // 5000ms
|
|
|
|
// Topics count (1)
|
|
msgBody = append(msgBody, 0, 0, 0, 1)
|
|
|
|
// Topic name
|
|
topicNameBytes := []byte(topicName)
|
|
msgBody = append(msgBody, byte(len(topicNameBytes)>>8), byte(len(topicNameBytes)))
|
|
msgBody = append(msgBody, topicNameBytes...)
|
|
|
|
// Partitions count (1)
|
|
msgBody = append(msgBody, 0, 0, 0, 1)
|
|
|
|
// Partition 0
|
|
msgBody = append(msgBody, 0, 0, 0, 0) // partition ID = 0
|
|
|
|
// Record set (simple test record)
|
|
testRecord := buildSimpleRecordSet("test-key", "test-value")
|
|
recordSetSize := make([]byte, 4)
|
|
binary.BigEndian.PutUint32(recordSetSize, uint32(len(testRecord)))
|
|
msgBody = append(msgBody, recordSetSize...)
|
|
msgBody = append(msgBody, testRecord...)
|
|
|
|
// Send request
|
|
sizeBytes := make([]byte, 4)
|
|
binary.BigEndian.PutUint32(sizeBytes, uint32(len(msgBody)))
|
|
request := append(sizeBytes, msgBody...)
|
|
|
|
fmt.Printf("Sending Produce request: %d bytes\n", len(request))
|
|
|
|
if _, err := conn.Write(request); err != nil {
|
|
return fmt.Errorf("write request: %w", err)
|
|
}
|
|
|
|
// Read response size
|
|
var responseSizeBytes [4]byte
|
|
if _, err := conn.Read(responseSizeBytes[:]); err != nil {
|
|
return fmt.Errorf("read response size: %w", err)
|
|
}
|
|
|
|
responseSize := binary.BigEndian.Uint32(responseSizeBytes[:])
|
|
|
|
// Read response body
|
|
responseBody := make([]byte, responseSize)
|
|
if _, err := conn.Read(responseBody); err != nil {
|
|
return fmt.Errorf("read response body: %w", err)
|
|
}
|
|
|
|
fmt.Printf("Produce response: %d bytes\n", responseSize)
|
|
|
|
// Check if the response indicates success (simplified check)
|
|
if responseSize > 8 {
|
|
// Extract correlation ID and basic error code
|
|
correlationResp := binary.BigEndian.Uint32(responseBody[0:4])
|
|
if correlationResp == correlationID {
|
|
fmt.Printf("✅ Produce request correlation ID matches: %d\n", correlationResp)
|
|
}
|
|
|
|
// Look for error codes in the response
|
|
if len(responseBody) > 20 {
|
|
// Skip to where partition error code should be (rough estimate)
|
|
errorCode := binary.BigEndian.Uint16(responseBody[16:18])
|
|
if errorCode == 0 {
|
|
fmt.Printf("✅ Produce request succeeded (error code: 0)\n")
|
|
} else {
|
|
fmt.Printf("⚠️ Produce request error code: %d\n", errorCode)
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func buildSimpleRecordSet(key, value string) []byte {
|
|
// Build a very simple Kafka record batch (v0 format for simplicity)
|
|
record := make([]byte, 0, 64)
|
|
|
|
// Record batch header (simplified v0 format)
|
|
record = append(record, 0, 0, 0, 0, 0, 0, 0, 0) // base offset
|
|
record = append(record, 0, 0, 0, 30) // batch length (estimated)
|
|
record = append(record, 0, 0, 0, 0) // partition leader epoch
|
|
record = append(record, 0) // magic byte (v0)
|
|
record = append(record, 0, 0, 0, 0) // CRC32 (simplified)
|
|
record = append(record, 0, 0) // attributes
|
|
record = append(record, 0, 0, 0, 1) // record count = 1
|
|
|
|
// Simple record: key_length + key + value_length + value
|
|
keyBytes := []byte(key)
|
|
valueBytes := []byte(value)
|
|
|
|
record = append(record, byte(len(keyBytes)>>8), byte(len(keyBytes)))
|
|
record = append(record, keyBytes...)
|
|
record = append(record, byte(len(valueBytes)>>8), byte(len(valueBytes)))
|
|
record = append(record, valueBytes...)
|
|
|
|
return record
|
|
}
|