Limit Metadata API to v4 to fix kafka-go client compatibility

PARTIAL FIX: Force kafka-go to use Metadata v4 instead of v6

## Issue Identified:
- kafka-go was using Metadata v6 due to ApiVersions advertising v0-v6
- Our Metadata v6 implementation has format issues causing client failures
- Sarama works because it uses Metadata v4, not v6

## Changes:
- Limited Metadata API max version from 6 to 4 in ApiVersions response
- Added debug test to isolate Metadata parsing issues
- kafka-go now uses Metadata v4 (same as working Sarama)

## Status:
- ✅ kafka-go now uses v4 instead of v6
- ❌ Still has metadata loops (deeper issue with response format)
- ✅ Produce operations work correctly
- ❌ ReadPartitions API still fails

## Next Steps:
- Investigate why kafka-go keeps requesting metadata even with v4
- Compare exact byte format between working Sarama and failing kafka-go
- May need to fix specific fields in Metadata v4 response format

This is progress toward full kafka-go compatibility but more investigation needed.

test/kafka/metadata_debug_test.go

@@ -1,243 +1,47 @@
 package kafka
 
 import (
-	"encoding/binary"
 	"fmt"
-	"net"
 	"testing"
 	"time"
 
+	"github.com/segmentio/kafka-go"
 	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/gateway"
 )
 
-// TestMetadataV1DebugCapture captures the exact bytes kafka-go sends and expects
-func TestMetadataV1DebugCapture(t *testing.T) {
-	// Start gateway server
-	gatewayServer := gateway.NewServer(gateway.Options{
-		Listen: ":0", // random port
-	})
-
+func TestMetadataV6Debug(t *testing.T) {
+	// Start gateway
+	gatewayServer := gateway.NewServer(gateway.Options{Listen: "127.0.0.1:0"})
 	go func() {
 		if err := gatewayServer.Start(); err != nil {
-			t.Errorf("Gateway server error: %v", err)
+			t.Errorf("Failed to start gateway: %v", err)
 		}
 	}()
 	defer gatewayServer.Close()
 
-	// Wait for server to start
 	time.Sleep(100 * time.Millisecond)
 
-	// Get the actual listening address
 	host, port := gatewayServer.GetListenerAddr()
-	brokerAddr := fmt.Sprintf("%s:%d", host, port)
-	t.Logf("Gateway running on %s", brokerAddr)
-
-	// Get handler and configure it
-	handler := gatewayServer.GetHandler()
-	handler.SetBrokerAddress(host, port)
-
-	// Add test topic
-	topicName := "debug-topic"
-	handler.AddTopicForTesting(topicName, 1)
+	addr := fmt.Sprintf("%s:%d", host, port)
+	topic := "metadata-debug-topic"
+	gatewayServer.GetHandler().AddTopicForTesting(topic, 1)
 
-	// Create raw TCP connection to manually send Metadata v1 request
-	conn, err := net.Dial("tcp", brokerAddr)
+	// Create a simple kafka-go client that just gets metadata
+	conn, err := kafka.Dial("tcp", addr)
 	if err != nil {
 		t.Fatalf("Failed to connect: %v", err)
 	}
 	defer conn.Close()
 
-	// Send ApiVersions request first
-	t.Log("=== Sending ApiVersions request ===")
-	apiVersionsRequest := []byte{
-		// Request header: api_key(2) + api_version(2) + correlation_id(4) + client_id_len(2) + client_id
-		0x00, 0x12, // api_key = 18 (ApiVersions)
-		0x00, 0x00, // api_version = 0
-		0x00, 0x00, 0x00, 0x01, // correlation_id = 1
-		0x00, 0x09, // client_id length = 9
-		'd', 'e', 'b', 'u', 'g', '-', 't', 'e', 's', 't', // client_id = "debug-test"
-	}
-	
-	// Prepend message length
-	messageLen := make([]byte, 4)
-	binary.BigEndian.PutUint32(messageLen, uint32(len(apiVersionsRequest)))
-	fullRequest := append(messageLen, apiVersionsRequest...)
-	
-	_, err = conn.Write(fullRequest)
-	if err != nil {
-		t.Fatalf("Failed to send ApiVersions: %v", err)
-	}
-
-	// Read ApiVersions response
-	responseLen := make([]byte, 4)
-	_, err = conn.Read(responseLen)
-	if err != nil {
-		t.Fatalf("Failed to read ApiVersions response length: %v", err)
-	}
-	
-	respLen := binary.BigEndian.Uint32(responseLen)
-	response := make([]byte, respLen)
-	_, err = conn.Read(response)
+	// Get metadata - this should work without loops
+	partitions, err := conn.ReadPartitions(topic)
 	if err != nil {
-		t.Fatalf("Failed to read ApiVersions response: %v", err)
+		t.Fatalf("Failed to read partitions: %v", err)
 	}
 
-	t.Logf("ApiVersions response (%d bytes): %x", len(response), response)
-
-	// Now send Metadata v1 request
-	t.Log("=== Sending Metadata v1 request ===")
-	metadataRequest := []byte{
-		// Request header: api_key(2) + api_version(2) + correlation_id(4) + client_id_len(2) + client_id
-		0x00, 0x03, // api_key = 3 (Metadata)
-		0x00, 0x01, // api_version = 1
-		0x00, 0x00, 0x00, 0x02, // correlation_id = 2
-		0x00, 0x09, // client_id length = 9
-		'd', 'e', 'b', 'u', 'g', '-', 't', 'e', 's', 't', // client_id = "debug-test"
-		// Metadata request body: topics_count(4) + topic_name_len(2) + topic_name
-		0x00, 0x00, 0x00, 0x01, // topics_count = 1
-		0x00, 0x0B, // topic_name length = 11
-		'd', 'e', 'b', 'u', 'g', '-', 't', 'o', 'p', 'i', 'c', // topic_name = "debug-topic"
+	t.Logf("Successfully read %d partitions for topic %s", len(partitions), topic)
+	for _, p := range partitions {
+		t.Logf("Partition %d: Leader=%d, Replicas=%v, ISR=%v", 
+			p.ID, p.Leader.ID, p.Replicas, p.Isr)
 	}
-	
-	// Prepend message length
-	messageLen = make([]byte, 4)
-	binary.BigEndian.PutUint32(messageLen, uint32(len(metadataRequest)))
-	fullRequest = append(messageLen, metadataRequest...)
-	
-	t.Logf("Sending Metadata v1 request (%d bytes): %x", len(fullRequest), fullRequest)
-	
-	_, err = conn.Write(fullRequest)
-	if err != nil {
-		t.Fatalf("Failed to send Metadata: %v", err)
-	}
-
-	// Read Metadata response
-	responseLen = make([]byte, 4)
-	_, err = conn.Read(responseLen)
-	if err != nil {
-		t.Fatalf("Failed to read Metadata response length: %v", err)
-	}
-	
-	respLen = binary.BigEndian.Uint32(responseLen)
-	response = make([]byte, respLen)
-	_, err = conn.Read(response)
-	if err != nil {
-		t.Fatalf("Failed to read Metadata response: %v", err)
-	}
-	
-	t.Logf("Metadata v1 response (%d bytes): %x", len(response), response)
-	
-	// Parse the response manually to understand the format
-	t.Log("=== Parsing Metadata v1 response ===")
-	offset := 0
-	
-	// Correlation ID (4 bytes)
-	correlationID := binary.BigEndian.Uint32(response[offset:offset+4])
-	offset += 4
-	t.Logf("Correlation ID: %d", correlationID)
-	
-	// Brokers array length (4 bytes)
-	brokersCount := binary.BigEndian.Uint32(response[offset:offset+4])
-	offset += 4
-	t.Logf("Brokers count: %d", brokersCount)
-	
-	// Parse each broker
-	for i := uint32(0); i < brokersCount; i++ {
-		// node_id (4 bytes)
-		nodeID := binary.BigEndian.Uint32(response[offset:offset+4])
-		offset += 4
-		t.Logf("Broker %d node_id: %d", i, nodeID)
-		
-		// host (STRING: 2 bytes length + bytes)
-		hostLen := binary.BigEndian.Uint16(response[offset:offset+2])
-		offset += 2
-		hostBytes := response[offset:offset+int(hostLen)]
-		offset += int(hostLen)
-		t.Logf("Broker %d host: %s", i, string(hostBytes))
-		
-		// port (4 bytes)
-		portNum := binary.BigEndian.Uint32(response[offset:offset+4])
-		offset += 4
-		t.Logf("Broker %d port: %d", i, portNum)
-		
-		// rack (STRING: 2 bytes length + bytes) - v1 addition
-		rackLen := binary.BigEndian.Uint16(response[offset:offset+2])
-		offset += 2
-		if rackLen > 0 {
-			rackBytes := response[offset:offset+int(rackLen)]
-			offset += int(rackLen)
-			t.Logf("Broker %d rack: %s", i, string(rackBytes))
-		} else {
-			t.Logf("Broker %d rack: (empty)", i)
-		}
-	}
-	
-	// Topics array length (4 bytes)
-	if offset < len(response) {
-		topicsCount := binary.BigEndian.Uint32(response[offset:offset+4])
-		offset += 4
-		t.Logf("Topics count: %d", topicsCount)
-		
-		// Parse each topic
-		for i := uint32(0); i < topicsCount && offset < len(response); i++ {
-			// error_code (2 bytes)
-			errorCode := binary.BigEndian.Uint16(response[offset:offset+2])
-			offset += 2
-			t.Logf("Topic %d error_code: %d", i, errorCode)
-			
-			// name (STRING: 2 bytes length + bytes)
-			nameLen := binary.BigEndian.Uint16(response[offset:offset+2])
-			offset += 2
-			nameBytes := response[offset:offset+int(nameLen)]
-			offset += int(nameLen)
-			t.Logf("Topic %d name: %s", i, string(nameBytes))
-			
-			// is_internal (1 byte) - v1 addition
-			if offset < len(response) {
-				isInternal := response[offset]
-				offset += 1
-				t.Logf("Topic %d is_internal: %d", i, isInternal)
-			}
-			
-			// partitions array length (4 bytes)
-			if offset+4 <= len(response) {
-				partitionsCount := binary.BigEndian.Uint32(response[offset:offset+4])
-				offset += 4
-				t.Logf("Topic %d partitions count: %d", i, partitionsCount)
-				
-				// Skip partition details for brevity
-				for j := uint32(0); j < partitionsCount && offset < len(response); j++ {
-					// error_code (2) + partition_id (4) + leader (4) + replicas (4+n*4) + isr (4+n*4)
-					if offset+2 <= len(response) {
-						partErrorCode := binary.BigEndian.Uint16(response[offset:offset+2])
-						offset += 2
-						t.Logf("  Partition %d error_code: %d", j, partErrorCode)
-					}
-					if offset+4 <= len(response) {
-						partitionID := binary.BigEndian.Uint32(response[offset:offset+4])
-						offset += 4
-						t.Logf("  Partition %d id: %d", j, partitionID)
-					}
-					if offset+4 <= len(response) {
-						leader := binary.BigEndian.Uint32(response[offset:offset+4])
-						offset += 4
-						t.Logf("  Partition %d leader: %d", j, leader)
-					}
-					// Skip replicas and isr arrays for brevity - just advance offset
-					if offset+4 <= len(response) {
-						replicasCount := binary.BigEndian.Uint32(response[offset:offset+4])
-						offset += 4 + int(replicasCount)*4
-						t.Logf("  Partition %d replicas count: %d", j, replicasCount)
-					}
-					if offset+4 <= len(response) {
-						isrCount := binary.BigEndian.Uint32(response[offset:offset+4])
-						offset += 4 + int(isrCount)*4
-						t.Logf("  Partition %d isr count: %d", j, isrCount)
-					}
-				}
-			}
-		}
-	}
-	
-	t.Logf("Parsed %d bytes, remaining: %d", offset, len(response)-offset)
 }

weed/mq/kafka/protocol/handler.go

@@ -329,10 +329,11 @@ func (h *Handler) handleApiVersions(correlationID uint32) ([]byte, error) {
 	response = append(response, 0, 3)  // max version 3
 
 	// API Key 3 (Metadata): api_key(2) + min_version(2) + max_version(2)
-	// kafka-go negotiates v1,v6 - try v6 since our v6 works with Sarama
+	// TEMPORARY FIX: Limit to v4 since v6 has format issues with kafka-go
+	// Sarama works with v4, kafka-go should also work with v4
 	response = append(response, 0, 3) // API key 3
 	response = append(response, 0, 0) // min version 0
-	response = append(response, 0, 6) // max version 6
+	response = append(response, 0, 4) // max version 4 (was 6)
 
 	// API Key 2 (ListOffsets): api_key(2) + min_version(2) + max_version(2)
 	response = append(response, 0, 2) // API key 2
@@ -1755,4 +1756,3 @@ func (h *Handler) IsSchemaEnabled() bool {
 func (h *Handler) IsBrokerIntegrationEnabled() bool {
 	return h.IsSchemaEnabled() && h.brokerClient != nil
 }
-