seaweedfs/test/kafka/network_capture_test.go

package kafka

import (
	"encoding/binary"
	"fmt"
	"io"
	"net"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/mq/kafka/gateway"
)

// TestNetworkCapture captures the exact bytes sent over the network
func TestNetworkCapture(t *testing.T) {
	// Start gateway
	gatewayServer := gateway.NewServer(gateway.Options{
		Listen: "127.0.0.1:0",
	})

	go gatewayServer.Start()
	defer gatewayServer.Close()

	// Wait for server to start
	time.Sleep(100 * time.Millisecond)

	host, port := gatewayServer.GetListenerAddr()
	addr := fmt.Sprintf("%s:%d", host, port)
	t.Logf("Gateway running on %s", addr)

	// Add test topic
	handler := gatewayServer.GetHandler()
	handler.AddTopicForTesting("capture-topic", 1)

	// Test: Capture exact network traffic
	testNetworkTraffic(addr, t)
}

func testNetworkTraffic(addr string, t *testing.T) {
	// Create raw TCP connection
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		t.Fatalf("Failed to dial: %v", err)
	}
	defer conn.Close()

	// Send ApiVersions request first
	t.Logf("=== Sending ApiVersions Request ===")
	apiVersionsReq := buildRawApiVersionsRequest()
	t.Logf("ApiVersions request (%d bytes): %x", len(apiVersionsReq), apiVersionsReq)

	if _, err := conn.Write(apiVersionsReq); err != nil {
		t.Fatalf("Failed to send ApiVersions: %v", err)
	}

	// Read ApiVersions response
	apiVersionsResp, err := readRawResponse(conn, t)
	if err != nil {
		t.Fatalf("Failed to read ApiVersions response: %v", err)
	}
	t.Logf("ApiVersions response (%d bytes): %x", len(apiVersionsResp), apiVersionsResp)

	// Send Metadata v1 request
	t.Logf("=== Sending Metadata v1 Request ===")
	metadataReq := buildRawMetadataV1Request([]string{"capture-topic"})
	t.Logf("Metadata request (%d bytes): %x", len(metadataReq), metadataReq)

	if _, err := conn.Write(metadataReq); err != nil {
		t.Fatalf("Failed to send Metadata: %v", err)
	}

	// Read Metadata response with detailed analysis
	metadataResp, err := readRawResponse(conn, t)
	if err != nil {
		t.Fatalf("Failed to read Metadata response: %v", err)
	}
	t.Logf("Metadata response (%d bytes): %x", len(metadataResp), metadataResp)

	// Analyze the response structure
	analyzeMetadataResponse(metadataResp, t)
}

func buildRawApiVersionsRequest() []byte {
	// Build ApiVersions request manually
	clientID := "test-client"

	// Calculate payload size: API key (2) + version (2) + correlation ID (4) + client ID length (2) + client ID
	payloadSize := 2 + 2 + 4 + 2 + len(clientID)

	req := make([]byte, 4) // Start with message size

	// Message size
	binary.BigEndian.PutUint32(req[0:4], uint32(payloadSize))

	// API key (ApiVersions = 18)
	req = append(req, 0, 18)
	// Version
	req = append(req, 0, 0)
	// Correlation ID
	req = append(req, 0, 0, 0, 1)
	// Client ID length
	clientIDLen := uint16(len(clientID))
	req = append(req, byte(clientIDLen>>8), byte(clientIDLen))
	// Client ID
	req = append(req, []byte(clientID)...)

	return req
}

func buildRawMetadataV1Request(topics []string) []byte {
	clientID := "test-client"

	// Calculate payload size: API key (2) + version (2) + correlation ID (4) + client ID length (2) + client ID + topics array
	payloadSize := 2 + 2 + 4 + 2 + len(clientID) + 4 // Base size + topics array length
	for _, topic := range topics {
		payloadSize += 2 + len(topic) // topic length (2) + topic name
	}

	req := make([]byte, 4) // Start with message size

	// Message size
	binary.BigEndian.PutUint32(req[0:4], uint32(payloadSize))

	// API key (Metadata = 3)
	req = append(req, 0, 3)
	// Version
	req = append(req, 0, 1)
	// Correlation ID
	req = append(req, 0, 0, 0, 2)
	// Client ID length
	clientIDLen := uint16(len(clientID))
	req = append(req, byte(clientIDLen>>8), byte(clientIDLen))
	// Client ID
	req = append(req, []byte(clientID)...)

	// Topics array
	topicsLen := uint32(len(topics))
	req = append(req, byte(topicsLen>>24), byte(topicsLen>>16), byte(topicsLen>>8), byte(topicsLen))
	for _, topic := range topics {
		topicLen := uint16(len(topic))
		req = append(req, byte(topicLen>>8), byte(topicLen))
		req = append(req, []byte(topic)...)
	}

	return req
}

func readRawResponse(conn net.Conn, t *testing.T) ([]byte, error) {
	// Read response size first
	sizeBuf := make([]byte, 4)
	if _, err := io.ReadFull(conn, sizeBuf); err != nil {
		return nil, fmt.Errorf("failed to read response size: %v", err)
	}

	size := binary.BigEndian.Uint32(sizeBuf)
	t.Logf("Response size header: %d bytes", size)

	// Read response data
	data := make([]byte, size)
	if _, err := io.ReadFull(conn, data); err != nil {
		return nil, fmt.Errorf("failed to read response data: %v", err)
	}

	return data, nil
}

func analyzeMetadataResponse(data []byte, t *testing.T) {
	t.Logf("=== Analyzing Metadata Response ===")

	if len(data) < 4 {
		t.Errorf("Response too short: %d bytes", len(data))
		return
	}

	offset := 0

	// Read correlation ID
	correlationID := binary.BigEndian.Uint32(data[offset : offset+4])
	offset += 4
	t.Logf("Correlation ID: %d", correlationID)

	// Read brokers count
	if offset+4 > len(data) {
		t.Errorf("Not enough data for brokers count at offset %d", offset)
		return
	}
	brokersCount := binary.BigEndian.Uint32(data[offset : offset+4])
	offset += 4
	t.Logf("Brokers count: %d", brokersCount)

	// Read each broker
	for i := 0; i < int(brokersCount); i++ {
		t.Logf("Reading broker %d at offset %d", i, offset)

		// Node ID
		if offset+4 > len(data) {
			t.Errorf("Not enough data for broker %d node ID", i)
			return
		}
		nodeID := binary.BigEndian.Uint32(data[offset : offset+4])
		offset += 4

		// Host
		if offset+2 > len(data) {
			t.Errorf("Not enough data for broker %d host length", i)
			return
		}
		hostLen := binary.BigEndian.Uint16(data[offset : offset+2])
		offset += 2

		if offset+int(hostLen) > len(data) {
			t.Errorf("Not enough data for broker %d host", i)
			return
		}
		host := string(data[offset : offset+int(hostLen)])
		offset += int(hostLen)

		// Port
		if offset+4 > len(data) {
			t.Errorf("Not enough data for broker %d port", i)
			return
		}
		port := binary.BigEndian.Uint32(data[offset : offset+4])
		offset += 4

		// Rack (v1 addition)
		if offset+2 > len(data) {
			t.Errorf("Not enough data for broker %d rack length", i)
			return
		}
		rackLen := binary.BigEndian.Uint16(data[offset : offset+2])
		offset += 2

		rack := ""
		if rackLen > 0 {
			if offset+int(rackLen) > len(data) {
				t.Errorf("Not enough data for broker %d rack", i)
				return
			}
			rack = string(data[offset : offset+int(rackLen)])
			offset += int(rackLen)
		}

		t.Logf("Broker %d: NodeID=%d, Host=%s, Port=%d, Rack=%s", i, nodeID, host, port, rack)
	}

	// Controller ID (v1 addition)
	if offset+4 > len(data) {
		t.Errorf("Not enough data for controller ID at offset %d", offset)
		return
	}
	controllerID := binary.BigEndian.Uint32(data[offset : offset+4])
	offset += 4
	t.Logf("Controller ID: %d", controllerID)

	// Topics count
	if offset+4 > len(data) {
		t.Errorf("Not enough data for topics count at offset %d", offset)
		return
	}
	topicsCount := binary.BigEndian.Uint32(data[offset : offset+4])
	offset += 4
	t.Logf("Topics count: %d", topicsCount)

	// Analyze remaining bytes
	remaining := len(data) - offset
	t.Logf("Remaining bytes after topics count: %d", remaining)
	t.Logf("Remaining data: %x", data[offset:])

	if remaining == 0 {
		t.Errorf("ERROR: No data remaining for topics! This might be the issue.")
		return
	}

	// Try to read first topic
	if topicsCount > 0 {
		t.Logf("Reading first topic at offset %d", offset)

		// Error code
		if offset+2 > len(data) {
			t.Errorf("Not enough data for topic error code")
			return
		}
		errorCode := binary.BigEndian.Uint16(data[offset : offset+2])
		offset += 2

		// Topic name
		if offset+2 > len(data) {
			t.Errorf("Not enough data for topic name length")
			return
		}
		nameLen := binary.BigEndian.Uint16(data[offset : offset+2])
		offset += 2

		if offset+int(nameLen) > len(data) {
			t.Errorf("Not enough data for topic name")
			return
		}
		name := string(data[offset : offset+int(nameLen)])
		offset += int(nameLen)

		t.Logf("Topic: ErrorCode=%d, Name=%s", errorCode, name)

		// Check remaining structure...
		remaining = len(data) - offset
		t.Logf("Remaining bytes after first topic name: %d", remaining)
	}
}