Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

fix kafka tests

pull/7231/head
chrislu 2 months ago
parent
c951fc03c1
commit
30b21abab9
9 changed files with 71 additions and 99 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 5
      test/kafka/api_sequence_test.go
  2. 9
      test/kafka/client_integration_test.go
  3. 6
      test/kafka/comprehensive_e2e_test.go
  4. 3
      test/kafka/consumer_test.go
  5. 9
      test/kafka/debug_connection_test.go
  6. 9
      test/kafka/persistent_offset_integration_test.go
  7. 3
      test/kafka/produce_consume_test.go
  8. 101
      test/kafka/raw_protocol_test.go
  9. 25
      test/kafka/seaweedmq_integration_test.go

5
test/kafka/api_sequence_test.go
View File

@ -14,8 +14,7 @@ import (
func TestKafkaGateway_APISequence(t *testing.T) {
// Start the gateway server
srv := gateway.NewServer(gateway.Options{
Listen: ":0",
UseSeaweedMQ: false,
Listen: ":0",
})
if err := srv.Start(); err != nil {
@ -31,7 +30,7 @@ func TestKafkaGateway_APISequence(t *testing.T) {
handler := srv.GetHandler()
handler.AddTopicForTesting(topicName, 1)
// Create a writer and try to write a single message
// Create a writer and try to write a single message
writer := &kafka.Writer{
Addr: kafka.TCP(brokerAddr),
Topic: topicName,

9
test/kafka/client_integration_test.go
View File

@ -16,8 +16,7 @@ import (
func TestKafkaGoClient_BasicProduceConsume(t *testing.T) {
// Start the gateway server
srv := gateway.NewServer(gateway.Options{
Listen: ":0", // Use random port
UseSeaweedMQ: false, // Use in-memory mode for testing
Listen: ":0", // Use random port
})
if err := srv.Start(); err != nil {
@ -83,8 +82,7 @@ func TestKafkaGoClient_BasicProduceConsume(t *testing.T) {
func TestKafkaGoClient_ConsumerGroups(t *testing.T) {
// Start the gateway server
srv := gateway.NewServer(gateway.Options{
Listen: ":0",
UseSeaweedMQ: false,
Listen: ":0",
})
if err := srv.Start(); err != nil {
@ -137,8 +135,7 @@ func TestKafkaGoClient_MultiplePartitions(t *testing.T) {
func TestKafkaGoClient_OffsetManagement(t *testing.T) {
// Start the gateway server
srv := gateway.NewServer(gateway.Options{
Listen: ":0",
UseSeaweedMQ: false,
Listen: ":0",
})
if err := srv.Start(); err != nil {

6
test/kafka/comprehensive_e2e_test.go
View File

@ -16,8 +16,7 @@ import (
func TestComprehensiveE2E(t *testing.T) {
// Start gateway
gatewayServer := gateway.NewServer(gateway.Options{
Listen: "127.0.0.1:0",
UseSeaweedMQ: false, // Use in-memory mode for testing
Listen: "127.0.0.1:0",
})
go func() {
@ -359,8 +358,7 @@ func testSaramaToKafkaGo(t *testing.T, addr, topic string) {
func TestOffsetManagement(t *testing.T) {
// Start gateway
gatewayServer := gateway.NewServer(gateway.Options{
Listen: "127.0.0.1:0",
UseSeaweedMQ: false,
Listen: "127.0.0.1:0",
})
go func() {

3
test/kafka/consumer_test.go
View File

@ -14,8 +14,7 @@ import (
func TestKafkaGoReader(t *testing.T) {
// Start the gateway server
srv := gateway.NewServer(gateway.Options{
Listen: ":0",
UseSeaweedMQ: false,
Listen: ":0",
})
if err := srv.Start(); err != nil {

9
test/kafka/debug_connection_test.go
View File

@ -13,8 +13,7 @@ import (
func TestGateway_BasicConnection(t *testing.T) {
// Start the gateway server
srv := gateway.NewServer(gateway.Options{
Listen: ":0",
UseSeaweedMQ: false,
Listen: ":0",
})
if err := srv.Start(); err != nil {
@ -39,8 +38,7 @@ func TestGateway_BasicConnection(t *testing.T) {
func TestGateway_ApiVersionsRequest(t *testing.T) {
// Start the gateway server
srv := gateway.NewServer(gateway.Options{
Listen: ":0",
UseSeaweedMQ: false,
Listen: ":0",
})
if err := srv.Start(); err != nil {
@ -141,8 +139,7 @@ func TestGateway_ApiVersionsRequest(t *testing.T) {
func TestGateway_CreateTopicsRequest(t *testing.T) {
// Start the gateway server
srv := gateway.NewServer(gateway.Options{
Listen: ":0",
UseSeaweedMQ: false,
Listen: ":0",
})
if err := srv.Start(); err != nil {

9
test/kafka/persistent_offset_integration_test.go
View File

@ -39,15 +39,13 @@ func TestPersistentOffsetIntegration(t *testing.T) {
func testOffsetPersistenceAndRecovery(t *testing.T, brokers []string) {
// Create offset storage
storage, err := offset.NewSeaweedMQStorage(brokers)
require.NoError(t, err)
storage := offset.NewSeaweedMQStorage()
defer storage.Close()
topicPartition := "test-persistence-topic-0"
// Create first ledger and add some entries
ledger1, err := offset.NewPersistentLedger(topicPartition, storage)
require.NoError(t, err)
ledger1 := offset.NewPersistentLedger(topicPartition, storage)
// Add test entries
testEntries := []struct {
@ -77,8 +75,7 @@ func testOffsetPersistenceAndRecovery(t *testing.T, brokers []string) {
time.Sleep(2 * time.Second)
// Create second ledger (simulating restart)
ledger2, err := offset.NewPersistentLedger(topicPartition, storage)
require.NoError(t, err)
ledger2 := offset.NewPersistentLedger(topicPartition, storage)
// Verify recovered state
assert.Equal(t, ledger1.GetHighWaterMark(), ledger2.GetHighWaterMark())

3
test/kafka/produce_consume_test.go
View File

@ -15,8 +15,7 @@ import (
func TestKafkaGoClient_DirectProduceConsume(t *testing.T) {
// Start the gateway server
srv := gateway.NewServer(gateway.Options{
Listen: ":0",
UseSeaweedMQ: false,
Listen: ":0",
})
if err := srv.Start(); err != nil {

101
test/kafka/raw_protocol_test.go
View File

@ -13,8 +13,7 @@ import (
func TestRawProduceRequest(t *testing.T) {
// Start the gateway server
srv := gateway.NewServer(gateway.Options{
Listen: ":0",
UseSeaweedMQ: false,
Listen: ":0",
})
if err := srv.Start(); err != nil {
@ -62,42 +61,42 @@ func TestRawProduceRequest(t *testing.T) {
func sendApiVersionsRequest(conn net.Conn) error {
// Build ApiVersions request
correlationID := uint32(1)
msgBody := make([]byte, 0, 32)
msgBody = append(msgBody, 0, 18) // API key 18 (ApiVersions)
msgBody = append(msgBody, 0, 0) // API version 0
// Correlation ID
correlationBytes := make([]byte, 4)
binary.BigEndian.PutUint32(correlationBytes, correlationID)
msgBody = append(msgBody, correlationBytes...)
// Client ID (empty)
msgBody = append(msgBody, 0, 0) // empty client ID
// Send request
sizeBytes := make([]byte, 4)
binary.BigEndian.PutUint32(sizeBytes, uint32(len(msgBody)))
request := append(sizeBytes, msgBody...)
if _, err := conn.Write(request); err != nil {
return fmt.Errorf("write request: %w", err)
}
// Read response size
var responseSizeBytes [4]byte
if _, err := conn.Read(responseSizeBytes[:]); err != nil {
return fmt.Errorf("read response size: %w", err)
}
responseSize := binary.BigEndian.Uint32(responseSizeBytes[:])
// Read response body
responseBody := make([]byte, responseSize)
if _, err := conn.Read(responseBody); err != nil {
return fmt.Errorf("read response body: %w", err)
}
fmt.Printf("ApiVersions response: %d bytes\n", responseSize)
return nil
}
@ -105,45 +104,45 @@ func sendApiVersionsRequest(conn net.Conn) error {
func sendMetadataRequest(conn net.Conn) error {
// Build Metadata request
correlationID := uint32(2)
msgBody := make([]byte, 0, 32)
msgBody = append(msgBody, 0, 3) // API key 3 (Metadata)
msgBody = append(msgBody, 0, 1) // API version 1
// Correlation ID
correlationBytes := make([]byte, 4)
binary.BigEndian.PutUint32(correlationBytes, correlationID)
msgBody = append(msgBody, correlationBytes...)
// Client ID (empty)
msgBody = append(msgBody, 0, 0) // empty client ID
// Topics array (empty = all topics)
msgBody = append(msgBody, 0xFF, 0xFF, 0xFF, 0xFF) // -1 = all topics
// Send request
sizeBytes := make([]byte, 4)
binary.BigEndian.PutUint32(sizeBytes, uint32(len(msgBody)))
request := append(sizeBytes, msgBody...)
if _, err := conn.Write(request); err != nil {
return fmt.Errorf("write request: %w", err)
}
// Read response size
var responseSizeBytes [4]byte
if _, err := conn.Read(responseSizeBytes[:]); err != nil {
return fmt.Errorf("read response size: %w", err)
}
responseSize := binary.BigEndian.Uint32(responseSizeBytes[:])
// Read response body
responseBody := make([]byte, responseSize)
if _, err := conn.Read(responseBody); err != nil {
return fmt.Errorf("read response body: %w", err)
}
fmt.Printf("Metadata response: %d bytes\n", responseSize)
return nil
}
@ -151,73 +150,73 @@ func sendMetadataRequest(conn net.Conn) error {
func sendProduceRequest(conn net.Conn, topicName string) error {
// Build simple Produce request
correlationID := uint32(3)
msgBody := make([]byte, 0, 128)
msgBody = append(msgBody, 0, 0) // API key 0 (Produce)
msgBody = append(msgBody, 0, 1) // API version 1
// Correlation ID
correlationBytes := make([]byte, 4)
binary.BigEndian.PutUint32(correlationBytes, correlationID)
msgBody = append(msgBody, correlationBytes...)
// Client ID (empty)
msgBody = append(msgBody, 0, 0) // empty client ID
// Acks (-1 = all replicas)
msgBody = append(msgBody, 0xFF, 0xFF) // -1
// Timeout (5000ms)
msgBody = append(msgBody, 0, 0, 0x13, 0x88) // 5000ms
// Topics count (1)
msgBody = append(msgBody, 0, 0, 0, 1)
// Topic name
topicNameBytes := []byte(topicName)
msgBody = append(msgBody, byte(len(topicNameBytes)>>8), byte(len(topicNameBytes)))
msgBody = append(msgBody, topicNameBytes...)
// Partitions count (1)
msgBody = append(msgBody, 0, 0, 0, 1)
// Partition 0
msgBody = append(msgBody, 0, 0, 0, 0) // partition ID = 0
// Record set (simple test record)
testRecord := buildSimpleRecordSet("test-key", "test-value")
recordSetSize := make([]byte, 4)
binary.BigEndian.PutUint32(recordSetSize, uint32(len(testRecord)))
msgBody = append(msgBody, recordSetSize...)
msgBody = append(msgBody, testRecord...)
// Send request
sizeBytes := make([]byte, 4)
binary.BigEndian.PutUint32(sizeBytes, uint32(len(msgBody)))
request := append(sizeBytes, msgBody...)
fmt.Printf("Sending Produce request: %d bytes\n", len(request))
if _, err := conn.Write(request); err != nil {
return fmt.Errorf("write request: %w", err)
}
// Read response size
var responseSizeBytes [4]byte
if _, err := conn.Read(responseSizeBytes[:]); err != nil {
return fmt.Errorf("read response size: %w", err)
}
responseSize := binary.BigEndian.Uint32(responseSizeBytes[:])
// Read response body
responseBody := make([]byte, responseSize)
if _, err := conn.Read(responseBody); err != nil {
return fmt.Errorf("read response body: %w", err)
}
fmt.Printf("Produce response: %d bytes\n", responseSize)
// Check if the response indicates success (simplified check)
if responseSize > 8 {
// Extract correlation ID and basic error code
@ -225,7 +224,7 @@ func sendProduceRequest(conn net.Conn, topicName string) error {
if correlationResp == correlationID {
fmt.Printf("✅ Produce request correlation ID matches: %d\n", correlationResp)
}
// Look for error codes in the response
if len(responseBody) > 20 {
// Skip to where partition error code should be (rough estimate)
@ -237,31 +236,31 @@ func sendProduceRequest(conn net.Conn, topicName string) error {
}
}
}
return nil
}
func buildSimpleRecordSet(key, value string) []byte {
// Build a very simple Kafka record batch (v0 format for simplicity)
record := make([]byte, 0, 64)
// Record batch header (simplified v0 format)
record = append(record, 0, 0, 0, 0, 0, 0, 0, 0) // base offset
record = append(record, 0, 0, 0, 30) // batch length (estimated)
record = append(record, 0, 0, 0, 0) // partition leader epoch
record = append(record, 0) // magic byte (v0)
record = append(record, 0, 0, 0, 0) // CRC32 (simplified)
record = append(record, 0, 0) // attributes
record = append(record, 0, 0, 0, 1) // record count = 1
record = append(record, 0, 0, 0, 30) // batch length (estimated)
record = append(record, 0, 0, 0, 0) // partition leader epoch
record = append(record, 0) // magic byte (v0)
record = append(record, 0, 0, 0, 0) // CRC32 (simplified)
record = append(record, 0, 0) // attributes
record = append(record, 0, 0, 0, 1) // record count = 1
// Simple record: key_length + key + value_length + value
keyBytes := []byte(key)
valueBytes := []byte(value)
record = append(record, byte(len(keyBytes)>>8), byte(len(keyBytes)))
record = append(record, keyBytes...)
record = append(record, byte(len(valueBytes)>>8), byte(len(valueBytes)))
record = append(record, valueBytes...)
return record
}

25
test/kafka/seaweedmq_integration_test.go
View File

@ -14,14 +14,9 @@ func TestSeaweedMQIntegration_E2E(t *testing.T) {
// Skip by default - requires real SeaweedMQ setup
t.Skip("Integration test requires real SeaweedMQ setup - run manually")
// Test configuration
agentAddress := "localhost:17777" // Default SeaweedMQ Agent address
// Start the gateway with SeaweedMQ backend
gatewayServer := gateway.NewServer(gateway.Options{
Listen: ":0", // random port
AgentAddress: agentAddress,
UseSeaweedMQ: true,
Listen: ":0", // random port
})
err := gatewayServer.Start()
@ -241,8 +236,7 @@ func TestSeaweedMQGateway_ModeSelection(t *testing.T) {
// Test in-memory mode (should always work)
t.Run("InMemoryMode", func(t *testing.T) {
server := gateway.NewServer(gateway.Options{
Listen: ":0",
UseSeaweedMQ: false,
Listen: ":0",
})
err := server.Start()
@ -262,9 +256,7 @@ func TestSeaweedMQGateway_ModeSelection(t *testing.T) {
// Test SeaweedMQ mode with invalid agent (should fall back)
t.Run("SeaweedMQModeFallback", func(t *testing.T) {
server := gateway.NewServer(gateway.Options{
Listen: ":0",
AgentAddress: "invalid:99999", // Invalid address
UseSeaweedMQ: true,
Listen: ":0",
})
err := server.Start()
@ -292,26 +284,21 @@ func TestSeaweedMQGateway_ConfigValidation(t *testing.T) {
{
name: "ValidInMemory",
options: gateway.Options{
Listen: ":0",
UseSeaweedMQ: false,
Listen: ":0",
},
shouldWork: true,
},
{
name: "ValidSeaweedMQWithAgent",
options: gateway.Options{
Listen: ":0",
AgentAddress: "localhost:17777",
UseSeaweedMQ: true,
Listen: ":0",
},
shouldWork: true, // May fail if no agent, but config is valid
},
{
name: "SeaweedMQWithoutAgent",
options: gateway.Options{
Listen: ":0",
UseSeaweedMQ: true,
// AgentAddress is empty
Listen: ":0",
},
shouldWork: true, // Should fall back to in-memory
},

Write Preview
Loading…
Cancel
Save