chrislu
4 months ago
45 changed files with 809 additions and 1054 deletions
-
2test/kafka/api_sequence_test.go
-
6test/kafka/client_integration_test.go
-
4test/kafka/comprehensive_e2e_test.go
-
2test/kafka/connection_close_debug_test.go
-
2test/kafka/connection_debug_test.go
-
2test/kafka/consumer_group_debug_test.go
-
6test/kafka/consumer_group_test.go
-
2test/kafka/consumer_only_test.go
-
2test/kafka/consumer_test.go
-
6test/kafka/debug_connection_test.go
-
2test/kafka/debug_consumer_group_test.go
-
2test/kafka/debug_produce_response_test.go
-
2test/kafka/debug_produce_v7_test.go
-
2test/kafka/debug_readpartitions_test.go
-
6test/kafka/e2e_test.go
-
4test/kafka/gateway_smoke_test.go
-
2test/kafka/joingroup_debug_test.go
-
2test/kafka/kafka_go_debug_test.go
-
2test/kafka/kafka_go_internal_debug_test.go
-
2test/kafka/kafka_go_metadata_test.go
-
22test/kafka/kafka_go_produce_only_test.go
-
2test/kafka/metadata_comparison_test.go
-
2test/kafka/metadata_debug_test.go
-
2test/kafka/metadata_v1_isolation_test.go
-
293test/kafka/mock_smq_handler.go
-
2test/kafka/network_capture_test.go
-
2test/kafka/parsing_debug_test.go
-
2test/kafka/produce_consume_cycle_test.go
-
2test/kafka/produce_consume_test.go
-
2test/kafka/raw_protocol_test.go
-
11test/kafka/sarama_e2e_test.go
-
21test/kafka/sarama_simple_test.go
-
14test/kafka/sarama_test.go
-
8test/kafka/seaweedmq_integration_test.go
-
23weed/mq/kafka/gateway/server.go
-
100weed/mq/kafka/gateway/server_test.go
-
37weed/mq/kafka/gateway/server_testing.go
-
9weed/mq/kafka/integration/seaweedmq_handler.go
-
31weed/mq/kafka/offset/ledger.go
-
4weed/mq/kafka/protocol/consumer_coordination_test.go
-
85weed/mq/kafka/protocol/fetch.go
-
266weed/mq/kafka/protocol/handler.go
-
558weed/mq/kafka/protocol/handler_test.go
-
299weed/mq/kafka/protocol/handler_testing.go
-
6weed/mq/kafka/protocol/offset_management_test.go
@ -0,0 +1,293 @@ |
|||||
|
package kafka |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"sync" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/kafka/offset" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb" |
||||
|
) |
||||
|
|
||||
|
// MockSMQHandler provides a realistic SeaweedMQ simulation for testing
|
||||
|
// It behaves like the real SeaweedMQ integration but uses in-memory storage
|
||||
|
type MockSMQHandler struct { |
||||
|
// Topic management
|
||||
|
topicsMu sync.RWMutex |
||||
|
topics map[string]*MockTopicInfo |
||||
|
|
||||
|
// Message storage - simulates SeaweedMQ's persistent storage
|
||||
|
messagesMu sync.RWMutex |
||||
|
messages map[string]map[int32][]*MockSMQRecord // topic -> partition -> []records
|
||||
|
|
||||
|
// Offset management - simulates Kafka offset ledgers
|
||||
|
ledgersMu sync.RWMutex |
||||
|
ledgers map[string]*offset.Ledger // topic-partition -> ledger
|
||||
|
|
||||
|
// Simulated SMQ timestamp tracking
|
||||
|
lastTimestamp map[string]map[int32]int64 // topic -> partition -> last_timestamp
|
||||
|
} |
||||
|
|
||||
|
// MockTopicInfo represents a Kafka topic in the mock SMQ environment
|
||||
|
type MockTopicInfo struct { |
||||
|
Name string |
||||
|
Partitions int32 |
||||
|
CreatedAt int64 |
||||
|
Schema *schema_pb.Topic |
||||
|
} |
||||
|
|
||||
|
// MockSMQRecord represents a record in the mock SeaweedMQ storage
|
||||
|
type MockSMQRecord struct { |
||||
|
Key []byte |
||||
|
Value []byte |
||||
|
Timestamp int64 // SeaweedMQ timestamp (nanoseconds)
|
||||
|
Offset int64 // Kafka offset for this partition
|
||||
|
} |
||||
|
|
||||
|
// Implement SMQRecord interface
|
||||
|
func (r *MockSMQRecord) GetKey() []byte { return r.Key } |
||||
|
func (r *MockSMQRecord) GetValue() []byte { return r.Value } |
||||
|
func (r *MockSMQRecord) GetTimestamp() int64 { return r.Timestamp } |
||||
|
func (r *MockSMQRecord) GetOffset() int64 { return r.Offset } |
||||
|
|
||||
|
// NewMockSMQHandler creates a new mock SeaweedMQ handler for testing
|
||||
|
func NewMockSMQHandler() *MockSMQHandler { |
||||
|
return &MockSMQHandler{ |
||||
|
topics: make(map[string]*MockTopicInfo), |
||||
|
messages: make(map[string]map[int32][]*MockSMQRecord), |
||||
|
ledgers: make(map[string]*offset.Ledger), |
||||
|
lastTimestamp: make(map[string]map[int32]int64), |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// TopicExists checks if a topic exists in the mock SMQ
|
||||
|
func (m *MockSMQHandler) TopicExists(topic string) bool { |
||||
|
m.topicsMu.RLock() |
||||
|
defer m.topicsMu.RUnlock() |
||||
|
_, exists := m.topics[topic] |
||||
|
return exists |
||||
|
} |
||||
|
|
||||
|
// ListTopics returns all topics in the mock SMQ
|
||||
|
func (m *MockSMQHandler) ListTopics() []string { |
||||
|
m.topicsMu.RLock() |
||||
|
defer m.topicsMu.RUnlock() |
||||
|
|
||||
|
topics := make([]string, 0, len(m.topics)) |
||||
|
for name := range m.topics { |
||||
|
topics = append(topics, name) |
||||
|
} |
||||
|
return topics |
||||
|
} |
||||
|
|
||||
|
// CreateTopic creates a new topic in the mock SMQ
|
||||
|
func (m *MockSMQHandler) CreateTopic(topic string, partitions int32) error { |
||||
|
m.topicsMu.Lock() |
||||
|
defer m.topicsMu.Unlock() |
||||
|
|
||||
|
if _, exists := m.topics[topic]; exists { |
||||
|
return fmt.Errorf("topic %s already exists", topic) |
||||
|
} |
||||
|
|
||||
|
// Create topic info
|
||||
|
m.topics[topic] = &MockTopicInfo{ |
||||
|
Name: topic, |
||||
|
Partitions: partitions, |
||||
|
CreatedAt: time.Now().UnixNano(), |
||||
|
Schema: &schema_pb.Topic{ |
||||
|
Name: topic, |
||||
|
// PartitionCount removed - not part of schema_pb.Topic
|
||||
|
}, |
||||
|
} |
||||
|
|
||||
|
// Initialize message storage for all partitions
|
||||
|
m.messagesMu.Lock() |
||||
|
m.messages[topic] = make(map[int32][]*MockSMQRecord) |
||||
|
for i := int32(0); i < partitions; i++ { |
||||
|
m.messages[topic][i] = make([]*MockSMQRecord, 0, 1000) |
||||
|
} |
||||
|
m.messagesMu.Unlock() |
||||
|
|
||||
|
// Initialize timestamp tracking
|
||||
|
if m.lastTimestamp[topic] == nil { |
||||
|
m.lastTimestamp[topic] = make(map[int32]int64) |
||||
|
} |
||||
|
for i := int32(0); i < partitions; i++ { |
||||
|
m.lastTimestamp[topic][i] = 0 |
||||
|
} |
||||
|
|
||||
|
fmt.Printf("MOCK SMQ: Created topic %s with %d partitions\n", topic, partitions) |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// DeleteTopic removes a topic from the mock SMQ
|
||||
|
func (m *MockSMQHandler) DeleteTopic(topic string) error { |
||||
|
m.topicsMu.Lock() |
||||
|
defer m.topicsMu.Unlock() |
||||
|
|
||||
|
if _, exists := m.topics[topic]; !exists { |
||||
|
return fmt.Errorf("topic %s does not exist", topic) |
||||
|
} |
||||
|
|
||||
|
delete(m.topics, topic) |
||||
|
|
||||
|
// Clean up messages
|
||||
|
m.messagesMu.Lock() |
||||
|
delete(m.messages, topic) |
||||
|
m.messagesMu.Unlock() |
||||
|
|
||||
|
// Clean up ledgers
|
||||
|
m.ledgersMu.Lock() |
||||
|
keysToDelete := make([]string, 0) |
||||
|
for key := range m.ledgers { |
||||
|
if key[:len(topic)+1] == topic+"-" { |
||||
|
keysToDelete = append(keysToDelete, key) |
||||
|
} |
||||
|
} |
||||
|
for _, key := range keysToDelete { |
||||
|
delete(m.ledgers, key) |
||||
|
} |
||||
|
m.ledgersMu.Unlock() |
||||
|
|
||||
|
fmt.Printf("MOCK SMQ: Deleted topic %s\n", topic) |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// GetOrCreateLedger gets or creates a Kafka offset ledger for a topic-partition
|
||||
|
func (m *MockSMQHandler) GetOrCreateLedger(topic string, partition int32) *offset.Ledger { |
||||
|
key := fmt.Sprintf("%s-%d", topic, partition) |
||||
|
|
||||
|
m.ledgersMu.Lock() |
||||
|
defer m.ledgersMu.Unlock() |
||||
|
|
||||
|
if ledger, exists := m.ledgers[key]; exists { |
||||
|
return ledger |
||||
|
} |
||||
|
|
||||
|
// Create new ledger
|
||||
|
ledger := offset.NewLedger() |
||||
|
m.ledgers[key] = ledger |
||||
|
|
||||
|
// Ensure topic is created
|
||||
|
if !m.TopicExists(topic) { |
||||
|
m.CreateTopic(topic, partition+1) // Ensure enough partitions
|
||||
|
} |
||||
|
|
||||
|
return ledger |
||||
|
} |
||||
|
|
||||
|
// GetLedger retrieves an existing ledger or returns nil
|
||||
|
func (m *MockSMQHandler) GetLedger(topic string, partition int32) *offset.Ledger { |
||||
|
key := fmt.Sprintf("%s-%d", topic, partition) |
||||
|
|
||||
|
m.ledgersMu.RLock() |
||||
|
defer m.ledgersMu.RUnlock() |
||||
|
|
||||
|
return m.ledgers[key] |
||||
|
} |
||||
|
|
||||
|
// ProduceRecord publishes a record to the mock SeaweedMQ and updates Kafka offset tracking
|
||||
|
func (m *MockSMQHandler) ProduceRecord(topic string, partition int32, key, value []byte) (int64, error) { |
||||
|
// Verify topic exists
|
||||
|
if !m.TopicExists(topic) { |
||||
|
return 0, fmt.Errorf("topic %s does not exist", topic) |
||||
|
} |
||||
|
|
||||
|
// Get current timestamp (simulate SeaweedMQ timestamp)
|
||||
|
timestamp := time.Now().UnixNano() |
||||
|
|
||||
|
// Get or create Kafka offset ledger
|
||||
|
ledger := m.GetOrCreateLedger(topic, partition) |
||||
|
|
||||
|
// Assign Kafka offset
|
||||
|
kafkaOffset := ledger.AssignOffsets(1) |
||||
|
|
||||
|
// Record in ledger (simulates Kafka offset -> SMQ timestamp mapping)
|
||||
|
messageSize := int32(len(value)) |
||||
|
if err := ledger.AppendRecord(kafkaOffset, timestamp, messageSize); err != nil { |
||||
|
return 0, fmt.Errorf("failed to append to ledger: %w", err) |
||||
|
} |
||||
|
|
||||
|
// Store message in mock SMQ storage (simulates persistent storage)
|
||||
|
m.messagesMu.Lock() |
||||
|
defer m.messagesMu.Unlock() |
||||
|
|
||||
|
if m.messages[topic] == nil { |
||||
|
m.messages[topic] = make(map[int32][]*MockSMQRecord) |
||||
|
} |
||||
|
if m.messages[topic][partition] == nil { |
||||
|
m.messages[topic][partition] = make([]*MockSMQRecord, 0, 1000) |
||||
|
} |
||||
|
|
||||
|
// Create message record (simulate SMQ storage)
|
||||
|
record := &MockSMQRecord{ |
||||
|
Key: append([]byte(nil), key...), // Deep copy key
|
||||
|
Value: append([]byte(nil), value...), // Deep copy value
|
||||
|
Timestamp: timestamp, |
||||
|
Offset: kafkaOffset, |
||||
|
} |
||||
|
|
||||
|
// Append to partition (simulate SMQ append-only log)
|
||||
|
m.messages[topic][partition] = append(m.messages[topic][partition], record) |
||||
|
|
||||
|
// Update last timestamp
|
||||
|
m.lastTimestamp[topic][partition] = timestamp |
||||
|
|
||||
|
fmt.Printf("MOCK SMQ: Stored record - topic:%s, partition:%d, kafka_offset:%d, smq_timestamp:%d, key:%s, value:%s\n", |
||||
|
topic, partition, kafkaOffset, timestamp, string(key), string(value)) |
||||
|
|
||||
|
return kafkaOffset, nil |
||||
|
} |
||||
|
|
||||
|
// GetStoredRecords retrieves records from mock SMQ storage starting from a given Kafka offset
|
||||
|
func (m *MockSMQHandler) GetStoredRecords(topic string, partition int32, fromKafkaOffset int64, maxRecords int) ([]offset.SMQRecord, error) { |
||||
|
m.messagesMu.RLock() |
||||
|
defer m.messagesMu.RUnlock() |
||||
|
|
||||
|
if m.messages[topic] == nil || m.messages[topic][partition] == nil { |
||||
|
return nil, nil // No messages
|
||||
|
} |
||||
|
|
||||
|
records := m.messages[topic][partition] |
||||
|
result := make([]offset.SMQRecord, 0, maxRecords) |
||||
|
|
||||
|
// Find records starting from the given Kafka offset
|
||||
|
for _, record := range records { |
||||
|
if record.Offset >= fromKafkaOffset && len(result) < maxRecords { |
||||
|
result = append(result, record) |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
fmt.Printf("MOCK SMQ: Retrieved %d records from topic:%s, partition:%d, from_offset:%d\n", |
||||
|
len(result), topic, partition, fromKafkaOffset) |
||||
|
|
||||
|
return result, nil |
||||
|
} |
||||
|
|
||||
|
// Close shuts down the mock SMQ handler
|
||||
|
func (m *MockSMQHandler) Close() error { |
||||
|
fmt.Printf("MOCK SMQ: Handler closed\n") |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// GetPartitionCount returns the number of partitions for a topic
|
||||
|
func (m *MockSMQHandler) GetPartitionCount(topic string) int32 { |
||||
|
m.topicsMu.RLock() |
||||
|
defer m.topicsMu.RUnlock() |
||||
|
|
||||
|
if topicInfo, exists := m.topics[topic]; exists { |
||||
|
return topicInfo.Partitions |
||||
|
} |
||||
|
return 0 |
||||
|
} |
||||
|
|
||||
|
// GetMessageCount returns the total number of messages stored for a topic-partition
|
||||
|
func (m *MockSMQHandler) GetMessageCount(topic string, partition int32) int { |
||||
|
m.messagesMu.RLock() |
||||
|
defer m.messagesMu.RUnlock() |
||||
|
|
||||
|
if m.messages[topic] == nil || m.messages[topic][partition] == nil { |
||||
|
return 0 |
||||
|
} |
||||
|
return len(m.messages[topic][partition]) |
||||
|
} |
||||
@ -1,100 +0,0 @@ |
|||||
package gateway |
|
||||
|
|
||||
import ( |
|
||||
"net" |
|
||||
"strings" |
|
||||
"testing" |
|
||||
"time" |
|
||||
) |
|
||||
|
|
||||
func TestServerStartAndClose(t *testing.T) { |
|
||||
// Skip this test as it requires a real SeaweedMQ Agent
|
|
||||
t.Skip("This test requires SeaweedMQ Agent integration - run manually with agent available") |
|
||||
|
|
||||
srv := NewServer(Options{ |
|
||||
Listen: ":0", |
|
||||
Masters: "localhost:9333", // Use masters instead of AgentAddress
|
|
||||
}) |
|
||||
if err := srv.Start(); err != nil { |
|
||||
t.Fatalf("start: %v", err) |
|
||||
} |
|
||||
// ensure listener is open and accepting
|
|
||||
// try to dial the actual chosen port
|
|
||||
// Find the actual address
|
|
||||
var addr string |
|
||||
if srv.ln == nil { |
|
||||
t.Fatalf("listener not set") |
|
||||
} |
|
||||
addr = srv.ln.Addr().String() |
|
||||
c, err := net.DialTimeout("tcp", addr, time.Second) |
|
||||
if err != nil { |
|
||||
t.Fatalf("dial: %v", err) |
|
||||
} |
|
||||
_ = c.Close() |
|
||||
if err := srv.Close(); err != nil { |
|
||||
t.Fatalf("close: %v", err) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
func TestGetListenerAddr(t *testing.T) { |
|
||||
// Skip this test as it requires a real SeaweedMQ Agent
|
|
||||
t.Skip("This test requires SeaweedMQ Agent integration - run manually with agent available") |
|
||||
|
|
||||
// Test with localhost binding - should return the actual address
|
|
||||
srv := NewServer(Options{ |
|
||||
Listen: "127.0.0.1:0", |
|
||||
Masters: "localhost:9333", // Would need real agent for this test
|
|
||||
}) |
|
||||
if err := srv.Start(); err != nil { |
|
||||
t.Fatalf("start: %v", err) |
|
||||
} |
|
||||
defer srv.Close() |
|
||||
|
|
||||
host, port := srv.GetListenerAddr() |
|
||||
if host != "127.0.0.1" { |
|
||||
t.Errorf("expected 127.0.0.1, got %s", host) |
|
||||
} |
|
||||
if port <= 0 { |
|
||||
t.Errorf("expected valid port, got %d", port) |
|
||||
} |
|
||||
|
|
||||
// Test IPv6 all interfaces binding - should resolve to non-loopback IP
|
|
||||
srv6 := NewServer(Options{ |
|
||||
Listen: "[::]:0", |
|
||||
Masters: "localhost:9333", // Would need real agent for this test
|
|
||||
}) |
|
||||
if err := srv6.Start(); err != nil { |
|
||||
t.Fatalf("start IPv6: %v", err) |
|
||||
} |
|
||||
defer srv6.Close() |
|
||||
|
|
||||
host6, port6 := srv6.GetListenerAddr() |
|
||||
// Should not be localhost when bound to all interfaces
|
|
||||
if host6 == "localhost" { |
|
||||
t.Errorf("IPv6 all interfaces should not resolve to localhost, got %s", host6) |
|
||||
} |
|
||||
if port6 <= 0 { |
|
||||
t.Errorf("expected valid port, got %d", port6) |
|
||||
} |
|
||||
t.Logf("IPv6 all interfaces resolved to: %s:%d", host6, port6) |
|
||||
} |
|
||||
|
|
||||
func TestResolveAdvertisedAddress(t *testing.T) { |
|
||||
addr := resolveAdvertisedAddress() |
|
||||
if addr == "" { |
|
||||
t.Error("resolveAdvertisedAddress returned empty string") |
|
||||
} |
|
||||
|
|
||||
// Should be a valid IP address
|
|
||||
ip := net.ParseIP(addr) |
|
||||
if ip == nil { |
|
||||
t.Errorf("resolveAdvertisedAddress returned invalid IP: %s", addr) |
|
||||
} |
|
||||
|
|
||||
// Should not be IPv6 (we prefer IPv4 for Kafka compatibility)
|
|
||||
if strings.Contains(addr, ":") { |
|
||||
t.Errorf("Expected IPv4 address, got IPv6: %s", addr) |
|
||||
} |
|
||||
|
|
||||
t.Logf("Resolved advertised address: %s", addr) |
|
||||
} |
|
||||
@ -0,0 +1,37 @@ |
|||||
|
package gateway |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
|
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/kafka/protocol" |
||||
|
) |
||||
|
|
||||
|
// NewTestServer creates a server for testing with in-memory handlers
|
||||
|
// This should ONLY be used for testing - never in production
|
||||
|
// WARNING: This function includes test-only components in production binary
|
||||
|
func NewTestServer(opts Options) *Server { |
||||
|
ctx, cancel := context.WithCancel(context.Background()) |
||||
|
|
||||
|
// Use test handler with storage capability
|
||||
|
handler := protocol.NewTestHandler() |
||||
|
|
||||
|
return &Server{ |
||||
|
opts: opts, |
||||
|
ctx: ctx, |
||||
|
cancel: cancel, |
||||
|
handler: handler, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// NewTestServerWithHandler creates a test server with a custom handler
|
||||
|
// This allows tests to inject specific handlers for different scenarios
|
||||
|
func NewTestServerWithHandler(opts Options, handler *protocol.Handler) *Server { |
||||
|
ctx, cancel := context.WithCancel(context.Background()) |
||||
|
|
||||
|
return &Server{ |
||||
|
opts: opts, |
||||
|
ctx: ctx, |
||||
|
cancel: cancel, |
||||
|
handler: handler, |
||||
|
} |
||||
|
} |
||||
@ -1,558 +0,0 @@ |
|||||
package protocol |
|
||||
|
|
||||
import ( |
|
||||
"context" |
|
||||
"encoding/binary" |
|
||||
"net" |
|
||||
"testing" |
|
||||
"time" |
|
||||
) |
|
||||
|
|
||||
func TestHandler_ApiVersions(t *testing.T) { |
|
||||
// Create handler
|
|
||||
h := NewTestHandler() |
|
||||
|
|
||||
// Create in-memory connection
|
|
||||
server, client := net.Pipe() |
|
||||
defer server.Close() |
|
||||
defer client.Close() |
|
||||
|
|
||||
// Handle connection in background
|
|
||||
done := make(chan error, 1) |
|
||||
go func() { |
|
||||
done <- h.HandleConn(context.Background(), server) |
|
||||
}() |
|
||||
|
|
||||
// Create ApiVersions request manually
|
|
||||
// Request format: api_key(2) + api_version(2) + correlation_id(4) + client_id_size(2) + client_id + body
|
|
||||
correlationID := uint32(12345) |
|
||||
clientID := "test-client" |
|
||||
|
|
||||
message := make([]byte, 0, 64) |
|
||||
message = append(message, 0, 18) // API key 18 (ApiVersions)
|
|
||||
message = append(message, 0, 0) // API version 0
|
|
||||
|
|
||||
// Correlation ID
|
|
||||
correlationIDBytes := make([]byte, 4) |
|
||||
binary.BigEndian.PutUint32(correlationIDBytes, correlationID) |
|
||||
message = append(message, correlationIDBytes...) |
|
||||
|
|
||||
// Client ID length and string
|
|
||||
clientIDLen := uint16(len(clientID)) |
|
||||
message = append(message, byte(clientIDLen>>8), byte(clientIDLen)) |
|
||||
message = append(message, []byte(clientID)...) |
|
||||
|
|
||||
// Empty request body for ApiVersions
|
|
||||
|
|
||||
// Write message size and data
|
|
||||
messageSize := uint32(len(message)) |
|
||||
sizeBuf := make([]byte, 4) |
|
||||
binary.BigEndian.PutUint32(sizeBuf, messageSize) |
|
||||
|
|
||||
if _, err := client.Write(sizeBuf); err != nil { |
|
||||
t.Fatalf("write size: %v", err) |
|
||||
} |
|
||||
if _, err := client.Write(message); err != nil { |
|
||||
t.Fatalf("write message: %v", err) |
|
||||
} |
|
||||
|
|
||||
// Read response size
|
|
||||
var respSizeBuf [4]byte |
|
||||
client.SetReadDeadline(time.Now().Add(5 * time.Second)) |
|
||||
if _, err := client.Read(respSizeBuf[:]); err != nil { |
|
||||
t.Fatalf("read response size: %v", err) |
|
||||
} |
|
||||
|
|
||||
respSize := binary.BigEndian.Uint32(respSizeBuf[:]) |
|
||||
if respSize == 0 || respSize > 1024*1024 { |
|
||||
t.Fatalf("invalid response size: %d", respSize) |
|
||||
} |
|
||||
|
|
||||
// Read response data
|
|
||||
respBuf := make([]byte, respSize) |
|
||||
if _, err := client.Read(respBuf); err != nil { |
|
||||
t.Fatalf("read response: %v", err) |
|
||||
} |
|
||||
|
|
||||
// Parse response: correlation_id(4) + error_code(2) + num_api_keys(4) + api_keys + throttle_time(4)
|
|
||||
if len(respBuf) < 14 { // minimum response size
|
|
||||
t.Fatalf("response too short: %d bytes", len(respBuf)) |
|
||||
} |
|
||||
|
|
||||
// Check correlation ID
|
|
||||
respCorrelationID := binary.BigEndian.Uint32(respBuf[0:4]) |
|
||||
if respCorrelationID != correlationID { |
|
||||
t.Errorf("correlation ID mismatch: got %d, want %d", respCorrelationID, correlationID) |
|
||||
} |
|
||||
|
|
||||
// Check error code
|
|
||||
errorCode := binary.BigEndian.Uint16(respBuf[4:6]) |
|
||||
if errorCode != 0 { |
|
||||
t.Errorf("expected no error, got error code: %d", errorCode) |
|
||||
} |
|
||||
|
|
||||
// Check number of API keys
|
|
||||
numAPIKeys := binary.BigEndian.Uint32(respBuf[6:10]) |
|
||||
if numAPIKeys != 14 { |
|
||||
t.Errorf("expected 14 API keys, got: %d", numAPIKeys) |
|
||||
} |
|
||||
|
|
||||
// Check API key details: api_key(2) + min_version(2) + max_version(2)
|
|
||||
if len(respBuf) < 52 { // need space for 7 API keys
|
|
||||
t.Fatalf("response too short for API key data") |
|
||||
} |
|
||||
|
|
||||
// First API key (ApiVersions)
|
|
||||
apiKey := binary.BigEndian.Uint16(respBuf[10:12]) |
|
||||
minVersion := binary.BigEndian.Uint16(respBuf[12:14]) |
|
||||
maxVersion := binary.BigEndian.Uint16(respBuf[14:16]) |
|
||||
|
|
||||
if apiKey != 18 { |
|
||||
t.Errorf("expected API key 18, got: %d", apiKey) |
|
||||
} |
|
||||
if minVersion != 0 { |
|
||||
t.Errorf("expected min version 0, got: %d", minVersion) |
|
||||
} |
|
||||
if maxVersion != 3 { |
|
||||
t.Errorf("expected max version 3, got: %d", maxVersion) |
|
||||
} |
|
||||
|
|
||||
// Second API key (Metadata)
|
|
||||
apiKey2 := binary.BigEndian.Uint16(respBuf[16:18]) |
|
||||
minVersion2 := binary.BigEndian.Uint16(respBuf[18:20]) |
|
||||
maxVersion2 := binary.BigEndian.Uint16(respBuf[20:22]) |
|
||||
|
|
||||
if apiKey2 != 3 { |
|
||||
t.Errorf("expected API key 3, got: %d", apiKey2) |
|
||||
} |
|
||||
if minVersion2 != 0 { |
|
||||
t.Errorf("expected min version 0, got: %d", minVersion2) |
|
||||
} |
|
||||
if maxVersion2 != 7 { |
|
||||
t.Errorf("expected max version 7, got: %d", maxVersion2) |
|
||||
} |
|
||||
|
|
||||
// Third API key (ListOffsets)
|
|
||||
apiKey3 := binary.BigEndian.Uint16(respBuf[22:24]) |
|
||||
minVersion3 := binary.BigEndian.Uint16(respBuf[24:26]) |
|
||||
maxVersion3 := binary.BigEndian.Uint16(respBuf[26:28]) |
|
||||
|
|
||||
if apiKey3 != 2 { |
|
||||
t.Errorf("expected API key 2, got: %d", apiKey3) |
|
||||
} |
|
||||
if minVersion3 != 0 { |
|
||||
t.Errorf("expected min version 0, got: %d", minVersion3) |
|
||||
} |
|
||||
if maxVersion3 != 2 { |
|
||||
t.Errorf("expected max version 2, got: %d", maxVersion3) |
|
||||
} |
|
||||
|
|
||||
// Fourth API key (CreateTopics)
|
|
||||
apiKey4 := binary.BigEndian.Uint16(respBuf[28:30]) |
|
||||
minVersion4 := binary.BigEndian.Uint16(respBuf[30:32]) |
|
||||
maxVersion4 := binary.BigEndian.Uint16(respBuf[32:34]) |
|
||||
|
|
||||
if apiKey4 != 19 { |
|
||||
t.Errorf("expected API key 19, got: %d", apiKey4) |
|
||||
} |
|
||||
if minVersion4 != 0 { |
|
||||
t.Errorf("expected min version 0, got: %d", minVersion4) |
|
||||
} |
|
||||
if maxVersion4 != 4 { |
|
||||
t.Errorf("expected max version 4, got: %d", maxVersion4) |
|
||||
} |
|
||||
|
|
||||
// Fifth API key (DeleteTopics)
|
|
||||
apiKey5 := binary.BigEndian.Uint16(respBuf[34:36]) |
|
||||
minVersion5 := binary.BigEndian.Uint16(respBuf[36:38]) |
|
||||
maxVersion5 := binary.BigEndian.Uint16(respBuf[38:40]) |
|
||||
|
|
||||
if apiKey5 != 20 { |
|
||||
t.Errorf("expected API key 20, got: %d", apiKey5) |
|
||||
} |
|
||||
if minVersion5 != 0 { |
|
||||
t.Errorf("expected min version 0, got: %d", minVersion5) |
|
||||
} |
|
||||
if maxVersion5 != 4 { |
|
||||
t.Errorf("expected max version 4, got: %d", maxVersion5) |
|
||||
} |
|
||||
|
|
||||
// Sixth API key (Produce)
|
|
||||
apiKey6 := binary.BigEndian.Uint16(respBuf[40:42]) |
|
||||
minVersion6 := binary.BigEndian.Uint16(respBuf[42:44]) |
|
||||
maxVersion6 := binary.BigEndian.Uint16(respBuf[44:46]) |
|
||||
|
|
||||
if apiKey6 != 0 { |
|
||||
t.Errorf("expected API key 0, got: %d", apiKey6) |
|
||||
} |
|
||||
if minVersion6 != 0 { |
|
||||
t.Errorf("expected min version 0, got: %d", minVersion6) |
|
||||
} |
|
||||
if maxVersion6 != 7 { |
|
||||
t.Errorf("expected max version 7, got: %d", maxVersion6) |
|
||||
} |
|
||||
|
|
||||
// Seventh API key (Fetch)
|
|
||||
apiKey7 := binary.BigEndian.Uint16(respBuf[46:48]) |
|
||||
minVersion7 := binary.BigEndian.Uint16(respBuf[48:50]) |
|
||||
maxVersion7 := binary.BigEndian.Uint16(respBuf[50:52]) |
|
||||
|
|
||||
if apiKey7 != 1 { |
|
||||
t.Errorf("expected API key 1, got: %d", apiKey7) |
|
||||
} |
|
||||
if minVersion7 != 0 { |
|
||||
t.Errorf("expected min version 0, got: %d", minVersion7) |
|
||||
} |
|
||||
if maxVersion7 != 7 { |
|
||||
t.Errorf("expected max version 7, got: %d", maxVersion7) |
|
||||
} |
|
||||
|
|
||||
// Close client to end handler
|
|
||||
client.Close() |
|
||||
|
|
||||
// Wait for handler to complete
|
|
||||
select { |
|
||||
case err := <-done: |
|
||||
if err != nil { |
|
||||
t.Errorf("handler error: %v", err) |
|
||||
} |
|
||||
case <-time.After(2 * time.Second): |
|
||||
t.Errorf("handler did not complete in time") |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
func TestHandler_handleApiVersions(t *testing.T) { |
|
||||
h := NewTestHandler() |
|
||||
correlationID := uint32(999) |
|
||||
|
|
||||
response, err := h.handleApiVersions(correlationID) |
|
||||
if err != nil { |
|
||||
t.Fatalf("handleApiVersions: %v", err) |
|
||||
} |
|
||||
|
|
||||
if len(response) < 90 { // minimum expected size (now has 13 API keys)
|
|
||||
t.Fatalf("response too short: %d bytes", len(response)) |
|
||||
} |
|
||||
|
|
||||
// Check correlation ID
|
|
||||
respCorrelationID := binary.BigEndian.Uint32(response[0:4]) |
|
||||
if respCorrelationID != correlationID { |
|
||||
t.Errorf("correlation ID: got %d, want %d", respCorrelationID, correlationID) |
|
||||
} |
|
||||
|
|
||||
// Check error code
|
|
||||
errorCode := binary.BigEndian.Uint16(response[4:6]) |
|
||||
if errorCode != 0 { |
|
||||
t.Errorf("error code: got %d, want 0", errorCode) |
|
||||
} |
|
||||
|
|
||||
// Check number of API keys
|
|
||||
numAPIKeys := binary.BigEndian.Uint32(response[6:10]) |
|
||||
if numAPIKeys != 14 { |
|
||||
t.Errorf("expected 14 API keys, got: %d", numAPIKeys) |
|
||||
} |
|
||||
|
|
||||
// Check first API key (ApiVersions)
|
|
||||
apiKey := binary.BigEndian.Uint16(response[10:12]) |
|
||||
if apiKey != 18 { |
|
||||
t.Errorf("first API key: got %d, want 18", apiKey) |
|
||||
} |
|
||||
|
|
||||
// Check second API key (Metadata)
|
|
||||
apiKey2 := binary.BigEndian.Uint16(response[16:18]) |
|
||||
if apiKey2 != 3 { |
|
||||
t.Errorf("second API key: got %d, want 3", apiKey2) |
|
||||
} |
|
||||
|
|
||||
// Check third API key (ListOffsets)
|
|
||||
apiKey3 := binary.BigEndian.Uint16(response[22:24]) |
|
||||
if apiKey3 != 2 { |
|
||||
t.Errorf("third API key: got %d, want 2", apiKey3) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
func TestHandler_handleMetadata(t *testing.T) { |
|
||||
h := NewTestHandler() |
|
||||
correlationID := uint32(456) |
|
||||
|
|
||||
// Empty request body for minimal test
|
|
||||
requestBody := []byte{} |
|
||||
|
|
||||
response, err := h.handleMetadata(correlationID, 0, requestBody) |
|
||||
if err != nil { |
|
||||
t.Fatalf("handleMetadata: %v", err) |
|
||||
} |
|
||||
|
|
||||
if len(response) < 31 { // minimum expected size for v0 (calculated)
|
|
||||
t.Fatalf("response too short: %d bytes", len(response)) |
|
||||
} |
|
||||
|
|
||||
// Check correlation ID
|
|
||||
respCorrelationID := binary.BigEndian.Uint32(response[0:4]) |
|
||||
if respCorrelationID != correlationID { |
|
||||
t.Errorf("correlation ID: got %d, want %d", respCorrelationID, correlationID) |
|
||||
} |
|
||||
|
|
||||
// Check brokers count
|
|
||||
brokersCount := binary.BigEndian.Uint32(response[4:8]) |
|
||||
if brokersCount != 1 { |
|
||||
t.Errorf("brokers count: got %d, want 1", brokersCount) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
func TestHandler_handleListOffsets(t *testing.T) { |
|
||||
h := NewTestHandler() |
|
||||
correlationID := uint32(123) |
|
||||
|
|
||||
// Build a simple ListOffsets v0 request body (header stripped): topics
|
|
||||
// topics_count(4) + topic + partitions
|
|
||||
topic := "test-topic" |
|
||||
|
|
||||
requestBody := make([]byte, 0, 64) |
|
||||
|
|
||||
// Topics count (1)
|
|
||||
requestBody = append(requestBody, 0, 0, 0, 1) |
|
||||
|
|
||||
// Topic name
|
|
||||
requestBody = append(requestBody, 0, byte(len(topic))) |
|
||||
requestBody = append(requestBody, []byte(topic)...) |
|
||||
|
|
||||
// Partitions count (2 partitions)
|
|
||||
requestBody = append(requestBody, 0, 0, 0, 2) |
|
||||
|
|
||||
// Partition 0: partition_id(4) + timestamp(8) - earliest
|
|
||||
requestBody = append(requestBody, 0, 0, 0, 0) // partition 0
|
|
||||
requestBody = append(requestBody, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE) // -2 (earliest)
|
|
||||
|
|
||||
// Partition 1: partition_id(4) + timestamp(8) - latest
|
|
||||
requestBody = append(requestBody, 0, 0, 0, 1) // partition 1
|
|
||||
requestBody = append(requestBody, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF) // -1 (latest)
|
|
||||
|
|
||||
response, err := h.handleListOffsets(correlationID, 0, requestBody) |
|
||||
if err != nil { |
|
||||
t.Fatalf("handleListOffsets: %v", err) |
|
||||
} |
|
||||
|
|
||||
if len(response) < 20 { // minimum expected size
|
|
||||
t.Fatalf("response too short: %d bytes", len(response)) |
|
||||
} |
|
||||
|
|
||||
// Check correlation ID
|
|
||||
respCorrelationID := binary.BigEndian.Uint32(response[0:4]) |
|
||||
if respCorrelationID != correlationID { |
|
||||
t.Errorf("correlation ID: got %d, want %d", respCorrelationID, correlationID) |
|
||||
} |
|
||||
|
|
||||
// For v0, throttle time is not present; topics count is next
|
|
||||
topicsCount := binary.BigEndian.Uint32(response[4:8]) |
|
||||
if topicsCount != 1 { |
|
||||
t.Errorf("topics count: got %d, want 1", topicsCount) |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
func TestHandler_ListOffsets_EndToEnd(t *testing.T) { |
|
||||
// Create handler
|
|
||||
h := NewTestHandler() |
|
||||
|
|
||||
// Create in-memory connection
|
|
||||
server, client := net.Pipe() |
|
||||
defer server.Close() |
|
||||
defer client.Close() |
|
||||
|
|
||||
// Handle connection in background
|
|
||||
done := make(chan error, 1) |
|
||||
go func() { |
|
||||
done <- h.HandleConn(context.Background(), server) |
|
||||
}() |
|
||||
|
|
||||
// Create ListOffsets request
|
|
||||
correlationID := uint32(555) |
|
||||
clientID := "listoffsets-test" |
|
||||
topic := "my-topic" |
|
||||
|
|
||||
message := make([]byte, 0, 128) |
|
||||
message = append(message, 0, 2) // API key 2 (ListOffsets)
|
|
||||
message = append(message, 0, 0) // API version 0
|
|
||||
|
|
||||
// Correlation ID
|
|
||||
correlationIDBytes := make([]byte, 4) |
|
||||
binary.BigEndian.PutUint32(correlationIDBytes, correlationID) |
|
||||
message = append(message, correlationIDBytes...) |
|
||||
|
|
||||
// Client ID length and string
|
|
||||
clientIDLen := uint16(len(clientID)) |
|
||||
message = append(message, byte(clientIDLen>>8), byte(clientIDLen)) |
|
||||
message = append(message, []byte(clientID)...) |
|
||||
|
|
||||
// Topics count (1)
|
|
||||
message = append(message, 0, 0, 0, 1) |
|
||||
|
|
||||
// Topic name
|
|
||||
topicLen := uint16(len(topic)) |
|
||||
message = append(message, byte(topicLen>>8), byte(topicLen)) |
|
||||
message = append(message, []byte(topic)...) |
|
||||
|
|
||||
// Partitions count (1)
|
|
||||
message = append(message, 0, 0, 0, 1) |
|
||||
|
|
||||
// Partition 0 requesting earliest offset
|
|
||||
message = append(message, 0, 0, 0, 0) // partition 0
|
|
||||
message = append(message, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE) // -2 (earliest)
|
|
||||
|
|
||||
// Write message size and data
|
|
||||
messageSize := uint32(len(message)) |
|
||||
sizeBuf := make([]byte, 4) |
|
||||
binary.BigEndian.PutUint32(sizeBuf, messageSize) |
|
||||
|
|
||||
if _, err := client.Write(sizeBuf); err != nil { |
|
||||
t.Fatalf("write size: %v", err) |
|
||||
} |
|
||||
if _, err := client.Write(message); err != nil { |
|
||||
t.Fatalf("write message: %v", err) |
|
||||
} |
|
||||
|
|
||||
// Read response size
|
|
||||
var respSizeBuf [4]byte |
|
||||
client.SetReadDeadline(time.Now().Add(5 * time.Second)) |
|
||||
if _, err := client.Read(respSizeBuf[:]); err != nil { |
|
||||
t.Fatalf("read response size: %v", err) |
|
||||
} |
|
||||
|
|
||||
respSize := binary.BigEndian.Uint32(respSizeBuf[:]) |
|
||||
if respSize == 0 || respSize > 1024*1024 { |
|
||||
t.Fatalf("invalid response size: %d", respSize) |
|
||||
} |
|
||||
|
|
||||
// Read response data
|
|
||||
respBuf := make([]byte, respSize) |
|
||||
if _, err := client.Read(respBuf); err != nil { |
|
||||
t.Fatalf("read response: %v", err) |
|
||||
} |
|
||||
|
|
||||
// Parse response: correlation_id(4) + topics
|
|
||||
if len(respBuf) < 20 { // minimum response size
|
|
||||
t.Fatalf("response too short: %d bytes", len(respBuf)) |
|
||||
} |
|
||||
|
|
||||
// Check correlation ID
|
|
||||
respCorrelationID := binary.BigEndian.Uint32(respBuf[0:4]) |
|
||||
if respCorrelationID != correlationID { |
|
||||
t.Errorf("correlation ID mismatch: got %d, want %d", respCorrelationID, correlationID) |
|
||||
} |
|
||||
|
|
||||
// Check topics count for v0 (no throttle time in v0)
|
|
||||
topicsCount := binary.BigEndian.Uint32(respBuf[4:8]) |
|
||||
if topicsCount != 1 { |
|
||||
t.Errorf("expected 1 topic, got: %d", topicsCount) |
|
||||
} |
|
||||
|
|
||||
// Close client to end handler
|
|
||||
client.Close() |
|
||||
|
|
||||
// Wait for handler to complete
|
|
||||
select { |
|
||||
case err := <-done: |
|
||||
if err != nil { |
|
||||
t.Errorf("handler error: %v", err) |
|
||||
} |
|
||||
case <-time.After(2 * time.Second): |
|
||||
t.Errorf("handler did not complete in time") |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
func TestHandler_Metadata_EndToEnd(t *testing.T) { |
|
||||
// Create handler
|
|
||||
h := NewTestHandler() |
|
||||
|
|
||||
// Create in-memory connection
|
|
||||
server, client := net.Pipe() |
|
||||
defer server.Close() |
|
||||
defer client.Close() |
|
||||
|
|
||||
// Handle connection in background
|
|
||||
done := make(chan error, 1) |
|
||||
go func() { |
|
||||
done <- h.HandleConn(context.Background(), server) |
|
||||
}() |
|
||||
|
|
||||
// Create Metadata request
|
|
||||
correlationID := uint32(789) |
|
||||
clientID := "metadata-test" |
|
||||
|
|
||||
message := make([]byte, 0, 64) |
|
||||
message = append(message, 0, 3) // API key 3 (Metadata)
|
|
||||
message = append(message, 0, 0) // API version 0
|
|
||||
|
|
||||
// Correlation ID
|
|
||||
correlationIDBytes := make([]byte, 4) |
|
||||
binary.BigEndian.PutUint32(correlationIDBytes, correlationID) |
|
||||
message = append(message, correlationIDBytes...) |
|
||||
|
|
||||
// Client ID length and string
|
|
||||
clientIDLen := uint16(len(clientID)) |
|
||||
message = append(message, byte(clientIDLen>>8), byte(clientIDLen)) |
|
||||
message = append(message, []byte(clientID)...) |
|
||||
|
|
||||
// Empty request body (all topics)
|
|
||||
message = append(message, 0xFF, 0xFF, 0xFF, 0xFF) // -1 = all topics
|
|
||||
|
|
||||
// Write message size and data
|
|
||||
messageSize := uint32(len(message)) |
|
||||
sizeBuf := make([]byte, 4) |
|
||||
binary.BigEndian.PutUint32(sizeBuf, messageSize) |
|
||||
|
|
||||
if _, err := client.Write(sizeBuf); err != nil { |
|
||||
t.Fatalf("write size: %v", err) |
|
||||
} |
|
||||
if _, err := client.Write(message); err != nil { |
|
||||
t.Fatalf("write message: %v", err) |
|
||||
} |
|
||||
|
|
||||
// Read response size
|
|
||||
var respSizeBuf [4]byte |
|
||||
client.SetReadDeadline(time.Now().Add(5 * time.Second)) |
|
||||
if _, err := client.Read(respSizeBuf[:]); err != nil { |
|
||||
t.Fatalf("read response size: %v", err) |
|
||||
} |
|
||||
|
|
||||
respSize := binary.BigEndian.Uint32(respSizeBuf[:]) |
|
||||
if respSize == 0 || respSize > 1024*1024 { |
|
||||
t.Fatalf("invalid response size: %d", respSize) |
|
||||
} |
|
||||
|
|
||||
// Read response data
|
|
||||
respBuf := make([]byte, respSize) |
|
||||
if _, err := client.Read(respBuf); err != nil { |
|
||||
t.Fatalf("read response: %v", err) |
|
||||
} |
|
||||
|
|
||||
// Parse response: correlation_id(4) + brokers + topics (v0 has no throttle time)
|
|
||||
if len(respBuf) < 31 { // minimum response size for v0
|
|
||||
t.Fatalf("response too short: %d bytes", len(respBuf)) |
|
||||
} |
|
||||
|
|
||||
// Check correlation ID
|
|
||||
respCorrelationID := binary.BigEndian.Uint32(respBuf[0:4]) |
|
||||
if respCorrelationID != correlationID { |
|
||||
t.Errorf("correlation ID mismatch: got %d, want %d", respCorrelationID, correlationID) |
|
||||
} |
|
||||
|
|
||||
// Check brokers count (immediately after correlation ID in v0)
|
|
||||
brokersCount := binary.BigEndian.Uint32(respBuf[4:8]) |
|
||||
if brokersCount != 1 { |
|
||||
t.Errorf("expected 1 broker, got: %d", brokersCount) |
|
||||
} |
|
||||
|
|
||||
// Close client to end handler
|
|
||||
client.Close() |
|
||||
|
|
||||
// Wait for handler to complete
|
|
||||
select { |
|
||||
case err := <-done: |
|
||||
if err != nil { |
|
||||
t.Errorf("handler error: %v", err) |
|
||||
} |
|
||||
case <-time.After(2 * time.Second): |
|
||||
t.Errorf("handler did not complete in time") |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,299 @@ |
|||||
|
package protocol |
||||
|
|
||||
|
import ( |
||||
|
"fmt" |
||||
|
"sync" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/kafka/consumer" |
||||
|
"github.com/seaweedfs/seaweedfs/weed/mq/kafka/offset" |
||||
|
) |
||||
|
|
||||
|
// MessageRecord represents a stored message (TEST ONLY)
|
||||
|
type MessageRecord struct { |
||||
|
Key []byte |
||||
|
Value []byte |
||||
|
Timestamp int64 |
||||
|
} |
||||
|
|
||||
|
// basicSeaweedMQHandler is a minimal in-memory implementation for testing (TEST ONLY)
|
||||
|
type basicSeaweedMQHandler struct { |
||||
|
topics map[string]bool |
||||
|
ledgers map[string]*offset.Ledger |
||||
|
// messages stores actual message content indexed by topic-partition-offset
|
||||
|
messages map[string]map[int32]map[int64]*MessageRecord // topic -> partition -> offset -> message
|
||||
|
mu sync.RWMutex |
||||
|
} |
||||
|
|
||||
|
// testSeaweedMQHandler is a minimal mock implementation for testing (TEST ONLY)
|
||||
|
type testSeaweedMQHandler struct { |
||||
|
topics map[string]bool |
||||
|
ledgers map[string]*offset.Ledger |
||||
|
mu sync.RWMutex |
||||
|
} |
||||
|
|
||||
|
// NewTestHandler creates a handler for testing purposes without requiring SeaweedMQ masters
|
||||
|
// This should ONLY be used in tests - uses basicSeaweedMQHandler for message storage simulation
|
||||
|
func NewTestHandler() *Handler { |
||||
|
return &Handler{ |
||||
|
groupCoordinator: consumer.NewGroupCoordinator(), |
||||
|
brokerHost: "localhost", |
||||
|
brokerPort: 9092, |
||||
|
seaweedMQHandler: &basicSeaweedMQHandler{ |
||||
|
topics: make(map[string]bool), |
||||
|
ledgers: make(map[string]*offset.Ledger), |
||||
|
messages: make(map[string]map[int32]map[int64]*MessageRecord), |
||||
|
}, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// NewSimpleTestHandler creates a minimal test handler without message storage
|
||||
|
// This should ONLY be used for basic protocol tests that don't need message content
|
||||
|
func NewSimpleTestHandler() *Handler { |
||||
|
return &Handler{ |
||||
|
groupCoordinator: consumer.NewGroupCoordinator(), |
||||
|
brokerHost: "localhost", |
||||
|
brokerPort: 9092, |
||||
|
seaweedMQHandler: &testSeaweedMQHandler{ |
||||
|
topics: make(map[string]bool), |
||||
|
ledgers: make(map[string]*offset.Ledger), |
||||
|
}, |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// ===== basicSeaweedMQHandler implementation (TEST ONLY) =====
|
||||
|
|
||||
|
func (b *basicSeaweedMQHandler) TopicExists(topic string) bool { |
||||
|
return b.topics[topic] |
||||
|
} |
||||
|
|
||||
|
func (b *basicSeaweedMQHandler) ListTopics() []string { |
||||
|
topics := make([]string, 0, len(b.topics)) |
||||
|
for topic := range b.topics { |
||||
|
topics = append(topics, topic) |
||||
|
} |
||||
|
return topics |
||||
|
} |
||||
|
|
||||
|
func (b *basicSeaweedMQHandler) CreateTopic(topic string, partitions int32) error { |
||||
|
b.topics[topic] = true |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (b *basicSeaweedMQHandler) DeleteTopic(topic string) error { |
||||
|
delete(b.topics, topic) |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (b *basicSeaweedMQHandler) GetOrCreateLedger(topic string, partition int32) *offset.Ledger { |
||||
|
b.mu.Lock() |
||||
|
defer b.mu.Unlock() |
||||
|
|
||||
|
key := fmt.Sprintf("%s-%d", topic, partition) |
||||
|
if ledger, exists := b.ledgers[key]; exists { |
||||
|
return ledger |
||||
|
} |
||||
|
|
||||
|
// Create new ledger
|
||||
|
ledger := offset.NewLedger() |
||||
|
b.ledgers[key] = ledger |
||||
|
|
||||
|
// Also create the topic if it doesn't exist
|
||||
|
b.topics[topic] = true |
||||
|
|
||||
|
return ledger |
||||
|
} |
||||
|
|
||||
|
func (b *basicSeaweedMQHandler) GetLedger(topic string, partition int32) *offset.Ledger { |
||||
|
b.mu.RLock() |
||||
|
defer b.mu.RUnlock() |
||||
|
|
||||
|
key := fmt.Sprintf("%s-%d", topic, partition) |
||||
|
if ledger, exists := b.ledgers[key]; exists { |
||||
|
return ledger |
||||
|
} |
||||
|
|
||||
|
// Return nil if ledger doesn't exist (topic doesn't exist)
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (b *basicSeaweedMQHandler) ProduceRecord(topicName string, partitionID int32, key, value []byte) (int64, error) { |
||||
|
// Get or create the ledger first (this will acquire and release the lock)
|
||||
|
ledger := b.GetOrCreateLedger(topicName, partitionID) |
||||
|
|
||||
|
// Now acquire the lock for the rest of the operation
|
||||
|
b.mu.Lock() |
||||
|
defer b.mu.Unlock() |
||||
|
|
||||
|
// Assign an offset and append the record
|
||||
|
offset := ledger.AssignOffsets(1) |
||||
|
timestamp := time.Now().UnixNano() |
||||
|
size := int32(len(value)) |
||||
|
|
||||
|
if err := ledger.AppendRecord(offset, timestamp, size); err != nil { |
||||
|
return 0, fmt.Errorf("failed to append record: %w", err) |
||||
|
} |
||||
|
|
||||
|
// Store the actual message content
|
||||
|
if b.messages[topicName] == nil { |
||||
|
b.messages[topicName] = make(map[int32]map[int64]*MessageRecord) |
||||
|
} |
||||
|
if b.messages[topicName][partitionID] == nil { |
||||
|
b.messages[topicName][partitionID] = make(map[int64]*MessageRecord) |
||||
|
} |
||||
|
|
||||
|
// Make copies of key and value to avoid referencing the original slices
|
||||
|
keyCopy := make([]byte, len(key)) |
||||
|
copy(keyCopy, key) |
||||
|
valueCopy := make([]byte, len(value)) |
||||
|
copy(valueCopy, value) |
||||
|
|
||||
|
b.messages[topicName][partitionID][offset] = &MessageRecord{ |
||||
|
Key: keyCopy, |
||||
|
Value: valueCopy, |
||||
|
Timestamp: timestamp, |
||||
|
} |
||||
|
|
||||
|
return offset, nil |
||||
|
} |
||||
|
|
||||
|
// GetStoredMessages retrieves stored messages for a topic-partition from a given offset (TEST ONLY)
|
||||
|
func (b *basicSeaweedMQHandler) GetStoredMessages(topicName string, partitionID int32, fromOffset int64, maxMessages int) []*MessageRecord { |
||||
|
b.mu.RLock() |
||||
|
defer b.mu.RUnlock() |
||||
|
|
||||
|
if b.messages[topicName] == nil || b.messages[topicName][partitionID] == nil { |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
partitionMessages := b.messages[topicName][partitionID] |
||||
|
var result []*MessageRecord |
||||
|
|
||||
|
// Collect messages starting from fromOffset
|
||||
|
for offset := fromOffset; offset < fromOffset+int64(maxMessages); offset++ { |
||||
|
if msg, exists := partitionMessages[offset]; exists { |
||||
|
result = append(result, msg) |
||||
|
} else { |
||||
|
// No more consecutive messages
|
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return result |
||||
|
} |
||||
|
|
||||
|
// BasicSMQRecord implements SMQRecord interface for basicSeaweedMQHandler (TEST ONLY)
|
||||
|
type BasicSMQRecord struct { |
||||
|
*MessageRecord |
||||
|
offset int64 |
||||
|
} |
||||
|
|
||||
|
func (r *BasicSMQRecord) GetKey() []byte { return r.Key } |
||||
|
func (r *BasicSMQRecord) GetValue() []byte { return r.Value } |
||||
|
func (r *BasicSMQRecord) GetTimestamp() int64 { return r.Timestamp } |
||||
|
func (r *BasicSMQRecord) GetOffset() int64 { return r.offset } |
||||
|
|
||||
|
// GetStoredRecords retrieves stored message records for basicSeaweedMQHandler (TEST ONLY)
|
||||
|
func (b *basicSeaweedMQHandler) GetStoredRecords(topic string, partition int32, fromOffset int64, maxRecords int) ([]offset.SMQRecord, error) { |
||||
|
messages := b.GetStoredMessages(topic, partition, fromOffset, maxRecords) |
||||
|
if len(messages) == 0 { |
||||
|
return nil, nil |
||||
|
} |
||||
|
|
||||
|
records := make([]offset.SMQRecord, len(messages)) |
||||
|
for i, msg := range messages { |
||||
|
records[i] = &BasicSMQRecord{ |
||||
|
MessageRecord: msg, |
||||
|
offset: fromOffset + int64(i), |
||||
|
} |
||||
|
} |
||||
|
return records, nil |
||||
|
} |
||||
|
|
||||
|
func (b *basicSeaweedMQHandler) Close() error { |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// ===== testSeaweedMQHandler implementation (TEST ONLY) =====
|
||||
|
|
||||
|
func (t *testSeaweedMQHandler) TopicExists(topic string) bool { |
||||
|
return t.topics[topic] |
||||
|
} |
||||
|
|
||||
|
func (t *testSeaweedMQHandler) ListTopics() []string { |
||||
|
var topics []string |
||||
|
for topic := range t.topics { |
||||
|
topics = append(topics, topic) |
||||
|
} |
||||
|
return topics |
||||
|
} |
||||
|
|
||||
|
func (t *testSeaweedMQHandler) CreateTopic(topic string, partitions int32) error { |
||||
|
t.topics[topic] = true |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (t *testSeaweedMQHandler) DeleteTopic(topic string) error { |
||||
|
delete(t.topics, topic) |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (t *testSeaweedMQHandler) GetOrCreateLedger(topic string, partition int32) *offset.Ledger { |
||||
|
t.mu.Lock() |
||||
|
defer t.mu.Unlock() |
||||
|
|
||||
|
// Mark topic as existing when creating ledger
|
||||
|
t.topics[topic] = true |
||||
|
|
||||
|
key := fmt.Sprintf("%s-%d", topic, partition) |
||||
|
if ledger, exists := t.ledgers[key]; exists { |
||||
|
return ledger |
||||
|
} |
||||
|
|
||||
|
ledger := offset.NewLedger() |
||||
|
t.ledgers[key] = ledger |
||||
|
return ledger |
||||
|
} |
||||
|
|
||||
|
func (t *testSeaweedMQHandler) GetLedger(topic string, partition int32) *offset.Ledger { |
||||
|
t.mu.RLock() |
||||
|
defer t.mu.RUnlock() |
||||
|
|
||||
|
key := fmt.Sprintf("%s-%d", topic, partition) |
||||
|
if ledger, exists := t.ledgers[key]; exists { |
||||
|
return ledger |
||||
|
} |
||||
|
|
||||
|
// Return nil if ledger doesn't exist (topic doesn't exist)
|
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (t *testSeaweedMQHandler) ProduceRecord(topicName string, partitionID int32, key, value []byte) (int64, error) { |
||||
|
// For testing, actually store the record in the ledger
|
||||
|
ledger := t.GetOrCreateLedger(topicName, partitionID) |
||||
|
|
||||
|
// Assign an offset and append the record
|
||||
|
offset := ledger.AssignOffsets(1) |
||||
|
timestamp := time.Now().UnixNano() |
||||
|
size := int32(len(value)) |
||||
|
|
||||
|
if err := ledger.AppendRecord(offset, timestamp, size); err != nil { |
||||
|
return 0, fmt.Errorf("failed to append record: %w", err) |
||||
|
} |
||||
|
|
||||
|
return offset, nil |
||||
|
} |
||||
|
|
||||
|
// GetStoredRecords for testSeaweedMQHandler - returns empty (no storage simulation)
|
||||
|
func (t *testSeaweedMQHandler) GetStoredRecords(topic string, partition int32, fromOffset int64, maxRecords int) ([]offset.SMQRecord, error) { |
||||
|
// Test handler doesn't simulate message storage, return empty
|
||||
|
return nil, nil |
||||
|
} |
||||
|
|
||||
|
func (t *testSeaweedMQHandler) Close() error { |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
// AddTopicForTesting moved to handler.go (available to production code for testing)
|
||||
|
|
||||
|
// GetStoredMessages is already defined in the basicSeaweedMQHandler implementation above
|
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue