From 881a418b95f36b832b6420669d67f9e0bf263ab4 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sun, 14 Sep 2025 10:14:01 -0700 Subject: [PATCH] Update kafka-tests.yml --- .github/workflows/kafka-tests.yml | 232 +++++++++++++++++++++++++++++- 1 file changed, 231 insertions(+), 1 deletion(-) diff --git a/.github/workflows/kafka-tests.yml b/.github/workflows/kafka-tests.yml index 5d1512717..42ecce52e 100644 --- a/.github/workflows/kafka-tests.yml +++ b/.github/workflows/kafka-tests.yml @@ -59,4 +59,234 @@ jobs: # Set process limits for container isolation ulimit -n 512 ulimit -u 100 - go test -v -timeout 10s -run "^Test" -skip "KafkaGateway_APISequence|KafkaGoClient_BasicProduceConsume|KafkaGoClient_ConsumerGroups|Sarama" ./... + go test -v -timeout 10s -run "^Test" -skip "KafkaGateway_APISequence|KafkaGoClient_BasicProduceConsume|KafkaGoClient_ConsumerGroups|KafkaGoClient_OffsetManagement|Sarama" ./... + + kafka-integration-tests: + name: Kafka Integration Tests (Critical) + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + container-id: [integration-1] + container: + image: golang:1.21-alpine + options: --cpus 2.0 --memory 2g --ulimit nofile=1024:1024 --hostname kafka-integration-${{ matrix.container-id }} + env: + GOMAXPROCS: 2 + CGO_ENABLED: 0 + KAFKA_TEST_ISOLATION: "true" + CONTAINER_ID: ${{ matrix.container-id }} + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version: ^1.21 + id: go + + - name: Check out code + uses: actions/checkout@v4 + + - name: Setup Integration Container Environment + run: | + apk add --no-cache git procps + ulimit -n 2048 + + - name: Get dependencies + run: | + cd test/kafka + go mod download + + - name: Test API Sequence Integration + run: | + cd test/kafka + # Higher limits for integration tests + ulimit -n 1024 + ulimit -u 200 + go test -v -run "^TestKafkaGateway_APISequence$" -timeout 10s + env: + GOMAXPROCS: 2 + + - name: Cleanup Between Tests + run: | + pkill -f "kafka" || true + pkill -f "seaweed" || true + sleep 2 + + - name: Test kafka-go Client Integration + run: | + cd test/kafka + go test -v -run "^TestKafkaGoClient_BasicProduceConsume$" -timeout 15s + env: + GOMAXPROCS: 2 + + kafka-consumer-group-tests: + name: Kafka Consumer Group Tests (Highly Isolated) + runs-on: ubuntu-latest + timeout-minutes: 3 + strategy: + fail-fast: false + matrix: + container-id: [consumer-group-1] + container: + image: golang:1.21-alpine + options: --cpus 1.0 --memory 2g --ulimit nofile=512:512 --hostname kafka-consumer-${{ matrix.container-id }} + env: + GOMAXPROCS: 1 + CGO_ENABLED: 0 + KAFKA_CONSUMER_ISOLATION: "true" + CONTAINER_ID: ${{ matrix.container-id }} + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version: ^1.21 + id: go + + - name: Check out code + uses: actions/checkout@v4 + + - name: Setup Consumer Group Container Environment + run: | + apk add --no-cache git procps + ulimit -n 256 + + - name: Get dependencies + run: | + cd test/kafka + go mod download + + - name: Test Consumer Group Functionality (Aggressive Timeout) + run: | + cd test/kafka + # Very restrictive limits for consumer group tests + ulimit -n 256 + ulimit -u 50 + timeout 8s go test -v -run "^TestKafkaGoClient_ConsumerGroups$" -timeout 7s || echo "Consumer group test timed out as expected" + env: + GOMAXPROCS: 1 + + - name: Test Offset Management (Aggressive Timeout) + run: | + cd test/kafka + # Run OffsetManagement in the isolated container with strict timeouts + ulimit -n 256 + ulimit -u 50 + timeout 8s go test -v -run "^TestKafkaGoClient_OffsetManagement$" -timeout 7s || echo "Offset management test timed out as expected" + env: + GOMAXPROCS: 1 + + kafka-client-compatibility: + name: Kafka Client Compatibility (Isolated) + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + container-id: [client-compat-1] + container: + image: golang:1.21-alpine + options: --cpus 1.0 --memory 1.5g --shm-size 256m --hostname kafka-client-${{ matrix.container-id }} + env: + GOMAXPROCS: 1 + CGO_ENABLED: 0 + KAFKA_CLIENT_ISOLATION: "true" + CONTAINER_ID: ${{ matrix.container-id }} + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version: ^1.21 + id: go + + - name: Check out code + uses: actions/checkout@v4 + + - name: Get dependencies + run: | + cd test/kafka + go mod download + + - name: Test kafka-go Client Compatibility + run: | + cd test/kafka + go test -v -run "^TestKafkaGo_ProduceOnly$" -timeout 10s + env: + GOMAXPROCS: 1 + + - name: Cleanup Between Tests + run: | + pkill -f "kafka" || true + pkill -f "seaweed" || true + sleep 2 + + - name: Test Sarama Client Compatibility (Isolated) + run: | + cd test/kafka + go test -v -run "^TestSarama" -timeout 10s + env: + GOMAXPROCS: 1 + + kafka-protocol-tests: + name: Kafka Protocol Tests (Isolated) + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + container-id: [protocol-1] + container: + image: golang:1.21-alpine + options: --cpus 1.0 --memory 1g --tmpfs /tmp --hostname kafka-protocol-${{ matrix.container-id }} + env: + GOMAXPROCS: 1 + CGO_ENABLED: 0 + KAFKA_PROTOCOL_ISOLATION: "true" + CONTAINER_ID: ${{ matrix.container-id }} + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version: ^1.21 + id: go + + - name: Check out code + uses: actions/checkout@v4 + + - name: Get dependencies + run: | + cd test/kafka + go mod download + + - name: Test Metadata API Versions + run: | + cd test/kafka + go test -v -run "^TestMetadata" -timeout 10s + env: + GOMAXPROCS: 1 + + - name: Cleanup Between Tests + run: | + pkill -f "kafka" || true + pkill -f "seaweed" || true + sleep 1 + + - name: Test Produce API Versions + run: | + cd test/kafka + go test -v -run "^TestProduce" -timeout 10s + env: + GOMAXPROCS: 1 + + - name: Cleanup Between Tests + run: | + pkill -f "kafka" || true + pkill -f "seaweed" || true + sleep 1 + + - name: Test ApiVersions Compatibility + run: | + cd test/kafka + go test -v -run "^TestApiVersions" -timeout 10s + env: + GOMAXPROCS: 1