seaweedfs/.github/workflows/kafka-tests.yml

name: "Kafka Gateway Tests"

on:
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]

concurrency:
  group: ${{ github.head_ref }}/kafka-tests
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  kafka-unit-tests:
    name: Kafka Unit Tests
    runs-on: ubuntu-latest
    timeout-minutes: 5
    strategy:
      fail-fast: false
      matrix:
        container-id: [unit-tests-1]
    container:
      image: golang:1.21-alpine
      options: --cpus 1.0 --memory 1g --hostname kafka-unit-${{ matrix.container-id }}
    env:
      GOMAXPROCS: 1
      CGO_ENABLED: 0
      CONTAINER_ID: ${{ matrix.container-id }}
    steps:
    - name: Set up Go 1.x
      uses: actions/setup-go@v5
      with:
        go-version: ^1.21
      id: go

    - name: Check out code
      uses: actions/checkout@v4

    - name: Setup Container Environment
      run: |
        apk add --no-cache git
        ulimit -n 1024
                
    - name: Get dependencies
      run: |
        cd test/kafka
        go mod download        

    - name: Run Kafka Gateway Unit Tests (Excluding Integration Tests)
      run: |
        cd test/kafka
        # Set process limits for container isolation
        ulimit -n 512
        ulimit -u 100
        go test -v -timeout 10s -run "^Test" -skip "KafkaGateway_APISequence|KafkaGoClient_BasicProduceConsume|Sarama" ./...        

  kafka-integration-tests:
    name: Kafka Integration Tests (Critical)
    runs-on: ubuntu-latest
    timeout-minutes: 5
    strategy:
      fail-fast: false
      matrix:
        container-id: [integration-1]
    container:
      image: golang:1.21-alpine
      options: --cpus 2.0 --memory 2g --ulimit nofile=1024:1024 --hostname kafka-integration-${{ matrix.container-id }}
    env:
      GOMAXPROCS: 2
      CGO_ENABLED: 0
      KAFKA_TEST_ISOLATION: "true"
      CONTAINER_ID: ${{ matrix.container-id }}
    steps:
    - name: Set up Go 1.x
      uses: actions/setup-go@v5
      with:
        go-version: ^1.21
      id: go

    - name: Check out code
      uses: actions/checkout@v4

    - name: Setup Integration Container Environment
      run: |
        apk add --no-cache git procps
        ulimit -n 2048
                
    - name: Get dependencies
      run: |
        cd test/kafka
        go mod download        

    - name: Test API Sequence Integration
      run: |
        cd test/kafka
        # Higher limits for integration tests
        ulimit -n 1024
        ulimit -u 200
        go test -v -run "^TestKafkaGateway_APISequence$" -timeout 10s        
      env:
        GOMAXPROCS: 2

    - name: Cleanup Between Tests
      run: |
        pkill -f "kafka" || true
        pkill -f "seaweed" || true
        sleep 2        

    - name: Test kafka-go Client Integration
      run: |
        cd test/kafka
        go test -v -run "^TestKafkaGoClient_BasicProduceConsume$" -timeout 15s        
      env:
        GOMAXPROCS: 1

  kafka-client-compatibility:
    name: Kafka Client Compatibility (Isolated)
    runs-on: ubuntu-latest
    timeout-minutes: 5
    strategy:
      fail-fast: false
      matrix:
        container-id: [client-compat-1]
    container:
      image: golang:1.21-alpine
      options: --cpus 1.0 --memory 1.5g --shm-size 256m --hostname kafka-client-${{ matrix.container-id }}
    env:
      GOMAXPROCS: 1
      CGO_ENABLED: 0
      KAFKA_CLIENT_ISOLATION: "true"
      CONTAINER_ID: ${{ matrix.container-id }}
    steps:
    - name: Set up Go 1.x
      uses: actions/setup-go@v5
      with:
        go-version: ^1.21
      id: go

    - name: Check out code
      uses: actions/checkout@v4

    - name: Get dependencies
      run: |
        cd test/kafka
        go mod download        

    - name: Test kafka-go Client Compatibility
      run: |
        cd test/kafka
        go test -v -run "^TestKafkaGo_ProduceOnly$" -timeout 10s        
      env:
        GOMAXPROCS: 1

    - name: Cleanup Between Tests
      run: |
        pkill -f "kafka" || true
        pkill -f "seaweed" || true
        sleep 2        

    - name: Test Sarama Client Compatibility (Isolated)
      run: |
        cd test/kafka
        go test -v -run "^TestSarama" -timeout 10s        
      env:
        GOMAXPROCS: 1

  kafka-protocol-tests:
    name: Kafka Protocol Tests (Isolated)
    runs-on: ubuntu-latest
    timeout-minutes: 5
    container:
      image: golang:1.21-alpine
      options: --cpus 1.0 --memory 1g --tmpfs /tmp
    env:
      GOMAXPROCS: 1
      CGO_ENABLED: 0
      KAFKA_PROTOCOL_ISOLATION: "true"
    steps:
    - name: Set up Go 1.x
      uses: actions/setup-go@v5
      with:
        go-version: ^1.21
      id: go

    - name: Check out code
      uses: actions/checkout@v4

    - name: Get dependencies
      run: |
        cd test/kafka
        go mod download        

    - name: Test Metadata API Versions
      run: |
        cd test/kafka
        go test -v -run "^TestMetadata" -timeout 10s        
      env:
        GOMAXPROCS: 1

    - name: Cleanup Between Tests
      run: |
        pkill -f "kafka" || true
        pkill -f "seaweed" || true
        sleep 1        

    - name: Test Produce API Versions
      run: |
        cd test/kafka
        go test -v -run "^TestProduce" -timeout 10s        
      env:
        GOMAXPROCS: 1

    - name: Cleanup Between Tests
      run: |
        pkill -f "kafka" || true
        pkill -f "seaweed" || true
        sleep 1        

    - name: Test ApiVersions Compatibility
      run: |
        cd test/kafka
        go test -v -run "^TestApiVersions" -timeout 10s        
      env:
        GOMAXPROCS: 1