Browse Source

refactor: run Spark tests fully in Docker with bridge network

Better approach than mixing host and container networks.

Changes to docker-compose.yml:
- Remove 'network_mode: host' from spark-tests container
- Add spark-tests to seaweedfs-spark bridge network
- Update SEAWEEDFS_FILER_HOST from 'localhost' to 'seaweedfs-filer'
- Add depends_on to ensure services are healthy before tests
- Update volume publicUrl from 'localhost:8080' to 'seaweedfs-volume:8080'

Changes to workflow:
- Remove separate build and test steps
- Run tests via 'docker compose up spark-tests'
- Use --abort-on-container-exit and --exit-code-from for proper exit codes
- Simpler: one step instead of two

Benefits:
✓ All components use Docker DNS (seaweedfs-master, seaweedfs-volume, seaweedfs-filer)
✓ No host/container network split or DNS resolution issues
✓ Consistent with how other SeaweedFS integration tests work
✓ Tests are fully containerized and reproducible
✓ Volume server accessible via seaweedfs-volume:8080 for all clients
✓ Automatic volume creation works (master can reach volume via gRPC)
✓ Data writes work (Spark can reach volume via Docker network)

This matches the architecture of other integration tests and is cleaner.
pull/7526/head
chrislu 6 days ago
parent
commit
a481a345ac
  1. 18
      .github/workflows/spark-integration-tests.yml
  2. 10
      test/java/spark/docker-compose.yml

18
.github/workflows/spark-integration-tests.yml

@ -185,23 +185,11 @@ jobs:
curl -s http://localhost:9333/dir/status | jq '.' || echo "jq not available" curl -s http://localhost:9333/dir/status | jq '.' || echo "jq not available"
echo "✓ Volume server registered, volumes will be created on-demand during tests" echo "✓ Volume server registered, volumes will be created on-demand during tests"
- name: Build Spark integration tests
- name: Run Spark integration tests in Docker
working-directory: test/java/spark working-directory: test/java/spark
run: | run: |
echo "Building Spark integration tests..."
mvn clean package -DskipTests
echo "✓ Build completed"
- name: Run Spark integration tests
working-directory: test/java/spark
env:
SEAWEEDFS_TEST_ENABLED: true
SEAWEEDFS_FILER_HOST: localhost
SEAWEEDFS_FILER_PORT: 8888
SEAWEEDFS_FILER_GRPC_PORT: 18888
run: |
echo "Running Spark integration tests..."
mvn test -B
echo "Running Spark integration tests in Docker container..."
docker compose up --abort-on-container-exit --exit-code-from spark-tests spark-tests
echo "✓ Tests completed" echo "✓ Tests completed"
- name: Upload test results - name: Upload test results

10
test/java/spark/docker-compose.yml

@ -27,7 +27,7 @@ services:
ports: ports:
- "8080:8080" - "8080:8080"
- "18080:18080" - "18080:18080"
command: "volume -mserver=seaweedfs-master:9333 -ip=seaweedfs-volume -ip.bind=0.0.0.0 -port=8080 -port.grpc=18080 -publicUrl=localhost:8080 -max=100 -dir=/data -preStopSeconds=1"
command: "volume -mserver=seaweedfs-master:9333 -ip=seaweedfs-volume -ip.bind=0.0.0.0 -port=8080 -port.grpc=18080 -publicUrl=seaweedfs-volume:8080 -max=100 -dir=/data -preStopSeconds=1"
volumes: volumes:
- seaweedfs-volume-data:/data - seaweedfs-volume-data:/data
depends_on: depends_on:
@ -69,18 +69,22 @@ services:
spark-tests: spark-tests:
image: maven:3.9-eclipse-temurin-17 image: maven:3.9-eclipse-temurin-17
container_name: seaweedfs-spark-tests container_name: seaweedfs-spark-tests
network_mode: "host"
volumes: volumes:
- .:/workspace - .:/workspace
- ~/.m2:/root/.m2 - ~/.m2:/root/.m2
working_dir: /workspace working_dir: /workspace
environment: environment:
- SEAWEEDFS_TEST_ENABLED=true - SEAWEEDFS_TEST_ENABLED=true
- SEAWEEDFS_FILER_HOST=localhost
- SEAWEEDFS_FILER_HOST=seaweedfs-filer
- SEAWEEDFS_FILER_PORT=8888 - SEAWEEDFS_FILER_PORT=8888
- SEAWEEDFS_FILER_GRPC_PORT=18888 - SEAWEEDFS_FILER_GRPC_PORT=18888
- HADOOP_HOME=/tmp - HADOOP_HOME=/tmp
command: sh -c "sleep 30 && mvn test" command: sh -c "sleep 30 && mvn test"
depends_on:
seaweedfs-filer:
condition: service_healthy
networks:
- seaweedfs-spark
mem_limit: 4g mem_limit: 4g
cpus: 2 cpus: 2

Loading…
Cancel
Save