From a481a345acf1c04c8ef19dde1ab975a59f7416c9 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sat, 22 Nov 2025 23:21:12 -0800 Subject: [PATCH] refactor: run Spark tests fully in Docker with bridge network MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Better approach than mixing host and container networks. Changes to docker-compose.yml: - Remove 'network_mode: host' from spark-tests container - Add spark-tests to seaweedfs-spark bridge network - Update SEAWEEDFS_FILER_HOST from 'localhost' to 'seaweedfs-filer' - Add depends_on to ensure services are healthy before tests - Update volume publicUrl from 'localhost:8080' to 'seaweedfs-volume:8080' Changes to workflow: - Remove separate build and test steps - Run tests via 'docker compose up spark-tests' - Use --abort-on-container-exit and --exit-code-from for proper exit codes - Simpler: one step instead of two Benefits: ✓ All components use Docker DNS (seaweedfs-master, seaweedfs-volume, seaweedfs-filer) ✓ No host/container network split or DNS resolution issues ✓ Consistent with how other SeaweedFS integration tests work ✓ Tests are fully containerized and reproducible ✓ Volume server accessible via seaweedfs-volume:8080 for all clients ✓ Automatic volume creation works (master can reach volume via gRPC) ✓ Data writes work (Spark can reach volume via Docker network) This matches the architecture of other integration tests and is cleaner. --- .github/workflows/spark-integration-tests.yml | 18 +++--------------- test/java/spark/docker-compose.yml | 10 +++++++--- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/.github/workflows/spark-integration-tests.yml b/.github/workflows/spark-integration-tests.yml index b9de013b6..285312202 100644 --- a/.github/workflows/spark-integration-tests.yml +++ b/.github/workflows/spark-integration-tests.yml @@ -185,23 +185,11 @@ jobs: curl -s http://localhost:9333/dir/status | jq '.' || echo "jq not available" echo "✓ Volume server registered, volumes will be created on-demand during tests" - - name: Build Spark integration tests + - name: Run Spark integration tests in Docker working-directory: test/java/spark run: | - echo "Building Spark integration tests..." - mvn clean package -DskipTests - echo "✓ Build completed" - - - name: Run Spark integration tests - working-directory: test/java/spark - env: - SEAWEEDFS_TEST_ENABLED: true - SEAWEEDFS_FILER_HOST: localhost - SEAWEEDFS_FILER_PORT: 8888 - SEAWEEDFS_FILER_GRPC_PORT: 18888 - run: | - echo "Running Spark integration tests..." - mvn test -B + echo "Running Spark integration tests in Docker container..." + docker compose up --abort-on-container-exit --exit-code-from spark-tests spark-tests echo "✓ Tests completed" - name: Upload test results diff --git a/test/java/spark/docker-compose.yml b/test/java/spark/docker-compose.yml index 713bf68a1..a170d7464 100644 --- a/test/java/spark/docker-compose.yml +++ b/test/java/spark/docker-compose.yml @@ -27,7 +27,7 @@ services: ports: - "8080:8080" - "18080:18080" - command: "volume -mserver=seaweedfs-master:9333 -ip=seaweedfs-volume -ip.bind=0.0.0.0 -port=8080 -port.grpc=18080 -publicUrl=localhost:8080 -max=100 -dir=/data -preStopSeconds=1" + command: "volume -mserver=seaweedfs-master:9333 -ip=seaweedfs-volume -ip.bind=0.0.0.0 -port=8080 -port.grpc=18080 -publicUrl=seaweedfs-volume:8080 -max=100 -dir=/data -preStopSeconds=1" volumes: - seaweedfs-volume-data:/data depends_on: @@ -69,18 +69,22 @@ services: spark-tests: image: maven:3.9-eclipse-temurin-17 container_name: seaweedfs-spark-tests - network_mode: "host" volumes: - .:/workspace - ~/.m2:/root/.m2 working_dir: /workspace environment: - SEAWEEDFS_TEST_ENABLED=true - - SEAWEEDFS_FILER_HOST=localhost + - SEAWEEDFS_FILER_HOST=seaweedfs-filer - SEAWEEDFS_FILER_PORT=8888 - SEAWEEDFS_FILER_GRPC_PORT=18888 - HADOOP_HOME=/tmp command: sh -c "sleep 30 && mvn test" + depends_on: + seaweedfs-filer: + condition: service_healthy + networks: + - seaweedfs-spark mem_limit: 4g cpus: 2