From e253030d2c9b59f4181ea80343833850b85ccf30 Mon Sep 17 00:00:00 2001
From: chrislu <chris.lu@gmail.com>
Date: Sat, 22 Nov 2025 22:39:09 -0800
Subject: [PATCH] ci: add volume cleanup and verification steps

- Add 'docker compose down -v' before starting services to clean up stale volumes
- Prevents accumulation of data/buckets from previous test runs
- Add volume registration verification after service startup
- Check that volume server has registered with master and volumes are available
- Helps diagnose 'No writable volumes' errors
- Shows volume count and waits up to 30 seconds for volumes to be created
- Both spark-tests and spark-example jobs updated with same improvements
---
 .github/workflows/spark-integration-tests.yml | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/.github/workflows/spark-integration-tests.yml b/.github/workflows/spark-integration-tests.yml
index 21845b62d..8ac508d7d 100644
--- a/.github/workflows/spark-integration-tests.yml
+++ b/.github/workflows/spark-integration-tests.yml
@@ -139,6 +139,9 @@ jobs:
     - name: Start SeaweedFS services
       working-directory: test/java/spark
       run: |
+        echo "Cleaning up any existing Docker Compose resources..."
+        docker compose down -v || true
+        
         echo "Starting SeaweedFS with Docker Compose..."
         docker compose build --no-cache
         docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer
@@ -176,6 +179,24 @@ jobs:
         echo "Verifying SeaweedFS services..."
         curl -f http://localhost:9333/cluster/status || exit 1
         curl -f http://localhost:8888/ || exit 1
+        
+        # Check volume server registration and volume availability
+        echo "Checking volume server status..."
+        curl -s http://localhost:9333/dir/status | jq '.' || echo "jq not available"
+        echo "Waiting for volume server to register and create volumes..."
+        for i in {1..15}; do
+          VOLUME_COUNT=$(curl -s http://localhost:9333/dir/status | jq -r '.Topology.DataCenters[0].Racks[0].DataNodes[0].Volumes // [] | length' 2>/dev/null || echo "0")
+          echo "Attempt $i/15: Volume count = $VOLUME_COUNT"
+          if [ "$VOLUME_COUNT" != "0" ] && [ "$VOLUME_COUNT" != "null" ]; then
+            echo "✓ Volume server has $VOLUME_COUNT volumes registered"
+            break
+          fi
+          if [ $i -eq 15 ]; then
+            echo "⚠️ No volumes created yet, but continuing (volumes may be created on-demand)"
+          fi
+          sleep 2
+        done
+        
         echo "✓ All SeaweedFS services are healthy"
 
     - name: Build Spark integration tests
@@ -303,6 +324,9 @@ jobs:
     - name: Start SeaweedFS services
       working-directory: test/java/spark
       run: |
+        echo "Cleaning up any existing Docker Compose resources..."
+        docker compose down -v || true
+        
         echo "Starting SeaweedFS with Docker Compose..."
         docker compose build --no-cache
         docker compose up -d seaweedfs-master seaweedfs-volume seaweedfs-filer
@@ -337,6 +361,24 @@ jobs:
         echo "Verifying SeaweedFS services..."
         curl -f http://localhost:9333/cluster/status || exit 1
         curl -f http://localhost:8888/ || exit 1
+        
+        # Check volume server registration and volume availability
+        echo "Checking volume server status..."
+        curl -s http://localhost:9333/dir/status | jq '.' || echo "jq not available"
+        echo "Waiting for volume server to register and create volumes..."
+        for i in {1..15}; do
+          VOLUME_COUNT=$(curl -s http://localhost:9333/dir/status | jq -r '.Topology.DataCenters[0].Racks[0].DataNodes[0].Volumes // [] | length' 2>/dev/null || echo "0")
+          echo "Attempt $i/15: Volume count = $VOLUME_COUNT"
+          if [ "$VOLUME_COUNT" != "0" ] && [ "$VOLUME_COUNT" != "null" ]; then
+            echo "✓ Volume server has $VOLUME_COUNT volumes registered"
+            break
+          fi
+          if [ $i -eq 15 ]; then
+            echo "⚠️ No volumes created yet, but continuing (volumes may be created on-demand)"
+          fi
+          sleep 2
+        done
+        
         echo "✓ All SeaweedFS services are healthy"
 
     - name: Build project