From 9b324f6b1b64be5eb3b25345feef9a2dc255ad31 Mon Sep 17 00:00:00 2001
From: chrislu <chris.lu@gmail.com>
Date: Mon, 25 Aug 2025 14:21:50 -0700
Subject: [PATCH] always run keycloak tests

---
 .github/workflows/s3-iam-tests.yml | 157 +++++++++++++++++------
 test/s3/iam/setup_keycloak.sh      | 196 +++++++++++++++++++++++++++++
 2 files changed, 312 insertions(+), 41 deletions(-)
 create mode 100755 test/s3/iam/setup_keycloak.sh

diff --git a/.github/workflows/s3-iam-tests.yml b/.github/workflows/s3-iam-tests.yml
index e501b0c64..8f7aeddc9 100644
--- a/.github/workflows/s3-iam-tests.yml
+++ b/.github/workflows/s3-iam-tests.yml
@@ -80,7 +80,7 @@ jobs:
     timeout-minutes: 25
     strategy:
       matrix:
-        test-type: ["basic", "advanced", "policy-enforcement"]
+        test-type: ["basic", "advanced", "policy-enforcement", "keycloak-integration"]
         
     steps:
       - name: Check out code
@@ -129,6 +129,38 @@ jobs:
               make clean setup start-services wait-for-services
               go test -v -timeout 15m -run "TestS3IAMPolicyEnforcement|TestS3IAMBucketPolicy|TestS3IAMContextual" ./...
               ;;
+            "keycloak-integration")
+              echo "Running Keycloak integration tests..."
+              # Start Keycloak container
+              docker run -d \
+                --name keycloak \
+                -p 8080:8080 \
+                -e KEYCLOAK_ADMIN=admin \
+                -e KEYCLOAK_ADMIN_PASSWORD=admin \
+                -e KC_HTTP_ENABLED=true \
+                -e KC_HOSTNAME_STRICT=false \
+                -e KC_HOSTNAME_STRICT_HTTPS=false \
+                quay.io/keycloak/keycloak:26.0 \
+                start-dev
+              
+              # Wait for Keycloak
+              timeout 180 bash -c 'until curl -s http://localhost:8080/realms/master > /dev/null; do sleep 5; echo "Waiting for Keycloak..."; done'
+              
+              # Setup Keycloak realm and users
+              chmod +x setup_keycloak.sh
+              ./setup_keycloak.sh
+              
+              # Start SeaweedFS services
+              make clean setup start-services wait-for-services
+              
+              # Run Keycloak tests
+              export KEYCLOAK_URL="http://localhost:8080"
+              go test -v -timeout 15m -run "TestKeycloak" ./...
+              
+              # Cleanup Keycloak
+              docker stop keycloak || true
+              docker rm keycloak || true
+              ;;
           esac
           
           # Always cleanup
@@ -169,12 +201,7 @@ jobs:
     name: S3 IAM Keycloak Integration
     runs-on: ubuntu-22.04
     timeout-minutes: 30
-    # Only run on master branch pushes or when Keycloak-related files change
-    if: |
-      github.event_name == 'push' && github.ref == 'refs/heads/master' ||
-      contains(github.event.pull_request.changed_files, 'test/s3/iam/keycloak') ||
-      contains(github.event.pull_request.changed_files, 'test/s3/iam/docker-compose') ||
-      contains(github.event.pull_request.changed_files, 'test/s3/iam/s3_keycloak')
+    # Always run Keycloak integration tests
     
     steps:
       - name: Check out code
@@ -186,36 +213,49 @@ jobs:
           go-version-file: 'go.mod'
         id: go
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build SeaweedFS IAM Image
-        working-directory: test/s3/iam
+      - name: Install SeaweedFS
+        working-directory: weed
         run: |
-          echo "Building custom SeaweedFS image with IAM support..."
-          docker build -f Dockerfile.s3 -t seaweedfs-iam:latest ../../..
+          go install -buildvcs=false
 
-      - name: Start Keycloak and SeaweedFS Services
-        working-directory: test/s3/iam
+      - name: Start Keycloak with Docker
         run: |
-          echo "Starting services with Docker Compose..."
-          docker compose up -d
+          echo "Starting Keycloak container..."
+          docker run -d \
+            --name keycloak \
+            -p 8080:8080 \
+            -e KEYCLOAK_ADMIN=admin \
+            -e KEYCLOAK_ADMIN_PASSWORD=admin \
+            -e KC_HTTP_ENABLED=true \
+            -e KC_HOSTNAME_STRICT=false \
+            -e KC_HOSTNAME_STRICT_HTTPS=false \
+            quay.io/keycloak/keycloak:26.0 \
+            start-dev
           
           echo "Waiting for Keycloak to be ready..."
-          timeout 120 bash -c 'until curl -s http://localhost:8080/realms/seaweedfs-test/.well-known/openid-configuration > /dev/null; do sleep 5; done' || {
+          timeout 180 bash -c 'until curl -s http://localhost:8080/realms/master > /dev/null; do sleep 5; echo "Waiting for Keycloak..."; done' || {
             echo "Keycloak failed to start"
-            docker compose logs keycloak
+            docker logs keycloak
             exit 1
           }
           
-          echo "Waiting for SeaweedFS S3 API to be ready..."
-          timeout 60 bash -c 'until curl -s http://localhost:8333 > /dev/null 2>&1; do sleep 3; done' || {
-            echo "SeaweedFS S3 API failed to start"
-            docker compose logs s3
-            exit 1
-          }
+          echo "Keycloak is ready"
+
+      - name: Setup Keycloak Realm and Users
+        working-directory: test/s3/iam
+        run: |
+          echo "Setting up Keycloak realm and test users..."
+          chmod +x setup_keycloak.sh
+          ./setup_keycloak.sh
+
+      - name: Start SeaweedFS Services
+        working-directory: test/s3/iam
+        run: |
+          echo "Starting SeaweedFS services..."
+          export WEED_BINARY=$(which weed)
+          make clean setup start-services wait-for-services
           
-          echo "All services are ready"
+          echo "SeaweedFS services are ready"
 
       - name: Run Keycloak Integration Tests
         timeout-minutes: 20
@@ -228,45 +268,80 @@ jobs:
           export S3_ENDPOINT="http://localhost:8333"
           
           # Give services extra time to fully initialize
-          sleep 15
+          sleep 10
+          
+          # Verify services are accessible
+          echo "=== Verifying Service Accessibility ==="
+          curl -f http://localhost:8080/realms/master || {
+            echo "❌ Keycloak not accessible"
+            docker logs keycloak --tail=50
+            exit 1
+          }
+          
+          curl -f http://localhost:8333 || {
+            echo "❌ SeaweedFS S3 API not accessible"
+            cat weed-s3.log || true
+            exit 1
+          }
           
           # Run Keycloak-specific tests
+          echo "=== Running Keycloak Tests ==="
           go test -v -timeout 15m -run "TestKeycloak" ./... || {
             echo "❌ Keycloak integration tests failed"
             echo "=== Service Logs ==="
-            docker compose logs --tail=100
+            echo "--- Keycloak logs ---"
+            docker logs keycloak --tail=100
+            echo "--- SeaweedFS logs ---"
+            cat weed-s3.log 2>/dev/null || echo "No S3 log found"
+            cat weed-master.log 2>/dev/null || echo "No master log found"
+            cat weed-filer.log 2>/dev/null || echo "No filer log found"
             exit 1
           }
 
-      - name: Show Docker logs on failure
+      - name: Show service logs on failure
         if: failure()
         working-directory: test/s3/iam
         run: |
-          echo "=== Docker Compose Service Logs ==="
-          docker compose logs --tail=200
+          echo "=== Keycloak Container Logs ==="
+          docker logs keycloak --tail=200 || true
+          
+          echo "=== SeaweedFS Service Logs ==="
+          echo "--- S3 API Log ---"
+          tail -100 weed-s3.log 2>/dev/null || echo "No S3 log found"
+          echo "--- Master Log ---"
+          tail -100 weed-master.log 2>/dev/null || echo "No master log found"
+          echo "--- Filer Log ---"
+          tail -100 weed-filer.log 2>/dev/null || echo "No filer log found"
+          echo "--- Volume Log ---"
+          tail -100 weed-volume.log 2>/dev/null || echo "No volume log found"
           
           echo "=== Container Status ==="
-          docker compose ps
+          docker ps -a
           
           echo "=== Network Information ==="
-          docker network ls
-          curl -v http://localhost:8080/realms/seaweedfs-test/.well-known/openid-configuration || true
+          netstat -tlnp | grep -E "(8080|8333|8888|9333)" || true
+          curl -v http://localhost:8080/realms/master || true
           curl -v http://localhost:8333 || true
 
-      - name: Cleanup Docker Services
+      - name: Cleanup Services
         if: always()
         working-directory: test/s3/iam
         run: |
-          echo "Stopping Docker Compose services..."
-          docker compose down -v --remove-orphans
-          docker image prune -f
+          echo "Stopping SeaweedFS services..."
+          make stop-services || true
+          
+          echo "Stopping Keycloak container..."
+          docker stop keycloak || true
+          docker rm keycloak || true
 
-      - name: Upload Docker logs on failure
+      - name: Upload service logs on failure
         if: failure()
         uses: actions/upload-artifact@v4
         with:
           name: s3-iam-keycloak-logs
-          path: test/s3/iam/docker-*.log
+          path: |
+            test/s3/iam/weed-*.log
+            test/s3/iam/*.log
           retention-days: 5
 
   # Distributed IAM tests
diff --git a/test/s3/iam/setup_keycloak.sh b/test/s3/iam/setup_keycloak.sh
new file mode 100755
index 000000000..fd19e9805
--- /dev/null
+++ b/test/s3/iam/setup_keycloak.sh
@@ -0,0 +1,196 @@
+#!/bin/bash
+
+# Keycloak Setup Script for CI/CD
+# This script sets up a Keycloak realm with test users and roles for SeaweedFS S3 IAM testing
+
+set -e
+
+KEYCLOAK_URL="${KEYCLOAK_URL:-http://localhost:8080}"
+ADMIN_USER="${KEYCLOAK_ADMIN:-admin}"
+ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD:-admin}"
+REALM_NAME="seaweedfs-test"
+CLIENT_ID="seaweedfs-s3"
+CLIENT_SECRET="seaweedfs-s3-secret"
+
+echo "🔧 Setting up Keycloak realm and users for SeaweedFS S3 IAM testing..."
+echo "Keycloak URL: $KEYCLOAK_URL"
+
+# Function to get admin access token
+get_admin_token() {
+    curl -s -X POST "$KEYCLOAK_URL/realms/master/protocol/openid-connect/token" \
+        -H "Content-Type: application/x-www-form-urlencoded" \
+        -d "username=$ADMIN_USER" \
+        -d "password=$ADMIN_PASSWORD" \
+        -d "grant_type=password" \
+        -d "client_id=admin-cli" | jq -r '.access_token'
+}
+
+# Function to check if realm exists
+realm_exists() {
+    local token=$1
+    curl -s -H "Authorization: Bearer $token" \
+        "$KEYCLOAK_URL/admin/realms/$REALM_NAME" \
+        -o /dev/null -w "%{http_code}" | grep -q "200"
+}
+
+# Function to create realm
+create_realm() {
+    local token=$1
+    echo "📝 Creating realm: $REALM_NAME"
+    
+    curl -s -X POST "$KEYCLOAK_URL/admin/realms" \
+        -H "Authorization: Bearer $token" \
+        -H "Content-Type: application/json" \
+        -d '{
+            "realm": "'$REALM_NAME'",
+            "enabled": true,
+            "displayName": "SeaweedFS Test Realm",
+            "accessTokenLifespan": 3600,
+            "sslRequired": "none"
+        }'
+}
+
+# Function to create client
+create_client() {
+    local token=$1
+    echo "📝 Creating client: $CLIENT_ID"
+    
+    curl -s -X POST "$KEYCLOAK_URL/admin/realms/$REALM_NAME/clients" \
+        -H "Authorization: Bearer $token" \
+        -H "Content-Type: application/json" \
+        -d '{
+            "clientId": "'$CLIENT_ID'",
+            "enabled": true,
+            "publicClient": false,
+            "secret": "'$CLIENT_SECRET'",
+            "directAccessGrantsEnabled": true,
+            "serviceAccountsEnabled": true,
+            "standardFlowEnabled": true,
+            "implicitFlowEnabled": false,
+            "redirectUris": ["*"],
+            "webOrigins": ["*"]
+        }'
+}
+
+# Function to create role
+create_role() {
+    local token=$1
+    local role_name=$2
+    local role_description=$3
+    
+    echo "📝 Creating role: $role_name"
+    curl -s -X POST "$KEYCLOAK_URL/admin/realms/$REALM_NAME/roles" \
+        -H "Authorization: Bearer $token" \
+        -H "Content-Type: application/json" \
+        -d '{
+            "name": "'$role_name'",
+            "description": "'$role_description'"
+        }'
+}
+
+# Function to create user
+create_user() {
+    local token=$1
+    local username=$2
+    local password=$3
+    local email=$4
+    local first_name=$5
+    local last_name=$6
+    local roles=$7
+    
+    echo "📝 Creating user: $username"
+    
+    # Create user
+    curl -s -X POST "$KEYCLOAK_URL/admin/realms/$REALM_NAME/users" \
+        -H "Authorization: Bearer $token" \
+        -H "Content-Type: application/json" \
+        -d '{
+            "username": "'$username'",
+            "email": "'$email'",
+            "firstName": "'$first_name'",
+            "lastName": "'$last_name'",
+            "enabled": true,
+            "emailVerified": true,
+            "credentials": [{
+                "type": "password",
+                "value": "'$password'",
+                "temporary": false
+            }]
+        }'
+    
+    # Get user ID
+    local user_id=$(curl -s -H "Authorization: Bearer $token" \
+        "$KEYCLOAK_URL/admin/realms/$REALM_NAME/users?username=$username" | \
+        jq -r '.[0].id')
+    
+    # Assign roles
+    if [ -n "$roles" ]; then
+        echo "📝 Assigning roles to $username: $roles"
+        IFS=',' read -ra ROLE_ARRAY <<< "$roles"
+        for role in "${ROLE_ARRAY[@]}"; do
+            # Get role representation
+            local role_rep=$(curl -s -H "Authorization: Bearer $token" \
+                "$KEYCLOAK_URL/admin/realms/$REALM_NAME/roles/$role")
+            
+            # Assign role to user
+            curl -s -X POST "$KEYCLOAK_URL/admin/realms/$REALM_NAME/users/$user_id/role-mappings/realm" \
+                -H "Authorization: Bearer $token" \
+                -H "Content-Type: application/json" \
+                -d "[$role_rep]"
+        done
+    fi
+}
+
+# Main setup process
+main() {
+    echo "🚀 Starting Keycloak setup..."
+    
+    # Wait for Keycloak to be ready
+    echo "⏳ Waiting for Keycloak to be ready..."
+    timeout 120 bash -c "until curl -s $KEYCLOAK_URL/realms/master > /dev/null; do sleep 2; done" || {
+        echo "❌ Keycloak is not ready after 120 seconds"
+        exit 1
+    }
+    
+    # Get admin token
+    echo "🔑 Getting admin access token..."
+    ADMIN_TOKEN=$(get_admin_token)
+    if [ -z "$ADMIN_TOKEN" ] || [ "$ADMIN_TOKEN" = "null" ]; then
+        echo "❌ Failed to get admin access token"
+        exit 1
+    fi
+    
+    # Create realm if it doesn't exist
+    if ! realm_exists "$ADMIN_TOKEN"; then
+        create_realm "$ADMIN_TOKEN"
+        sleep 2
+    else
+        echo "✅ Realm $REALM_NAME already exists"
+    fi
+    
+    # Create client
+    create_client "$ADMIN_TOKEN"
+    sleep 1
+    
+    # Create roles
+    create_role "$ADMIN_TOKEN" "s3-admin" "SeaweedFS S3 Administrator"
+    create_role "$ADMIN_TOKEN" "s3-read-only" "SeaweedFS S3 Read-Only User"
+    create_role "$ADMIN_TOKEN" "s3-write-only" "SeaweedFS S3 Write-Only User"
+    sleep 1
+    
+    # Create test users
+    create_user "$ADMIN_TOKEN" "admin-user" "admin123" "admin@seaweedfs.test" "Admin" "User" "s3-admin"
+    create_user "$ADMIN_TOKEN" "read-user" "read123" "read@seaweedfs.test" "Read" "User" "s3-read-only"
+    create_user "$ADMIN_TOKEN" "write-user" "write123" "write@seaweedfs.test" "Write" "User" "s3-write-only"
+    
+    echo "✅ Keycloak setup completed successfully!"
+    echo "🔗 Realm: $KEYCLOAK_URL/realms/$REALM_NAME"
+    echo "👥 Test users created:"
+    echo "   - admin-user (password: admin123) - s3-admin role"
+    echo "   - read-user (password: read123) - s3-read-only role"
+    echo "   - write-user (password: write123) - s3-write-only role"
+    echo "🔑 Client: $CLIENT_ID (secret: $CLIENT_SECRET)"
+}
+
+# Run main function
+main "$@"