diff --git a/test/s3/iam/Dockerfile.debug b/test/s3/iam/Dockerfile.debug new file mode 100644 index 000000000..2d3120a76 --- /dev/null +++ b/test/s3/iam/Dockerfile.debug @@ -0,0 +1,22 @@ +# Debug version of SeaweedFS with role mapping fixes +FROM alpine:latest + +# Install required packages +RUN apk add --no-cache \ + ca-certificates \ + curl \ + wget \ + bash + +# Copy the debug weed binary +COPY weed-debug /usr/bin/weed +RUN chmod +x /usr/bin/weed + +# Create working directory +WORKDIR /data + +# Expose common ports +EXPOSE 8080 8333 8888 9333 18080 18333 18888 19333 + +# Default command +CMD ["weed"] diff --git a/test/s3/iam/KEYCLOAK.md b/test/s3/iam/KEYCLOAK.md deleted file mode 100644 index 189373d39..000000000 --- a/test/s3/iam/KEYCLOAK.md +++ /dev/null @@ -1,252 +0,0 @@ -# Keycloak Integration for SeaweedFS S3 IAM Tests - -This document describes the integration of [Keycloak](https://github.com/keycloak/keycloak) as a real OIDC provider for SeaweedFS S3 IAM integration tests. - -## Overview - -The integration tests support both **mock OIDC** and **real Keycloak authentication**: - -- **Mock OIDC** (default): Fast, no dependencies, generates test JWT tokens locally -- **Keycloak OIDC** (optional): Real-world authentication using Keycloak as OIDC provider - -The test framework automatically detects if Keycloak is available and switches modes accordingly. - -## Architecture - -``` -┌─────────────┐ JWT Token ┌──────────────────┐ S3 API ┌─────────────────┐ -│ Keycloak │ ────────────► │ SeaweedFS S3 │ ────────► │ SeaweedFS │ -│ OIDC │ (Bearer) │ Gateway + IAM │ │ Storage │ -│ Provider │ │ │ │ │ -└─────────────┘ └──────────────────┘ └─────────────────┘ -``` - -1. **Test** authenticates user with Keycloak using username/password -2. **Keycloak** returns JWT access token with user roles and claims -3. **Test** creates S3 client with JWT Bearer token authentication -4. **SeaweedFS S3 Gateway** validates JWT token and enforces IAM policies -5. **S3 operations** are authorized based on user roles and attached policies - -## Quick Start - -### Option 1: Docker Compose (Recommended) - -Start everything with Docker Compose including Keycloak: - -```bash -cd test/s3/iam -make docker-test -``` - -This will: -- Start Keycloak with pre-configured realm and users -- Start SeaweedFS services (master, volume, filer, S3 gateway) -- Run Keycloak integration tests -- Clean up all services - -### Option 2: Manual Setup - -1. Start Keycloak manually: -```bash -docker run -p 8080:8080 \ - -e KEYCLOAK_ADMIN=admin \ - -e KEYCLOAK_ADMIN_PASSWORD=admin123 \ - -v $(pwd)/keycloak-realm.json:/opt/keycloak/data/import/realm.json \ - quay.io/keycloak/keycloak:26.0.7 start-dev --import-realm -``` - -2. Start SeaweedFS services: -```bash -make start-services -``` - -3. Run tests with Keycloak: -```bash -export KEYCLOAK_URL="http://localhost:8080" -make test-quick -``` - -## Configuration - -### Keycloak Realm Configuration - -The test realm (`seaweedfs-test`) includes: - -**Client:** -- **Client ID**: `seaweedfs-s3` -- **Client Secret**: `seaweedfs-s3-secret` -- **Direct Access**: Enabled (for username/password authentication) - -**Roles:** -- `s3-admin`: Full S3 access -- `s3-read-only`: Read-only S3 access -- `s3-read-write`: Read-write S3 access - -**Test Users:** -- `admin-user` (password: `admin123`) → `s3-admin` role -- `read-user` (password: `read123`) → `s3-read-only` role -- `write-user` (password: `write123`) → `s3-read-write` role - -### SeaweedFS IAM Configuration - -The IAM system maps Keycloak roles to SeaweedFS IAM roles: - -```json -{ - "roles": [ - { - "roleName": "S3AdminRole", - "trustPolicy": { - "Principal": { "Federated": "keycloak-oidc" }, - "Action": ["sts:AssumeRoleWithWebIdentity"], - "Condition": { "StringEquals": { "roles": "s3-admin" } } - }, - "attachedPolicies": ["S3AdminPolicy"] - } - ] -} -``` - -## Test Structure - -### Framework Detection - -The test framework automatically detects Keycloak availability: - -```go -// Check if Keycloak is running -framework.useKeycloak = framework.isKeycloakAvailable(keycloakURL) - -if framework.useKeycloak { - // Use real Keycloak authentication - token, err = framework.getKeycloakToken(username) -} else { - // Fall back to mock JWT tokens - token, err = framework.generateSTSSessionToken(username, roleName, time.Hour) -} -``` - -### Test Categories - -**Keycloak-Specific Tests** (`TestKeycloak*`): -- `TestKeycloakAuthentication`: Real authentication flow -- `TestKeycloakRoleMapping`: Role mapping from Keycloak to S3 policies -- `TestKeycloakTokenExpiration`: JWT token lifecycle -- `TestKeycloakS3Operations`: End-to-end S3 operations with real auth - -**General Tests** (work with both modes): -- `TestS3IAMAuthentication`: Basic authentication tests -- `TestS3IAMPolicyEnforcement`: Policy enforcement tests -- All other integration tests - -## Environment Variables - -- `KEYCLOAK_URL`: Keycloak base URL (default: `http://localhost:8080`) -- `S3_ENDPOINT`: SeaweedFS S3 endpoint (default: `http://localhost:8333`) - -## Docker Services - -The Docker Compose setup includes: - -```yaml -services: - keycloak: # Keycloak OIDC provider - seaweedfs-master: # SeaweedFS master server - seaweedfs-volume: # SeaweedFS volume server - seaweedfs-filer: # SeaweedFS filer server - seaweedfs-s3: # SeaweedFS S3 gateway with IAM -``` - -All services include health checks and proper dependencies. - -## Authentication Flow - -1. **Test requests authentication**: - ```go - tokenResp, err := keycloakClient.AuthenticateUser("admin-user", "admin123") - ``` - -2. **Keycloak returns JWT token** with claims: - ```json - { - "sub": "user-id", - "preferred_username": "admin-user", - "roles": ["s3-admin"], - "iss": "http://keycloak:8080/realms/seaweedfs-test" - } - ``` - -3. **S3 client sends Bearer token**: - ```http - GET / HTTP/1.1 - Authorization: Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9... - ``` - -4. **SeaweedFS validates token** and checks policies: - - Validate JWT signature with Keycloak JWKS - - Extract roles from token claims - - Map roles to IAM roles via trust policies - - Enforce attached IAM policies for S3 operations - -## Troubleshooting - -### Keycloak Not Available - -If Keycloak is not running, tests automatically fall back to mock mode: - -``` -Using mock OIDC server for testing -``` - -### Token Validation Errors - -Check that: -- Keycloak realm configuration matches `iam_config_docker.json` -- JWT signing algorithms are compatible (RS256/HS256) -- Trust policies correctly reference the Keycloak provider - -### Service Dependencies - -Docker Compose includes health checks. Monitor with: - -```bash -make docker-logs -``` - -### Authentication Failures - -Enable debug logging: -```bash -export KEYCLOAK_URL="http://localhost:8080" -go test -v -run "TestKeycloak" ./... -``` - -## Extending the Integration - -### Adding New Roles - -1. Update `keycloak-realm.json` with new roles -2. Add corresponding IAM role in `iam_config_docker.json` -3. Create trust policy mapping the Keycloak role -4. Define appropriate IAM policies for the role - -### Adding New Test Users - -1. Add user to `keycloak-realm.json` with credentials and roles -2. Add password mapping in `getTestUserPassword()` -3. Create tests for the new user's permissions - -### Custom OIDC Providers - -The framework can be extended to support other OIDC providers by: -1. Implementing the provider in the IAM integration system -2. Adding provider configuration to IAM config -3. Updating test framework authentication methods - -## Benefits - -- **Real-world validation**: Tests against actual OIDC provider -- **Production-like environment**: Mirrors real deployment scenarios -- **Comprehensive coverage**: Role mapping, token validation, policy enforcement -- **Automatic fallback**: Works without Keycloak dependencies -- **Easy CI/CD**: Docker Compose makes automation simple diff --git a/test/s3/iam/Makefile b/test/s3/iam/Makefile index 7a2f103de..47b26c579 100644 --- a/test/s3/iam/Makefile +++ b/test/s3/iam/Makefile @@ -75,7 +75,7 @@ start-services: ## Start SeaweedFS services for testing @sleep 2 @echo "Starting S3 API server with IAM..." - @$(WEED_BINARY) s3 -port=$(S3_PORT) \ + @$(WEED_BINARY) -v=3 s3 -port=$(S3_PORT) \ -filer=localhost:$(FILER_PORT) \ -config=test_config.json \ -iam.config=$(shell pwd)/iam_config.json > weed-s3.log 2>&1 & \ diff --git a/test/s3/iam/Makefile.docker b/test/s3/iam/Makefile.docker new file mode 100644 index 000000000..4f3a66921 --- /dev/null +++ b/test/s3/iam/Makefile.docker @@ -0,0 +1,156 @@ +# Makefile for SeaweedFS S3 IAM Integration Tests with Docker Compose +.PHONY: help docker-build docker-up docker-down docker-logs docker-test docker-clean docker-status docker-keycloak-setup + +# Default target +.DEFAULT_GOAL := help + +# Docker Compose configuration +COMPOSE_FILE := docker-compose.yml +PROJECT_NAME := seaweedfs-iam-test + +help: ## Show this help message + @echo "SeaweedFS S3 IAM Integration Tests - Docker Compose" + @echo "" + @echo "Available commands:" + @echo "" + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + @echo "" + @echo "Environment:" + @echo " COMPOSE_FILE: $(COMPOSE_FILE)" + @echo " PROJECT_NAME: $(PROJECT_NAME)" + +docker-build: ## Build custom SeaweedFS image if needed + @echo "🔨 Building custom SeaweedFS image with debug changes..." + @if [ -f ../../../weed-debug ]; then \ + echo "Using debug binary for Docker image..."; \ + cp ../../../weed-debug ./weed-debug; \ + docker build -t seaweedfs-debug:latest -f Dockerfile.debug .; \ + rm -f ./weed-debug; \ + else \ + echo "No debug binary found, using standard image"; \ + fi + +docker-up: ## Start all services with Docker Compose + @echo "🚀 Starting SeaweedFS S3 IAM integration environment..." + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) up -d + @echo "" + @echo "✅ Environment started! Services will be available at:" + @echo " 🔐 Keycloak: http://localhost:8080 (admin/admin)" + @echo " 🗄️ S3 API: http://localhost:8333" + @echo " 📁 Filer: http://localhost:8888" + @echo " 🎯 Master: http://localhost:9333" + @echo "" + @echo "⏳ Waiting for all services to be healthy..." + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) ps + +docker-down: ## Stop and remove all containers + @echo "🛑 Stopping SeaweedFS S3 IAM integration environment..." + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) down -v + @echo "✅ Environment stopped and cleaned up" + +docker-restart: docker-down docker-up ## Restart the entire environment + +docker-logs: ## Show logs from all services + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) logs -f + +docker-logs-s3: ## Show logs from S3 service only + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) logs -f weed-s3 + +docker-logs-keycloak: ## Show logs from Keycloak service only + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) logs -f keycloak + +docker-status: ## Check status of all services + @echo "📊 Service Status:" + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) ps + @echo "" + @echo "🏥 Health Checks:" + @docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | grep $(PROJECT_NAME) || true + +docker-test: ## Run integration tests against Docker environment + @echo "🧪 Running SeaweedFS S3 IAM integration tests..." + @echo "⏳ Waiting for services to be ready..." + @timeout 120 bash -c 'until curl -s http://localhost:8333 > /dev/null 2>&1; do sleep 2; done' + @timeout 120 bash -c 'until curl -s http://localhost:8080/health/ready > /dev/null 2>&1; do sleep 2; done' + @echo "✅ Services are ready" + @echo "" + @KEYCLOAK_URL=http://localhost:8080 go test -v -timeout 10m ./... + +docker-test-single: ## Run a single test (use TEST_NAME=TestName) + @if [ -z "$(TEST_NAME)" ]; then \ + echo "❌ Please specify TEST_NAME, e.g., make docker-test-single TEST_NAME=TestKeycloakAuthentication"; \ + exit 1; \ + fi + @echo "🧪 Running single test: $(TEST_NAME)" + @KEYCLOAK_URL=http://localhost:8080 go test -v -run "$(TEST_NAME)" -timeout 5m ./... + +docker-keycloak-setup: ## Manually run Keycloak setup (usually automatic) + @echo "🔧 Running Keycloak setup manually..." + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) run --rm keycloak-setup + +docker-clean: ## Clean up everything (containers, volumes, images) + @echo "🧹 Cleaning up Docker environment..." + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) down -v --remove-orphans + @docker system prune -f + @echo "✅ Cleanup complete" + +docker-shell-s3: ## Get shell access to S3 container + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) exec weed-s3 sh + +docker-shell-keycloak: ## Get shell access to Keycloak container + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) exec keycloak bash + +docker-debug: ## Show debug information + @echo "🔍 Docker Environment Debug Information" + @echo "" + @echo "📋 Docker Compose Config:" + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) config + @echo "" + @echo "📊 Container Status:" + @docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) ps + @echo "" + @echo "🌐 Network Information:" + @docker network ls | grep $(PROJECT_NAME) || echo "No networks found" + @echo "" + @echo "💾 Volume Information:" + @docker volume ls | grep $(PROJECT_NAME) || echo "No volumes found" + +# Quick test targets +docker-test-auth: ## Quick test of authentication only + @KEYCLOAK_URL=http://localhost:8080 go test -v -run "TestKeycloakAuthentication" -timeout 2m ./... + +docker-test-roles: ## Quick test of role mapping only + @KEYCLOAK_URL=http://localhost:8080 go test -v -run "TestKeycloakRoleMapping" -timeout 2m ./... + +docker-test-s3ops: ## Quick test of S3 operations only + @KEYCLOAK_URL=http://localhost:8080 go test -v -run "TestKeycloakS3Operations" -timeout 2m ./... + +# Development workflow +docker-dev: docker-down docker-up docker-test ## Complete dev workflow: down -> up -> test + +# Show service URLs for easy access +docker-urls: ## Display all service URLs + @echo "🌐 Service URLs:" + @echo "" + @echo " 🔐 Keycloak Admin: http://localhost:8080 (admin/admin)" + @echo " 🔐 Keycloak Realm: http://localhost:8080/realms/seaweedfs-test" + @echo " 📁 S3 API: http://localhost:8333" + @echo " 📂 Filer UI: http://localhost:8888" + @echo " 🎯 Master UI: http://localhost:9333" + @echo " 💾 Volume Server: http://localhost:8080" + @echo "" + @echo " 📖 Test Users:" + @echo " • admin-user (password: adminuser123) - s3-admin role" + @echo " • read-user (password: readuser123) - s3-read-only role" + @echo " • write-user (password: writeuser123) - s3-read-write role" + @echo " • write-only-user (password: writeonlyuser123) - s3-write-only role" + +# Wait targets for CI/CD +docker-wait-healthy: ## Wait for all services to be healthy + @echo "⏳ Waiting for all services to be healthy..." + @timeout 300 bash -c ' \ + while [ $$(docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) ps | grep -c "healthy") -lt 4 ]; do \ + echo "Waiting for services to be healthy..."; \ + sleep 5; \ + done \ + ' + @echo "✅ All services are healthy" diff --git a/test/s3/iam/README-Docker.md b/test/s3/iam/README-Docker.md new file mode 100644 index 000000000..39036b449 --- /dev/null +++ b/test/s3/iam/README-Docker.md @@ -0,0 +1,231 @@ +# SeaweedFS S3 IAM Integration with Docker Compose + +This directory contains a complete Docker Compose setup for testing SeaweedFS S3 IAM integration with Keycloak OIDC authentication. + +## 🚀 Quick Start + +1. **Start the environment:** + ```bash + make -f Makefile.docker docker-up + ``` + +2. **Run the tests:** + ```bash + make -f Makefile.docker docker-test + ``` + +3. **Stop the environment:** + ```bash + make -f Makefile.docker docker-down + ``` + +## 📋 What's Included + +The Docker Compose setup includes: + +- **🔐 Keycloak** - Identity provider with OIDC support +- **🎯 SeaweedFS Master** - Metadata management +- **💾 SeaweedFS Volume** - Data storage +- **📁 SeaweedFS Filer** - File system interface +- **📊 SeaweedFS S3** - S3-compatible API with IAM integration +- **🔧 Keycloak Setup** - Automated realm and user configuration + +## 🌐 Service URLs + +After starting with `docker-up`, services are available at: + +| Service | URL | Credentials | +|---------|-----|-------------| +| 🔐 Keycloak Admin | http://localhost:8080 | admin/admin | +| 📊 S3 API | http://localhost:8333 | JWT tokens | +| 📁 Filer | http://localhost:8888 | - | +| 🎯 Master | http://localhost:9333 | - | + +## 👥 Test Users + +The setup automatically creates test users in Keycloak: + +| Username | Password | Role | Permissions | +|----------|----------|------|-------------| +| admin-user | adminuser123 | s3-admin | Full S3 access | +| read-user | readuser123 | s3-read-only | Read-only access | +| write-user | writeuser123 | s3-read-write | Read and write | +| write-only-user | writeonlyuser123 | s3-write-only | Write only | + +## 🧪 Running Tests + +### All Tests +```bash +make -f Makefile.docker docker-test +``` + +### Specific Test Categories +```bash +# Authentication tests only +make -f Makefile.docker docker-test-auth + +# Role mapping tests only +make -f Makefile.docker docker-test-roles + +# S3 operations tests only +make -f Makefile.docker docker-test-s3ops +``` + +### Single Test +```bash +make -f Makefile.docker docker-test-single TEST_NAME=TestKeycloakAuthentication +``` + +## 🔧 Development Workflow + +### Complete workflow (recommended) +```bash +make -f Makefile.docker docker-dev +``` +This runs: down → up → test + +### Manual steps +```bash +# Start services +make -f Makefile.docker docker-up + +# Watch logs +make -f Makefile.docker docker-logs + +# Check status +make -f Makefile.docker docker-status + +# Run tests +make -f Makefile.docker docker-test + +# Stop services +make -f Makefile.docker docker-down +``` + +## 🔍 Debugging + +### View logs +```bash +# All services +make -f Makefile.docker docker-logs + +# S3 service only (includes role mapping debug) +make -f Makefile.docker docker-logs-s3 + +# Keycloak only +make -f Makefile.docker docker-logs-keycloak +``` + +### Get shell access +```bash +# S3 container +make -f Makefile.docker docker-shell-s3 + +# Keycloak container +make -f Makefile.docker docker-shell-keycloak +``` + +### Debug environment +```bash +make -f Makefile.docker docker-debug +``` + +## 📁 File Structure + +``` +seaweedfs/test/s3/iam/ +├── docker-compose.yml # Main Docker Compose configuration +├── Makefile.docker # Docker-specific Makefile +├── Dockerfile.debug # Debug SeaweedFS image +├── setup_keycloak_docker.sh # Keycloak setup for containers +├── README-Docker.md # This file +├── iam_config.json # IAM configuration (auto-generated) +├── test_config.json # S3 service configuration +└── *_test.go # Go integration tests +``` + +## 🔄 Configuration + +### IAM Configuration +The `setup_keycloak_docker.sh` script automatically generates `iam_config.json` with: + +- **OIDC Provider**: Keycloak configuration with proper container networking +- **Role Mapping**: Maps Keycloak roles to SeaweedFS IAM roles +- **Policies**: Defines S3 permissions for each role +- **Trust Relationships**: Allows Keycloak users to assume SeaweedFS roles + +### Role Mapping Rules +```json +{ + "claim": "roles", + "value": "s3-admin", + "role": "arn:seaweed:iam::role/KeycloakAdminRole" +} +``` + +## 🐛 Troubleshooting + +### Services not starting +```bash +# Check service status +make -f Makefile.docker docker-status + +# View logs for specific service +docker-compose -p seaweedfs-iam-test logs +``` + +### Keycloak setup issues +```bash +# Re-run Keycloak setup manually +make -f Makefile.docker docker-keycloak-setup + +# Check Keycloak logs +make -f Makefile.docker docker-logs-keycloak +``` + +### Role mapping not working +```bash +# Check S3 logs for role mapping debug messages +make -f Makefile.docker docker-logs-s3 | grep -i "role\|claim\|mapping" +``` + +### Port conflicts +If ports are already in use, modify `docker-compose.yml`: +```yaml +ports: + - "8081:8080" # Change external port +``` + +## 🧹 Cleanup + +```bash +# Stop containers and remove volumes +make -f Makefile.docker docker-down + +# Complete cleanup (containers, volumes, images) +make -f Makefile.docker docker-clean +``` + +## 🎯 Key Features + +- **Isolated Environment**: No conflicts with local services +- **Consistent Networking**: Services communicate via Docker network +- **Automated Setup**: Keycloak realm and users created automatically +- **Debug Logging**: Verbose logging enabled for troubleshooting +- **Health Checks**: Proper service dependency management +- **Volume Persistence**: Data persists between restarts (until docker-down) + +## 🚦 CI/CD Integration + +For automated testing: + +```bash +# Wait for all services to be healthy +make -f Makefile.docker docker-wait-healthy + +# Run tests with proper cleanup +make -f Makefile.docker docker-up +make -f Makefile.docker docker-wait-healthy +make -f Makefile.docker docker-test +make -f Makefile.docker docker-down +``` diff --git a/test/s3/iam/docker-compose-simple.yml b/test/s3/iam/docker-compose-simple.yml new file mode 100644 index 000000000..9e3b91e42 --- /dev/null +++ b/test/s3/iam/docker-compose-simple.yml @@ -0,0 +1,22 @@ +version: '3.8' + +services: + # Keycloak Identity Provider + keycloak: + image: quay.io/keycloak/keycloak:26.0.7 + container_name: keycloak-test-simple + ports: + - "8080:8080" + environment: + KC_BOOTSTRAP_ADMIN_USERNAME: admin + KC_BOOTSTRAP_ADMIN_PASSWORD: admin + KC_HTTP_ENABLED: "true" + KC_HOSTNAME_STRICT: "false" + KC_HOSTNAME_STRICT_HTTPS: "false" + command: start-dev + networks: + - test-network + +networks: + test-network: + driver: bridge diff --git a/test/s3/iam/docker-compose.yml b/test/s3/iam/docker-compose.yml index 2b8d08f6b..c7df89a97 100644 --- a/test/s3/iam/docker-compose.yml +++ b/test/s3/iam/docker-compose.yml @@ -1,121 +1,162 @@ version: '3.8' services: - # Keycloak OIDC Provider + # Keycloak Identity Provider keycloak: image: quay.io/keycloak/keycloak:26.0.7 container_name: keycloak-iam-test + hostname: keycloak environment: - KEYCLOAK_ADMIN: admin - KEYCLOAK_ADMIN_PASSWORD: admin123 - KC_HTTP_PORT: 8080 - KC_HOSTNAME_STRICT: false - KC_HOSTNAME_STRICT_HTTPS: false - KC_HTTP_ENABLED: true - KC_HEALTH_ENABLED: true + KC_BOOTSTRAP_ADMIN_USERNAME: admin + KC_BOOTSTRAP_ADMIN_PASSWORD: admin + KC_HTTP_ENABLED: "true" + KC_HOSTNAME_STRICT: "false" + KC_HOSTNAME_STRICT_HTTPS: "false" + KC_HTTP_RELATIVE_PATH: / ports: - "8080:8080" - command: - - start-dev - - --import-realm - volumes: - - ./keycloak-realm.json:/opt/keycloak/data/import/realm.json:ro + command: start-dev + networks: + - seaweedfs-iam healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/health/ready"] - interval: 30s - timeout: 10s + interval: 10s + timeout: 5s retries: 5 start_period: 60s - networks: - - seaweedfs-iam # SeaweedFS Master - seaweedfs-master: + weed-master: image: chrislusf/seaweedfs:latest - container_name: seaweedfs-master-iam + container_name: weed-master + hostname: weed-master ports: - "9333:9333" - "19333:19333" - command: master -ip=seaweedfs-master -mdir=/data -volumeSizeLimitMB=50 + command: "master -ip=weed-master -port=9333 -mdir=/data" volumes: - - seaweedfs-master-data:/data + - master-data:/data + networks: + - seaweedfs-iam healthcheck: - test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:9333/cluster/status"] + test: ["CMD", "wget", "-q", "--spider", "http://localhost:9333/cluster/status"] interval: 10s timeout: 5s retries: 3 - networks: - - seaweedfs-iam + start_period: 10s # SeaweedFS Volume Server - seaweedfs-volume: + weed-volume: image: chrislusf/seaweedfs:latest - container_name: seaweedfs-volume-iam + container_name: weed-volume + hostname: weed-volume ports: - "8080:8080" - command: volume -ip=seaweedfs-volume -port=8080 -mserver=seaweedfs-master:9333 -dir=/data + - "18080:18080" + command: "volume -ip=weed-volume -port=8080 -dir=/data -mserver=weed-master:9333 -dataCenter=dc1 -rack=rack1" volumes: - - seaweedfs-volume-data:/data + - volume-data:/data + networks: + - seaweedfs-iam depends_on: - seaweedfs-master: + weed-master: condition: service_healthy healthcheck: - test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8080/status"] + test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/status"] interval: 10s timeout: 5s retries: 3 - networks: - - seaweedfs-iam + start_period: 10s # SeaweedFS Filer - seaweedfs-filer: + weed-filer: image: chrislusf/seaweedfs:latest - container_name: seaweedfs-filer-iam + container_name: weed-filer + hostname: weed-filer ports: - "8888:8888" - "18888:18888" - command: filer -ip=seaweedfs-filer -master=seaweedfs-master:9333 + command: "filer -ip=weed-filer -port=8888 -master=weed-master:9333 -defaultStoreDir=/data" + volumes: + - filer-data:/data + networks: + - seaweedfs-iam depends_on: - seaweedfs-master: + weed-master: + condition: service_healthy + weed-volume: condition: service_healthy healthcheck: - test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8888/"] + test: ["CMD", "wget", "-q", "--spider", "http://localhost:8888/status"] interval: 10s timeout: 5s retries: 3 - networks: - - seaweedfs-iam + start_period: 10s - # SeaweedFS S3 Gateway with IAM - seaweedfs-s3: - build: - context: ../../.. - dockerfile: test/s3/iam/Dockerfile.s3 - container_name: seaweedfs-s3-iam + # SeaweedFS S3 API with IAM + weed-s3: + image: ${SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:latest} + container_name: weed-s3 + hostname: weed-s3 ports: - "8333:8333" environment: - - KEYCLOAK_URL=http://keycloak:8080 - command: s3 -port=8333 -filer=seaweedfs-filer:8888 -iam.config=/etc/seaweedfs/iam_config.json + WEED_FILER: "weed-filer:8888" + WEED_IAM_CONFIG: "/config/iam_config.json" + WEED_S3_CONFIG: "/config/test_config.json" + GLOG_v: "3" + command: > + sh -c " + echo 'Waiting for dependencies...' && + sleep 10 && + echo 'Starting S3 API with IAM...' && + weed -v=3 s3 -ip=weed-s3 -port=8333 + -filer=weed-filer:8888 + -config=/config/test_config.json + -iam.config=/config/iam_config.json + " volumes: - - ./iam_config_docker.json:/etc/seaweedfs/iam_config.json:ro + - ./iam_config.json:/config/iam_config.json:ro + - ./test_config.json:/config/test_config.json:ro + networks: + - seaweedfs-iam depends_on: - keycloak: + weed-filer: condition: service_healthy - seaweedfs-filer: + keycloak: condition: service_healthy healthcheck: - test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8333/"] + test: ["CMD", "wget", "-q", "--spider", "http://localhost:8333"] interval: 10s timeout: 5s - retries: 3 + retries: 5 + start_period: 30s + + # Keycloak Setup Service + keycloak-setup: + image: alpine/curl:latest + container_name: keycloak-setup + volumes: + - ./setup_keycloak_docker.sh:/setup.sh:ro + - .:/workspace:rw + working_dir: /workspace networks: - seaweedfs-iam + depends_on: + keycloak: + condition: service_healthy + command: > + sh -c " + apk add --no-cache bash jq && + chmod +x /setup.sh && + /setup.sh + " volumes: - seaweedfs-master-data: - seaweedfs-volume-data: + master-data: + volume-data: + filer-data: networks: seaweedfs-iam: - driver: bridge + driver: bridge \ No newline at end of file diff --git a/test/s3/iam/iam_config.github.json b/test/s3/iam/iam_config.github.json deleted file mode 100644 index d931f8389..000000000 --- a/test/s3/iam/iam_config.github.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "sts": { - "tokenDuration": 3600000000000, - "maxSessionLength": 43200000000000, - "issuer": "seaweedfs-sts", - "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc=" - }, - "providers": [ - { - "name": "test-oidc", - "type": "mock", - "config": { - "issuer": "test-oidc-issuer", - "clientId": "test-oidc-client" - } - }, - { - "name": "keycloak", - "type": "oidc", - "enabled": true, - "config": { - "issuer": "http://localhost:8080/realms/seaweedfs-test", - "clientId": "seaweedfs-s3", - "clientSecret": "seaweedfs-s3-secret", - "jwksUri": "http://localhost:8080/realms/seaweedfs-test/protocol/openid-connect/certs", - "userInfoUri": "http://localhost:8080/realms/seaweedfs-test/protocol/openid-connect/userinfo", - "scopes": ["openid", "profile", "email"], - "claimsMapping": { - "username": "preferred_username", - "email": "email", - "name": "name" - }, - "roleMapping": { - "rules": [ - { - "claim": "roles", - "value": "s3-admin", - "role": "arn:seaweed:iam::role/KeycloakAdminRole" - }, - { - "claim": "roles", - "value": "s3-read-only", - "role": "arn:seaweed:iam::role/KeycloakReadOnlyRole" - }, - { - "claim": "roles", - "value": "s3-write-only", - "role": "arn:seaweed:iam::role/KeycloakWriteOnlyRole" - }, - { - "claim": "roles", - "value": "s3-read-write", - "role": "arn:seaweed:iam::role/KeycloakReadWriteRole" - } - ], - "defaultRole": "arn:seaweed:iam::role/KeycloakReadOnlyRole" - } - } - } - ], - "policy": { - "defaultEffect": "Deny", - "storeType": "memory" - }, - "roles": [ - { - "roleName": "TestAdminRole", - "roleArn": "arn:seaweed:iam::role/TestAdminRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "test-oidc" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3AdminPolicy"], - "description": "Admin role for testing" - }, - { - "roleName": "TestReadOnlyRole", - "roleArn": "arn:seaweed:iam::role/TestReadOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "test-oidc" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3ReadOnlyPolicy"], - "description": "Read-only role for testing" - }, - { - "roleName": "TestWriteOnlyRole", - "roleArn": "arn:seaweed:iam::role/TestWriteOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "test-oidc" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3WriteOnlyPolicy"], - "description": "Write-only role for testing" - }, - { - "roleName": "KeycloakAdminRole", - "roleArn": "arn:seaweed:iam::role/KeycloakAdminRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "keycloak" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3AdminPolicy"], - "description": "Admin role for Keycloak users" - }, - { - "roleName": "KeycloakReadOnlyRole", - "roleArn": "arn:seaweed:iam::role/KeycloakReadOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "keycloak" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3ReadOnlyPolicy"], - "description": "Read-only role for Keycloak users" - }, - { - "roleName": "KeycloakWriteOnlyRole", - "roleArn": "arn:seaweed:iam::role/KeycloakWriteOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "keycloak" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3WriteOnlyPolicy"], - "description": "Write-only role for Keycloak users" - }, - { - "roleName": "KeycloakReadWriteRole", - "roleArn": "arn:seaweed:iam::role/KeycloakReadWriteRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "keycloak" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3ReadWritePolicy"], - "description": "Read-write role for Keycloak users" - } - ], - "policies": [ - { - "name": "S3AdminPolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:*"], - "Resource": ["*"] - }, - { - "Effect": "Allow", - "Action": ["sts:ValidateSession"], - "Resource": ["*"] - } - ] - } - }, - { - "name": "S3ReadOnlyPolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Allow", - "Action": ["sts:ValidateSession"], - "Resource": ["*"] - } - ] - } - }, - { - "name": "S3WriteOnlyPolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:*" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Deny", - "Action": [ - "s3:GetObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Allow", - "Action": ["sts:ValidateSession"], - "Resource": ["*"] - } - ] - } - }, - { - "name": "S3ReadWritePolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:*" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Allow", - "Action": ["sts:ValidateSession"], - "Resource": ["*"] - } - ] - } - } - ] -} diff --git a/test/s3/iam/iam_config.json b/test/s3/iam/iam_config.json deleted file mode 100644 index d931f8389..000000000 --- a/test/s3/iam/iam_config.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "sts": { - "tokenDuration": 3600000000000, - "maxSessionLength": 43200000000000, - "issuer": "seaweedfs-sts", - "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc=" - }, - "providers": [ - { - "name": "test-oidc", - "type": "mock", - "config": { - "issuer": "test-oidc-issuer", - "clientId": "test-oidc-client" - } - }, - { - "name": "keycloak", - "type": "oidc", - "enabled": true, - "config": { - "issuer": "http://localhost:8080/realms/seaweedfs-test", - "clientId": "seaweedfs-s3", - "clientSecret": "seaweedfs-s3-secret", - "jwksUri": "http://localhost:8080/realms/seaweedfs-test/protocol/openid-connect/certs", - "userInfoUri": "http://localhost:8080/realms/seaweedfs-test/protocol/openid-connect/userinfo", - "scopes": ["openid", "profile", "email"], - "claimsMapping": { - "username": "preferred_username", - "email": "email", - "name": "name" - }, - "roleMapping": { - "rules": [ - { - "claim": "roles", - "value": "s3-admin", - "role": "arn:seaweed:iam::role/KeycloakAdminRole" - }, - { - "claim": "roles", - "value": "s3-read-only", - "role": "arn:seaweed:iam::role/KeycloakReadOnlyRole" - }, - { - "claim": "roles", - "value": "s3-write-only", - "role": "arn:seaweed:iam::role/KeycloakWriteOnlyRole" - }, - { - "claim": "roles", - "value": "s3-read-write", - "role": "arn:seaweed:iam::role/KeycloakReadWriteRole" - } - ], - "defaultRole": "arn:seaweed:iam::role/KeycloakReadOnlyRole" - } - } - } - ], - "policy": { - "defaultEffect": "Deny", - "storeType": "memory" - }, - "roles": [ - { - "roleName": "TestAdminRole", - "roleArn": "arn:seaweed:iam::role/TestAdminRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "test-oidc" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3AdminPolicy"], - "description": "Admin role for testing" - }, - { - "roleName": "TestReadOnlyRole", - "roleArn": "arn:seaweed:iam::role/TestReadOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "test-oidc" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3ReadOnlyPolicy"], - "description": "Read-only role for testing" - }, - { - "roleName": "TestWriteOnlyRole", - "roleArn": "arn:seaweed:iam::role/TestWriteOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "test-oidc" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3WriteOnlyPolicy"], - "description": "Write-only role for testing" - }, - { - "roleName": "KeycloakAdminRole", - "roleArn": "arn:seaweed:iam::role/KeycloakAdminRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "keycloak" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3AdminPolicy"], - "description": "Admin role for Keycloak users" - }, - { - "roleName": "KeycloakReadOnlyRole", - "roleArn": "arn:seaweed:iam::role/KeycloakReadOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "keycloak" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3ReadOnlyPolicy"], - "description": "Read-only role for Keycloak users" - }, - { - "roleName": "KeycloakWriteOnlyRole", - "roleArn": "arn:seaweed:iam::role/KeycloakWriteOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "keycloak" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3WriteOnlyPolicy"], - "description": "Write-only role for Keycloak users" - }, - { - "roleName": "KeycloakReadWriteRole", - "roleArn": "arn:seaweed:iam::role/KeycloakReadWriteRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "keycloak" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3ReadWritePolicy"], - "description": "Read-write role for Keycloak users" - } - ], - "policies": [ - { - "name": "S3AdminPolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:*"], - "Resource": ["*"] - }, - { - "Effect": "Allow", - "Action": ["sts:ValidateSession"], - "Resource": ["*"] - } - ] - } - }, - { - "name": "S3ReadOnlyPolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Allow", - "Action": ["sts:ValidateSession"], - "Resource": ["*"] - } - ] - } - }, - { - "name": "S3WriteOnlyPolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:*" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Deny", - "Action": [ - "s3:GetObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Allow", - "Action": ["sts:ValidateSession"], - "Resource": ["*"] - } - ] - } - }, - { - "name": "S3ReadWritePolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:*" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Allow", - "Action": ["sts:ValidateSession"], - "Resource": ["*"] - } - ] - } - } - ] -} diff --git a/test/s3/iam/iam_config.local.json b/test/s3/iam/iam_config.local.json deleted file mode 100644 index e87bbca96..000000000 --- a/test/s3/iam/iam_config.local.json +++ /dev/null @@ -1,346 +0,0 @@ -{ - "sts": { - "tokenDuration": 3600000000000, - "maxSessionLength": 43200000000000, - "issuer": "seaweedfs-sts", - "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc=" - }, - "providers": [ - { - "name": "test-oidc", - "type": "mock", - "config": { - "issuer": "test-oidc-issuer", - "clientId": "test-oidc-client" - } - }, - { - "name": "keycloak", - "type": "oidc", - "enabled": true, - "config": { - "issuer": "http://localhost:8090/realms/seaweedfs-test", - "clientId": "seaweedfs-s3", - "clientSecret": "seaweedfs-s3-secret", - "jwksUri": "http://localhost:8090/realms/seaweedfs-test/protocol/openid-connect/certs", - "userInfoUri": "http://localhost:8090/realms/seaweedfs-test/protocol/openid-connect/userinfo", - "scopes": [ - "openid", - "profile", - "email" - ], - "claimsMapping": { - "username": "preferred_username", - "email": "email", - "name": "name" - }, - "roleMapping": { - "rules": [ - { - "claim": "roles", - "value": "s3-admin", - "role": "arn:seaweed:iam::role/KeycloakAdminRole" - }, - { - "claim": "roles", - "value": "s3-read-only", - "role": "arn:seaweed:iam::role/KeycloakReadOnlyRole" - }, - { - "claim": "roles", - "value": "s3-write-only", - "role": "arn:seaweed:iam::role/KeycloakWriteOnlyRole" - }, - { - "claim": "roles", - "value": "s3-read-write", - "role": "arn:seaweed:iam::role/KeycloakReadWriteRole" - } - ], - "defaultRole": "arn:seaweed:iam::role/KeycloakReadOnlyRole" - } - } - } - ], - "policy": { - "defaultEffect": "Deny", - "storeType": "memory" - }, - "roles": [ - { - "roleName": "TestAdminRole", - "roleArn": "arn:seaweed:iam::role/TestAdminRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "test-oidc" - }, - "Action": [ - "sts:AssumeRoleWithWebIdentity" - ] - } - ] - }, - "attachedPolicies": [ - "S3AdminPolicy" - ], - "description": "Admin role for testing" - }, - { - "roleName": "TestReadOnlyRole", - "roleArn": "arn:seaweed:iam::role/TestReadOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "test-oidc" - }, - "Action": [ - "sts:AssumeRoleWithWebIdentity" - ] - } - ] - }, - "attachedPolicies": [ - "S3ReadOnlyPolicy" - ], - "description": "Read-only role for testing" - }, - { - "roleName": "TestWriteOnlyRole", - "roleArn": "arn:seaweed:iam::role/TestWriteOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "test-oidc" - }, - "Action": [ - "sts:AssumeRoleWithWebIdentity" - ] - } - ] - }, - "attachedPolicies": [ - "S3WriteOnlyPolicy" - ], - "description": "Write-only role for testing" - }, - { - "roleName": "KeycloakAdminRole", - "roleArn": "arn:seaweed:iam::role/KeycloakAdminRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "keycloak" - }, - "Action": [ - "sts:AssumeRoleWithWebIdentity" - ] - } - ] - }, - "attachedPolicies": [ - "S3AdminPolicy" - ], - "description": "Admin role for Keycloak users" - }, - { - "roleName": "KeycloakReadOnlyRole", - "roleArn": "arn:seaweed:iam::role/KeycloakReadOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "keycloak" - }, - "Action": [ - "sts:AssumeRoleWithWebIdentity" - ] - } - ] - }, - "attachedPolicies": [ - "S3ReadOnlyPolicy" - ], - "description": "Read-only role for Keycloak users" - }, - { - "roleName": "KeycloakWriteOnlyRole", - "roleArn": "arn:seaweed:iam::role/KeycloakWriteOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "keycloak" - }, - "Action": [ - "sts:AssumeRoleWithWebIdentity" - ] - } - ] - }, - "attachedPolicies": [ - "S3WriteOnlyPolicy" - ], - "description": "Write-only role for Keycloak users" - }, - { - "roleName": "KeycloakReadWriteRole", - "roleArn": "arn:seaweed:iam::role/KeycloakReadWriteRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "keycloak" - }, - "Action": [ - "sts:AssumeRoleWithWebIdentity" - ] - } - ] - }, - "attachedPolicies": [ - "S3ReadWritePolicy" - ], - "description": "Read-write role for Keycloak users" - } - ], - "policies": [ - { - "name": "S3AdminPolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:*" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "sts:ValidateSession" - ], - "Resource": [ - "*" - ] - } - ] - } - }, - { - "name": "S3ReadOnlyPolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "sts:ValidateSession" - ], - "Resource": [ - "*" - ] - } - ] - } - }, - { - "name": "S3WriteOnlyPolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:*" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Deny", - "Action": [ - "s3:GetObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "sts:ValidateSession" - ], - "Resource": [ - "*" - ] - } - ] - } - }, - { - "name": "S3ReadWritePolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:*" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "sts:ValidateSession" - ], - "Resource": [ - "*" - ] - } - ] - } - } - ] -} diff --git a/test/s3/iam/keycloak-realm.json b/test/s3/iam/keycloak-realm.json deleted file mode 100644 index 6ccbc76b9..000000000 --- a/test/s3/iam/keycloak-realm.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "realm": "seaweedfs-test", - "enabled": true, - "displayName": "SeaweedFS Test Realm", - "accessTokenLifespan": 3600, - "accessTokenLifespanForImplicitFlow": 3600, - "ssoSessionIdleTimeout": 3600, - "ssoSessionMaxLifespan": 36000, - "clients": [ - { - "clientId": "seaweedfs-s3", - "enabled": true, - "protocol": "openid-connect", - "publicClient": false, - "secret": "seaweedfs-s3-secret", - "directAccessGrantsEnabled": true, - "serviceAccountsEnabled": true, - "standardFlowEnabled": true, - "implicitFlowEnabled": false, - "redirectUris": ["*"], - "webOrigins": ["*"], - "protocolMappers": [ - { - "name": "role-mapper", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-realm-role-mapper", - "config": { - "claim.name": "roles", - "jsonType.label": "String", - "multivalued": "true", - "userinfo.token.claim": "true", - "id.token.claim": "true", - "access.token.claim": "true" - } - }, - { - "name": "username-mapper", - "protocol": "openid-connect", - "protocolMapper": "oidc-usermodel-property-mapper", - "config": { - "claim.name": "preferred_username", - "user.attribute": "username", - "jsonType.label": "String", - "userinfo.token.claim": "true", - "id.token.claim": "true", - "access.token.claim": "true" - } - } - ] - } - ], - "roles": { - "realm": [ - { - "name": "s3-admin", - "description": "S3 Administrator role with full access" - }, - { - "name": "s3-read-only", - "description": "S3 Read-only role" - }, - { - "name": "s3-read-write", - "description": "S3 Read-write role" - } - ] - }, - "users": [ - { - "username": "admin-user", - "enabled": true, - "firstName": "Admin", - "lastName": "User", - "email": "admin@seaweedfs.test", - "emailVerified": true, - "credentials": [ - { - "type": "password", - "value": "admin123", - "temporary": false - } - ], - "realmRoles": ["s3-admin"], - "attributes": { - "department": ["engineering"], - "location": ["datacenter-1"] - } - }, - { - "username": "read-user", - "enabled": true, - "firstName": "Read", - "lastName": "User", - "email": "read@seaweedfs.test", - "emailVerified": true, - "credentials": [ - { - "type": "password", - "value": "read123", - "temporary": false - } - ], - "realmRoles": ["s3-read-only"], - "attributes": { - "department": ["analytics"], - "location": ["datacenter-2"] - } - }, - { - "username": "write-user", - "enabled": true, - "firstName": "Write", - "lastName": "User", - "email": "write@seaweedfs.test", - "emailVerified": true, - "credentials": [ - { - "type": "password", - "value": "write123", - "temporary": false - } - ], - "realmRoles": ["s3-read-write"], - "attributes": { - "department": ["operations"], - "location": ["datacenter-1"] - } - } - ], - "identityProviders": [], - "identityProviderMappers": [], - "requiredActions": [], - "browserFlow": "browser", - "registrationFlow": "registration", - "directGrantFlow": "direct grant", - "resetCredentialsFlow": "reset credentials", - "clientAuthenticationFlow": "clients" -} diff --git a/test/s3/iam/setup_all_tests.sh b/test/s3/iam/setup_all_tests.sh deleted file mode 100755 index 49ea34dca..000000000 --- a/test/s3/iam/setup_all_tests.sh +++ /dev/null @@ -1,621 +0,0 @@ -#!/bin/bash - -# SeaweedFS S3 IAM Complete Test Setup Script -# This script enables all previously skipped tests by setting up the required environment - -set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Configuration -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SEAWEEDFS_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" -TEST_DIR="$SCRIPT_DIR" - -# Service ports -KEYCLOAK_PORT=8080 -S3_PORT=8333 -FILER_PORT=8888 -MASTER_PORT=9333 -VOLUME_PORT=8080 - -# Test configuration -export KEYCLOAK_URL="http://localhost:$KEYCLOAK_PORT" -export S3_ENDPOINT="http://localhost:$S3_PORT" -export TEST_TIMEOUT="60m" -export CGO_ENABLED=0 - -echo -e "${BLUE}🚀 SeaweedFS S3 IAM Complete Test Setup${NC}" -echo -e "${BLUE}======================================${NC}" - -# Function to check if a service is running -check_service() { - local service_name="$1" - local url="$2" - local max_attempts=30 - local attempt=1 - - echo -e "${YELLOW}⏳ Waiting for $service_name to be ready...${NC}" - - while [ $attempt -le $max_attempts ]; do - if curl -s "$url" > /dev/null 2>&1; then - echo -e "${GREEN}✅ $service_name is ready${NC}" - return 0 - fi - echo -n "." - sleep 2 - ((attempt++)) - done - - echo -e "${RED}❌ $service_name failed to start after $((max_attempts * 2)) seconds${NC}" - return 1 -} - -# Function to setup Keycloak -setup_keycloak() { - echo -e "${BLUE}🔐 Setting up Keycloak for IAM integration tests...${NC}" - - # Check if Keycloak is already running - if curl -s "http://localhost:$KEYCLOAK_PORT/health/ready" > /dev/null 2>&1; then - echo -e "${GREEN}✅ Keycloak is already running${NC}" - return 0 - fi - - # Check if Docker is available - if ! command -v docker &> /dev/null; then - echo -e "${RED}❌ Docker is required for Keycloak setup${NC}" - echo -e "${YELLOW}💡 Install Docker or run Keycloak manually${NC}" - return 1 - fi - - # Start Keycloak with Docker - echo -e "${YELLOW}🐳 Starting Keycloak container...${NC}" - - # Stop any existing Keycloak container - docker stop keycloak-iam-test 2>/dev/null || true - docker rm keycloak-iam-test 2>/dev/null || true - - # Start new Keycloak container with correct environment variables for 26.0 - docker run -d \ - --name keycloak-iam-test \ - -p $KEYCLOAK_PORT:8080 \ - -e KC_BOOTSTRAP_ADMIN_USERNAME=admin \ - -e KC_BOOTSTRAP_ADMIN_PASSWORD=admin \ - -e KC_HTTP_ENABLED=true \ - -e KC_HOSTNAME_STRICT=false \ - -e KC_HOSTNAME_STRICT_HTTPS=false \ - -e KC_HEALTH_ENABLED=true \ - quay.io/keycloak/keycloak:26.0 start-dev - - # Wait for Keycloak to be ready - if check_service "Keycloak" "http://localhost:$KEYCLOAK_PORT/health/ready"; then - echo -e "${GREEN}✅ Keycloak setup complete${NC}" - return 0 - else - echo -e "${RED}❌ Keycloak setup failed${NC}" - return 1 - fi -} - -# Function to setup distributed environment -setup_distributed_environment() { - echo -e "${BLUE}🌐 Setting up distributed test environment...${NC}" - - # Create distributed configuration - if [ ! -f "$TEST_DIR/iam_config_distributed.json" ]; then - echo -e "${YELLOW}📝 Creating distributed IAM configuration...${NC}" - cat > "$TEST_DIR/iam_config_distributed.json" << 'EOF' -{ - "sts": { - "tokenDuration": 3600000000000, - "maxSessionLength": 43200000000000, - "issuer": "seaweedfs-sts", - "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc=", - "sessionStoreType": "filer", - "sessionStoreConfig": { - "filerAddress": "localhost:8888", - "basePath": "/seaweedfs/iam/sessions" - } - }, - "identityProviders": [ - { - "name": "test-oidc", - "type": "mock", - "config": { - "issuer": "test-oidc-issuer" - } - } - ], - "policy": { - "defaultEffect": "Deny", - "storeType": "filer", - "storeConfig": { - "filerAddress": "localhost:8888", - "basePath": "/seaweedfs/iam/policies" - } - }, - "roleStore": { - "storeType": "filer", - "storeConfig": { - "filerAddress": "localhost:8888", - "basePath": "/seaweedfs/iam/roles" - } - }, - "roles": [ - { - "roleName": "TestAdminRole", - "roleArn": "arn:seaweed:iam::role/TestAdminRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "test-oidc" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3AdminPolicy"], - "description": "Admin role for distributed testing" - }, - { - "roleName": "TestReadOnlyRole", - "roleArn": "arn:seaweed:iam::role/TestReadOnlyRole", - "trustPolicy": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "test-oidc" - }, - "Action": ["sts:AssumeRoleWithWebIdentity"] - } - ] - }, - "attachedPolicies": ["S3ReadOnlyPolicy"], - "description": "Read-only role for distributed testing" - } - ], - "policies": [ - { - "name": "S3AdminPolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:*"], - "Resource": ["*"] - }, - { - "Effect": "Allow", - "Action": ["sts:ValidateSession"], - "Resource": ["*"] - } - ] - } - }, - { - "name": "S3ReadOnlyPolicy", - "document": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:seaweed:s3:::*", - "arn:seaweed:s3:::*/*" - ] - }, - { - "Effect": "Allow", - "Action": ["sts:ValidateSession"], - "Resource": ["*"] - } - ] - } - } - ] -} -EOF - fi - - echo -e "${GREEN}✅ Distributed environment configuration ready${NC}" -} - -# Function to create performance test runner -create_performance_test_runner() { - echo -e "${BLUE}🏁 Creating performance test runner...${NC}" - - cat > "$TEST_DIR/run_performance_tests.sh" << 'EOF' -#!/bin/bash - -# Performance Test Runner for SeaweedFS S3 IAM - -set -e - -# Colors -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -echo -e "${YELLOW}🏁 Running S3 IAM Performance Tests${NC}" - -# Enable performance tests -export ENABLE_PERFORMANCE_TESTS=true -export TEST_TIMEOUT=60m - -# Run benchmarks -echo -e "${YELLOW}📊 Running benchmarks...${NC}" -go test -bench=. -benchmem -timeout=$TEST_TIMEOUT ./... - -# Run performance tests -echo -e "${YELLOW}🧪 Running performance test suite...${NC}" -go test -v -timeout=$TEST_TIMEOUT -run "TestS3IAMPerformanceTests" ./... - -echo -e "${GREEN}✅ Performance tests completed${NC}" -EOF - - chmod +x "$TEST_DIR/run_performance_tests.sh" - echo -e "${GREEN}✅ Performance test runner created${NC}" -} - -# Function to create stress test runner -create_stress_test_runner() { - echo -e "${BLUE}💪 Creating stress test runner...${NC}" - - cat > "$TEST_DIR/run_stress_tests.sh" << 'EOF' -#!/bin/bash - -# Stress Test Runner for SeaweedFS S3 IAM - -set -e - -# Colors -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' - -echo -e "${YELLOW}💪 Running S3 IAM Stress Tests${NC}" - -# Enable stress tests -export ENABLE_STRESS_TESTS=true -export TEST_TIMEOUT=60m - -# Run stress tests multiple times -STRESS_ITERATIONS=5 - -echo -e "${YELLOW}🔄 Running stress tests with $STRESS_ITERATIONS iterations...${NC}" - -for i in $(seq 1 $STRESS_ITERATIONS); do - echo -e "${YELLOW}📊 Iteration $i/$STRESS_ITERATIONS${NC}" - - if ! go test -v -timeout=$TEST_TIMEOUT -run "TestS3IAMDistributedTests.*concurrent" ./... -count=1; then - echo -e "${RED}❌ Stress test failed on iteration $i${NC}" - exit 1 - fi - - # Brief pause between iterations - sleep 2 -done - -echo -e "${GREEN}✅ All stress test iterations completed successfully${NC}" -EOF - - chmod +x "$TEST_DIR/run_stress_tests.sh" - echo -e "${GREEN}✅ Stress test runner created${NC}" -} - -# Function to create versioning stress test setup -setup_versioning_stress_tests() { - echo -e "${BLUE}📚 Setting up S3 versioning stress tests...${NC}" - - # Navigate to versioning test directory - VERSIONING_DIR="$SEAWEEDFS_ROOT/test/s3/versioning" - - if [ ! -d "$VERSIONING_DIR" ]; then - echo -e "${RED}❌ Versioning test directory not found: $VERSIONING_DIR${NC}" - return 1 - fi - - # Create versioning stress test enabler - cat > "$VERSIONING_DIR/enable_stress_tests.sh" << 'EOF' -#!/bin/bash - -# Enable S3 Versioning Stress Tests - -set -e - -# Colors -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -echo -e "${YELLOW}📚 Enabling S3 Versioning Stress Tests${NC}" - -# Disable short mode to enable stress tests -export ENABLE_STRESS_TESTS=true - -# Run versioning stress tests -echo -e "${YELLOW}🧪 Running versioning stress tests...${NC}" -make test-versioning-stress - -echo -e "${GREEN}✅ Versioning stress tests completed${NC}" -EOF - - chmod +x "$VERSIONING_DIR/enable_stress_tests.sh" - echo -e "${GREEN}✅ Versioning stress test setup complete${NC}" -} - -# Function to create master test runner -create_master_test_runner() { - echo -e "${BLUE}🎯 Creating master test runner...${NC}" - - cat > "$TEST_DIR/run_all_tests.sh" << 'EOF' -#!/bin/bash - -# Master Test Runner - Enables and runs all previously skipped tests - -set -e - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -echo -e "${BLUE}🎯 SeaweedFS S3 IAM Complete Test Suite${NC}" -echo -e "${BLUE}=====================================${NC}" - -# Set environment variables to enable all tests -export ENABLE_DISTRIBUTED_TESTS=true -export ENABLE_PERFORMANCE_TESTS=true -export ENABLE_STRESS_TESTS=true -export KEYCLOAK_URL="http://localhost:8080" -export S3_ENDPOINT="http://localhost:8333" -export TEST_TIMEOUT=60m -export CGO_ENABLED=0 - -# Function to run test category -run_test_category() { - local category="$1" - local test_pattern="$2" - local description="$3" - - echo -e "${YELLOW}🧪 Running $description...${NC}" - - if go test -v -timeout=$TEST_TIMEOUT -run "$test_pattern" ./...; then - echo -e "${GREEN}✅ $description completed successfully${NC}" - return 0 - else - echo -e "${RED}❌ $description failed${NC}" - return 1 - fi -} - -# Track results -TOTAL_CATEGORIES=0 -PASSED_CATEGORIES=0 - -# 1. Standard IAM Integration Tests -echo -e "\n${BLUE}1. Standard IAM Integration Tests${NC}" -TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1)) -if run_test_category "standard" "TestS3IAM(?!.*Distributed|.*Performance)" "Standard IAM Integration Tests"; then - PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1)) -fi - -# 2. Keycloak Integration Tests (if Keycloak is available) -echo -e "\n${BLUE}2. Keycloak Integration Tests${NC}" -TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1)) -if curl -s "http://localhost:8080/health/ready" > /dev/null 2>&1; then - if run_test_category "keycloak" "TestKeycloak" "Keycloak Integration Tests"; then - PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1)) - fi -else - echo -e "${YELLOW}⚠️ Keycloak not available, skipping Keycloak tests${NC}" - echo -e "${YELLOW}💡 Run './setup_all_tests.sh' to start Keycloak${NC}" -fi - -# 3. Distributed Tests -echo -e "\n${BLUE}3. Distributed IAM Tests${NC}" -TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1)) -if run_test_category "distributed" "TestS3IAMDistributedTests" "Distributed IAM Tests"; then - PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1)) -fi - -# 4. Performance Tests -echo -e "\n${BLUE}4. Performance Tests${NC}" -TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1)) -if run_test_category "performance" "TestS3IAMPerformanceTests" "Performance Tests"; then - PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1)) -fi - -# 5. Benchmarks -echo -e "\n${BLUE}5. Benchmark Tests${NC}" -TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1)) -if go test -bench=. -benchmem -timeout=$TEST_TIMEOUT ./...; then - echo -e "${GREEN}✅ Benchmark tests completed successfully${NC}" - PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1)) -else - echo -e "${RED}❌ Benchmark tests failed${NC}" -fi - -# 6. Versioning Stress Tests -echo -e "\n${BLUE}6. S3 Versioning Stress Tests${NC}" -TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1)) -if [ -f "../versioning/enable_stress_tests.sh" ]; then - if (cd ../versioning && ./enable_stress_tests.sh); then - echo -e "${GREEN}✅ Versioning stress tests completed successfully${NC}" - PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1)) - else - echo -e "${RED}❌ Versioning stress tests failed${NC}" - fi -else - echo -e "${YELLOW}⚠️ Versioning stress tests not available${NC}" -fi - -# Summary -echo -e "\n${BLUE}📊 Test Summary${NC}" -echo -e "${BLUE}===============${NC}" -echo -e "Total test categories: $TOTAL_CATEGORIES" -echo -e "Passed: ${GREEN}$PASSED_CATEGORIES${NC}" -echo -e "Failed: ${RED}$((TOTAL_CATEGORIES - PASSED_CATEGORIES))${NC}" - -if [ $PASSED_CATEGORIES -eq $TOTAL_CATEGORIES ]; then - echo -e "\n${GREEN}🎉 All test categories passed!${NC}" - exit 0 -else - echo -e "\n${RED}❌ Some test categories failed${NC}" - exit 1 -fi -EOF - - chmod +x "$TEST_DIR/run_all_tests.sh" - echo -e "${GREEN}✅ Master test runner created${NC}" -} - -# Function to update Makefile with new targets -update_makefile() { - echo -e "${BLUE}📝 Updating Makefile with new test targets...${NC}" - - # Add new targets to Makefile - cat >> "$TEST_DIR/Makefile" << 'EOF' - -# New test targets for previously skipped tests - -test-distributed: ## Run distributed IAM tests - @echo "🌐 Running distributed IAM tests..." - @export ENABLE_DISTRIBUTED_TESTS=true && go test -v -timeout $(TEST_TIMEOUT) -run "TestS3IAMDistributedTests" ./... - -test-performance: ## Run performance tests - @echo "🏁 Running performance tests..." - @export ENABLE_PERFORMANCE_TESTS=true && go test -v -timeout $(TEST_TIMEOUT) -run "TestS3IAMPerformanceTests" ./... - -test-stress: ## Run stress tests - @echo "💪 Running stress tests..." - @export ENABLE_STRESS_TESTS=true && ./run_stress_tests.sh - -test-versioning-stress: ## Run S3 versioning stress tests - @echo "📚 Running versioning stress tests..." - @cd ../versioning && ./enable_stress_tests.sh - -test-keycloak-full: docker-up ## Run complete Keycloak integration tests - @echo "🔐 Running complete Keycloak integration tests..." - @export KEYCLOAK_URL="http://localhost:8080" && \ - export S3_ENDPOINT="http://localhost:8333" && \ - sleep 15 && \ - go test -v -timeout $(TEST_TIMEOUT) -run "TestKeycloak" ./... - @make docker-down - -test-all-previously-skipped: ## Run all previously skipped tests - @echo "🎯 Running all previously skipped tests..." - @./run_all_tests.sh - -setup-all-tests: ## Setup environment for all tests (including Keycloak) - @echo "🚀 Setting up complete test environment..." - @./setup_all_tests.sh - -# Update help target -help: ## Show this help message - @echo "SeaweedFS S3 IAM Integration Tests" - @echo "" - @echo "Usage:" - @echo " make [target]" - @echo "" - @echo "Standard Targets:" - @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-25s %s\n", $$1, $$2}' $(MAKEFILE_LIST) | head -20 - @echo "" - @echo "New Test Targets (Previously Skipped):" - @echo " test-distributed Run distributed IAM tests" - @echo " test-performance Run performance tests" - @echo " test-stress Run stress tests" - @echo " test-versioning-stress Run S3 versioning stress tests" - @echo " test-keycloak-full Run complete Keycloak integration tests" - @echo " test-all-previously-skipped Run all previously skipped tests" - @echo " setup-all-tests Setup environment for all tests" - @echo "" - @echo "Docker Compose Targets:" - @echo " docker-test Run tests with Docker Compose including Keycloak" - @echo " docker-up Start all services with Docker Compose" - @echo " docker-down Stop all Docker Compose services" - @echo " docker-logs Show logs from all services" - -EOF - - echo -e "${GREEN}✅ Makefile updated with new targets${NC}" -} - -# Main execution -main() { - echo -e "${BLUE}🔧 Starting complete test setup...${NC}" - - # Setup distributed environment - setup_distributed_environment - - # Create test functions - - # Create test runners - create_performance_test_runner - create_stress_test_runner - create_master_test_runner - - # Setup versioning stress tests - setup_versioning_stress_tests - - # Update Makefile - update_makefile - - # Setup Keycloak (optional) - echo -e "\n${YELLOW}🔐 Setting up Keycloak...${NC}" - if setup_keycloak; then - echo -e "${GREEN}✅ Keycloak container running${NC}" - # Configure realm, client, roles, and users idempotently - if [ -f "$TEST_DIR/setup_keycloak.sh" ]; then - echo -e "${YELLOW}🧩 Configuring Keycloak realm and test users...${NC}" - bash "$TEST_DIR/setup_keycloak.sh" - fi - echo -e "${GREEN}✅ Keycloak setup successful${NC}" - else - echo -e "${YELLOW}⚠️ Keycloak setup failed or skipped${NC}" - echo -e "${YELLOW}💡 You can run Keycloak manually or skip Keycloak tests${NC}" - fi - - echo -e "\n${GREEN}🎉 Complete test setup finished!${NC}" - echo -e "\n${BLUE}📋 Available commands:${NC}" - echo -e " ${YELLOW}./run_all_tests.sh${NC} - Run all previously skipped tests" - echo -e " ${YELLOW}make test-distributed${NC} - Run distributed tests only" - echo -e " ${YELLOW}make test-performance${NC} - Run performance tests only" - echo -e " ${YELLOW}make test-stress${NC} - Run stress tests only" - echo -e " ${YELLOW}make test-keycloak-full${NC} - Run complete Keycloak tests" - echo -e " ${YELLOW}make test-versioning-stress${NC} - Run versioning stress tests" - echo -e " ${YELLOW}make help${NC} - Show all available targets" - - echo -e "\n${BLUE}🔍 Environment Status:${NC}" - echo -e " Keycloak: $(curl -s http://localhost:8080/health/ready > /dev/null 2>&1 && echo -e "${GREEN}✅ Running${NC}" || echo -e "${RED}❌ Not running${NC}")" - # For IAM-enabled S3 API, any response (including 403) indicates the service is running - echo -e " S3 API: $(curl -s http://localhost:8333 > /dev/null 2>&1 && echo -e "${GREEN}✅ Running (IAM-protected)${NC}" || echo -e "${RED}❌ Not running${NC}")" - - if ! curl -s http://localhost:8333 > /dev/null 2>&1; then - echo -e "\n${YELLOW}💡 To start SeaweedFS services:${NC}" - echo -e " ${YELLOW}make start-services wait-for-services${NC}" - fi -} - -# Run main function -main "$@" diff --git a/test/s3/iam/setup_keycloak.sh b/test/s3/iam/setup_keycloak.sh deleted file mode 100755 index 0f59e165b..000000000 --- a/test/s3/iam/setup_keycloak.sh +++ /dev/null @@ -1,377 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -KEYCLOAK_IMAGE="quay.io/keycloak/keycloak:26.0.7" -CONTAINER_NAME="keycloak-iam-test" -KEYCLOAK_PORT="8080" # Default external port -KEYCLOAK_INTERNAL_PORT="8080" # Internal container port (always 8080) -KEYCLOAK_URL="http://localhost:${KEYCLOAK_PORT}" - -# Realm and test fixtures expected by tests -REALM_NAME="seaweedfs-test" -CLIENT_ID="seaweedfs-s3" -CLIENT_SECRET="seaweedfs-s3-secret" -ROLE_ADMIN="s3-admin" -ROLE_READONLY="s3-read-only" -ROLE_WRITEONLY="s3-write-only" -ROLE_READWRITE="s3-read-write" - -# User credentials (compatible with older bash versions) -get_user_password() { - case "$1" in - "admin-user") echo "admin123" ;; - "read-user") echo "read123" ;; - "write-user") echo "readwrite123" ;; - "write-only-user") echo "writeonly123" ;; - *) echo "" ;; - esac -} - -# List of users to create -USERS="admin-user read-user write-user write-only-user" - -echo -e "${BLUE}🔧 Setting up Keycloak realm and users for SeaweedFS S3 IAM testing...${NC}" - -ensure_container() { - # Check for any existing Keycloak container and detect its port - local keycloak_containers=$(docker ps --format '{{.Names}}\t{{.Ports}}' | grep -E "(keycloak|quay.io/keycloak)") - - if [[ -n "$keycloak_containers" ]]; then - # Parse the first available Keycloak container - CONTAINER_NAME=$(echo "$keycloak_containers" | head -1 | awk '{print $1}') - - # Extract the external port from the port mapping using sed (compatible with older bash) - local port_mapping=$(echo "$keycloak_containers" | head -1 | awk '{print $2}') - local extracted_port=$(echo "$port_mapping" | sed -n 's/.*:\([0-9]*\)->8080.*/\1/p') - if [[ -n "$extracted_port" ]]; then - KEYCLOAK_PORT="$extracted_port" - KEYCLOAK_URL="http://localhost:${KEYCLOAK_PORT}" - echo -e "${GREEN}✅ Using existing container '${CONTAINER_NAME}' on port ${KEYCLOAK_PORT}${NC}" - return 0 - fi - fi - - # Fallback: check for specific container names - if docker ps --format '{{.Names}}' | grep -q '^keycloak$'; then - CONTAINER_NAME="keycloak" - # Try to detect port for 'keycloak' container using docker port command - local ports=$(docker port keycloak 8080 2>/dev/null | head -1) - if [[ -n "$ports" ]]; then - local extracted_port=$(echo "$ports" | sed -n 's/.*:\([0-9]*\)$/\1/p') - if [[ -n "$extracted_port" ]]; then - KEYCLOAK_PORT="$extracted_port" - KEYCLOAK_URL="http://localhost:${KEYCLOAK_PORT}" - fi - fi - echo -e "${GREEN}✅ Using existing container '${CONTAINER_NAME}' on port ${KEYCLOAK_PORT}${NC}" - return 0 - fi - if docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then - echo -e "${GREEN}✅ Using existing container '${CONTAINER_NAME}'${NC}" - return 0 - fi - echo -e "${YELLOW}🐳 Starting Keycloak container (${KEYCLOAK_IMAGE})...${NC}" - docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true - docker run -d --name "${CONTAINER_NAME}" -p "${KEYCLOAK_PORT}:8080" \ - -e KEYCLOAK_ADMIN=admin \ - -e KEYCLOAK_ADMIN_PASSWORD=admin \ - -e KC_HTTP_ENABLED=true \ - -e KC_HOSTNAME_STRICT=false \ - -e KC_HOSTNAME_STRICT_HTTPS=false \ - -e KC_HEALTH_ENABLED=true \ - "${KEYCLOAK_IMAGE}" start-dev >/dev/null -} - -wait_ready() { - echo -e "${YELLOW}⏳ Waiting for Keycloak to be ready...${NC}" - for i in $(seq 1 120); do - if curl -sf "${KEYCLOAK_URL}/health/ready" >/dev/null; then - echo -e "${GREEN}✅ Keycloak health check passed${NC}" - return 0 - fi - if curl -sf "${KEYCLOAK_URL}/realms/master" >/dev/null; then - echo -e "${GREEN}✅ Keycloak master realm accessible${NC}" - return 0 - fi - sleep 2 - done - echo -e "${RED}❌ Keycloak did not become ready in time${NC}" - exit 1 -} - -kcadm() { - # Always authenticate before each command to ensure context - # Try different admin passwords that might be used in different environments - # GitHub Actions uses "admin", local testing might use "admin123" - local admin_passwords=("admin" "admin123" "password") - local auth_success=false - - for pwd in "${admin_passwords[@]}"; do - if docker exec -i "${CONTAINER_NAME}" /opt/keycloak/bin/kcadm.sh config credentials --server "http://localhost:${KEYCLOAK_INTERNAL_PORT}" --realm master --user admin --password "$pwd" >/dev/null 2>&1; then - auth_success=true - break - fi - done - - if [[ "$auth_success" == false ]]; then - echo -e "${RED}❌ Failed to authenticate with any known admin password${NC}" - return 1 - fi - - docker exec -i "${CONTAINER_NAME}" /opt/keycloak/bin/kcadm.sh "$@" -} - -admin_login() { - # This is now handled by each kcadm() call - echo "Logging into http://localhost:${KEYCLOAK_INTERNAL_PORT} as user admin of realm master" -} - -ensure_realm() { - if kcadm get realms | grep -q "${REALM_NAME}"; then - echo -e "${GREEN}✅ Realm '${REALM_NAME}' already exists${NC}" - else - echo -e "${YELLOW}📝 Creating realm '${REALM_NAME}'...${NC}" - if kcadm create realms -s realm="${REALM_NAME}" -s enabled=true 2>/dev/null; then - echo -e "${GREEN}✅ Realm created${NC}" - else - # Check if it exists now (might have been created by another process) - if kcadm get realms | grep -q "${REALM_NAME}"; then - echo -e "${GREEN}✅ Realm '${REALM_NAME}' already exists (created concurrently)${NC}" - else - echo -e "${RED}❌ Failed to create realm '${REALM_NAME}'${NC}" - return 1 - fi - fi - fi -} - -ensure_client() { - local id - id=$(kcadm get clients -r "${REALM_NAME}" -q clientId="${CLIENT_ID}" | jq -r '.[0].id // empty') - if [[ -n "${id}" ]]; then - echo -e "${GREEN}✅ Client '${CLIENT_ID}' already exists${NC}" - else - echo -e "${YELLOW}📝 Creating client '${CLIENT_ID}'...${NC}" - kcadm create clients -r "${REALM_NAME}" \ - -s clientId="${CLIENT_ID}" \ - -s protocol=openid-connect \ - -s publicClient=false \ - -s serviceAccountsEnabled=true \ - -s directAccessGrantsEnabled=true \ - -s standardFlowEnabled=true \ - -s implicitFlowEnabled=false \ - -s secret="${CLIENT_SECRET}" >/dev/null - echo -e "${GREEN}✅ Client created${NC}" - fi - - # Create and configure role mapper for the client - configure_role_mapper "${CLIENT_ID}" -} - -ensure_role() { - local role="$1" - if kcadm get roles -r "${REALM_NAME}" | jq -r '.[].name' | grep -qx "${role}"; then - echo -e "${GREEN}✅ Role '${role}' exists${NC}" - else - echo -e "${YELLOW}📝 Creating role '${role}'...${NC}" - kcadm create roles -r "${REALM_NAME}" -s name="${role}" >/dev/null - fi -} - -ensure_user() { - local username="$1" password="$2" - local uid - uid=$(kcadm get users -r "${REALM_NAME}" -q username="${username}" | jq -r '.[0].id // empty') - if [[ -z "${uid}" ]]; then - echo -e "${YELLOW}📝 Creating user '${username}'...${NC}" - uid=$(kcadm create users -r "${REALM_NAME}" \ - -s username="${username}" \ - -s enabled=true \ - -s email="${username}@seaweedfs.test" \ - -s emailVerified=true \ - -s firstName="${username}" \ - -s lastName="User" \ - -i) - else - echo -e "${GREEN}✅ User '${username}' exists${NC}" - fi - echo -e "${YELLOW}🔑 Setting password for '${username}'...${NC}" - kcadm set-password -r "${REALM_NAME}" --userid "${uid}" --new-password "${password}" --temporary=false >/dev/null -} - -assign_role() { - local username="$1" role="$2" - local uid rid - uid=$(kcadm get users -r "${REALM_NAME}" -q username="${username}" | jq -r '.[0].id') - rid=$(kcadm get roles -r "${REALM_NAME}" | jq -r ".[] | select(.name==\"${role}\") | .id") - # Check if role already assigned - if kcadm get "users/${uid}/role-mappings/realm" -r "${REALM_NAME}" | jq -r '.[].name' | grep -qx "${role}"; then - echo -e "${GREEN}✅ User '${username}' already has role '${role}'${NC}" - return 0 - fi - echo -e "${YELLOW}➕ Assigning role '${role}' to '${username}'...${NC}" - kcadm add-roles -r "${REALM_NAME}" --uid "${uid}" --rolename "${role}" >/dev/null -} - -configure_role_mapper() { - echo -e "${YELLOW}🔧 Configuring role mapper for client '${CLIENT_ID}'...${NC}" - - # Get client's internal ID - local internal_id - internal_id=$(kcadm get clients -r "${REALM_NAME}" -q clientId="${CLIENT_ID}" | jq -r '.[0].id // empty') - - if [[ -z "${internal_id}" ]]; then - echo -e "${RED}❌ Could not find client ${client_id} to configure role mapper${NC}" - return 1 - fi - - # Check if a realm roles mapper already exists for this client - local existing_mapper - existing_mapper=$(kcadm get "clients/${internal_id}/protocol-mappers/models" -r "${REALM_NAME}" | jq -r '.[] | select(.name=="realm roles" and .protocolMapper=="oidc-usermodel-realm-role-mapper") | .id // empty') - - if [[ -n "${existing_mapper}" ]]; then - echo -e "${GREEN}✅ Realm roles mapper already exists${NC}" - else - echo -e "${YELLOW}📝 Creating realm roles mapper...${NC}" - - # Create protocol mapper for realm roles - kcadm create "clients/${internal_id}/protocol-mappers/models" -r "${REALM_NAME}" \ - -s name="realm roles" \ - -s protocol="openid-connect" \ - -s protocolMapper="oidc-usermodel-realm-role-mapper" \ - -s consentRequired=false \ - -s 'config."multivalued"=true' \ - -s 'config."userinfo.token.claim"=true' \ - -s 'config."id.token.claim"=true' \ - -s 'config."access.token.claim"=true' \ - -s 'config."claim.name"=roles' \ - -s 'config."jsonType.label"=String' >/dev/null || { - echo -e "${RED}❌ Failed to create realm roles mapper${NC}" - return 1 - } - - echo -e "${GREEN}✅ Realm roles mapper created${NC}" - fi -} - -main() { - command -v docker >/dev/null || { echo -e "${RED}❌ Docker is required${NC}"; exit 1; } - command -v jq >/dev/null || { echo -e "${RED}❌ jq is required${NC}"; exit 1; } - - ensure_container - echo "Keycloak URL: ${KEYCLOAK_URL}" - wait_ready - admin_login - ensure_realm - ensure_client - configure_role_mapper - ensure_role "${ROLE_ADMIN}" - ensure_role "${ROLE_READONLY}" - ensure_role "${ROLE_WRITEONLY}" - ensure_role "${ROLE_READWRITE}" - - for u in $USERS; do - ensure_user "$u" "$(get_user_password "$u")" - done - - assign_role admin-user "${ROLE_ADMIN}" - assign_role read-user "${ROLE_READONLY}" - assign_role write-user "${ROLE_READWRITE}" - - # Also create a dedicated write-only user for testing - ensure_user write-only-user "$(get_user_password write-only-user)" - assign_role write-only-user "${ROLE_WRITEONLY}" - - # Copy the appropriate IAM configuration for this environment - setup_iam_config - - # Validate the setup by testing authentication and role inclusion - echo -e "${YELLOW}🔍 Validating setup by testing admin-user authentication and role mapping...${NC}" - sleep 2 - - local validation_result=$(curl -s -w "%{http_code}" -X POST "http://localhost:${KEYCLOAK_PORT}/realms/${REALM_NAME}/protocol/openid-connect/token" \ - -H "Content-Type: application/x-www-form-urlencoded" \ - -d "grant_type=password" \ - -d "client_id=${CLIENT_ID}" \ - -d "client_secret=${CLIENT_SECRET}" \ - -d "username=admin-user" \ - -d "password=admin123" \ - -d "scope=openid profile email" \ - -o /tmp/auth_test_response.json) - - if [[ "${validation_result: -3}" == "200" ]]; then - echo -e "${GREEN}✅ Authentication validation successful${NC}" - - # Extract and decode JWT token to check for roles - local access_token=$(cat /tmp/auth_test_response.json | jq -r '.access_token // empty') - if [[ -n "${access_token}" ]]; then - # Decode JWT payload (second part) and check for roles - local payload=$(echo "${access_token}" | cut -d'.' -f2) - # Add padding if needed for base64 decode - while [[ $((${#payload} % 4)) -ne 0 ]]; do - payload="${payload}=" - done - - local decoded=$(echo "${payload}" | base64 -d 2>/dev/null || echo "{}") - local roles=$(echo "${decoded}" | jq -r '.roles // empty' 2>/dev/null || echo "") - - if [[ -n "${roles}" && "${roles}" != "null" ]]; then - echo -e "${GREEN}✅ JWT token includes roles: ${roles}${NC}" - else - echo -e "${YELLOW}⚠️ JWT token does not include 'roles' claim${NC}" - echo -e "${YELLOW}Decoded payload sample:${NC}" - echo "${decoded}" | jq '.' 2>/dev/null || echo "${decoded}" - fi - fi - else - echo -e "${RED}❌ Authentication validation failed with HTTP ${validation_result: -3}${NC}" - echo -e "${YELLOW}Response body:${NC}" - cat /tmp/auth_test_response.json 2>/dev/null || echo "No response body" - echo -e "${YELLOW}This may indicate a setup issue that needs to be resolved${NC}" - fi - rm -f /tmp/auth_test_response.json - - echo -e "${GREEN}✅ Keycloak test realm '${REALM_NAME}' configured${NC}" -} - -setup_iam_config() { - echo -e "${BLUE}🔧 Setting up IAM configuration for detected environment${NC}" - - # Change to script directory to ensure config files are found - local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - cd "$script_dir" - - # Choose the appropriate config based on detected port - local config_source - if [[ "${KEYCLOAK_PORT}" == "8080" ]]; then - config_source="iam_config.github.json" - echo " Using GitHub Actions configuration (port 8080)" - else - config_source="iam_config.local.json" - echo " Using local development configuration (port ${KEYCLOAK_PORT})" - fi - - # Verify source config exists - if [[ ! -f "$config_source" ]]; then - echo -e "${RED}❌ Config file $config_source not found in $script_dir${NC}" - exit 1 - fi - - # Copy the appropriate config - cp "$config_source" "iam_config.json" - - local detected_issuer=$(cat iam_config.json | jq -r '.providers[] | select(.name=="keycloak") | .config.issuer') - echo -e "${GREEN}✅ IAM configuration set successfully${NC}" - echo " - Using config: $config_source" - echo " - Keycloak issuer: $detected_issuer" -} - -main "$@" diff --git a/test/s3/iam/setup_keycloak_docker.sh b/test/s3/iam/setup_keycloak_docker.sh new file mode 100755 index 000000000..6ec1485b8 --- /dev/null +++ b/test/s3/iam/setup_keycloak_docker.sh @@ -0,0 +1,404 @@ +#!/bin/bash +set -e + +# Keycloak configuration for Docker environment +KEYCLOAK_URL="http://keycloak:8080" +KEYCLOAK_ADMIN_USER="admin" +KEYCLOAK_ADMIN_PASSWORD="admin" +REALM_NAME="seaweedfs-test" +CLIENT_ID="seaweedfs-s3" +CLIENT_SECRET="seaweedfs-s3-secret" + +echo "🔧 Setting up Keycloak realm and users for SeaweedFS S3 IAM testing..." +echo "Keycloak URL: $KEYCLOAK_URL" + +# Wait for Keycloak to be ready +echo "⏳ Waiting for Keycloak to be ready..." +timeout 120 bash -c ' + until curl -f "$0/health/ready" > /dev/null 2>&1; do + echo "Waiting for Keycloak..." + sleep 5 + done + echo "✅ Keycloak health check passed" +' "$KEYCLOAK_URL" + +# Download kcadm.sh if not available +if ! command -v kcadm.sh &> /dev/null; then + echo "📥 Downloading Keycloak admin CLI..." + wget -q https://github.com/keycloak/keycloak/releases/download/26.0.7/keycloak-26.0.7.tar.gz + tar -xzf keycloak-26.0.7.tar.gz + export PATH="$PWD/keycloak-26.0.7/bin:$PATH" +fi + +# Wait a bit more for admin user initialization +echo "⏳ Waiting for admin user to be fully initialized..." +sleep 10 + +# Function to execute kcadm commands with retry and multiple password attempts +kcadm() { + local max_retries=3 + local retry_count=0 + local passwords=("admin" "admin123" "password") + + while [ $retry_count -lt $max_retries ]; do + for password in "${passwords[@]}"; do + if kcadm.sh "$@" --server "$KEYCLOAK_URL" --realm master --user "$KEYCLOAK_ADMIN_USER" --password "$password" 2>/dev/null; then + return 0 + fi + done + retry_count=$((retry_count + 1)) + echo "🔄 Retry $retry_count of $max_retries..." + sleep 5 + done + + echo "❌ Failed to execute kcadm command after $max_retries retries" + return 1 +} + +# Create realm +echo "📝 Creating realm '$REALM_NAME'..." +kcadm create realms -s realm="$REALM_NAME" -s enabled=true || echo "Realm may already exist" +echo "✅ Realm created" + +# Create OIDC client +echo "📝 Creating client '$CLIENT_ID'..." +CLIENT_UUID=$(kcadm create clients -r "$REALM_NAME" \ + -s clientId="$CLIENT_ID" \ + -s secret="$CLIENT_SECRET" \ + -s enabled=true \ + -s serviceAccountsEnabled=true \ + -s standardFlowEnabled=true \ + -s directAccessGrantsEnabled=true \ + -s 'redirectUris=["*"]' \ + -s 'webOrigins=["*"]' \ + -i 2>/dev/null || echo "existing-client") + +if [ "$CLIENT_UUID" != "existing-client" ]; then + echo "✅ Client created with ID: $CLIENT_UUID" +else + echo "✅ Using existing client" + CLIENT_UUID=$(kcadm get clients -r "$REALM_NAME" -q clientId="$CLIENT_ID" --fields id --format csv --noquotes | tail -n +2) +fi + +# Configure protocol mapper for roles +echo "🔧 Configuring role mapper for client '$CLIENT_ID'..." +MAPPER_CONFIG='{ + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-realm-role-mapper", + "name": "realm-roles", + "config": { + "claim.name": "roles", + "jsonType.label": "String", + "multivalued": "true", + "usermodel.realmRoleMapping.rolePrefix": "" + } +}' + +kcadm create clients/"$CLIENT_UUID"/protocol-mappers/models -r "$REALM_NAME" -b "$MAPPER_CONFIG" 2>/dev/null || echo "✅ Role mapper already exists" +echo "✅ Realm roles mapper configured" + +# Create realm roles +echo "📝 Creating realm roles..." +for role in "s3-admin" "s3-read-only" "s3-write-only" "s3-read-write"; do + kcadm create roles -r "$REALM_NAME" -s name="$role" 2>/dev/null || echo "Role $role may already exist" +done + +# Create users with roles +declare -A USERS=( + ["admin-user"]="s3-admin" + ["read-user"]="s3-read-only" + ["write-user"]="s3-read-write" + ["write-only-user"]="s3-write-only" +) + +for username in "${!USERS[@]}"; do + role="${USERS[$username]}" + password="${username//[^a-zA-Z]/}123" # e.g., "admin-user" -> "adminuser123" + + echo "📝 Creating user '$username'..." + kcadm create users -r "$REALM_NAME" \ + -s username="$username" \ + -s enabled=true \ + -s firstName="Test" \ + -s lastName="User" \ + -s email="$username@test.com" 2>/dev/null || echo "User $username may already exist" + + echo "🔑 Setting password for '$username'..." + kcadm set-password -r "$REALM_NAME" --username "$username" --new-password "$password" + + echo "➕ Assigning role '$role' to '$username'..." + kcadm add-roles -r "$REALM_NAME" --uusername "$username" --rolename "$role" +done + +# Create IAM configuration for Docker environment +echo "🔧 Setting up IAM configuration for Docker environment..." +cat > iam_config.json << 'EOF' +{ + "sts": { + "tokenDuration": 3600000000000, + "maxSessionLength": 43200000000000, + "issuer": "seaweedfs-sts", + "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc=" + }, + "providers": [ + { + "name": "keycloak", + "type": "oidc", + "enabled": true, + "config": { + "issuer": "http://keycloak:8080/realms/seaweedfs-test", + "clientId": "seaweedfs-s3", + "clientSecret": "seaweedfs-s3-secret", + "jwksUri": "http://keycloak:8080/realms/seaweedfs-test/protocol/openid-connect/certs", + "userInfoUri": "http://keycloak:8080/realms/seaweedfs-test/protocol/openid-connect/userinfo", + "scopes": ["openid", "profile", "email"], + "claimsMapping": { + "username": "preferred_username", + "email": "email", + "name": "name" + }, + "roleMapping": { + "rules": [ + { + "claim": "roles", + "value": "s3-admin", + "role": "arn:seaweed:iam::role/KeycloakAdminRole" + }, + { + "claim": "roles", + "value": "s3-read-only", + "role": "arn:seaweed:iam::role/KeycloakReadOnlyRole" + }, + { + "claim": "roles", + "value": "s3-write-only", + "role": "arn:seaweed:iam::role/KeycloakWriteOnlyRole" + }, + { + "claim": "roles", + "value": "s3-read-write", + "role": "arn:seaweed:iam::role/KeycloakReadWriteRole" + } + ], + "defaultRole": "arn:seaweed:iam::role/KeycloakReadOnlyRole" + } + } + } + ], + "policy": { + "defaultEffect": "Deny", + "storeType": "memory" + }, + "roles": [ + { + "roleName": "KeycloakAdminRole", + "roleArn": "arn:seaweed:iam::role/KeycloakAdminRole", + "trustPolicy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "keycloak" + }, + "Action": ["sts:AssumeRoleWithWebIdentity"] + } + ] + }, + "attachedPolicies": ["S3AdminPolicy"], + "description": "Admin role for Keycloak users" + }, + { + "roleName": "KeycloakReadOnlyRole", + "roleArn": "arn:seaweed:iam::role/KeycloakReadOnlyRole", + "trustPolicy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "keycloak" + }, + "Action": ["sts:AssumeRoleWithWebIdentity"] + } + ] + }, + "attachedPolicies": ["S3ReadOnlyPolicy"], + "description": "Read-only role for Keycloak users" + }, + { + "roleName": "KeycloakWriteOnlyRole", + "roleArn": "arn:seaweed:iam::role/KeycloakWriteOnlyRole", + "trustPolicy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "keycloak" + }, + "Action": ["sts:AssumeRoleWithWebIdentity"] + } + ] + }, + "attachedPolicies": ["S3WriteOnlyPolicy"], + "description": "Write-only role for Keycloak users" + }, + { + "roleName": "KeycloakReadWriteRole", + "roleArn": "arn:seaweed:iam::role/KeycloakReadWriteRole", + "trustPolicy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "keycloak" + }, + "Action": ["sts:AssumeRoleWithWebIdentity"] + } + ] + }, + "attachedPolicies": ["S3ReadWritePolicy"], + "description": "Read-write role for Keycloak users" + } + ], + "policies": [ + { + "name": "S3AdminPolicy", + "document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:*"], + "Resource": ["*"] + }, + { + "Effect": "Allow", + "Action": ["sts:ValidateSession"], + "Resource": ["*"] + } + ] + } + }, + { + "name": "S3ReadOnlyPolicy", + "document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:seaweed:s3:::*", + "arn:seaweed:s3:::*/*" + ] + }, + { + "Effect": "Allow", + "Action": ["sts:ValidateSession"], + "Resource": ["*"] + } + ] + } + }, + { + "name": "S3WriteOnlyPolicy", + "document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:*"], + "Resource": [ + "arn:seaweed:s3:::*", + "arn:seaweed:s3:::*/*" + ] + }, + { + "Effect": "Deny", + "Action": [ + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:seaweed:s3:::*", + "arn:seaweed:s3:::*/*" + ] + }, + { + "Effect": "Allow", + "Action": ["sts:ValidateSession"], + "Resource": ["*"] + } + ] + } + }, + { + "name": "S3ReadWritePolicy", + "document": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:*"], + "Resource": [ + "arn:seaweed:s3:::*", + "arn:seaweed:s3:::*/*" + ] + }, + { + "Effect": "Allow", + "Action": ["sts:ValidateSession"], + "Resource": ["*"] + } + ] + } + } + ] +} +EOF + +# Validate setup by testing authentication +echo "🔍 Validating setup by testing admin-user authentication and role mapping..." +KEYCLOAK_TOKEN_URL="http://keycloak:8080/realms/$REALM_NAME/protocol/openid-connect/token" + +# Get access token for admin-user +ACCESS_TOKEN=$(curl -s -X POST "$KEYCLOAK_TOKEN_URL" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=password" \ + -d "client_id=$CLIENT_ID" \ + -d "client_secret=$CLIENT_SECRET" \ + -d "username=admin-user" \ + -d "password=adminuser123" \ + -d "scope=openid profile email" | jq -r '.access_token') + +if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then + echo "❌ Failed to obtain access token" + exit 1 +fi + +echo "✅ Authentication validation successful" + +# Decode and check JWT claims +PAYLOAD=$(echo "$ACCESS_TOKEN" | cut -d'.' -f2) +# Add padding for base64 decode +while [ $((${#PAYLOAD} % 4)) -ne 0 ]; do + PAYLOAD="${PAYLOAD}=" +done + +CLAIMS=$(echo "$PAYLOAD" | base64 -d 2>/dev/null | jq .) +ROLES=$(echo "$CLAIMS" | jq -r '.roles[]?') + +if [ -n "$ROLES" ]; then + echo "✅ JWT token includes roles: [$(echo "$ROLES" | tr '\n' ',' | sed 's/,$//' | sed 's/,/, /g')]" +else + echo "⚠️ No roles found in JWT token" +fi + +echo "✅ Keycloak test realm '$REALM_NAME' configured for Docker environment" +echo "🐳 Setup complete! You can now run: docker-compose up -d" diff --git a/weed/iam/oidc/oidc_provider.go b/weed/iam/oidc/oidc_provider.go index 006e76e57..b30d8e00d 100644 --- a/weed/iam/oidc/oidc_provider.go +++ b/weed/iam/oidc/oidc_provider.go @@ -156,6 +156,16 @@ func (p *OIDCProvider) Authenticate(ctx context.Context, token string) (*provide displayName, _ := claims.GetClaimString("name") groups, _ := claims.GetClaimStringSlice("groups") + // Debug: Log available claims + glog.V(3).Infof("Available claims: %+v", claims.Claims) + if rolesFromClaims, exists := claims.GetClaimStringSlice("roles"); exists { + glog.V(3).Infof("Roles claim found as string slice: %v", rolesFromClaims) + } else if roleFromClaims, exists := claims.GetClaimString("roles"); exists { + glog.V(3).Infof("Roles claim found as string: %s", roleFromClaims) + } else { + glog.V(3).Infof("No roles claim found in token") + } + // Map claims to roles using configured role mapping roles := p.mapClaimsToRolesWithConfig(claims) @@ -373,25 +383,36 @@ func (p *OIDCProvider) mapClaimsToRoles(claims *providers.TokenClaims) []string // mapClaimsToRolesWithConfig maps token claims to roles using configured role mapping func (p *OIDCProvider) mapClaimsToRolesWithConfig(claims *providers.TokenClaims) []string { + glog.V(3).Infof("mapClaimsToRolesWithConfig: RoleMapping is nil? %t", p.config.RoleMapping == nil) + if p.config.RoleMapping == nil { + glog.V(2).Infof("No role mapping configured for provider %s, using legacy mapping", p.name) // Fallback to legacy mapping if no role mapping configured return p.mapClaimsToRoles(claims) } + glog.V(3).Infof("Applying %d role mapping rules", len(p.config.RoleMapping.Rules)) roles := []string{} // Apply role mapping rules - for _, rule := range p.config.RoleMapping.Rules { + for i, rule := range p.config.RoleMapping.Rules { + glog.V(3).Infof("Rule %d: claim=%s, value=%s, role=%s", i, rule.Claim, rule.Value, rule.Role) + if rule.Matches(claims) { + glog.V(2).Infof("Rule %d matched! Adding role: %s", i, rule.Role) roles = append(roles, rule.Role) + } else { + glog.V(3).Infof("Rule %d did not match", i) } } // Use default role if no rules matched if len(roles) == 0 && p.config.RoleMapping.DefaultRole != "" { + glog.V(2).Infof("No rules matched, using default role: %s", p.config.RoleMapping.DefaultRole) roles = []string{p.config.RoleMapping.DefaultRole} } + glog.V(2).Infof("Role mapping result: %v", roles) return roles } diff --git a/weed/iam/providers/provider.go b/weed/iam/providers/provider.go index 74d63dd46..f014dc8db 100644 --- a/weed/iam/providers/provider.go +++ b/weed/iam/providers/provider.go @@ -6,6 +6,8 @@ import ( "net/mail" "path/filepath" "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" ) // IdentityProvider defines the interface for external identity providers @@ -189,22 +191,30 @@ type MappingRule struct { // Matches checks if a rule matches the given claims func (r *MappingRule) Matches(claims *TokenClaims) bool { if r.Claim == "" || r.Value == "" { + glog.V(3).Infof("Rule invalid: claim=%s, value=%s", r.Claim, r.Value) return false } claimValue, exists := claims.GetClaimString(r.Claim) if !exists { + glog.V(3).Infof("Claim '%s' not found as string, trying as string slice", r.Claim) // Try as string slice if claimSlice, sliceExists := claims.GetClaimStringSlice(r.Claim); sliceExists { + glog.V(3).Infof("Claim '%s' found as string slice: %v", r.Claim, claimSlice) for _, val := range claimSlice { + glog.V(3).Infof("Checking if '%s' matches rule value '%s'", val, r.Value) if r.matchValue(val) { + glog.V(3).Infof("Match found: '%s' matches '%s'", val, r.Value) return true } } + } else { + glog.V(3).Infof("Claim '%s' not found in any format", r.Claim) } return false } + glog.V(3).Infof("Claim '%s' found as string: '%s'", r.Claim, claimValue) return r.matchValue(claimValue) } @@ -212,7 +222,9 @@ func (r *MappingRule) Matches(claims *TokenClaims) bool { func (r *MappingRule) matchValue(value string) bool { matched, err := filepath.Match(r.Value, value) if err != nil { + glog.V(3).Infof("Error matching '%s' against pattern '%s': %v", value, r.Value, err) return false } + glog.V(3).Infof("Pattern match result: '%s' matches '%s' = %t", value, r.Value, matched) return matched }