diff --git a/.github/workflows/kms-tests.yml b/.github/workflows/kms-tests.yml new file mode 100644 index 000000000..9e58cc80f --- /dev/null +++ b/.github/workflows/kms-tests.yml @@ -0,0 +1,140 @@ +name: "KMS Tests" + +on: + pull_request: + paths: + - 'weed/kms/**' + - 'weed/s3api/s3_sse_*.go' + - 'weed/s3api/s3api_object_handlers.go' + - 'weed/s3api/s3api_object_handlers_put.go' + - 'test/kms/**' + - '.github/workflows/kms-tests.yml' + push: + branches: [ master, main ] + paths: + - 'weed/kms/**' + - 'weed/s3api/s3_sse_*.go' + - 'weed/s3api/s3api_object_handlers.go' + - 'weed/s3api/s3api_object_handlers_put.go' + - 'test/kms/**' + +concurrency: + group: ${{ github.head_ref || github.ref }}-kms-tests + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + working-directory: weed + +jobs: + kms-provider-tests: + name: KMS Provider Integration Tests + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run KMS provider integration tests + timeout-minutes: 15 + working-directory: test/kms + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + docker --version + + make test-provider-ci + + - name: Show OpenBao logs on failure + if: failure() + run: | + echo "=== OpenBao Container Logs ===" + docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao container found" + echo "=== Setup Logs ===" + cat /tmp/openbao-ci-setup.log 2>/dev/null || echo "No setup log found" + + - name: Cleanup + if: always() + working-directory: test/kms + run: | + make stop-openbao-ci || true + + s3-kms-e2e-tests: + name: S3 KMS End-to-End Tests + runs-on: ubuntu-22.04 + timeout-minutes: 25 + + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 KMS end-to-end tests + timeout-minutes: 20 + working-directory: test/kms + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + docker --version + aws --version + + make test-s3-kms-ci + + - name: Show logs on failure + if: failure() + working-directory: test/kms + run: | + echo "=== OpenBao Container Logs ===" + cat /tmp/openbao-ci-container.log 2>/dev/null || docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao logs found" + echo "=== SeaweedFS Server Logs ===" + tail -100 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found" + echo "=== Setup Logs ===" + cat /tmp/weed-kms-ci-setup.log 2>/dev/null || echo "No weed setup log" + echo "=== Process Information ===" + ps aux | grep -E "(weed|test)" || true + + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v7 + with: + name: s3-kms-e2e-logs + path: | + /tmp/seaweedfs-kms-mini.log + /tmp/openbao-ci-container.log + /tmp/weed-kms-ci-setup.log + retention-days: 3 + + - name: Cleanup + if: always() + working-directory: test/kms + run: | + make stop-seaweedfs-ci || true + make stop-openbao-ci || true + make clean-ci || true diff --git a/test/kms/Makefile b/test/kms/Makefile index bfbe51ec9..a46aa19b2 100644 --- a/test/kms/Makefile +++ b/test/kms/Makefile @@ -7,6 +7,14 @@ SEAWEEDFS_S3_ENDPOINT ?= http://127.0.0.1:8333 TEST_TIMEOUT ?= 5m DOCKER_COMPOSE ?= docker-compose +# CI configuration (build weed from source, run natively) +SEAWEEDFS_BINARY ?= weed +S3_PORT ?= 8333 +ACCESS_KEY ?= some_access_key1 +SECRET_KEY ?= some_secret_key1 +TEST_DIR := $(shell pwd) +SEAWEEDFS_ROOT := $(shell cd ../.. && pwd) + # Colors for output BLUE := \033[36m GREEN := \033[32m @@ -14,7 +22,10 @@ YELLOW := \033[33m RED := \033[31m NC := \033[0m # No Color -.PHONY: help setup test test-unit test-integration test-e2e clean logs status +.PHONY: help setup test test-unit test-integration test-e2e clean logs status \ + build-weed check-binary start-openbao-ci stop-openbao-ci \ + start-seaweedfs-ci stop-seaweedfs-ci clean-ci \ + test-provider-ci test-s3-kms-ci help: ## Show this help message @echo "$(BLUE)SeaweedFS KMS Integration Testing$(NC)" @@ -124,16 +135,146 @@ check-env: ## Check test environment setup @echo "TEST_TIMEOUT: $(TEST_TIMEOUT)" @make install-deps -# CI targets -ci-test: ## Run tests in CI environment +# CI targets (docker-compose based) +ci-test: ## Run tests in CI environment (docker-compose) @echo "$(YELLOW)Running CI tests...$(NC)" @make setup @make test-unit @make test-integration @make clean -ci-e2e: ## Run end-to-end tests in CI +ci-e2e: ## Run end-to-end tests in CI (docker-compose) @echo "$(YELLOW)Running CI end-to-end tests...$(NC)" @make setup-seaweedfs @make test-e2e @make clean + +#################################################### +# GitHub Actions CI targets (build weed from source) +#################################################### + +build-weed: ## Build SeaweedFS binary from source + @echo "Building SeaweedFS binary..." + @cd $(SEAWEEDFS_ROOT)/weed && go install -buildvcs=false + @echo "$(GREEN)SeaweedFS binary built successfully$(NC)" + +check-binary: + @if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1; then \ + echo "$(RED)Error: SeaweedFS binary '$(SEAWEEDFS_BINARY)' not found in PATH$(NC)"; \ + exit 1; \ + fi + +start-openbao-ci: ## Start OpenBao via Docker for CI + @echo "Starting OpenBao for CI..." + @docker rm -f openbao-ci 2>/dev/null || true + @docker run -d --name openbao-ci \ + -p 8200:8200 \ + -e BAO_DEV_ROOT_TOKEN_ID=$(OPENBAO_TOKEN) \ + -e BAO_DEV_LISTEN_ADDRESS=0.0.0.0:8200 \ + ghcr.io/openbao/openbao:latest \ + bao server -dev -dev-root-token-id=$(OPENBAO_TOKEN) -dev-listen-address=0.0.0.0:8200 + @echo "Waiting for OpenBao to be ready..." + @for i in $$(seq 1 30); do \ + if curl -s $(OPENBAO_ADDR)/v1/sys/health > /dev/null 2>&1; then \ + echo "$(GREEN)OpenBao is ready$(NC)"; \ + break; \ + fi; \ + if [ $$i -eq 30 ]; then \ + echo "$(RED)Timeout waiting for OpenBao$(NC)"; \ + docker logs openbao-ci 2>&1 | tail -20; \ + exit 1; \ + fi; \ + sleep 2; \ + done + @chmod +x setup_openbao.sh + @OPENBAO_ADDR=$(OPENBAO_ADDR) OPENBAO_TOKEN=$(OPENBAO_TOKEN) ./setup_openbao.sh + +stop-openbao-ci: ## Stop OpenBao CI container + @echo "Stopping OpenBao..." + @docker stop openbao-ci 2>/dev/null || true + @docker rm openbao-ci 2>/dev/null || true + +start-seaweedfs-ci: check-binary ## Start weed mini with OpenBao KMS config + @echo "Starting SeaweedFS with OpenBao KMS..." + @mkdir -p /tmp/seaweedfs-test-kms + @rm -f /tmp/seaweedfs-kms-*.log || true + @sed -e 's/ACCESS_KEY_PLACEHOLDER/$(ACCESS_KEY)/g' \ + -e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \ + -e 's|OPENBAO_ADDR_PLACEHOLDER|$(OPENBAO_ADDR)|g' \ + -e 's/OPENBAO_TOKEN_PLACEHOLDER/$(OPENBAO_TOKEN)/g' \ + s3-config-openbao-template.json > /tmp/seaweedfs-kms-s3.json + @# Start weed mini from the data dir to avoid picking up test/kms/filer.toml + @# (filer.toml is read from "." first, and the one here has Docker-only paths) + @cd /tmp/seaweedfs-test-kms && \ + AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \ + -dir=/tmp/seaweedfs-test-kms \ + -s3.port=$(S3_PORT) \ + -s3.config=/tmp/seaweedfs-kms-s3.json \ + -ip=127.0.0.1 \ + > /tmp/seaweedfs-kms-mini.log 2>&1 & echo $$! > /tmp/weed-kms-mini.pid + @echo "Checking S3 service is ready..." + @for i in $$(seq 1 30); do \ + if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ + echo "$(GREEN)S3 service is ready$(NC)"; \ + break; \ + fi; \ + if [ $$i -eq 30 ]; then \ + echo "$(RED)Timeout waiting for S3 service$(NC)"; \ + echo "=== Last 50 lines of server log ==="; \ + tail -50 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"; \ + exit 1; \ + fi; \ + sleep 1; \ + done + +stop-seaweedfs-ci: ## Stop weed mini (CI-safe) + @echo "Stopping SeaweedFS..." + @if [ -f /tmp/weed-kms-mini.pid ]; then \ + kill $$(cat /tmp/weed-kms-mini.pid) 2>/dev/null || true; \ + rm -f /tmp/weed-kms-mini.pid; \ + fi + @if command -v lsof >/dev/null 2>&1; then \ + lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ + fi + @sleep 2 + +clean-ci: ## Clean up CI test artifacts + @rm -rf /tmp/seaweedfs-test-kms + @rm -f /tmp/seaweedfs-kms-*.log /tmp/seaweedfs-kms-s3.json /tmp/weed-kms-mini.pid + +# Run KMS provider Go integration tests (GitHub Actions compatible) +test-provider-ci: build-weed ## Run KMS provider tests with OpenBao in CI + @echo "$(YELLOW)Starting KMS provider integration tests...$(NC)" + @if $(MAKE) start-openbao-ci > /tmp/openbao-ci-setup.log 2>&1; then \ + echo "$(GREEN)OpenBao started and configured$(NC)"; \ + trap '$(MAKE) -C $(TEST_DIR) stop-openbao-ci || true' EXIT; \ + cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/kms/... || exit 1; \ + echo "$(GREEN)KMS provider tests completed successfully$(NC)"; \ + else \ + echo "$(RED)Failed to start OpenBao$(NC)"; \ + cat /tmp/openbao-ci-setup.log 2>/dev/null || true; \ + exit 1; \ + fi + +# Run S3 KMS end-to-end tests (GitHub Actions compatible) +test-s3-kms-ci: build-weed ## Run S3 KMS e2e tests with weed mini + OpenBao in CI + @echo "$(YELLOW)Starting S3 KMS end-to-end tests...$(NC)" + @if $(MAKE) start-openbao-ci > /tmp/openbao-ci-setup.log 2>&1; then \ + echo "$(GREEN)OpenBao started and configured$(NC)"; \ + trap 'docker logs openbao-ci > /tmp/openbao-ci-container.log 2>&1 || true; $(MAKE) -C $(TEST_DIR) stop-seaweedfs-ci || true; $(MAKE) -C $(TEST_DIR) stop-openbao-ci || true; $(MAKE) -C $(TEST_DIR) clean-ci || true' EXIT; \ + if $(MAKE) start-seaweedfs-ci > /tmp/weed-kms-ci-setup.log 2>&1; then \ + echo "$(GREEN)SeaweedFS started with OpenBao KMS$(NC)"; \ + sleep 3; \ + chmod +x test_s3_kms.sh; \ + SEAWEEDFS_S3_ENDPOINT=http://127.0.0.1:$(S3_PORT) ACCESS_KEY=$(ACCESS_KEY) SECRET_KEY=$(SECRET_KEY) ./test_s3_kms.sh || exit 1; \ + echo "$(GREEN)S3 KMS e2e tests completed successfully$(NC)"; \ + else \ + echo "$(RED)Failed to start SeaweedFS$(NC)"; \ + cat /tmp/weed-kms-ci-setup.log 2>/dev/null || true; \ + exit 1; \ + fi; \ + else \ + echo "$(RED)Failed to start OpenBao$(NC)"; \ + cat /tmp/openbao-ci-setup.log 2>/dev/null || true; \ + exit 1; \ + fi diff --git a/test/kms/s3-config-openbao-template.json b/test/kms/s3-config-openbao-template.json new file mode 100644 index 000000000..cba583dbb --- /dev/null +++ b/test/kms/s3-config-openbao-template.json @@ -0,0 +1,27 @@ +{ + "identities": [ + { + "name": "admin", + "credentials": [ + { + "accessKey": "ACCESS_KEY_PLACEHOLDER", + "secretKey": "SECRET_KEY_PLACEHOLDER" + } + ], + "actions": ["Admin", "Read", "List", "Tagging", "Write"] + } + ], + "kms": { + "default_provider": "openbao", + "providers": { + "openbao": { + "type": "openbao", + "address": "OPENBAO_ADDR_PLACEHOLDER", + "token": "OPENBAO_TOKEN_PLACEHOLDER", + "transit_path": "transit", + "cache_enabled": true, + "cache_ttl": "20m" + } + } + } +} diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index bb160fa55..2625df8be 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -365,14 +365,20 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader } } - // If SSE-S3 was applied by bucket default, prepare metadata (if not already done) + // If bucket default encryption was applied, serialize the metadata (SSE-S3 and SSE-KMS are mutually exclusive) + var metaErr error if sseS3Key != nil && len(sseS3Metadata) == 0 { - var metaErr error sseS3Metadata, metaErr = SerializeSSES3Metadata(sseS3Key) if metaErr != nil { glog.Errorf("Failed to serialize SSE-S3 metadata for bucket default encryption: %v", metaErr) return "", s3err.ErrInternalError, SSEResponseMetadata{} } + } else if sseKMSKey != nil && len(sseKMSMetadata) == 0 { + sseKMSMetadata, metaErr = SerializeSSEKMSMetadata(sseKMSKey) + if metaErr != nil { + glog.Errorf("Failed to serialize SSE-KMS metadata for bucket default encryption: %v", metaErr) + return "", s3err.ErrInternalError, SSEResponseMetadata{} + } } } else { glog.V(4).Infof("putToFiler: explicit encryption already applied, skipping bucket default encryption")