fix: serialize SSE-KMS metadata when bucket default encryption applies KMS (#8780)

* fix: serialize SSE-KMS metadata when bucket default encryption applies KMS When a bucket has default SSE-KMS encryption enabled and a file is uploaded without explicit SSE headers, the encryption was applied correctly but the SSE-KMS metadata (x-seaweedfs-sse-kms-key) was not serialized. This caused downloads to fail with "empty SSE-KMS metadata" because the entry's Extended map stored an empty byte slice. The existing code already handled this for SSE-S3 bucket defaults (SerializeSSES3Metadata) but was missing the equivalent call to SerializeSSEKMSMetadata for the KMS path. Fixes seaweedfs/seaweedfs#8776 * ci: add KMS integration tests to GitHub Actions Add a kms-tests.yml workflow that runs on changes to KMS/SSE code with two jobs: 1. KMS provider tests: starts OpenBao via Docker, runs Go integration tests in test/kms/ against a real KMS backend 2. S3 KMS e2e tests: starts OpenBao + weed mini built from source, runs test_s3_kms.sh which covers bucket-default SSE-KMS upload/download (the exact scenario from #8776) Supporting changes: - test/kms/Makefile: add CI targets (test-provider-ci, test-s3-kms-ci) that manage OpenBao via plain Docker and run weed from source - test/kms/s3-config-openbao-template.json: S3 config template with OpenBao KMS provider for weed mini * refactor: combine SSE-S3 and SSE-KMS metadata serialization into else-if SSE-S3 and SSE-KMS bucket default encryption are mutually exclusive, so use a single if/else-if block instead of two independent if blocks. * Update .github/workflows/kms-tests.yml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * fix(ci): start weed mini from data dir to avoid Docker filer.toml weed mini reads filer.toml from the current working directory first. When running from test/kms/, it picked up the Docker-targeted filer.toml which has dir="/data/filerdb" (a path that doesn't exist in CI), causing a fatal crash at filer store initialization. Fix by cd-ing to the data directory before starting weed mini. Also improve log visibility on failure. --------- Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
19 hours ago · 17028fbf59
4 changed files with 320 additions and 6 deletions
--- a/.github/workflows/kms-tests.yml
+++ b/.github/workflows/kms-tests.yml
@ -0,0 +1,140 @@
+name: "KMS Tests"
+
+on:
+  pull_request:
+    paths:
+      - 'weed/kms/**'
+      - 'weed/s3api/s3_sse_*.go'
+      - 'weed/s3api/s3api_object_handlers.go'
+      - 'weed/s3api/s3api_object_handlers_put.go'
+      - 'test/kms/**'
+      - '.github/workflows/kms-tests.yml'
+  push:
+    branches: [ master, main ]
+    paths:
+      - 'weed/kms/**'
+      - 'weed/s3api/s3_sse_*.go'
+      - 'weed/s3api/s3api_object_handlers.go'
+      - 'weed/s3api/s3api_object_handlers_put.go'
+      - 'test/kms/**'
+
+concurrency:
+  group: ${{ github.head_ref || github.ref }}-kms-tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    working-directory: weed
+
+jobs:
+  kms-provider-tests:
+    name: KMS Provider Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false
+
+      - name: Run KMS provider integration tests
+        timeout-minutes: 15
+        working-directory: test/kms
+        run: |
+          set -x
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          docker --version
+
+          make test-provider-ci
+
+      - name: Show OpenBao logs on failure
+        if: failure()
+        run: |
+          echo "=== OpenBao Container Logs ==="
+          docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao container found"
+          echo "=== Setup Logs ==="
+          cat /tmp/openbao-ci-setup.log 2>/dev/null || echo "No setup log found"
+
+      - name: Cleanup
+        if: always()
+        working-directory: test/kms
+        run: |
+          make stop-openbao-ci || true
+
+  s3-kms-e2e-tests:
+    name: S3 KMS End-to-End Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 25
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false
+
+      - name: Run S3 KMS end-to-end tests
+        timeout-minutes: 20
+        working-directory: test/kms
+        run: |
+          set -x
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          docker --version
+          aws --version
+
+          make test-s3-kms-ci
+
+      - name: Show logs on failure
+        if: failure()
+        working-directory: test/kms
+        run: |
+          echo "=== OpenBao Container Logs ==="
+          cat /tmp/openbao-ci-container.log 2>/dev/null || docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao logs found"
+          echo "=== SeaweedFS Server Logs ==="
+          tail -100 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"
+          echo "=== Setup Logs ==="
+          cat /tmp/weed-kms-ci-setup.log 2>/dev/null || echo "No weed setup log"
+          echo "=== Process Information ==="
+          ps aux | grep -E "(weed|test)" || true
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: s3-kms-e2e-logs
+          path: |
+            /tmp/seaweedfs-kms-mini.log
+            /tmp/openbao-ci-container.log
+            /tmp/weed-kms-ci-setup.log
+          retention-days: 3
+
+      - name: Cleanup
+        if: always()
+        working-directory: test/kms
+        run: |
+          make stop-seaweedfs-ci || true
+          make stop-openbao-ci || true
+          make clean-ci || true
--- a/test/kms/Makefile
+++ b/test/kms/Makefile
@ -7,6 +7,14 @@ SEAWEEDFS_S3_ENDPOINT ?= http://127.0.0.1:8333
 TEST_TIMEOUT ?= 5m
 DOCKER_COMPOSE ?= docker-compose

+# CI configuration (build weed from source, run natively)
+SEAWEEDFS_BINARY ?= weed
+S3_PORT ?= 8333
+ACCESS_KEY ?= some_access_key1
+SECRET_KEY ?= some_secret_key1
+TEST_DIR := $(shell pwd)
+SEAWEEDFS_ROOT := $(shell cd ../.. && pwd)
+
 # Colors for output
 BLUE := \033[36m
 GREEN := \033[32m
@ -14,7 +22,10 @@ YELLOW := \033[33m
 RED := \033[31m
 NC := \033[0m # No Color

-.PHONY: help setup test test-unit test-integration test-e2e clean logs status
+.PHONY: help setup test test-unit test-integration test-e2e clean logs status \
+       build-weed check-binary start-openbao-ci stop-openbao-ci \
+       start-seaweedfs-ci stop-seaweedfs-ci clean-ci \
+       test-provider-ci test-s3-kms-ci

 help: ## Show this help message
 	@echo "$(BLUE)SeaweedFS KMS Integration Testing$(NC)"
@ -124,16 +135,146 @@ check-env: ## Check test environment setup
 	@echo "TEST_TIMEOUT: $(TEST_TIMEOUT)"
 	@make install-deps

-# CI targets
-ci-test: ## Run tests in CI environment
+# CI targets (docker-compose based)
+ci-test: ## Run tests in CI environment (docker-compose)
 	@echo "$(YELLOW)Running CI tests...$(NC)"
 	@make setup
 	@make test-unit
 	@make test-integration
 	@make clean

-ci-e2e: ## Run end-to-end tests in CI
+ci-e2e: ## Run end-to-end tests in CI (docker-compose)
 	@echo "$(YELLOW)Running CI end-to-end tests...$(NC)"
 	@make setup-seaweedfs
 	@make test-e2e
 	@make clean
+
+####################################################
+# GitHub Actions CI targets (build weed from source)
+####################################################
+
+build-weed: ## Build SeaweedFS binary from source
+	@echo "Building SeaweedFS binary..."
+	@cd $(SEAWEEDFS_ROOT)/weed && go install -buildvcs=false
+	@echo "$(GREEN)SeaweedFS binary built successfully$(NC)"
+
+check-binary:
+	@if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1; then \
+		echo "$(RED)Error: SeaweedFS binary '$(SEAWEEDFS_BINARY)' not found in PATH$(NC)"; \
+		exit 1; \
+	fi
+
+start-openbao-ci: ## Start OpenBao via Docker for CI
+	@echo "Starting OpenBao for CI..."
+	@docker rm -f openbao-ci 2>/dev/null || true
+	@docker run -d --name openbao-ci \
+		-p 8200:8200 \
+		-e BAO_DEV_ROOT_TOKEN_ID=$(OPENBAO_TOKEN) \
+		-e BAO_DEV_LISTEN_ADDRESS=0.0.0.0:8200 \
+		ghcr.io/openbao/openbao:latest \
+		bao server -dev -dev-root-token-id=$(OPENBAO_TOKEN) -dev-listen-address=0.0.0.0:8200
+	@echo "Waiting for OpenBao to be ready..."
+	@for i in $$(seq 1 30); do \
+		if curl -s $(OPENBAO_ADDR)/v1/sys/health > /dev/null 2>&1; then \
+			echo "$(GREEN)OpenBao is ready$(NC)"; \
+			break; \
+		fi; \
+		if [ $$i -eq 30 ]; then \
+			echo "$(RED)Timeout waiting for OpenBao$(NC)"; \
+			docker logs openbao-ci 2>&1 | tail -20; \
+			exit 1; \
+		fi; \
+		sleep 2; \
+	done
+	@chmod +x setup_openbao.sh
+	@OPENBAO_ADDR=$(OPENBAO_ADDR) OPENBAO_TOKEN=$(OPENBAO_TOKEN) ./setup_openbao.sh
+
+stop-openbao-ci: ## Stop OpenBao CI container
+	@echo "Stopping OpenBao..."
+	@docker stop openbao-ci 2>/dev/null || true
+	@docker rm openbao-ci 2>/dev/null || true
+
+start-seaweedfs-ci: check-binary ## Start weed mini with OpenBao KMS config
+	@echo "Starting SeaweedFS with OpenBao KMS..."
+	@mkdir -p /tmp/seaweedfs-test-kms
+	@rm -f /tmp/seaweedfs-kms-*.log || true
+	@sed -e 's/ACCESS_KEY_PLACEHOLDER/$(ACCESS_KEY)/g' \
+	     -e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \
+	     -e 's|OPENBAO_ADDR_PLACEHOLDER|$(OPENBAO_ADDR)|g' \
+	     -e 's/OPENBAO_TOKEN_PLACEHOLDER/$(OPENBAO_TOKEN)/g' \
+	     s3-config-openbao-template.json > /tmp/seaweedfs-kms-s3.json
+	@# Start weed mini from the data dir to avoid picking up test/kms/filer.toml
+	@# (filer.toml is read from "." first, and the one here has Docker-only paths)
+	@cd /tmp/seaweedfs-test-kms && \
+		AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
+		-dir=/tmp/seaweedfs-test-kms \
+		-s3.port=$(S3_PORT) \
+		-s3.config=/tmp/seaweedfs-kms-s3.json \
+		-ip=127.0.0.1 \
+		> /tmp/seaweedfs-kms-mini.log 2>&1 & echo $$! > /tmp/weed-kms-mini.pid
+	@echo "Checking S3 service is ready..."
+	@for i in $$(seq 1 30); do \
+		if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \
+			echo "$(GREEN)S3 service is ready$(NC)"; \
+			break; \
+		fi; \
+		if [ $$i -eq 30 ]; then \
+			echo "$(RED)Timeout waiting for S3 service$(NC)"; \
+			echo "=== Last 50 lines of server log ==="; \
+			tail -50 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"; \
+			exit 1; \
+		fi; \
+		sleep 1; \
+	done
+
+stop-seaweedfs-ci: ## Stop weed mini (CI-safe)
+	@echo "Stopping SeaweedFS..."
+	@if [ -f /tmp/weed-kms-mini.pid ]; then \
+		kill $$(cat /tmp/weed-kms-mini.pid) 2>/dev/null || true; \
+		rm -f /tmp/weed-kms-mini.pid; \
+	fi
+	@if command -v lsof >/dev/null 2>&1; then \
+		lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
+	fi
+	@sleep 2
+
+clean-ci: ## Clean up CI test artifacts
+	@rm -rf /tmp/seaweedfs-test-kms
+	@rm -f /tmp/seaweedfs-kms-*.log /tmp/seaweedfs-kms-s3.json /tmp/weed-kms-mini.pid
+
+# Run KMS provider Go integration tests (GitHub Actions compatible)
+test-provider-ci: build-weed ## Run KMS provider tests with OpenBao in CI
+	@echo "$(YELLOW)Starting KMS provider integration tests...$(NC)"
+	@if $(MAKE) start-openbao-ci > /tmp/openbao-ci-setup.log 2>&1; then \
+		echo "$(GREEN)OpenBao started and configured$(NC)"; \
+		trap '$(MAKE) -C $(TEST_DIR) stop-openbao-ci || true' EXIT; \
+		cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/kms/... || exit 1; \
+		echo "$(GREEN)KMS provider tests completed successfully$(NC)"; \
+	else \
+		echo "$(RED)Failed to start OpenBao$(NC)"; \
+		cat /tmp/openbao-ci-setup.log 2>/dev/null || true; \
+		exit 1; \
+	fi
+
+# Run S3 KMS end-to-end tests (GitHub Actions compatible)
+test-s3-kms-ci: build-weed ## Run S3 KMS e2e tests with weed mini + OpenBao in CI
+	@echo "$(YELLOW)Starting S3 KMS end-to-end tests...$(NC)"
+	@if $(MAKE) start-openbao-ci > /tmp/openbao-ci-setup.log 2>&1; then \
+		echo "$(GREEN)OpenBao started and configured$(NC)"; \
+		trap 'docker logs openbao-ci > /tmp/openbao-ci-container.log 2>&1 || true; $(MAKE) -C $(TEST_DIR) stop-seaweedfs-ci || true; $(MAKE) -C $(TEST_DIR) stop-openbao-ci || true; $(MAKE) -C $(TEST_DIR) clean-ci || true' EXIT; \
+		if $(MAKE) start-seaweedfs-ci > /tmp/weed-kms-ci-setup.log 2>&1; then \
+			echo "$(GREEN)SeaweedFS started with OpenBao KMS$(NC)"; \
+			sleep 3; \
+			chmod +x test_s3_kms.sh; \
+			SEAWEEDFS_S3_ENDPOINT=http://127.0.0.1:$(S3_PORT) ACCESS_KEY=$(ACCESS_KEY) SECRET_KEY=$(SECRET_KEY) ./test_s3_kms.sh || exit 1; \
+			echo "$(GREEN)S3 KMS e2e tests completed successfully$(NC)"; \
+		else \
+			echo "$(RED)Failed to start SeaweedFS$(NC)"; \
+			cat /tmp/weed-kms-ci-setup.log 2>/dev/null || true; \
+			exit 1; \
+		fi; \
+	else \
+		echo "$(RED)Failed to start OpenBao$(NC)"; \
+		cat /tmp/openbao-ci-setup.log 2>/dev/null || true; \
+		exit 1; \
+	fi
--- a/test/kms/s3-config-openbao-template.json
+++ b/test/kms/s3-config-openbao-template.json
@ -0,0 +1,27 @@
+{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "ACCESS_KEY_PLACEHOLDER",
+          "secretKey": "SECRET_KEY_PLACEHOLDER"
+        }
+      ],
+      "actions": ["Admin", "Read", "List", "Tagging", "Write"]
+    }
+  ],
+  "kms": {
+    "default_provider": "openbao",
+    "providers": {
+      "openbao": {
+        "type": "openbao",
+        "address": "OPENBAO_ADDR_PLACEHOLDER",
+        "token": "OPENBAO_TOKEN_PLACEHOLDER",
+        "transit_path": "transit",
+        "cache_enabled": true,
+        "cache_ttl": "20m"
+      }
+    }
+  }
+}
--- a/weed/s3api/s3api_object_handlers_put.go
+++ b/weed/s3api/s3api_object_handlers_put.go
@ -365,14 +365,20 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
 			}
 		}

-		// If SSE-S3 was applied by bucket default, prepare metadata (if not already done)
+		// If bucket default encryption was applied, serialize the metadata (SSE-S3 and SSE-KMS are mutually exclusive)
+		var metaErr error
 		if sseS3Key != nil && len(sseS3Metadata) == 0 {
-			var metaErr error
 			sseS3Metadata, metaErr = SerializeSSES3Metadata(sseS3Key)
 			if metaErr != nil {
 				glog.Errorf("Failed to serialize SSE-S3 metadata for bucket default encryption: %v", metaErr)
 				return "", s3err.ErrInternalError, SSEResponseMetadata{}
 			}
+		} else if sseKMSKey != nil && len(sseKMSMetadata) == 0 {
+			sseKMSMetadata, metaErr = SerializeSSEKMSMetadata(sseKMSKey)
+			if metaErr != nil {
+				glog.Errorf("Failed to serialize SSE-KMS metadata for bucket default encryption: %v", metaErr)
+				return "", s3err.ErrInternalError, SSEResponseMetadata{}
+			}
 		}
 	} else {
 		glog.V(4).Infof("putToFiler: explicit encryption already applied, skipping bucket default encryption")