Browse Source
fix: serialize SSE-KMS metadata when bucket default encryption applies KMS (#8780)
fix: serialize SSE-KMS metadata when bucket default encryption applies KMS (#8780)
* fix: serialize SSE-KMS metadata when bucket default encryption applies KMS When a bucket has default SSE-KMS encryption enabled and a file is uploaded without explicit SSE headers, the encryption was applied correctly but the SSE-KMS metadata (x-seaweedfs-sse-kms-key) was not serialized. This caused downloads to fail with "empty SSE-KMS metadata" because the entry's Extended map stored an empty byte slice. The existing code already handled this for SSE-S3 bucket defaults (SerializeSSES3Metadata) but was missing the equivalent call to SerializeSSEKMSMetadata for the KMS path. Fixes seaweedfs/seaweedfs#8776 * ci: add KMS integration tests to GitHub Actions Add a kms-tests.yml workflow that runs on changes to KMS/SSE code with two jobs: 1. KMS provider tests: starts OpenBao via Docker, runs Go integration tests in test/kms/ against a real KMS backend 2. S3 KMS e2e tests: starts OpenBao + weed mini built from source, runs test_s3_kms.sh which covers bucket-default SSE-KMS upload/download (the exact scenario from #8776) Supporting changes: - test/kms/Makefile: add CI targets (test-provider-ci, test-s3-kms-ci) that manage OpenBao via plain Docker and run weed from source - test/kms/s3-config-openbao-template.json: S3 config template with OpenBao KMS provider for weed mini * refactor: combine SSE-S3 and SSE-KMS metadata serialization into else-if SSE-S3 and SSE-KMS bucket default encryption are mutually exclusive, so use a single if/else-if block instead of two independent if blocks. * Update .github/workflows/kms-tests.yml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> * fix(ci): start weed mini from data dir to avoid Docker filer.toml weed mini reads filer.toml from the current working directory first. When running from test/kms/, it picked up the Docker-targeted filer.toml which has dir="/data/filerdb" (a path that doesn't exist in CI), causing a fatal crash at filer store initialization. Fix by cd-ing to the data directory before starting weed mini. Also improve log visibility on failure. --------- Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>pull/8788/head
Chris Lu
1 day ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 320 additions and 6 deletions
-
140.github/workflows/kms-tests.yml
-
149test/kms/Makefile
-
27test/kms/s3-config-openbao-template.json
-
10weed/s3api/s3api_object_handlers_put.go
@ -0,0 +1,140 @@ |
|||||
|
name: "KMS Tests" |
||||
|
|
||||
|
on: |
||||
|
pull_request: |
||||
|
paths: |
||||
|
- 'weed/kms/**' |
||||
|
- 'weed/s3api/s3_sse_*.go' |
||||
|
- 'weed/s3api/s3api_object_handlers.go' |
||||
|
- 'weed/s3api/s3api_object_handlers_put.go' |
||||
|
- 'test/kms/**' |
||||
|
- '.github/workflows/kms-tests.yml' |
||||
|
push: |
||||
|
branches: [ master, main ] |
||||
|
paths: |
||||
|
- 'weed/kms/**' |
||||
|
- 'weed/s3api/s3_sse_*.go' |
||||
|
- 'weed/s3api/s3api_object_handlers.go' |
||||
|
- 'weed/s3api/s3api_object_handlers_put.go' |
||||
|
- 'test/kms/**' |
||||
|
|
||||
|
concurrency: |
||||
|
group: ${{ github.head_ref || github.ref }}-kms-tests |
||||
|
cancel-in-progress: true |
||||
|
|
||||
|
permissions: |
||||
|
contents: read |
||||
|
|
||||
|
defaults: |
||||
|
run: |
||||
|
working-directory: weed |
||||
|
|
||||
|
jobs: |
||||
|
kms-provider-tests: |
||||
|
name: KMS Provider Integration Tests |
||||
|
runs-on: ubuntu-22.04 |
||||
|
timeout-minutes: 20 |
||||
|
|
||||
|
steps: |
||||
|
- name: Check out code |
||||
|
uses: actions/checkout@v6 |
||||
|
|
||||
|
- name: Set up Go |
||||
|
uses: actions/setup-go@v6 |
||||
|
with: |
||||
|
go-version-file: 'go.mod' |
||||
|
id: go |
||||
|
|
||||
|
- name: Install SeaweedFS |
||||
|
run: | |
||||
|
go install -buildvcs=false |
||||
|
|
||||
|
- name: Run KMS provider integration tests |
||||
|
timeout-minutes: 15 |
||||
|
working-directory: test/kms |
||||
|
run: | |
||||
|
set -x |
||||
|
echo "=== System Information ===" |
||||
|
uname -a |
||||
|
free -h |
||||
|
docker --version |
||||
|
|
||||
|
make test-provider-ci |
||||
|
|
||||
|
- name: Show OpenBao logs on failure |
||||
|
if: failure() |
||||
|
run: | |
||||
|
echo "=== OpenBao Container Logs ===" |
||||
|
docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao container found" |
||||
|
echo "=== Setup Logs ===" |
||||
|
cat /tmp/openbao-ci-setup.log 2>/dev/null || echo "No setup log found" |
||||
|
|
||||
|
- name: Cleanup |
||||
|
if: always() |
||||
|
working-directory: test/kms |
||||
|
run: | |
||||
|
make stop-openbao-ci || true |
||||
|
|
||||
|
s3-kms-e2e-tests: |
||||
|
name: S3 KMS End-to-End Tests |
||||
|
runs-on: ubuntu-22.04 |
||||
|
timeout-minutes: 25 |
||||
|
|
||||
|
steps: |
||||
|
- name: Check out code |
||||
|
uses: actions/checkout@v6 |
||||
|
|
||||
|
- name: Set up Go |
||||
|
uses: actions/setup-go@v6 |
||||
|
with: |
||||
|
go-version-file: 'go.mod' |
||||
|
id: go |
||||
|
|
||||
|
- name: Install SeaweedFS |
||||
|
run: | |
||||
|
go install -buildvcs=false |
||||
|
|
||||
|
- name: Run S3 KMS end-to-end tests |
||||
|
timeout-minutes: 20 |
||||
|
working-directory: test/kms |
||||
|
run: | |
||||
|
set -x |
||||
|
echo "=== System Information ===" |
||||
|
uname -a |
||||
|
free -h |
||||
|
docker --version |
||||
|
aws --version |
||||
|
|
||||
|
make test-s3-kms-ci |
||||
|
|
||||
|
- name: Show logs on failure |
||||
|
if: failure() |
||||
|
working-directory: test/kms |
||||
|
run: | |
||||
|
echo "=== OpenBao Container Logs ===" |
||||
|
cat /tmp/openbao-ci-container.log 2>/dev/null || docker logs openbao-ci 2>&1 | tail -50 || echo "No OpenBao logs found" |
||||
|
echo "=== SeaweedFS Server Logs ===" |
||||
|
tail -100 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found" |
||||
|
echo "=== Setup Logs ===" |
||||
|
cat /tmp/weed-kms-ci-setup.log 2>/dev/null || echo "No weed setup log" |
||||
|
echo "=== Process Information ===" |
||||
|
ps aux | grep -E "(weed|test)" || true |
||||
|
|
||||
|
- name: Upload test logs on failure |
||||
|
if: failure() |
||||
|
uses: actions/upload-artifact@v7 |
||||
|
with: |
||||
|
name: s3-kms-e2e-logs |
||||
|
path: | |
||||
|
/tmp/seaweedfs-kms-mini.log |
||||
|
/tmp/openbao-ci-container.log |
||||
|
/tmp/weed-kms-ci-setup.log |
||||
|
retention-days: 3 |
||||
|
|
||||
|
- name: Cleanup |
||||
|
if: always() |
||||
|
working-directory: test/kms |
||||
|
run: | |
||||
|
make stop-seaweedfs-ci || true |
||||
|
make stop-openbao-ci || true |
||||
|
make clean-ci || true |
||||
@ -0,0 +1,27 @@ |
|||||
|
{ |
||||
|
"identities": [ |
||||
|
{ |
||||
|
"name": "admin", |
||||
|
"credentials": [ |
||||
|
{ |
||||
|
"accessKey": "ACCESS_KEY_PLACEHOLDER", |
||||
|
"secretKey": "SECRET_KEY_PLACEHOLDER" |
||||
|
} |
||||
|
], |
||||
|
"actions": ["Admin", "Read", "List", "Tagging", "Write"] |
||||
|
} |
||||
|
], |
||||
|
"kms": { |
||||
|
"default_provider": "openbao", |
||||
|
"providers": { |
||||
|
"openbao": { |
||||
|
"type": "openbao", |
||||
|
"address": "OPENBAO_ADDR_PLACEHOLDER", |
||||
|
"token": "OPENBAO_TOKEN_PLACEHOLDER", |
||||
|
"transit_path": "transit", |
||||
|
"cache_enabled": true, |
||||
|
"cache_ttl": "20m" |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue