From e1551d36b03290b98c4951a59ec2db80ae1b0415 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sat, 15 Nov 2025 11:24:46 -0800 Subject: [PATCH] s3: Fix SSE multipart upload metadata preservation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Critical fix for SSE multipart upload test failures (SSE-C and SSE-KMS): **Root Cause - Incomplete SSE Metadata Copying**: The old code only tried to copy 'SeaweedFSSSEKMSKey' from the first part to the completed object. This had TWO bugs: 1. **Wrong Constant Name** (Key Mismatch Bug): - Storage uses: SeaweedFSSSEKMSKeyHeader = 'X-SeaweedFS-SSE-KMS-Key' - Old code read: SeaweedFSSSEKMSKey = 'x-seaweedfs-sse-kms-key' - Result: SSE-KMS metadata was NEVER copied → 500 errors 2. **Missing SSE-C and SSE-S3 Headers**: - SSE-C requires: IV, Algorithm, KeyMD5 - SSE-S3 requires: encrypted key data + standard headers - Old code: copied nothing for SSE-C/SSE-S3 → decryption failures **Fix - Complete SSE Header Preservation**: Now copies ALL SSE headers from first part to completed object: - SSE-C: SeaweedFSSSEIV, CustomerAlgorithm, CustomerKeyMD5 - SSE-KMS: SeaweedFSSSEKMSKeyHeader, AwsKmsKeyId, ServerSideEncryption - SSE-S3: SeaweedFSSSES3Key, ServerSideEncryption Applied consistently to all 3 code paths: 1. Versioned buckets (creates version file) 2. Suspended versioning (creates main object with null versionId) 3. Non-versioned buckets (creates main object) **Why This Is Correct**: The headers copied EXACTLY match what putToFiler stores during part upload (lines 496-521 in s3api_object_handlers_put.go). This ensures detectPrimarySSEType() can correctly identify encrypted multipart objects and trigger inline decryption with proper metadata. Fixes: TestSSEMultipartUploadIntegration (SSE-C and SSE-KMS subtests) --- weed/s3api/filer_multipart.go | 84 +++++++++++++++++++++++++++-------- 1 file changed, 66 insertions(+), 18 deletions(-) diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index dc71a6371..69e7d1e5b 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -340,15 +340,31 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl } } - // Preserve SSE-KMS metadata from the first part (if any) - // SSE-KMS metadata is stored in individual parts, not the upload directory + // Preserve ALL SSE metadata from the first part (if any) + // SSE metadata is stored in individual parts, not the upload directory if len(completedPartNumbers) > 0 && len(partEntries[completedPartNumbers[0]]) > 0 { firstPartEntry := partEntries[completedPartNumbers[0]][0] if firstPartEntry.Extended != nil { - // Copy SSE-KMS metadata from the first part - if kmsMetadata, exists := firstPartEntry.Extended[s3_constants.SeaweedFSSSEKMSKey]; exists { - versionEntry.Extended[s3_constants.SeaweedFSSSEKMSKey] = kmsMetadata - glog.V(3).Infof("completeMultipartUpload: preserved SSE-KMS metadata from first part (versioned)") + // Copy ALL SSE-related headers (not just SeaweedFSSSEKMSKey) + // This is critical for detectPrimarySSEType to work correctly + sseKeys := []string{ + // SSE-C headers + s3_constants.SeaweedFSSSEIV, + s3_constants.AmzServerSideEncryptionCustomerAlgorithm, + s3_constants.AmzServerSideEncryptionCustomerKeyMD5, + // SSE-KMS headers + s3_constants.SeaweedFSSSEKMSKeyHeader, + s3_constants.AmzServerSideEncryptionAwsKmsKeyId, + // SSE-S3 headers + s3_constants.SeaweedFSSSES3Key, + // Common SSE header (for SSE-KMS and SSE-S3) + s3_constants.AmzServerSideEncryption, + } + for _, key := range sseKeys { + if value, exists := firstPartEntry.Extended[key]; exists { + versionEntry.Extended[key] = value + glog.V(4).Infof("completeMultipartUpload: copied SSE header %s from first part (versioned)", key) + } } } } @@ -404,15 +420,31 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl } } - // Preserve SSE-KMS metadata from the first part (if any) - // SSE-KMS metadata is stored in individual parts, not the upload directory + // Preserve ALL SSE metadata from the first part (if any) + // SSE metadata is stored in individual parts, not the upload directory if len(completedPartNumbers) > 0 && len(partEntries[completedPartNumbers[0]]) > 0 { firstPartEntry := partEntries[completedPartNumbers[0]][0] if firstPartEntry.Extended != nil { - // Copy SSE-KMS metadata from the first part - if kmsMetadata, exists := firstPartEntry.Extended[s3_constants.SeaweedFSSSEKMSKey]; exists { - entry.Extended[s3_constants.SeaweedFSSSEKMSKey] = kmsMetadata - glog.V(3).Infof("completeMultipartUpload: preserved SSE-KMS metadata from first part (suspended versioning)") + // Copy ALL SSE-related headers (not just SeaweedFSSSEKMSKey) + // This is critical for detectPrimarySSEType to work correctly + sseKeys := []string{ + // SSE-C headers + s3_constants.SeaweedFSSSEIV, + s3_constants.AmzServerSideEncryptionCustomerAlgorithm, + s3_constants.AmzServerSideEncryptionCustomerKeyMD5, + // SSE-KMS headers + s3_constants.SeaweedFSSSEKMSKeyHeader, + s3_constants.AmzServerSideEncryptionAwsKmsKeyId, + // SSE-S3 headers + s3_constants.SeaweedFSSSES3Key, + // Common SSE header (for SSE-KMS and SSE-S3) + s3_constants.AmzServerSideEncryption, + } + for _, key := range sseKeys { + if value, exists := firstPartEntry.Extended[key]; exists { + entry.Extended[key] = value + glog.V(4).Infof("completeMultipartUpload: copied SSE header %s from first part (suspended versioning)", key) + } } } } @@ -459,15 +491,31 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl } } - // Preserve SSE-KMS metadata from the first part (if any) - // SSE-KMS metadata is stored in individual parts, not the upload directory + // Preserve ALL SSE metadata from the first part (if any) + // SSE metadata is stored in individual parts, not the upload directory if len(completedPartNumbers) > 0 && len(partEntries[completedPartNumbers[0]]) > 0 { firstPartEntry := partEntries[completedPartNumbers[0]][0] if firstPartEntry.Extended != nil { - // Copy SSE-KMS metadata from the first part - if kmsMetadata, exists := firstPartEntry.Extended[s3_constants.SeaweedFSSSEKMSKey]; exists { - entry.Extended[s3_constants.SeaweedFSSSEKMSKey] = kmsMetadata - glog.V(3).Infof("completeMultipartUpload: preserved SSE-KMS metadata from first part") + // Copy ALL SSE-related headers (not just SeaweedFSSSEKMSKey) + // This is critical for detectPrimarySSEType to work correctly + sseKeys := []string{ + // SSE-C headers + s3_constants.SeaweedFSSSEIV, + s3_constants.AmzServerSideEncryptionCustomerAlgorithm, + s3_constants.AmzServerSideEncryptionCustomerKeyMD5, + // SSE-KMS headers + s3_constants.SeaweedFSSSEKMSKeyHeader, + s3_constants.AmzServerSideEncryptionAwsKmsKeyId, + // SSE-S3 headers + s3_constants.SeaweedFSSSES3Key, + // Common SSE header (for SSE-KMS and SSE-S3) + s3_constants.AmzServerSideEncryption, + } + for _, key := range sseKeys { + if value, exists := firstPartEntry.Extended[key]; exists { + entry.Extended[key] = value + glog.V(4).Infof("completeMultipartUpload: copied SSE header %s from first part (non-versioned)", key) + } } } }