diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 6a6acce6a..2edbd3a0a 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -1457,6 +1457,22 @@ func writeZeroBytes(w io.Writer, n int64) error { } // decryptSSECChunkView decrypts a specific chunk view with SSE-C +// +// IV Handling for SSE-C: +// ---------------------- +// SSE-C multipart encryption (see lines 2772-2781) differs fundamentally from SSE-KMS/SSE-S3: +// +// 1. Encryption: CreateSSECEncryptedReader generates a RANDOM IV per part/chunk +// - Each part starts with a fresh random IV +// - CTR counter starts from 0 for each part: counter₀, counter₁, counter₂, ... +// - PartOffset is stored in metadata but NOT applied during encryption +// +// 2. Decryption: Use the stored IV directly WITHOUT offset adjustment +// - The stored IV already represents the start of this part's encryption +// - Applying calculateIVWithOffset would shift to counterₙ, misaligning the keystream +// - Result: XOR with wrong keystream = corrupted plaintext +// +// This contrasts with SSE-KMS/SSE-S3 which use: base IV + calculateIVWithOffset(ChunkOffset) func (s3a *S3ApiServer) decryptSSECChunkView(ctx context.Context, fileChunk *filer_pb.FileChunk, chunkView *filer.ChunkView, customerKey *SSECustomerKey) (io.Reader, error) { // For multipart SSE-C, each chunk has its own IV in chunk.SseMetadata if fileChunk.GetSseType() == filer_pb.SSEType_SSE_C && len(fileChunk.GetSseMetadata()) > 0 { @@ -1476,35 +1492,16 @@ func (s3a *S3ApiServer) decryptSSECChunkView(ctx context.Context, fileChunk *fil return nil, fmt.Errorf("failed to fetch full chunk: %w", err) } - // Calculate IV using PartOffset - // PartOffset is the position of this chunk within its part's encrypted stream - var adjustedIV []byte - var ivSkip int - if ssecMetadata.PartOffset > 0 { - adjustedIV, ivSkip = calculateIVWithOffset(chunkIV, ssecMetadata.PartOffset) - } else { - adjustedIV = chunkIV - ivSkip = 0 - } - - // Decrypt the full chunk - decryptedReader, decryptErr := CreateSSECDecryptedReader(fullChunkReader, customerKey, adjustedIV) + // CRITICAL: Use stored IV directly WITHOUT offset adjustment + // The stored IV is the random IV used at encryption time for this specific part + // SSE-C does NOT apply calculateIVWithOffset during encryption, so we must not apply it during decryption + // (See documentation above and at lines 2772-2781 for detailed explanation) + decryptedReader, decryptErr := CreateSSECDecryptedReader(fullChunkReader, customerKey, chunkIV) if decryptErr != nil { fullChunkReader.Close() return nil, fmt.Errorf("failed to create decrypted reader: %w", decryptErr) } - // CRITICAL: Skip intra-block bytes from CTR decryption (non-block-aligned offset handling) - if ivSkip > 0 { - _, err = io.CopyN(io.Discard, decryptedReader, int64(ivSkip)) - if err != nil { - if closer, ok := decryptedReader.(io.Closer); ok { - closer.Close() - } - return nil, fmt.Errorf("failed to skip intra-block bytes (%d): %w", ivSkip, err) - } - } - // Skip to the position we need in the decrypted stream if chunkView.OffsetInChunk > 0 { _, err = io.CopyN(io.Discard, decryptedReader, chunkView.OffsetInChunk) @@ -1531,6 +1528,23 @@ func (s3a *S3ApiServer) decryptSSECChunkView(ctx context.Context, fileChunk *fil } // decryptSSEKMSChunkView decrypts a specific chunk view with SSE-KMS +// +// IV Handling for SSE-KMS: +// ------------------------ +// SSE-KMS (and SSE-S3) use a fundamentally different IV scheme than SSE-C: +// +// 1. Encryption: Uses a BASE IV + offset calculation +// - Base IV is generated once for the entire object +// - For each chunk at position N: adjustedIV = calculateIVWithOffset(baseIV, N) +// - This shifts the CTR counter to counterₙ where n = N/16 +// - ChunkOffset is stored in metadata and IS applied during encryption +// +// 2. Decryption: Apply the same offset calculation +// - Use calculateIVWithOffset(baseIV, ChunkOffset) to reconstruct the encryption IV +// - Also handle ivSkip for non-block-aligned offsets (intra-block positioning) +// - This ensures decryption uses the same CTR counter sequence as encryption +// +// This contrasts with SSE-C which uses random IVs without offset calculation. func (s3a *S3ApiServer) decryptSSEKMSChunkView(ctx context.Context, fileChunk *filer_pb.FileChunk, chunkView *filer.ChunkView) (io.Reader, error) { if fileChunk.GetSseType() == filer_pb.SSEType_SSE_KMS && len(fileChunk.GetSseMetadata()) > 0 { sseKMSKey, err := DeserializeSSEKMSMetadata(fileChunk.GetSseMetadata()) @@ -1544,7 +1558,9 @@ func (s3a *S3ApiServer) decryptSSEKMSChunkView(ctx context.Context, fileChunk *f return nil, fmt.Errorf("failed to fetch full chunk: %w", err) } - // Calculate IV using ChunkOffset (same as PartOffset in SSE-C) + // IMPORTANT: Calculate adjusted IV using ChunkOffset + // SSE-KMS uses base IV + offset calculation (unlike SSE-C which uses random IVs) + // This reconstructs the same IV that was used during encryption var adjustedIV []byte var ivSkip int if sseKMSKey.ChunkOffset > 0 { @@ -1600,6 +1616,25 @@ func (s3a *S3ApiServer) decryptSSEKMSChunkView(ctx context.Context, fileChunk *f } // decryptSSES3ChunkView decrypts a specific chunk view with SSE-S3 +// +// IV Handling for SSE-S3: +// ----------------------- +// SSE-S3 uses the same BASE IV + offset scheme as SSE-KMS, but with a subtle difference: +// +// 1. Encryption: Uses BASE IV + offset, but stores the ADJUSTED IV +// - Base IV is generated once for the entire object +// - For each chunk at position N: adjustedIV, skip = calculateIVWithOffset(baseIV, N) +// - The ADJUSTED IV (not base IV) is stored in chunk metadata +// - ChunkOffset calculation is performed during encryption +// +// 2. Decryption: Use the stored adjusted IV directly +// - The stored IV is already block-aligned and ready to use +// - No need to call calculateIVWithOffset again (unlike SSE-KMS) +// - Decrypt full chunk from start, then skip to OffsetInChunk in plaintext +// +// This differs from: +// - SSE-C: Uses random IV per chunk, no offset calculation +// - SSE-KMS: Stores base IV, requires calculateIVWithOffset during decryption func (s3a *S3ApiServer) decryptSSES3ChunkView(ctx context.Context, fileChunk *filer_pb.FileChunk, chunkView *filer.ChunkView, entry *filer_pb.Entry) (io.Reader, error) { // For multipart SSE-S3, each chunk has its own IV in chunk.SseMetadata if fileChunk.GetSseType() == filer_pb.SSEType_SSE_S3 && len(fileChunk.GetSseMetadata()) > 0 { @@ -1617,15 +1652,11 @@ func (s3a *S3ApiServer) decryptSSES3ChunkView(ctx context.Context, fileChunk *fi return nil, fmt.Errorf("failed to fetch full chunk: %w", err) } - // Use the chunk's IV directly (already adjusted for block offset during encryption) - // Note: SSE-S3 encryption flow: - // 1. Upload: CreateSSES3EncryptedReaderWithBaseIV(reader, key, baseIV, partOffset) - // calls calculateIVWithOffset(baseIV, partOffset) → (blockAlignedIV, skip) - // The blockAlignedIV is stored in chunk metadata - // 2. Download: We decrypt the FULL chunk from offset 0 using that blockAlignedIV - // Then skip to chunkView.OffsetInChunk in the PLAINTEXT (not ciphertext) - // This differs from SSE-C which stores base IV + PartOffset and calculates IV during decryption - // No ivSkip needed here because we're decrypting from chunk start (offset 0) + // IMPORTANT: Use the stored IV directly - it's already block-aligned + // During encryption, CreateSSES3EncryptedReaderWithBaseIV called: + // adjustedIV, skip := calculateIVWithOffset(baseIV, partOffset) + // and stored the adjustedIV in metadata. We use it as-is for decryption. + // No need to call calculateIVWithOffset again (unlike SSE-KMS which stores base IV). iv := chunkSSES3Metadata.IV glog.V(4).Infof("Decrypting multipart SSE-S3 chunk %s with chunk-specific IV length=%d", diff --git a/weed/s3api/s3api_sse_decrypt_test.go b/weed/s3api/s3api_sse_decrypt_test.go new file mode 100644 index 000000000..043271bf4 --- /dev/null +++ b/weed/s3api/s3api_sse_decrypt_test.go @@ -0,0 +1,190 @@ +package s3api + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "io" + "testing" +) + +// TestSSECDecryptChunkView_NoOffsetAdjustment verifies that SSE-C decryption +// does NOT apply calculateIVWithOffset, preventing the critical bug where +// offset adjustment would cause CTR stream misalignment and data corruption. +func TestSSECDecryptChunkView_NoOffsetAdjustment(t *testing.T) { + // Setup: Create test data + plaintext := []byte("This is a test message for SSE-C decryption without offset adjustment") + customerKey := &SSECustomerKey{ + Key: make([]byte, 32), // 256-bit key + KeyMD5: "test-key-md5", + } + // Generate random AES key + if _, err := rand.Read(customerKey.Key); err != nil { + t.Fatalf("Failed to generate random key: %v", err) + } + + // Generate random IV for this "part" + randomIV := make([]byte, aes.BlockSize) + if _, err := rand.Read(randomIV); err != nil { + t.Fatalf("Failed to generate random IV: %v", err) + } + + // Encrypt the plaintext using the random IV (simulating SSE-C multipart upload) + // This is what CreateSSECEncryptedReader does - uses the IV directly without offset + block, err := aes.NewCipher(customerKey.Key) + if err != nil { + t.Fatalf("Failed to create cipher: %v", err) + } + ciphertext := make([]byte, len(plaintext)) + stream := cipher.NewCTR(block, randomIV) + stream.XORKeyStream(ciphertext, plaintext) + + partOffset := int64(1024) // Non-zero offset that should NOT be applied during SSE-C decryption + + // TEST: Decrypt using stored IV directly (correct behavior) + decryptedReaderCorrect, err := CreateSSECDecryptedReader( + io.NopCloser(bytes.NewReader(ciphertext)), + customerKey, + randomIV, // Use stored IV directly - CORRECT + ) + if err != nil { + t.Fatalf("Failed to create decrypted reader (correct): %v", err) + } + decryptedCorrect, err := io.ReadAll(decryptedReaderCorrect) + if err != nil { + t.Fatalf("Failed to read decrypted data (correct): %v", err) + } + + // Verify correct decryption + if !bytes.Equal(decryptedCorrect, plaintext) { + t.Errorf("Correct decryption failed:\nExpected: %s\nGot: %s", plaintext, decryptedCorrect) + } else { + t.Logf("✓ Correct decryption (using stored IV directly) successful") + } + + // ANTI-TEST: Decrypt using offset-adjusted IV (incorrect behavior - the bug) + adjustedIV, ivSkip := calculateIVWithOffset(randomIV, partOffset) + decryptedReaderWrong, err := CreateSSECDecryptedReader( + io.NopCloser(bytes.NewReader(ciphertext)), + customerKey, + adjustedIV, // Use adjusted IV - WRONG + ) + if err != nil { + t.Fatalf("Failed to create decrypted reader (wrong): %v", err) + } + + // Skip ivSkip bytes (as the buggy code would do) + if ivSkip > 0 { + io.CopyN(io.Discard, decryptedReaderWrong, int64(ivSkip)) + } + + decryptedWrong, err := io.ReadAll(decryptedReaderWrong) + if err != nil { + t.Fatalf("Failed to read decrypted data (wrong): %v", err) + } + + // Verify that offset adjustment produces DIFFERENT (corrupted) output + if bytes.Equal(decryptedWrong, plaintext) { + t.Errorf("CRITICAL: Offset-adjusted IV produced correct plaintext! This shouldn't happen for SSE-C.") + } else { + t.Logf("✓ Verified: Offset-adjusted IV produces corrupted data (as expected for SSE-C)") + maxLen := 20 + if len(plaintext) < maxLen { + maxLen = len(plaintext) + } + t.Logf(" Plaintext: %q", plaintext[:maxLen]) + maxLen2 := 20 + if len(decryptedWrong) < maxLen2 { + maxLen2 = len(decryptedWrong) + } + t.Logf(" Corrupted: %q", decryptedWrong[:maxLen2]) + } +} + +// TestSSEKMSDecryptChunkView_RequiresOffsetAdjustment verifies that SSE-KMS +// decryption DOES require calculateIVWithOffset, unlike SSE-C. +func TestSSEKMSDecryptChunkView_RequiresOffsetAdjustment(t *testing.T) { + // Setup: Create test data + plaintext := []byte("This is a test message for SSE-KMS decryption with offset adjustment") + + // Generate base IV and key + baseIV := make([]byte, aes.BlockSize) + key := make([]byte, 32) + if _, err := rand.Read(baseIV); err != nil { + t.Fatalf("Failed to generate base IV: %v", err) + } + if _, err := rand.Read(key); err != nil { + t.Fatalf("Failed to generate key: %v", err) + } + + chunkOffset := int64(2048) // Simulate chunk at offset 2048 + + // Encrypt using base IV + offset (simulating SSE-KMS multipart upload) + adjustedIV, ivSkip := calculateIVWithOffset(baseIV, chunkOffset) + block, err := aes.NewCipher(key) + if err != nil { + t.Fatalf("Failed to create cipher: %v", err) + } + + ciphertext := make([]byte, len(plaintext)) + stream := cipher.NewCTR(block, adjustedIV) + + // Skip ivSkip bytes in the encryption stream if needed + if ivSkip > 0 { + dummy := make([]byte, ivSkip) + stream.XORKeyStream(dummy, dummy) + } + stream.XORKeyStream(ciphertext, plaintext) + + // TEST: Decrypt using base IV + offset adjustment (correct for SSE-KMS) + adjustedIVDecrypt, ivSkipDecrypt := calculateIVWithOffset(baseIV, chunkOffset) + blockDecrypt, err := aes.NewCipher(key) + if err != nil { + t.Fatalf("Failed to create cipher for decryption: %v", err) + } + + decrypted := make([]byte, len(ciphertext)) + streamDecrypt := cipher.NewCTR(blockDecrypt, adjustedIVDecrypt) + + // Skip ivSkip bytes in the decryption stream + if ivSkipDecrypt > 0 { + dummy := make([]byte, ivSkipDecrypt) + streamDecrypt.XORKeyStream(dummy, dummy) + } + streamDecrypt.XORKeyStream(decrypted, ciphertext) + + // Verify correct decryption with offset adjustment + if !bytes.Equal(decrypted, plaintext) { + t.Errorf("SSE-KMS decryption with offset adjustment failed:\nExpected: %s\nGot: %s", plaintext, decrypted) + } else { + t.Logf("✓ SSE-KMS decryption with offset adjustment successful") + } + + // ANTI-TEST: Decrypt using base IV directly (incorrect for SSE-KMS) + blockWrong, err := aes.NewCipher(key) + if err != nil { + t.Fatalf("Failed to create cipher for wrong decryption: %v", err) + } + + decryptedWrong := make([]byte, len(ciphertext)) + streamWrong := cipher.NewCTR(blockWrong, baseIV) // Use base IV directly - WRONG for SSE-KMS + streamWrong.XORKeyStream(decryptedWrong, ciphertext) + + // Verify that NOT using offset adjustment produces corrupted output + if bytes.Equal(decryptedWrong, plaintext) { + t.Errorf("CRITICAL: Base IV without offset produced correct plaintext! SSE-KMS requires offset adjustment.") + } else { + t.Logf("✓ Verified: Base IV without offset produces corrupted data (as expected for SSE-KMS)") + } +} + +// TestSSEDecryptionDifferences documents the key differences between SSE types +func TestSSEDecryptionDifferences(t *testing.T) { + t.Log("SSE-C: Random IV per part → Use stored IV DIRECTLY (no offset)") + t.Log("SSE-KMS: Base IV + offset → MUST call calculateIVWithOffset(baseIV, offset)") + t.Log("SSE-S3: Base IV + offset → Stores ADJUSTED IV, use directly") + + // This test documents the critical differences and serves as executable documentation +} +