Browse Source

sse tests

pull/7481/head
chrislu 3 weeks ago
parent
commit
e4a882cc93
  1. 99
      weed/s3api/s3api_object_handlers.go
  2. 190
      weed/s3api/s3api_sse_decrypt_test.go

99
weed/s3api/s3api_object_handlers.go

@ -1457,6 +1457,22 @@ func writeZeroBytes(w io.Writer, n int64) error {
}
// decryptSSECChunkView decrypts a specific chunk view with SSE-C
//
// IV Handling for SSE-C:
// ----------------------
// SSE-C multipart encryption (see lines 2772-2781) differs fundamentally from SSE-KMS/SSE-S3:
//
// 1. Encryption: CreateSSECEncryptedReader generates a RANDOM IV per part/chunk
// - Each part starts with a fresh random IV
// - CTR counter starts from 0 for each part: counter₀, counter₁, counter₂, ...
// - PartOffset is stored in metadata but NOT applied during encryption
//
// 2. Decryption: Use the stored IV directly WITHOUT offset adjustment
// - The stored IV already represents the start of this part's encryption
// - Applying calculateIVWithOffset would shift to counterₙ, misaligning the keystream
// - Result: XOR with wrong keystream = corrupted plaintext
//
// This contrasts with SSE-KMS/SSE-S3 which use: base IV + calculateIVWithOffset(ChunkOffset)
func (s3a *S3ApiServer) decryptSSECChunkView(ctx context.Context, fileChunk *filer_pb.FileChunk, chunkView *filer.ChunkView, customerKey *SSECustomerKey) (io.Reader, error) {
// For multipart SSE-C, each chunk has its own IV in chunk.SseMetadata
if fileChunk.GetSseType() == filer_pb.SSEType_SSE_C && len(fileChunk.GetSseMetadata()) > 0 {
@ -1476,35 +1492,16 @@ func (s3a *S3ApiServer) decryptSSECChunkView(ctx context.Context, fileChunk *fil
return nil, fmt.Errorf("failed to fetch full chunk: %w", err)
}
// Calculate IV using PartOffset
// PartOffset is the position of this chunk within its part's encrypted stream
var adjustedIV []byte
var ivSkip int
if ssecMetadata.PartOffset > 0 {
adjustedIV, ivSkip = calculateIVWithOffset(chunkIV, ssecMetadata.PartOffset)
} else {
adjustedIV = chunkIV
ivSkip = 0
}
// Decrypt the full chunk
decryptedReader, decryptErr := CreateSSECDecryptedReader(fullChunkReader, customerKey, adjustedIV)
// CRITICAL: Use stored IV directly WITHOUT offset adjustment
// The stored IV is the random IV used at encryption time for this specific part
// SSE-C does NOT apply calculateIVWithOffset during encryption, so we must not apply it during decryption
// (See documentation above and at lines 2772-2781 for detailed explanation)
decryptedReader, decryptErr := CreateSSECDecryptedReader(fullChunkReader, customerKey, chunkIV)
if decryptErr != nil {
fullChunkReader.Close()
return nil, fmt.Errorf("failed to create decrypted reader: %w", decryptErr)
}
// CRITICAL: Skip intra-block bytes from CTR decryption (non-block-aligned offset handling)
if ivSkip > 0 {
_, err = io.CopyN(io.Discard, decryptedReader, int64(ivSkip))
if err != nil {
if closer, ok := decryptedReader.(io.Closer); ok {
closer.Close()
}
return nil, fmt.Errorf("failed to skip intra-block bytes (%d): %w", ivSkip, err)
}
}
// Skip to the position we need in the decrypted stream
if chunkView.OffsetInChunk > 0 {
_, err = io.CopyN(io.Discard, decryptedReader, chunkView.OffsetInChunk)
@ -1531,6 +1528,23 @@ func (s3a *S3ApiServer) decryptSSECChunkView(ctx context.Context, fileChunk *fil
}
// decryptSSEKMSChunkView decrypts a specific chunk view with SSE-KMS
//
// IV Handling for SSE-KMS:
// ------------------------
// SSE-KMS (and SSE-S3) use a fundamentally different IV scheme than SSE-C:
//
// 1. Encryption: Uses a BASE IV + offset calculation
// - Base IV is generated once for the entire object
// - For each chunk at position N: adjustedIV = calculateIVWithOffset(baseIV, N)
// - This shifts the CTR counter to counterₙ where n = N/16
// - ChunkOffset is stored in metadata and IS applied during encryption
//
// 2. Decryption: Apply the same offset calculation
// - Use calculateIVWithOffset(baseIV, ChunkOffset) to reconstruct the encryption IV
// - Also handle ivSkip for non-block-aligned offsets (intra-block positioning)
// - This ensures decryption uses the same CTR counter sequence as encryption
//
// This contrasts with SSE-C which uses random IVs without offset calculation.
func (s3a *S3ApiServer) decryptSSEKMSChunkView(ctx context.Context, fileChunk *filer_pb.FileChunk, chunkView *filer.ChunkView) (io.Reader, error) {
if fileChunk.GetSseType() == filer_pb.SSEType_SSE_KMS && len(fileChunk.GetSseMetadata()) > 0 {
sseKMSKey, err := DeserializeSSEKMSMetadata(fileChunk.GetSseMetadata())
@ -1544,7 +1558,9 @@ func (s3a *S3ApiServer) decryptSSEKMSChunkView(ctx context.Context, fileChunk *f
return nil, fmt.Errorf("failed to fetch full chunk: %w", err)
}
// Calculate IV using ChunkOffset (same as PartOffset in SSE-C)
// IMPORTANT: Calculate adjusted IV using ChunkOffset
// SSE-KMS uses base IV + offset calculation (unlike SSE-C which uses random IVs)
// This reconstructs the same IV that was used during encryption
var adjustedIV []byte
var ivSkip int
if sseKMSKey.ChunkOffset > 0 {
@ -1600,6 +1616,25 @@ func (s3a *S3ApiServer) decryptSSEKMSChunkView(ctx context.Context, fileChunk *f
}
// decryptSSES3ChunkView decrypts a specific chunk view with SSE-S3
//
// IV Handling for SSE-S3:
// -----------------------
// SSE-S3 uses the same BASE IV + offset scheme as SSE-KMS, but with a subtle difference:
//
// 1. Encryption: Uses BASE IV + offset, but stores the ADJUSTED IV
// - Base IV is generated once for the entire object
// - For each chunk at position N: adjustedIV, skip = calculateIVWithOffset(baseIV, N)
// - The ADJUSTED IV (not base IV) is stored in chunk metadata
// - ChunkOffset calculation is performed during encryption
//
// 2. Decryption: Use the stored adjusted IV directly
// - The stored IV is already block-aligned and ready to use
// - No need to call calculateIVWithOffset again (unlike SSE-KMS)
// - Decrypt full chunk from start, then skip to OffsetInChunk in plaintext
//
// This differs from:
// - SSE-C: Uses random IV per chunk, no offset calculation
// - SSE-KMS: Stores base IV, requires calculateIVWithOffset during decryption
func (s3a *S3ApiServer) decryptSSES3ChunkView(ctx context.Context, fileChunk *filer_pb.FileChunk, chunkView *filer.ChunkView, entry *filer_pb.Entry) (io.Reader, error) {
// For multipart SSE-S3, each chunk has its own IV in chunk.SseMetadata
if fileChunk.GetSseType() == filer_pb.SSEType_SSE_S3 && len(fileChunk.GetSseMetadata()) > 0 {
@ -1617,15 +1652,11 @@ func (s3a *S3ApiServer) decryptSSES3ChunkView(ctx context.Context, fileChunk *fi
return nil, fmt.Errorf("failed to fetch full chunk: %w", err)
}
// Use the chunk's IV directly (already adjusted for block offset during encryption)
// Note: SSE-S3 encryption flow:
// 1. Upload: CreateSSES3EncryptedReaderWithBaseIV(reader, key, baseIV, partOffset)
// calls calculateIVWithOffset(baseIV, partOffset) → (blockAlignedIV, skip)
// The blockAlignedIV is stored in chunk metadata
// 2. Download: We decrypt the FULL chunk from offset 0 using that blockAlignedIV
// Then skip to chunkView.OffsetInChunk in the PLAINTEXT (not ciphertext)
// This differs from SSE-C which stores base IV + PartOffset and calculates IV during decryption
// No ivSkip needed here because we're decrypting from chunk start (offset 0)
// IMPORTANT: Use the stored IV directly - it's already block-aligned
// During encryption, CreateSSES3EncryptedReaderWithBaseIV called:
// adjustedIV, skip := calculateIVWithOffset(baseIV, partOffset)
// and stored the adjustedIV in metadata. We use it as-is for decryption.
// No need to call calculateIVWithOffset again (unlike SSE-KMS which stores base IV).
iv := chunkSSES3Metadata.IV
glog.V(4).Infof("Decrypting multipart SSE-S3 chunk %s with chunk-specific IV length=%d",

190
weed/s3api/s3api_sse_decrypt_test.go

@ -0,0 +1,190 @@
package s3api
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"io"
"testing"
)
// TestSSECDecryptChunkView_NoOffsetAdjustment verifies that SSE-C decryption
// does NOT apply calculateIVWithOffset, preventing the critical bug where
// offset adjustment would cause CTR stream misalignment and data corruption.
func TestSSECDecryptChunkView_NoOffsetAdjustment(t *testing.T) {
// Setup: Create test data
plaintext := []byte("This is a test message for SSE-C decryption without offset adjustment")
customerKey := &SSECustomerKey{
Key: make([]byte, 32), // 256-bit key
KeyMD5: "test-key-md5",
}
// Generate random AES key
if _, err := rand.Read(customerKey.Key); err != nil {
t.Fatalf("Failed to generate random key: %v", err)
}
// Generate random IV for this "part"
randomIV := make([]byte, aes.BlockSize)
if _, err := rand.Read(randomIV); err != nil {
t.Fatalf("Failed to generate random IV: %v", err)
}
// Encrypt the plaintext using the random IV (simulating SSE-C multipart upload)
// This is what CreateSSECEncryptedReader does - uses the IV directly without offset
block, err := aes.NewCipher(customerKey.Key)
if err != nil {
t.Fatalf("Failed to create cipher: %v", err)
}
ciphertext := make([]byte, len(plaintext))
stream := cipher.NewCTR(block, randomIV)
stream.XORKeyStream(ciphertext, plaintext)
partOffset := int64(1024) // Non-zero offset that should NOT be applied during SSE-C decryption
// TEST: Decrypt using stored IV directly (correct behavior)
decryptedReaderCorrect, err := CreateSSECDecryptedReader(
io.NopCloser(bytes.NewReader(ciphertext)),
customerKey,
randomIV, // Use stored IV directly - CORRECT
)
if err != nil {
t.Fatalf("Failed to create decrypted reader (correct): %v", err)
}
decryptedCorrect, err := io.ReadAll(decryptedReaderCorrect)
if err != nil {
t.Fatalf("Failed to read decrypted data (correct): %v", err)
}
// Verify correct decryption
if !bytes.Equal(decryptedCorrect, plaintext) {
t.Errorf("Correct decryption failed:\nExpected: %s\nGot: %s", plaintext, decryptedCorrect)
} else {
t.Logf("✓ Correct decryption (using stored IV directly) successful")
}
// ANTI-TEST: Decrypt using offset-adjusted IV (incorrect behavior - the bug)
adjustedIV, ivSkip := calculateIVWithOffset(randomIV, partOffset)
decryptedReaderWrong, err := CreateSSECDecryptedReader(
io.NopCloser(bytes.NewReader(ciphertext)),
customerKey,
adjustedIV, // Use adjusted IV - WRONG
)
if err != nil {
t.Fatalf("Failed to create decrypted reader (wrong): %v", err)
}
// Skip ivSkip bytes (as the buggy code would do)
if ivSkip > 0 {
io.CopyN(io.Discard, decryptedReaderWrong, int64(ivSkip))
}
decryptedWrong, err := io.ReadAll(decryptedReaderWrong)
if err != nil {
t.Fatalf("Failed to read decrypted data (wrong): %v", err)
}
// Verify that offset adjustment produces DIFFERENT (corrupted) output
if bytes.Equal(decryptedWrong, plaintext) {
t.Errorf("CRITICAL: Offset-adjusted IV produced correct plaintext! This shouldn't happen for SSE-C.")
} else {
t.Logf("✓ Verified: Offset-adjusted IV produces corrupted data (as expected for SSE-C)")
maxLen := 20
if len(plaintext) < maxLen {
maxLen = len(plaintext)
}
t.Logf(" Plaintext: %q", plaintext[:maxLen])
maxLen2 := 20
if len(decryptedWrong) < maxLen2 {
maxLen2 = len(decryptedWrong)
}
t.Logf(" Corrupted: %q", decryptedWrong[:maxLen2])
}
}
// TestSSEKMSDecryptChunkView_RequiresOffsetAdjustment verifies that SSE-KMS
// decryption DOES require calculateIVWithOffset, unlike SSE-C.
func TestSSEKMSDecryptChunkView_RequiresOffsetAdjustment(t *testing.T) {
// Setup: Create test data
plaintext := []byte("This is a test message for SSE-KMS decryption with offset adjustment")
// Generate base IV and key
baseIV := make([]byte, aes.BlockSize)
key := make([]byte, 32)
if _, err := rand.Read(baseIV); err != nil {
t.Fatalf("Failed to generate base IV: %v", err)
}
if _, err := rand.Read(key); err != nil {
t.Fatalf("Failed to generate key: %v", err)
}
chunkOffset := int64(2048) // Simulate chunk at offset 2048
// Encrypt using base IV + offset (simulating SSE-KMS multipart upload)
adjustedIV, ivSkip := calculateIVWithOffset(baseIV, chunkOffset)
block, err := aes.NewCipher(key)
if err != nil {
t.Fatalf("Failed to create cipher: %v", err)
}
ciphertext := make([]byte, len(plaintext))
stream := cipher.NewCTR(block, adjustedIV)
// Skip ivSkip bytes in the encryption stream if needed
if ivSkip > 0 {
dummy := make([]byte, ivSkip)
stream.XORKeyStream(dummy, dummy)
}
stream.XORKeyStream(ciphertext, plaintext)
// TEST: Decrypt using base IV + offset adjustment (correct for SSE-KMS)
adjustedIVDecrypt, ivSkipDecrypt := calculateIVWithOffset(baseIV, chunkOffset)
blockDecrypt, err := aes.NewCipher(key)
if err != nil {
t.Fatalf("Failed to create cipher for decryption: %v", err)
}
decrypted := make([]byte, len(ciphertext))
streamDecrypt := cipher.NewCTR(blockDecrypt, adjustedIVDecrypt)
// Skip ivSkip bytes in the decryption stream
if ivSkipDecrypt > 0 {
dummy := make([]byte, ivSkipDecrypt)
streamDecrypt.XORKeyStream(dummy, dummy)
}
streamDecrypt.XORKeyStream(decrypted, ciphertext)
// Verify correct decryption with offset adjustment
if !bytes.Equal(decrypted, plaintext) {
t.Errorf("SSE-KMS decryption with offset adjustment failed:\nExpected: %s\nGot: %s", plaintext, decrypted)
} else {
t.Logf("✓ SSE-KMS decryption with offset adjustment successful")
}
// ANTI-TEST: Decrypt using base IV directly (incorrect for SSE-KMS)
blockWrong, err := aes.NewCipher(key)
if err != nil {
t.Fatalf("Failed to create cipher for wrong decryption: %v", err)
}
decryptedWrong := make([]byte, len(ciphertext))
streamWrong := cipher.NewCTR(blockWrong, baseIV) // Use base IV directly - WRONG for SSE-KMS
streamWrong.XORKeyStream(decryptedWrong, ciphertext)
// Verify that NOT using offset adjustment produces corrupted output
if bytes.Equal(decryptedWrong, plaintext) {
t.Errorf("CRITICAL: Base IV without offset produced correct plaintext! SSE-KMS requires offset adjustment.")
} else {
t.Logf("✓ Verified: Base IV without offset produces corrupted data (as expected for SSE-KMS)")
}
}
// TestSSEDecryptionDifferences documents the key differences between SSE types
func TestSSEDecryptionDifferences(t *testing.T) {
t.Log("SSE-C: Random IV per part → Use stored IV DIRECTLY (no offset)")
t.Log("SSE-KMS: Base IV + offset → MUST call calculateIVWithOffset(baseIV, offset)")
t.Log("SSE-S3: Base IV + offset → Stores ADJUSTED IV, use directly")
// This test documents the critical differences and serves as executable documentation
}
Loading…
Cancel
Save