sse tests

weed/s3api/s3api_object_handlers.go

@@ -1457,6 +1457,22 @@ func writeZeroBytes(w io.Writer, n int64) error {
 }
 
 // decryptSSECChunkView decrypts a specific chunk view with SSE-C
+//
+// IV Handling for SSE-C:
+// ----------------------
+// SSE-C multipart encryption (see lines 2772-2781) differs fundamentally from SSE-KMS/SSE-S3:
+//
+// 1. Encryption: CreateSSECEncryptedReader generates a RANDOM IV per part/chunk
+//    - Each part starts with a fresh random IV
+//    - CTR counter starts from 0 for each part: counter₀, counter₁, counter₂, ...
+//    - PartOffset is stored in metadata but NOT applied during encryption
+//
+// 2. Decryption: Use the stored IV directly WITHOUT offset adjustment
+//    - The stored IV already represents the start of this part's encryption
+//    - Applying calculateIVWithOffset would shift to counterₙ, misaligning the keystream
+//    - Result: XOR with wrong keystream = corrupted plaintext
+//
+// This contrasts with SSE-KMS/SSE-S3 which use: base IV + calculateIVWithOffset(ChunkOffset)
 func (s3a *S3ApiServer) decryptSSECChunkView(ctx context.Context, fileChunk *filer_pb.FileChunk, chunkView *filer.ChunkView, customerKey *SSECustomerKey) (io.Reader, error) {
 	// For multipart SSE-C, each chunk has its own IV in chunk.SseMetadata
 	if fileChunk.GetSseType() == filer_pb.SSEType_SSE_C && len(fileChunk.GetSseMetadata()) > 0 {
@@ -1476,35 +1492,16 @@ func (s3a *S3ApiServer) decryptSSECChunkView(ctx context.Context, fileChunk *fil
 			return nil, fmt.Errorf("failed to fetch full chunk: %w", err)
 		}
 
-		// Calculate IV using PartOffset
-		// PartOffset is the position of this chunk within its part's encrypted stream
-		var adjustedIV []byte
-		var ivSkip int
-		if ssecMetadata.PartOffset > 0 {
-			adjustedIV, ivSkip = calculateIVWithOffset(chunkIV, ssecMetadata.PartOffset)
-		} else {
-			adjustedIV = chunkIV
-			ivSkip = 0
-		}
-
-		// Decrypt the full chunk
-		decryptedReader, decryptErr := CreateSSECDecryptedReader(fullChunkReader, customerKey, adjustedIV)
+		// CRITICAL: Use stored IV directly WITHOUT offset adjustment
+		// The stored IV is the random IV used at encryption time for this specific part
+		// SSE-C does NOT apply calculateIVWithOffset during encryption, so we must not apply it during decryption
+		// (See documentation above and at lines 2772-2781 for detailed explanation)
+		decryptedReader, decryptErr := CreateSSECDecryptedReader(fullChunkReader, customerKey, chunkIV)
 		if decryptErr != nil {
 			fullChunkReader.Close()
 			return nil, fmt.Errorf("failed to create decrypted reader: %w", decryptErr)
 		}
 
-		// CRITICAL: Skip intra-block bytes from CTR decryption (non-block-aligned offset handling)
-		if ivSkip > 0 {
-			_, err = io.CopyN(io.Discard, decryptedReader, int64(ivSkip))
-			if err != nil {
-				if closer, ok := decryptedReader.(io.Closer); ok {
-					closer.Close()
-				}
-				return nil, fmt.Errorf("failed to skip intra-block bytes (%d): %w", ivSkip, err)
-			}
-		}
-
 		// Skip to the position we need in the decrypted stream
 		if chunkView.OffsetInChunk > 0 {
 			_, err = io.CopyN(io.Discard, decryptedReader, chunkView.OffsetInChunk)
@@ -1531,6 +1528,23 @@ func (s3a *S3ApiServer) decryptSSECChunkView(ctx context.Context, fileChunk *fil
 }
 
 // decryptSSEKMSChunkView decrypts a specific chunk view with SSE-KMS
+//
+// IV Handling for SSE-KMS:
+// ------------------------
+// SSE-KMS (and SSE-S3) use a fundamentally different IV scheme than SSE-C:
+//
+// 1. Encryption: Uses a BASE IV + offset calculation
+//    - Base IV is generated once for the entire object
+//    - For each chunk at position N: adjustedIV = calculateIVWithOffset(baseIV, N)
+//    - This shifts the CTR counter to counterₙ where n = N/16
+//    - ChunkOffset is stored in metadata and IS applied during encryption
+//
+// 2. Decryption: Apply the same offset calculation
+//    - Use calculateIVWithOffset(baseIV, ChunkOffset) to reconstruct the encryption IV
+//    - Also handle ivSkip for non-block-aligned offsets (intra-block positioning)
+//    - This ensures decryption uses the same CTR counter sequence as encryption
+//
+// This contrasts with SSE-C which uses random IVs without offset calculation.
 func (s3a *S3ApiServer) decryptSSEKMSChunkView(ctx context.Context, fileChunk *filer_pb.FileChunk, chunkView *filer.ChunkView) (io.Reader, error) {
 	if fileChunk.GetSseType() == filer_pb.SSEType_SSE_KMS && len(fileChunk.GetSseMetadata()) > 0 {
 		sseKMSKey, err := DeserializeSSEKMSMetadata(fileChunk.GetSseMetadata())
@@ -1544,7 +1558,9 @@ func (s3a *S3ApiServer) decryptSSEKMSChunkView(ctx context.Context, fileChunk *f
 			return nil, fmt.Errorf("failed to fetch full chunk: %w", err)
 		}
 
-		// Calculate IV using ChunkOffset (same as PartOffset in SSE-C)
+		// IMPORTANT: Calculate adjusted IV using ChunkOffset
+		// SSE-KMS uses base IV + offset calculation (unlike SSE-C which uses random IVs)
+		// This reconstructs the same IV that was used during encryption
 		var adjustedIV []byte
 		var ivSkip int
 		if sseKMSKey.ChunkOffset > 0 {
@@ -1600,6 +1616,25 @@ func (s3a *S3ApiServer) decryptSSEKMSChunkView(ctx context.Context, fileChunk *f
 }
 
 // decryptSSES3ChunkView decrypts a specific chunk view with SSE-S3
+//
+// IV Handling for SSE-S3:
+// -----------------------
+// SSE-S3 uses the same BASE IV + offset scheme as SSE-KMS, but with a subtle difference:
+//
+// 1. Encryption: Uses BASE IV + offset, but stores the ADJUSTED IV
+//    - Base IV is generated once for the entire object
+//    - For each chunk at position N: adjustedIV, skip = calculateIVWithOffset(baseIV, N)
+//    - The ADJUSTED IV (not base IV) is stored in chunk metadata
+//    - ChunkOffset calculation is performed during encryption
+//
+// 2. Decryption: Use the stored adjusted IV directly
+//    - The stored IV is already block-aligned and ready to use
+//    - No need to call calculateIVWithOffset again (unlike SSE-KMS)
+//    - Decrypt full chunk from start, then skip to OffsetInChunk in plaintext
+//
+// This differs from:
+//    - SSE-C: Uses random IV per chunk, no offset calculation
+//    - SSE-KMS: Stores base IV, requires calculateIVWithOffset during decryption
 func (s3a *S3ApiServer) decryptSSES3ChunkView(ctx context.Context, fileChunk *filer_pb.FileChunk, chunkView *filer.ChunkView, entry *filer_pb.Entry) (io.Reader, error) {
 	// For multipart SSE-S3, each chunk has its own IV in chunk.SseMetadata
 	if fileChunk.GetSseType() == filer_pb.SSEType_SSE_S3 && len(fileChunk.GetSseMetadata()) > 0 {
@@ -1617,15 +1652,11 @@ func (s3a *S3ApiServer) decryptSSES3ChunkView(ctx context.Context, fileChunk *fi
 			return nil, fmt.Errorf("failed to fetch full chunk: %w", err)
 		}
 
-		// Use the chunk's IV directly (already adjusted for block offset during encryption)
-		// Note: SSE-S3 encryption flow:
-		// 1. Upload: CreateSSES3EncryptedReaderWithBaseIV(reader, key, baseIV, partOffset)
-		//    calls calculateIVWithOffset(baseIV, partOffset) → (blockAlignedIV, skip)
-		//    The blockAlignedIV is stored in chunk metadata
-		// 2. Download: We decrypt the FULL chunk from offset 0 using that blockAlignedIV
-		//    Then skip to chunkView.OffsetInChunk in the PLAINTEXT (not ciphertext)
-		// This differs from SSE-C which stores base IV + PartOffset and calculates IV during decryption
-		// No ivSkip needed here because we're decrypting from chunk start (offset 0)
+		// IMPORTANT: Use the stored IV directly - it's already block-aligned
+		// During encryption, CreateSSES3EncryptedReaderWithBaseIV called:
+		//   adjustedIV, skip := calculateIVWithOffset(baseIV, partOffset)
+		// and stored the adjustedIV in metadata. We use it as-is for decryption.
+		// No need to call calculateIVWithOffset again (unlike SSE-KMS which stores base IV).
 		iv := chunkSSES3Metadata.IV
 
 		glog.V(4).Infof("Decrypting multipart SSE-S3 chunk %s with chunk-specific IV length=%d",

weed/s3api/s3api_sse_decrypt_test.go

@@ -0,0 +1,190 @@
+package s3api
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"io"
+	"testing"
+)
+
+// TestSSECDecryptChunkView_NoOffsetAdjustment verifies that SSE-C decryption
+// does NOT apply calculateIVWithOffset, preventing the critical bug where
+// offset adjustment would cause CTR stream misalignment and data corruption.
+func TestSSECDecryptChunkView_NoOffsetAdjustment(t *testing.T) {
+	// Setup: Create test data
+	plaintext := []byte("This is a test message for SSE-C decryption without offset adjustment")
+	customerKey := &SSECustomerKey{
+		Key:    make([]byte, 32), // 256-bit key
+		KeyMD5: "test-key-md5",
+	}
+	// Generate random AES key
+	if _, err := rand.Read(customerKey.Key); err != nil {
+		t.Fatalf("Failed to generate random key: %v", err)
+	}
+
+	// Generate random IV for this "part"
+	randomIV := make([]byte, aes.BlockSize)
+	if _, err := rand.Read(randomIV); err != nil {
+		t.Fatalf("Failed to generate random IV: %v", err)
+	}
+
+	// Encrypt the plaintext using the random IV (simulating SSE-C multipart upload)
+	// This is what CreateSSECEncryptedReader does - uses the IV directly without offset
+	block, err := aes.NewCipher(customerKey.Key)
+	if err != nil {
+		t.Fatalf("Failed to create cipher: %v", err)
+	}
+	ciphertext := make([]byte, len(plaintext))
+	stream := cipher.NewCTR(block, randomIV)
+	stream.XORKeyStream(ciphertext, plaintext)
+
+	partOffset := int64(1024) // Non-zero offset that should NOT be applied during SSE-C decryption
+
+	// TEST: Decrypt using stored IV directly (correct behavior)
+	decryptedReaderCorrect, err := CreateSSECDecryptedReader(
+		io.NopCloser(bytes.NewReader(ciphertext)),
+		customerKey,
+		randomIV, // Use stored IV directly - CORRECT
+	)
+	if err != nil {
+		t.Fatalf("Failed to create decrypted reader (correct): %v", err)
+	}
+	decryptedCorrect, err := io.ReadAll(decryptedReaderCorrect)
+	if err != nil {
+		t.Fatalf("Failed to read decrypted data (correct): %v", err)
+	}
+
+	// Verify correct decryption
+	if !bytes.Equal(decryptedCorrect, plaintext) {
+		t.Errorf("Correct decryption failed:\nExpected: %s\nGot: %s", plaintext, decryptedCorrect)
+	} else {
+		t.Logf("✓ Correct decryption (using stored IV directly) successful")
+	}
+
+	// ANTI-TEST: Decrypt using offset-adjusted IV (incorrect behavior - the bug)
+	adjustedIV, ivSkip := calculateIVWithOffset(randomIV, partOffset)
+	decryptedReaderWrong, err := CreateSSECDecryptedReader(
+		io.NopCloser(bytes.NewReader(ciphertext)),
+		customerKey,
+		adjustedIV, // Use adjusted IV - WRONG
+	)
+	if err != nil {
+		t.Fatalf("Failed to create decrypted reader (wrong): %v", err)
+	}
+	
+	// Skip ivSkip bytes (as the buggy code would do)
+	if ivSkip > 0 {
+		io.CopyN(io.Discard, decryptedReaderWrong, int64(ivSkip))
+	}
+	
+	decryptedWrong, err := io.ReadAll(decryptedReaderWrong)
+	if err != nil {
+		t.Fatalf("Failed to read decrypted data (wrong): %v", err)
+	}
+
+	// Verify that offset adjustment produces DIFFERENT (corrupted) output
+	if bytes.Equal(decryptedWrong, plaintext) {
+		t.Errorf("CRITICAL: Offset-adjusted IV produced correct plaintext! This shouldn't happen for SSE-C.")
+	} else {
+		t.Logf("✓ Verified: Offset-adjusted IV produces corrupted data (as expected for SSE-C)")
+		maxLen := 20
+		if len(plaintext) < maxLen {
+			maxLen = len(plaintext)
+		}
+		t.Logf("  Plaintext:  %q", plaintext[:maxLen])
+		maxLen2 := 20
+		if len(decryptedWrong) < maxLen2 {
+			maxLen2 = len(decryptedWrong)
+		}
+		t.Logf("  Corrupted:  %q", decryptedWrong[:maxLen2])
+	}
+}
+
+// TestSSEKMSDecryptChunkView_RequiresOffsetAdjustment verifies that SSE-KMS
+// decryption DOES require calculateIVWithOffset, unlike SSE-C.
+func TestSSEKMSDecryptChunkView_RequiresOffsetAdjustment(t *testing.T) {
+	// Setup: Create test data
+	plaintext := []byte("This is a test message for SSE-KMS decryption with offset adjustment")
+	
+	// Generate base IV and key
+	baseIV := make([]byte, aes.BlockSize)
+	key := make([]byte, 32)
+	if _, err := rand.Read(baseIV); err != nil {
+		t.Fatalf("Failed to generate base IV: %v", err)
+	}
+	if _, err := rand.Read(key); err != nil {
+		t.Fatalf("Failed to generate key: %v", err)
+	}
+
+	chunkOffset := int64(2048) // Simulate chunk at offset 2048
+
+	// Encrypt using base IV + offset (simulating SSE-KMS multipart upload)
+	adjustedIV, ivSkip := calculateIVWithOffset(baseIV, chunkOffset)
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		t.Fatalf("Failed to create cipher: %v", err)
+	}
+	
+	ciphertext := make([]byte, len(plaintext))
+	stream := cipher.NewCTR(block, adjustedIV)
+	
+	// Skip ivSkip bytes in the encryption stream if needed
+	if ivSkip > 0 {
+		dummy := make([]byte, ivSkip)
+		stream.XORKeyStream(dummy, dummy)
+	}
+	stream.XORKeyStream(ciphertext, plaintext)
+
+	// TEST: Decrypt using base IV + offset adjustment (correct for SSE-KMS)
+	adjustedIVDecrypt, ivSkipDecrypt := calculateIVWithOffset(baseIV, chunkOffset)
+	blockDecrypt, err := aes.NewCipher(key)
+	if err != nil {
+		t.Fatalf("Failed to create cipher for decryption: %v", err)
+	}
+	
+	decrypted := make([]byte, len(ciphertext))
+	streamDecrypt := cipher.NewCTR(blockDecrypt, adjustedIVDecrypt)
+	
+	// Skip ivSkip bytes in the decryption stream
+	if ivSkipDecrypt > 0 {
+		dummy := make([]byte, ivSkipDecrypt)
+		streamDecrypt.XORKeyStream(dummy, dummy)
+	}
+	streamDecrypt.XORKeyStream(decrypted, ciphertext)
+
+	// Verify correct decryption with offset adjustment
+	if !bytes.Equal(decrypted, plaintext) {
+		t.Errorf("SSE-KMS decryption with offset adjustment failed:\nExpected: %s\nGot: %s", plaintext, decrypted)
+	} else {
+		t.Logf("✓ SSE-KMS decryption with offset adjustment successful")
+	}
+
+	// ANTI-TEST: Decrypt using base IV directly (incorrect for SSE-KMS)
+	blockWrong, err := aes.NewCipher(key)
+	if err != nil {
+		t.Fatalf("Failed to create cipher for wrong decryption: %v", err)
+	}
+	
+	decryptedWrong := make([]byte, len(ciphertext))
+	streamWrong := cipher.NewCTR(blockWrong, baseIV) // Use base IV directly - WRONG for SSE-KMS
+	streamWrong.XORKeyStream(decryptedWrong, ciphertext)
+
+	// Verify that NOT using offset adjustment produces corrupted output
+	if bytes.Equal(decryptedWrong, plaintext) {
+		t.Errorf("CRITICAL: Base IV without offset produced correct plaintext! SSE-KMS requires offset adjustment.")
+	} else {
+		t.Logf("✓ Verified: Base IV without offset produces corrupted data (as expected for SSE-KMS)")
+	}
+}
+
+// TestSSEDecryptionDifferences documents the key differences between SSE types
+func TestSSEDecryptionDifferences(t *testing.T) {
+	t.Log("SSE-C:   Random IV per part → Use stored IV DIRECTLY (no offset)")
+	t.Log("SSE-KMS: Base IV + offset → MUST call calculateIVWithOffset(baseIV, offset)")
+	t.Log("SSE-S3:  Base IV + offset → Stores ADJUSTED IV, use directly")
+	
+	// This test documents the critical differences and serves as executable documentation
+}
+