From b1d2a4cdcff0522674dca671c1597b980c65d204 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sun, 16 Nov 2025 18:36:38 -0800 Subject: [PATCH] fix offset --- weed/s3api/s3api_object_handlers.go | 44 +++++++++---------------- weed/s3api/s3api_object_handlers_put.go | 18 ++++------ 2 files changed, 22 insertions(+), 40 deletions(-) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 8546b9375..302aab8bc 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -772,12 +772,14 @@ func (s3a *S3ApiServer) createLookupFileIdFunction() func(context.Context, strin } if locs, found := resp.LocationsMap[vid]; found { for _, loc := range locs.Locations { - // Ensure URL has http:// scheme prefix for proper URL parsing - urls = append(urls, "http://"+loc.Url) + // Build complete URL with volume server address and fileId + // Format: http://host:port/volumeId,fileId + urls = append(urls, "http://"+loc.Url+"/"+fileId) } } return nil }) + glog.V(3).Infof("createLookupFileIdFunction: fileId=%s, resolved urls=%v", fileId, urls) return urls, err } } @@ -1224,20 +1226,14 @@ func (s3a *S3ApiServer) decryptSSECChunkView(ctx context.Context, fileChunk *fil // Decrypt with CTR IV offset adjustment // CTR mode: IV for block N = base_IV + (N / 16) - // PartOffset in metadata distinguishes single-part vs multipart: - // - Single-part: PartOffset = 0 for all chunks (one encrypted stream starting at 0) - // - Multipart: PartOffset = chunk position within part during upload - var ivOffset int64 - if ssecMetadata.PartOffset == 0 { - // Single-part upload: use absolute file position - ivOffset = chunkView.ViewOffset - } else { - // Multipart upload: use position within the part's encrypted stream - // After assembly, chunk.Offset is in final file, so we calculate position within part - ivOffset = chunkView.ViewOffset - fileChunk.Offset - } - glog.V(3).Infof("decryptSSECChunkView: chunk=%s, fileChunk.Offset=%d, chunkView.ViewOffset=%d, chunkView.ViewSize=%d, metadata.PartOffset=%d, ivOffset=%d", - chunkView.FileId, fileChunk.Offset, chunkView.ViewOffset, chunkView.ViewSize, ssecMetadata.PartOffset, ivOffset) + // PartOffset stores the position within the encrypted stream (which always starts at 0) + // After multipart assembly, fileChunk.Offset is the position in the final file + // Formula: ivOffset = PartOffset + (ViewOffset - fileChunk.Offset) + // - PartOffset: where this chunk is in its encrypted stream + // - (ViewOffset - fileChunk.Offset): offset within this chunk + ivOffset := ssecMetadata.PartOffset + (chunkView.ViewOffset - fileChunk.Offset) + glog.V(3).Infof("decryptSSECChunkView: chunk=%s, fileChunk.Offset=%d, chunkView.ViewOffset=%d, metadata.PartOffset=%d, ivOffset=%d", + chunkView.FileId, fileChunk.Offset, chunkView.ViewOffset, ssecMetadata.PartOffset, ivOffset) adjustedIV := adjustCTRIV(chunkIV, ivOffset) return CreateSSECDecryptedReader(encryptedReader, customerKey, adjustedIV) } @@ -1265,18 +1261,10 @@ func (s3a *S3ApiServer) decryptSSEKMSChunkView(ctx context.Context, fileChunk *f return nil, fmt.Errorf("failed to fetch chunk data: %w", err) } - // Decrypt with CTR IV offset adjustment - // ChunkOffset in KMS metadata distinguishes single-part vs multipart (same logic as SSE-C) - // - Single-part: ChunkOffset = 0 for all chunks (one encrypted stream starting at 0) - // - Multipart: ChunkOffset = chunk position within part during upload - var ivOffset int64 - if sseKMSKey.ChunkOffset == 0 { - // Single-part upload: use absolute file position - ivOffset = chunkView.ViewOffset - } else { - // Multipart upload: use position within the part's encrypted stream - ivOffset = chunkView.ViewOffset - fileChunk.Offset - } + // Decrypt with CTR IV offset adjustment (same logic as SSE-C) + // ChunkOffset stores the position within the encrypted stream (which always starts at 0) + // Formula: ivOffset = ChunkOffset + (ViewOffset - fileChunk.Offset) + ivOffset := sseKMSKey.ChunkOffset + (chunkView.ViewOffset - fileChunk.Offset) adjustedIV := adjustCTRIV(sseKMSKey.IV, ivOffset) adjustedKey := &SSEKMSKey{ KeyID: sseKMSKey.KeyID, diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index 1e34d53d6..d4bdfcea4 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -390,17 +390,11 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader for _, chunk := range chunkResult.FileChunks { chunk.SseType = filer_pb.SSEType_SSE_C if len(sseIV) > 0 { - // For single-part uploads (partNumber == 0), all chunks belong to one encrypted stream starting at 0 - // For multipart uploads (partNumber > 0), each part is encrypted independently starting at 0 - var partOffset int64 - if partNumber == 0 { - // Single-part: use 0 since the entire object is one encrypted stream starting at position 0 - partOffset = 0 - } else { - // Multipart: use chunk.Offset which represents position within this part's encrypted stream - partOffset = chunk.Offset - } - + // PartOffset tracks position within the encrypted stream + // Since ALL uploads (single-part and multipart parts) encrypt starting from offset 0, + // PartOffset = chunk.Offset represents where this chunk is in that encrypted stream + // - Single-part: chunk.Offset is position in the file's encrypted stream + // - Multipart: chunk.Offset is position in this part's encrypted stream ssecMetadataStruct := struct { Algorithm string `json:"algorithm"` IV string `json:"iv"` @@ -410,7 +404,7 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader Algorithm: "AES256", IV: base64.StdEncoding.EncodeToString(sseIV), KeyMD5: customerKey.KeyMD5, - PartOffset: partOffset, + PartOffset: chunk.Offset, // Position within the encrypted stream (always encrypted from 0) } if ssecMetadata, serErr := json.Marshal(ssecMetadataStruct); serErr == nil { chunk.SseMetadata = ssecMetadata