diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index d897894bc..6eedfc8f3 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -53,10 +53,11 @@ func (iam *IdentityAccessManagement) reqSignatureV4Verify(r *http.Request) (*Ide // Constants specific to this file const ( - emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" - streamingUnsignedPayload = "STREAMING-UNSIGNED-PAYLOAD-TRAILER" - unsignedPayload = "UNSIGNED-PAYLOAD" + emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + streamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" + streamingUnsignedPayload = "STREAMING-UNSIGNED-PAYLOAD-TRAILER" + unsignedPayload = "UNSIGNED-PAYLOAD" // Limit for IAM/STS request body size to prevent DoS attacks iamRequestBodyLimit = 10 * (1 << 20) // 10 MiB ) diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go index f841c3e1e..ca58ecec0 100644 --- a/weed/s3api/chunked_reader_v4.go +++ b/weed/s3api/chunked_reader_v4.go @@ -53,8 +53,8 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr // This check ensures we only proceed for streaming uploads. switch authInfo.HashedPayload { - case streamingContentSHA256: - glog.V(3).Infof("streaming content sha256") + case streamingContentSHA256, streamingContentSHA256Trailer: + glog.V(3).Infof("streaming content sha256 (with trailer: %v)", authInfo.HashedPayload == streamingContentSHA256Trailer) case streamingUnsignedPayload: glog.V(3).Infof("streaming unsigned payload") default: @@ -87,9 +87,9 @@ func (iam *IdentityAccessManagement) newChunkedReader(req *http.Request) (io.Rea var errCode s3err.ErrorCode switch contentSha256Header { - // Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' - case streamingContentSHA256: - glog.V(3).Infof("streaming content sha256") + // Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' or 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER' + case streamingContentSHA256, streamingContentSHA256Trailer: + glog.V(3).Infof("streaming content sha256 (with trailer: %v)", contentSha256Header == streamingContentSHA256Trailer) credential, seedSignature, region, service, seedDate, errCode = iam.calculateSeedSignature(req) if errCode != s3err.ErrNone { return nil, errCode diff --git a/weed/s3api/chunked_reader_v4_test.go b/weed/s3api/chunked_reader_v4_test.go index b797bf340..7a5fcf551 100644 --- a/weed/s3api/chunked_reader_v4_test.go +++ b/weed/s3api/chunked_reader_v4_test.go @@ -234,6 +234,96 @@ func TestSignedStreamingUpload(t *testing.T) { assert.Equal(t, chunk1Data+chunk2Data, string(data)) } +// TestSignedStreamingUploadWithTrailer tests streaming uploads with signed chunks and trailers +// This tests the STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER content-sha256 header value +// which is used by AWS SDK v2 when checksum validation is enabled +func TestSignedStreamingUploadWithTrailer(t *testing.T) { + iam := setupIam() + + // Create a simple streaming upload with 1 chunk and a trailer + chunk1Data := "hello world\n" + + // Use current time for signatures + now := time.Now().UTC() + amzDate := now.Format(iso8601Format) + dateStamp := now.Format(yyyymmdd) + + // Calculate seed signature + scope := dateStamp + "/" + defaultRegion + "/s3/aws4_request" + + // Build canonical request for seed signature + hashedPayload := "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" + canonicalHeaders := "content-encoding:aws-chunked\n" + + "host:s3.amazonaws.com\n" + + "x-amz-content-sha256:" + hashedPayload + "\n" + + "x-amz-date:" + amzDate + "\n" + + "x-amz-decoded-content-length:12\n" + + "x-amz-trailer:x-amz-checksum-crc32\n" + signedHeaders := "content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-trailer" + + canonicalRequest := "PUT\n" + + "/test-bucket/test-object\n" + + "\n" + + canonicalHeaders + "\n" + + signedHeaders + "\n" + + hashedPayload + + canonicalRequestHash := getSHA256Hash([]byte(canonicalRequest)) + stringToSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + canonicalRequestHash + + signingKey := getSigningKey(defaultSecretAccessKey, dateStamp, defaultRegion, "s3") + seedSignature := getSignature(signingKey, stringToSign) + + // Calculate chunk signatures + chunk1Hash := getSHA256Hash([]byte(chunk1Data)) + chunk1StringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" + + seedSignature + "\n" + emptySHA256 + "\n" + chunk1Hash + chunk1Signature := getSignature(signingKey, chunk1StringToSign) + + // Final chunk (0 bytes) + finalStringToSign := "AWS4-HMAC-SHA256-PAYLOAD\n" + amzDate + "\n" + scope + "\n" + + chunk1Signature + "\n" + emptySHA256 + "\n" + emptySHA256 + finalSignature := getSignature(signingKey, finalStringToSign) + + // Calculate CRC32 checksum for trailer + writer := crc32.NewIEEE() + writer.Write([]byte(chunk1Data)) + checksum := writer.Sum(nil) + base64EncodedChecksum := base64.StdEncoding.EncodeToString(checksum) + + // Build the chunked payload with trailer + payload := fmt.Sprintf("c;chunk-signature=%s\r\n%s\r\n", chunk1Signature, chunk1Data) + + fmt.Sprintf("0;chunk-signature=%s\r\n", finalSignature) + + "x-amz-checksum-crc32:" + base64EncodedChecksum + "\n\r\n" + + "\r\n" + + // Create the request + req, err := http.NewRequest("PUT", "http://s3.amazonaws.com/test-bucket/test-object", + bytes.NewReader([]byte(payload))) + assert.NoError(t, err) + + req.Header.Set("Host", "s3.amazonaws.com") + req.Header.Set("x-amz-date", amzDate) + req.Header.Set("x-amz-content-sha256", hashedPayload) + req.Header.Set("Content-Encoding", "aws-chunked") + req.Header.Set("x-amz-decoded-content-length", "12") + req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32") + + authHeader := fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", + defaultAccessKeyId, scope, signedHeaders, seedSignature) + req.Header.Set("Authorization", authHeader) + + // Test the chunked reader + reader, errCode := iam.newChunkedReader(req) + assert.Equal(t, s3err.ErrNone, errCode) + assert.NotNil(t, reader) + + // Read and verify the payload + data, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Equal(t, chunk1Data, string(data)) +} + // TestSignedStreamingUploadInvalidSignature tests that invalid chunk signatures are rejected // This is a negative test case to ensure signature validation is actually working func TestSignedStreamingUploadInvalidSignature(t *testing.T) { diff --git a/weed/s3api/s3api_auth.go b/weed/s3api/s3api_auth.go index e946b1284..95c195d49 100644 --- a/weed/s3api/s3api_auth.go +++ b/weed/s3api/s3api_auth.go @@ -48,8 +48,12 @@ func isRequestPostPolicySignatureV4(r *http.Request) bool { } // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation. +// Supports both with and without trailer variants: +// - STREAMING-AWS4-HMAC-SHA256-PAYLOAD (original) +// - STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER (with trailing checksums) func isRequestSignStreamingV4(r *http.Request) bool { - return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 && + contentSha256 := r.Header.Get("x-amz-content-sha256") + return (contentSha256 == streamingContentSHA256 || contentSha256 == streamingContentSHA256Trailer) && r.Method == http.MethodPut }