From a7b964af96cd6bc07fc89571cc5f3f8048464cb3 Mon Sep 17 00:00:00 2001 From: Tom Crasset <25140344+tcrasset@users.noreply.github.com> Date: Fri, 7 Feb 2025 19:54:31 +0100 Subject: [PATCH] add s3 signature tests and prepare implementation of STREAMING-UNSIGNED-PAYLOAD-TRAILER (#6525) * add tests for s3 signature * add test for newSignV4ChunkedReader.Read() * add glog import --- weed/s3api/auth_credentials.go | 7 ++ weed/s3api/auth_signature_v4.go | 5 +- weed/s3api/auto_signature_v4_test.go | 73 +++++++++++++++- weed/s3api/chunked_reader_v4.go | 14 +++- weed/s3api/chunked_reader_v4_test.go | 107 ++++++++++++++++++++++++ weed/s3api/s3api_auth.go | 8 ++ weed/s3api/s3api_object_handlers_put.go | 2 +- 7 files changed, 208 insertions(+), 8 deletions(-) create mode 100644 weed/s3api/chunked_reader_v4_test.go diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index e80773993..1fb118d6f 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -364,6 +364,9 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) glog.V(3).Infof("post policy auth type") r.Header.Set(s3_constants.AmzAuthType, "PostPolicy") return identity, s3err.ErrNone + case authTypeStreamingUnsigned: + glog.V(3).Infof("unsigned streaming upload") + return identity, s3err.ErrNone case authTypeJWT: glog.V(3).Infof("jwt auth type") r.Header.Set(s3_constants.AmzAuthType, "Jwt") @@ -412,6 +415,10 @@ func (iam *IdentityAccessManagement) authUser(r *http.Request) (*Identity, s3err var authType string switch getRequestAuthType(r) { case authTypeStreamingSigned: + glog.V(3).Infof("signed streaming upload") + return identity, s3err.ErrNone + case authTypeStreamingUnsigned: + glog.V(3).Infof("unsigned streaming upload") return identity, s3err.ErrNone case authTypeUnknown: glog.V(3).Infof("unknown auth type") diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 47fb94a43..43ca851fc 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -56,9 +56,10 @@ const ( streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" signV4ChunkedAlgorithm = "AWS4-HMAC-SHA256-PAYLOAD" - // http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD" indicates that the + // http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD" or "STREAMING-UNSIGNED-PAYLOAD-TRAILER" indicates that the // client did not calculate sha256 of the payload. - unsignedPayload = "UNSIGNED-PAYLOAD" + unsignedPayload = "UNSIGNED-PAYLOAD" + streamingUnsignedPayload = "STREAMING-UNSIGNED-PAYLOAD-TRAILER" ) // Returns SHA256 for calculating canonical-request. diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index bb67c35c2..86fbbd19e 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -8,8 +8,6 @@ import ( "encoding/hex" "errors" "fmt" - "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "io" "net/http" "net/url" @@ -21,7 +19,11 @@ import ( "time" "unicode/utf8" + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/stretchr/testify/assert" ) // TestIsRequestPresignedSignatureV4 - Test validates the logic for presign signature version v4 detection. @@ -288,6 +290,73 @@ var ignoredHeaders = map[string]bool{ "User-Agent": true, } +// Tests the test helper with an example from the AWS Doc. +// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html +// This time it's a PUT request uploading the file with content "Welcome to Amazon S3." +func TestGetStringToSignPUT(t *testing.T) { + + canonicalRequest := `PUT +/test%24file.text + +date:Fri, 24 May 2013 00:00:00 GMT +host:examplebucket.s3.amazonaws.com +x-amz-content-sha256:44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072 +x-amz-date:20130524T000000Z +x-amz-storage-class:REDUCED_REDUNDANCY + +date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class +44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072` + + date, err := time.Parse(iso8601Format, "20130524T000000Z") + + if err != nil { + t.Fatalf("Error parsing date: %v", err) + } + + scope := "20130524/us-east-1/s3/aws4_request" + stringToSign := getStringToSign(canonicalRequest, date, scope) + + expected := `AWS4-HMAC-SHA256 +20130524T000000Z +20130524/us-east-1/s3/aws4_request +9e0e90d9c76de8fa5b200d8c849cd5b8dc7a3be3951ddb7f6a76b4158342019d` + + assert.Equal(t, expected, stringToSign) +} + +// Tests the test helper with an example from the AWS Doc. +// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html +// The GET request example with empty string hash. +func TestGetStringToSignGETEmptyStringHash(t *testing.T) { + + canonicalRequest := `GET +/test.txt + +host:examplebucket.s3.amazonaws.com +range:bytes=0-9 +x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +x-amz-date:20130524T000000Z + +host;range;x-amz-content-sha256;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855` + + date, err := time.Parse(iso8601Format, "20130524T000000Z") + + if err != nil { + t.Fatalf("Error parsing date: %v", err) + } + + scope := "20130524/us-east-1/s3/aws4_request" + stringToSign := getStringToSign(canonicalRequest, date, scope) + + expected := `AWS4-HMAC-SHA256 +20130524T000000Z +20130524/us-east-1/s3/aws4_request +7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972` + + assert.Equal(t, expected, stringToSign) +} + // Sign given request using Signature V4. func signRequestV4(req *http.Request, accessKey, secretKey string) error { // Get hashed payload. diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go index 4bf74d025..a646e8875 100644 --- a/weed/s3api/chunked_reader_v4.go +++ b/weed/s3api/chunked_reader_v4.go @@ -29,6 +29,7 @@ import ( "net/http" "time" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" @@ -54,14 +55,21 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr return nil, "", "", time.Time{}, errCode } - // Payload streaming. - payload := streamingContentSHA256 + contentSha256Header := req.Header.Get("X-Amz-Content-Sha256") + switch contentSha256Header { // Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' - if payload != req.Header.Get("X-Amz-Content-Sha256") { + case streamingContentSHA256: + glog.V(3).Infof("streaming content sha256") + case streamingUnsignedPayload: + glog.V(3).Infof("streaming unsigned payload") + default: return nil, "", "", time.Time{}, s3err.ErrContentSHA256Mismatch } + // Payload streaming. + payload := contentSha256Header + // Extract all the signed headers along with its values. extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r) if errCode != s3err.ErrNone { diff --git a/weed/s3api/chunked_reader_v4_test.go b/weed/s3api/chunked_reader_v4_test.go new file mode 100644 index 000000000..16d4a3db3 --- /dev/null +++ b/weed/s3api/chunked_reader_v4_test.go @@ -0,0 +1,107 @@ +package s3api + +import ( + "bytes" + "io" + "net/http" + "strings" + "sync" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/stretchr/testify/assert" +) + +// This test will implement the following scenario: +// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming + +const ( + defaultTimestamp = "20130524T000000Z" + defaultBucketName = "examplebucket" + defaultAccessKeyId = "AKIAIOSFODNN7EXAMPLE" + defaultSecretAccessKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + defaultRegion = "us-east-1" +) + +func generatePayload() string { + chunk1 := "10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n" + + strings.Repeat("a", 65536) + "\r\n" + chunk2 := "400;chunk-signature=0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497\r\n" + + strings.Repeat("a", 1024) + "\r\n" + chunk3 := "0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n\r\n" + + payload := chunk1 + chunk2 + chunk3 + return payload +} + +func NewRequest() (*http.Request, error) { + payload := generatePayload() + req, err := http.NewRequest("PUT", "http://s3.amazonaws.com/examplebucket/chunkObject.txt", bytes.NewReader([]byte(payload))) + if err != nil { + return nil, err + } + + req.Header.Set("Host", "s3.amazonaws.com") + req.Header.Set("x-amz-date", defaultTimestamp) + req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9") + req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD") + req.Header.Set("Content-Encoding", "aws-chunked") + req.Header.Set("x-amz-decoded-content-length", "66560") + req.Header.Set("Content-Length", "66824") + + return req, nil +} + +func TestNewSignV4ChunkedReader(t *testing.T) { + req, err := NewRequest() + if err != nil { + t.Fatalf("Failed to create request: %v", err) + } + + // Create an IdentityAccessManagement instance + iam := IdentityAccessManagement{ + identities: []*Identity{}, + accessKeyIdent: map[string]*Identity{}, + accounts: map[string]*Account{}, + emailAccount: map[string]*Account{}, + hashes: map[string]*sync.Pool{}, + hashCounters: map[string]*int32{}, + identityAnonymous: nil, + domain: "", + isAuthEnabled: false, + } + + // Add default access keys and secrets + iam.identities = append(iam.identities, &Identity{ + Name: "default", + Credentials: []*Credential{ + { + AccessKey: defaultAccessKeyId, + SecretKey: defaultSecretAccessKey, + }, + }, + Actions: []Action{ + "Read", + "Write", + "List", + }, + }) + + iam.accessKeyIdent[defaultAccessKeyId] = iam.identities[0] + + // Call newSignV4ChunkedReader + reader, errCode := iam.newSignV4ChunkedReader(req) + assert.NotNil(t, reader) + assert.Equal(t, s3err.ErrNone, errCode) + + data, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("Failed to read data: %v", err) + } + + // The expected payload a long string of 'a's + expectedPayload := strings.Repeat("a", 66560) + assert.Equal(t, expectedPayload, string(data)) + +} diff --git a/weed/s3api/s3api_auth.go b/weed/s3api/s3api_auth.go index bf5cf5fab..8424e7d2c 100644 --- a/weed/s3api/s3api_auth.go +++ b/weed/s3api/s3api_auth.go @@ -53,6 +53,11 @@ func isRequestSignStreamingV4(r *http.Request) bool { r.Method == http.MethodPut } +func isRequestUnsignedStreaming(r *http.Request) bool { + return r.Header.Get("x-amz-content-sha256") == streamingUnsignedPayload && + r.Method == http.MethodPut +} + // Authorization type. type authType int @@ -64,6 +69,7 @@ const ( authTypePresignedV2 authTypePostPolicy authTypeStreamingSigned + authTypeStreamingUnsigned authTypeSigned authTypeSignedV2 authTypeJWT @@ -77,6 +83,8 @@ func getRequestAuthType(r *http.Request) authType { return authTypePresignedV2 } else if isRequestSignStreamingV4(r) { return authTypeStreamingSigned + } else if isRequestUnsignedStreaming(r) { + return authTypeStreamingUnsigned } else if isRequestSignatureV4(r) { return authTypeSigned } else if isRequestPresignedSignatureV4(r) { diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index cc21725ee..9a0f01f8a 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -52,7 +52,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) if s3a.iam.isEnabled() { var s3ErrCode s3err.ErrorCode switch rAuthType { - case authTypeStreamingSigned: + case authTypeStreamingSigned, authTypeStreamingUnsigned: dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r) case authTypeSignedV2, authTypePresignedV2: _, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)