add s3 signature tests and prepare implementation of STREAMING-UNSIGNED-PAYLOAD-TRAILER (#6525)

* add tests for s3 signature

* add test for newSignV4ChunkedReader.Read()

* add glog import

weed/s3api/auth_credentials.go

@@ -364,6 +364,9 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
 		glog.V(3).Infof("post policy auth type")
 		r.Header.Set(s3_constants.AmzAuthType, "PostPolicy")
 		return identity, s3err.ErrNone
+	case authTypeStreamingUnsigned:
+		glog.V(3).Infof("unsigned streaming upload")
+		return identity, s3err.ErrNone
 	case authTypeJWT:
 		glog.V(3).Infof("jwt auth type")
 		r.Header.Set(s3_constants.AmzAuthType, "Jwt")
@@ -412,6 +415,10 @@ func (iam *IdentityAccessManagement) authUser(r *http.Request) (*Identity, s3err
 	var authType string
 	switch getRequestAuthType(r) {
 	case authTypeStreamingSigned:
+		glog.V(3).Infof("signed streaming upload")
+		return identity, s3err.ErrNone
+	case authTypeStreamingUnsigned:
+		glog.V(3).Infof("unsigned streaming upload")
 		return identity, s3err.ErrNone
 	case authTypeUnknown:
 		glog.V(3).Infof("unknown auth type")

weed/s3api/auth_signature_v4.go

@@ -56,9 +56,10 @@ const (
 	streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
 	signV4ChunkedAlgorithm = "AWS4-HMAC-SHA256-PAYLOAD"
 
-	// http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD" indicates that the
+	// http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD" or "STREAMING-UNSIGNED-PAYLOAD-TRAILER" indicates that the
 	// client did not calculate sha256 of the payload.
-	unsignedPayload = "UNSIGNED-PAYLOAD"
+	unsignedPayload          = "UNSIGNED-PAYLOAD"
+	streamingUnsignedPayload = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
 )
 
 // Returns SHA256 for calculating canonical-request.

weed/s3api/auto_signature_v4_test.go

@@ -8,8 +8,6 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
-	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"io"
 	"net/http"
 	"net/url"
@@ -21,7 +19,11 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"github.com/stretchr/testify/assert"
 )
 
 // TestIsRequestPresignedSignatureV4 - Test validates the logic for presign signature version v4 detection.
@@ -288,6 +290,73 @@ var ignoredHeaders = map[string]bool{
 	"User-Agent":     true,
 }
 
+// Tests the test helper with an example from the AWS Doc.
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+// This time it's a PUT request uploading the file with content "Welcome to Amazon S3."
+func TestGetStringToSignPUT(t *testing.T) {
+
+	canonicalRequest := `PUT
+/test%24file.text
+
+date:Fri, 24 May 2013 00:00:00 GMT
+host:examplebucket.s3.amazonaws.com
+x-amz-content-sha256:44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072
+x-amz-date:20130524T000000Z
+x-amz-storage-class:REDUCED_REDUNDANCY
+
+date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class
+44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072`
+
+	date, err := time.Parse(iso8601Format, "20130524T000000Z")
+
+	if err != nil {
+		t.Fatalf("Error parsing date: %v", err)
+	}
+
+	scope := "20130524/us-east-1/s3/aws4_request"
+	stringToSign := getStringToSign(canonicalRequest, date, scope)
+
+	expected := `AWS4-HMAC-SHA256
+20130524T000000Z
+20130524/us-east-1/s3/aws4_request
+9e0e90d9c76de8fa5b200d8c849cd5b8dc7a3be3951ddb7f6a76b4158342019d`
+
+	assert.Equal(t, expected, stringToSign)
+}
+
+// Tests the test helper with an example from the AWS Doc.
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+// The GET request example with empty string hash.
+func TestGetStringToSignGETEmptyStringHash(t *testing.T) {
+
+	canonicalRequest := `GET
+/test.txt
+
+host:examplebucket.s3.amazonaws.com
+range:bytes=0-9
+x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+x-amz-date:20130524T000000Z
+
+host;range;x-amz-content-sha256;x-amz-date
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
+
+	date, err := time.Parse(iso8601Format, "20130524T000000Z")
+
+	if err != nil {
+		t.Fatalf("Error parsing date: %v", err)
+	}
+
+	scope := "20130524/us-east-1/s3/aws4_request"
+	stringToSign := getStringToSign(canonicalRequest, date, scope)
+
+	expected := `AWS4-HMAC-SHA256
+20130524T000000Z
+20130524/us-east-1/s3/aws4_request
+7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972`
+
+	assert.Equal(t, expected, stringToSign)
+}
+
 // Sign given request using Signature V4.
 func signRequestV4(req *http.Request, accessKey, secretKey string) error {
 	// Get hashed payload.

weed/s3api/chunked_reader_v4.go

@@ -29,6 +29,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 
@@ -54,14 +55,21 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
 		return nil, "", "", time.Time{}, errCode
 	}
 
-	// Payload streaming.
-	payload := streamingContentSHA256
+	contentSha256Header := req.Header.Get("X-Amz-Content-Sha256")
 
+	switch contentSha256Header {
 	// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
-	if payload != req.Header.Get("X-Amz-Content-Sha256") {
+	case streamingContentSHA256:
+		glog.V(3).Infof("streaming content sha256")
+	case streamingUnsignedPayload:
+		glog.V(3).Infof("streaming unsigned payload")
+	default:
 		return nil, "", "", time.Time{}, s3err.ErrContentSHA256Mismatch
 	}
 
+	// Payload streaming.
+	payload := contentSha256Header
+
 	// Extract all the signed headers along with its values.
 	extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r)
 	if errCode != s3err.ErrNone {

weed/s3api/chunked_reader_v4_test.go

@@ -0,0 +1,107 @@
+package s3api
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"github.com/stretchr/testify/assert"
+)
+
+// This test will implement the following scenario:
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming
+
+const (
+	defaultTimestamp       = "20130524T000000Z"
+	defaultBucketName      = "examplebucket"
+	defaultAccessKeyId     = "AKIAIOSFODNN7EXAMPLE"
+	defaultSecretAccessKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+	defaultRegion          = "us-east-1"
+)
+
+func generatePayload() string {
+	chunk1 := "10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648\r\n" +
+		strings.Repeat("a", 65536) + "\r\n"
+	chunk2 := "400;chunk-signature=0055627c9e194cb4542bae2aa5492e3c1575bbb81b612b7d234b86a503ef5497\r\n" +
+		strings.Repeat("a", 1024) + "\r\n"
+	chunk3 := "0;chunk-signature=b6c6ea8a5354eaf15b3cb7646744f4275b71ea724fed81ceb9323e279d449df9\r\n\r\n"
+
+	payload := chunk1 + chunk2 + chunk3
+	return payload
+}
+
+func NewRequest() (*http.Request, error) {
+	payload := generatePayload()
+	req, err := http.NewRequest("PUT", "http://s3.amazonaws.com/examplebucket/chunkObject.txt", bytes.NewReader([]byte(payload)))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Host", "s3.amazonaws.com")
+	req.Header.Set("x-amz-date", defaultTimestamp)
+	req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY")
+	req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;content-length;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9")
+	req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
+	req.Header.Set("Content-Encoding", "aws-chunked")
+	req.Header.Set("x-amz-decoded-content-length", "66560")
+	req.Header.Set("Content-Length", "66824")
+
+	return req, nil
+}
+
+func TestNewSignV4ChunkedReader(t *testing.T) {
+	req, err := NewRequest()
+	if err != nil {
+		t.Fatalf("Failed to create request: %v", err)
+	}
+
+	// Create an IdentityAccessManagement instance
+	iam := IdentityAccessManagement{
+		identities:        []*Identity{},
+		accessKeyIdent:    map[string]*Identity{},
+		accounts:          map[string]*Account{},
+		emailAccount:      map[string]*Account{},
+		hashes:            map[string]*sync.Pool{},
+		hashCounters:      map[string]*int32{},
+		identityAnonymous: nil,
+		domain:            "",
+		isAuthEnabled:     false,
+	}
+
+	// Add default access keys and secrets
+	iam.identities = append(iam.identities, &Identity{
+		Name: "default",
+		Credentials: []*Credential{
+			{
+				AccessKey: defaultAccessKeyId,
+				SecretKey: defaultSecretAccessKey,
+			},
+		},
+		Actions: []Action{
+			"Read",
+			"Write",
+			"List",
+		},
+	})
+
+	iam.accessKeyIdent[defaultAccessKeyId] = iam.identities[0]
+
+	// Call newSignV4ChunkedReader
+	reader, errCode := iam.newSignV4ChunkedReader(req)
+	assert.NotNil(t, reader)
+	assert.Equal(t, s3err.ErrNone, errCode)
+
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		t.Fatalf("Failed to read data: %v", err)
+	}
+
+	// The expected payload a long string of 'a's
+	expectedPayload := strings.Repeat("a", 66560)
+	assert.Equal(t, expectedPayload, string(data))
+
+}

weed/s3api/s3api_auth.go

@@ -53,6 +53,11 @@ func isRequestSignStreamingV4(r *http.Request) bool {
 		r.Method == http.MethodPut
 }
 
+func isRequestUnsignedStreaming(r *http.Request) bool {
+	return r.Header.Get("x-amz-content-sha256") == streamingUnsignedPayload &&
+		r.Method == http.MethodPut
+}
+
 // Authorization type.
 type authType int
 
@@ -64,6 +69,7 @@ const (
 	authTypePresignedV2
 	authTypePostPolicy
 	authTypeStreamingSigned
+	authTypeStreamingUnsigned
 	authTypeSigned
 	authTypeSignedV2
 	authTypeJWT
@@ -77,6 +83,8 @@ func getRequestAuthType(r *http.Request) authType {
 		return authTypePresignedV2
 	} else if isRequestSignStreamingV4(r) {
 		return authTypeStreamingSigned
+	} else if isRequestUnsignedStreaming(r) {
+		return authTypeStreamingUnsigned
 	} else if isRequestSignatureV4(r) {
 		return authTypeSigned
 	} else if isRequestPresignedSignatureV4(r) {

weed/s3api/s3api_object_handlers_put.go

@@ -52,7 +52,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 	if s3a.iam.isEnabled() {
 		var s3ErrCode s3err.ErrorCode
 		switch rAuthType {
-		case authTypeStreamingSigned:
+		case authTypeStreamingSigned, authTypeStreamingUnsigned:
 			dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r)
 		case authTypeSignedV2, authTypePresignedV2:
 			_, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)