diff --git a/weed/s3api/auth_signature_v2.go b/weed/s3api/auth_signature_v2.go index 3fdc321b1..4cdc07df0 100644 --- a/weed/s3api/auth_signature_v2.go +++ b/weed/s3api/auth_signature_v2.go @@ -22,16 +22,14 @@ import ( "crypto/sha1" "crypto/subtle" "encoding/base64" - "fmt" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" - "net" "net/http" - "net/url" - "path" "sort" "strconv" "strings" "time" + + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" ) // Whitelist resource list that will be used in query string for signature-V2 calculation. @@ -61,7 +59,7 @@ var resourceList = []string{ "website", } -// Verify if request has valid AWS Signature Version '2'. +// Verify if request has AWS Signature Version '2'. func (iam *IdentityAccessManagement) isReqAuthenticatedV2(r *http.Request) (*Identity, s3err.ErrorCode) { if isRequestSignatureV2(r) { return iam.doesSignV2Match(r) @@ -70,277 +68,186 @@ func (iam *IdentityAccessManagement) isReqAuthenticatedV2(r *http.Request) (*Ide } func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(formValues http.Header) s3err.ErrorCode { + accessKey := formValues.Get("AWSAccessKeyId") - _, cred, found := iam.lookupByAccessKey(accessKey) + if accessKey == "" { + return s3err.ErrMissingFields + } + + identity, cred, found := iam.lookupByAccessKey(accessKey) if !found { return s3err.ErrInvalidAccessKeyID } - policy := formValues.Get("Policy") - signature := formValues.Get("Signature") - if !compareSignatureV2(signature, calculateSignatureV2(policy, cred.SecretKey)) { - return s3err.ErrSignatureDoesNotMatch - } - return s3err.ErrNone -} - -// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature; -// Signature = Base64( HMAC-SHA1( YourSecretKey, UTF-8-Encoding-Of( StringToSign ) ) ); -// -// StringToSign = HTTP-Verb + "\n" + -// Content-Md5 + "\n" + -// Content-Type + "\n" + -// Date + "\n" + -// CanonicalizedProtocolHeaders + -// CanonicalizedResource; -// -// CanonicalizedResource = [ "/" + Bucket ] + -// + -// [ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; -// -// CanonicalizedProtocolHeaders = -// doesSignV2Match - Verify authorization header with calculated header in accordance with -// - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html -// returns true if matches, false otherwise. if error is not nil then it is always false - -func validateV2AuthHeader(v2Auth string) (accessKey string, errCode s3err.ErrorCode) { - if v2Auth == "" { - return "", s3err.ErrAuthHeaderEmpty - } - // Verify if the header algorithm is supported or not. - if !strings.HasPrefix(v2Auth, signV2Algorithm) { - return "", s3err.ErrSignatureVersionNotSupported + bucket := formValues.Get("bucket") + if !identity.canDo(s3_constants.ACTION_WRITE, bucket, "") { + return s3err.ErrAccessDenied } - // below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string). - // Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature - authFields := strings.Split(v2Auth, " ") - if len(authFields) != 2 { - return "", s3err.ErrMissingFields + policy := formValues.Get("Policy") + if policy == "" { + return s3err.ErrMissingFields } - // Then will be splitting on ":", this will separate `AWSAccessKeyId` and `Signature` string. - keySignFields := strings.Split(strings.TrimSpace(authFields[1]), ":") - if len(keySignFields) != 2 { - return "", s3err.ErrMissingFields + signature := formValues.Get("Signature") + if signature == "" { + return s3err.ErrMissingFields } - return keySignFields[0], s3err.ErrNone + if !compareSignatureV2(signature, calculateSignatureV2(policy, cred.SecretKey)) { + return s3err.ErrSignatureDoesNotMatch + } + return s3err.ErrNone } +// doesSignV2Match - Verify authorization header with calculated header in accordance with +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html +// +// returns ErrNone if the signature matches. func (iam *IdentityAccessManagement) doesSignV2Match(r *http.Request) (*Identity, s3err.ErrorCode) { v2Auth := r.Header.Get("Authorization") - - accessKey, apiError := validateV2AuthHeader(v2Auth) - if apiError != s3err.ErrNone { - return nil, apiError + accessKey, errCode := validateV2AuthHeader(v2Auth) + if errCode != s3err.ErrNone { + return nil, errCode } - // Access credentials. - // Validate if access key id same. - ident, cred, found := iam.lookupByAccessKey(accessKey) + identity, cred, found := iam.lookupByAccessKey(accessKey) if !found { return nil, s3err.ErrInvalidAccessKeyID } - // r.RequestURI will have raw encoded URI as sent by the client. - tokens := strings.SplitN(r.RequestURI, "?", 2) - encodedResource := tokens[0] - encodedQuery := "" - if len(tokens) == 2 { - encodedQuery = tokens[1] + bucket, object := s3_constants.GetBucketAndObject(r) + if !identity.canDo(s3_constants.ACTION_WRITE, bucket, object) { + return nil, s3err.ErrAccessDenied } - unescapedQueries, err := unescapeQueries(encodedQuery) - if err != nil { - return nil, s3err.ErrInvalidQueryParams - } - - encodedResource, err = getResource(encodedResource, r.Host, iam.domain) - if err != nil { - return nil, s3err.ErrInvalidRequest - } - - prefix := fmt.Sprintf("%s %s:", signV2Algorithm, cred.AccessKey) - if !strings.HasPrefix(v2Auth, prefix) { - return nil, s3err.ErrSignatureDoesNotMatch - } - v2Auth = v2Auth[len(prefix):] - expectedAuth := signatureV2(cred, r.Method, encodedResource, strings.Join(unescapedQueries, "&"), r.Header) + expectedAuth := signatureV2(cred, r.Method, r.URL.Path, r.URL.Query().Encode(), r.Header) if !compareSignatureV2(v2Auth, expectedAuth) { return nil, s3err.ErrSignatureDoesNotMatch } - return ident, s3err.ErrNone + return identity, s3err.ErrNone } -// doesPresignV2SignatureMatch - Verify query headers with presigned signature -// - http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationQueryStringAuth +// doesPresignV2SignatureMatch - Verify query headers with calculated header in accordance with +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html // -// returns ErrNone if matches. S3 errors otherwise. +// returns ErrNone if the signature matches. func (iam *IdentityAccessManagement) doesPresignV2SignatureMatch(r *http.Request) (*Identity, s3err.ErrorCode) { - - // r.RequestURI will have raw encoded URI as sent by the client. - tokens := strings.SplitN(r.RequestURI, "?", 2) - encodedResource := tokens[0] - encodedQuery := "" - if len(tokens) == 2 { - encodedQuery = tokens[1] + query := r.URL.Query() + expires := query.Get("Expires") + if expires == "" { + return nil, s3err.ErrMissingFields } - var ( - filteredQueries []string - gotSignature string - expires string - accessKey string - err error - ) - - var unescapedQueries []string - unescapedQueries, err = unescapeQueries(encodedQuery) + expireTimestamp, err := strconv.ParseInt(expires, 10, 64) if err != nil { - return nil, s3err.ErrInvalidQueryParams - } - - // Extract the necessary values from presigned query, construct a list of new filtered queries. - for _, query := range unescapedQueries { - keyval := strings.SplitN(query, "=", 2) - if len(keyval) != 2 { - return nil, s3err.ErrInvalidQueryParams - } - switch keyval[0] { - case "AWSAccessKeyId": - accessKey = keyval[1] - case "Signature": - gotSignature = keyval[1] - case "Expires": - expires = keyval[1] - default: - filteredQueries = append(filteredQueries, query) - } + return nil, s3err.ErrMalformedExpires } - // Invalid values returns error. - if accessKey == "" || gotSignature == "" || expires == "" { - return nil, s3err.ErrInvalidQueryParams + if time.Unix(expireTimestamp, 0).Before(time.Now().UTC()) { + return nil, s3err.ErrExpiredPresignRequest } - // Validate if access key id same. - ident, cred, found := iam.lookupByAccessKey(accessKey) - if !found { + accessKey := query.Get("AWSAccessKeyId") + if accessKey == "" { return nil, s3err.ErrInvalidAccessKeyID } - // Make sure the request has not expired. - expiresInt, err := strconv.ParseInt(expires, 10, 64) - if err != nil { - return nil, s3err.ErrMalformedExpires + signature := query.Get("Signature") + if signature == "" { + return nil, s3err.ErrMissingFields } - // Check if the presigned URL has expired. - if expiresInt < time.Now().UTC().Unix() { - return nil, s3err.ErrExpiredPresignRequest + identity, cred, found := iam.lookupByAccessKey(accessKey) + if !found { + return nil, s3err.ErrInvalidAccessKeyID } - encodedResource, err = getResource(encodedResource, r.Host, iam.domain) - if err != nil { - return nil, s3err.ErrInvalidRequest + bucket, object := s3_constants.GetBucketAndObject(r) + if !identity.canDo(s3_constants.ACTION_READ, bucket, object) { + return nil, s3err.ErrAccessDenied } - expectedSignature := preSignatureV2(cred, r.Method, encodedResource, strings.Join(filteredQueries, "&"), r.Header, expires) - if !compareSignatureV2(gotSignature, expectedSignature) { + expectedSignature := preSignatureV2(cred, r.Method, r.URL.Path, r.URL.Query().Encode(), r.Header, expires) + if !compareSignatureV2(signature, expectedSignature) { return nil, s3err.ErrSignatureDoesNotMatch } - - return ident, s3err.ErrNone + return identity, s3err.ErrNone } -// Escape encodedQuery string into unescaped list of query params, returns error -// if any while unescaping the values. -func unescapeQueries(encodedQuery string) (unescapedQueries []string, err error) { - for _, query := range strings.Split(encodedQuery, "&") { - var unescapedQuery string - unescapedQuery, err = url.QueryUnescape(query) - if err != nil { - return nil, err - } - unescapedQueries = append(unescapedQueries, unescapedQuery) +// validateV2AuthHeader validates AWS Signature Version '2' authentication header. +func validateV2AuthHeader(v2Auth string) (accessKey string, errCode s3err.ErrorCode) { + if v2Auth == "" { + return "", s3err.ErrAuthHeaderEmpty } - return unescapedQueries, nil -} -// Returns "/bucketName/objectName" for path-style or virtual-host-style requests. -func getResource(path string, host string, domain string) (string, error) { - if domain == "" { - return path, nil - } - // If virtual-host-style is enabled construct the "resource" properly. - if strings.Contains(host, ":") { - // In bucket.mydomain.com:9000, strip out :9000 - var err error - if host, _, err = net.SplitHostPort(host); err != nil { - return "", err - } + // Signature V2 authorization header format: + // Authorization: AWS AKIAIOSFODNN7EXAMPLE:frJIUN8DYpKDtOLCwo//yllqDzg= + if !strings.HasPrefix(v2Auth, signV2Algorithm) { + return "", s3err.ErrSignatureVersionNotSupported } - if !strings.HasSuffix(host, "."+domain) { - return path, nil + + // Strip off the Algorithm prefix. + v2Auth = v2Auth[len(signV2Algorithm):] + authFields := strings.Split(v2Auth, ":") + if len(authFields) != 2 { + return "", s3err.ErrMissingFields } - bucket := strings.TrimSuffix(host, "."+domain) - return "/" + pathJoin(bucket, path), nil -} -// pathJoin - like path.Join() but retains trailing "/" of the last element -func pathJoin(elem ...string) string { - trailingSlash := "" - if len(elem) > 0 { - if strings.HasSuffix(elem[len(elem)-1], "/") { - trailingSlash = "/" - } + // The first field is Access Key ID. + if authFields[0] == "" { + return "", s3err.ErrInvalidAccessKeyID + } + + // The second field is signature. + if authFields[1] == "" { + return "", s3err.ErrMissingFields } - return path.Join(elem...) + trailingSlash + + return authFields[0], s3err.ErrNone } -// Return the signature v2 of a given request. +// signatureV2 - calculates signature version 2 for request. func signatureV2(cred *Credential, method string, encodedResource string, encodedQuery string, headers http.Header) string { stringToSign := getStringToSignV2(method, encodedResource, encodedQuery, headers, "") signature := calculateSignatureV2(stringToSign, cred.SecretKey) - return signature + return signV2Algorithm + cred.AccessKey + ":" + signature } -// Return string to sign under two different conditions. -// - if expires string is set then string to sign includes date instead of the Date header. -// - if expires string is empty then string to sign includes date header instead. +// getStringToSignV2 - string to sign in accordance with +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html func getStringToSignV2(method string, encodedResource, encodedQuery string, headers http.Header, expires string) string { canonicalHeaders := canonicalizedAmzHeadersV2(headers) if len(canonicalHeaders) > 0 { canonicalHeaders += "\n" } - date := expires // Date is set to expires date for presign operations. - if date == "" { - // If expires date is empty then request header Date is used. - date = headers.Get("Date") - } - // From the Amazon docs: // // StringToSign = HTTP-Verb + "\n" + - // Content-Md5 + "\n" + + // Content-MD5 + "\n" + // Content-Type + "\n" + - // Date/Expires + "\n" + - // CanonicalizedProtocolHeaders + + // Date + "\n" + + // CanonicalizedAmzHeaders + // CanonicalizedResource; - stringToSign := strings.Join([]string{ - method, - headers.Get("Content-MD5"), - headers.Get("Content-Type"), - date, - canonicalHeaders, - }, "\n") - - return stringToSign + canonicalizedResourceV2(encodedResource, encodedQuery) + stringToSign := method + "\n" + stringToSign += headers.Get("Content-Md5") + "\n" + stringToSign += headers.Get("Content-Type") + "\n" + + if expires != "" { + stringToSign += expires + "\n" + } else { + stringToSign += headers.Get("Date") + "\n" + if v := headers.Get("x-amz-date"); v != "" { + stringToSign = strings.Replace(stringToSign, headers.Get("Date")+"\n", "\n", -1) + } + } + stringToSign += canonicalHeaders + stringToSign += canonicalizedResourceV2(encodedResource, encodedQuery) + return stringToSign } -// Return canonical resource string. +// canonicalizedResourceV2 - canonicalize the resource string for signature V2. func canonicalizedResourceV2(encodedResource, encodedQuery string) string { queries := strings.Split(encodedQuery, "&") keyval := make(map[string]string) @@ -356,28 +263,26 @@ func canonicalizedResourceV2(encodedResource, encodedQuery string) string { } var canonicalQueries []string - for _, key := range resourceList { - val, ok := keyval[key] - if !ok { - continue - } - if val == "" { - canonicalQueries = append(canonicalQueries, key) - continue + for _, resource := range resourceList { + if val, ok := keyval[resource]; ok { + if val == "" { + canonicalQueries = append(canonicalQueries, resource) + continue + } + canonicalQueries = append(canonicalQueries, resource+"="+val) } - canonicalQueries = append(canonicalQueries, key+"="+val) } - // The queries will be already sorted as resourceList is sorted, if canonicalQueries - // is empty strings.Join returns empty. - canonicalQuery := strings.Join(canonicalQueries, "&") - if canonicalQuery != "" { - return encodedResource + "?" + canonicalQuery + // The queries will be already sorted as resourceList is sorted. + if len(canonicalQueries) == 0 { + return encodedResource } - return encodedResource + + // If queries are present then the canonicalized resource is set to encodedResource + "?" + strings.Join(canonicalQueries, "&") + return encodedResource + "?" + strings.Join(canonicalQueries, "&") } -// Return canonical headers. +// canonicalizedAmzHeadersV2 - canonicalize the x-amz-* headers for signature V2. func canonicalizedAmzHeadersV2(headers http.Header) string { var keys []string keyval := make(map[string]string) @@ -390,6 +295,7 @@ func canonicalizedAmzHeadersV2(headers http.Header) string { keyval[lkey] = strings.Join(headers[key], ",") } sort.Strings(keys) + var canonicalHeaders []string for _, key := range keys { canonicalHeaders = append(canonicalHeaders, key+":"+keyval[key]) @@ -397,6 +303,7 @@ func canonicalizedAmzHeadersV2(headers http.Header) string { return strings.Join(canonicalHeaders, "\n") } +// calculateSignatureV2 - calculates signature version 2. func calculateSignatureV2(stringToSign string, secret string) string { hm := hmac.New(sha1.New, []byte(secret)) hm.Write([]byte(stringToSign)) diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 0ddbaa917..8d9011f0d 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -23,21 +23,15 @@ import ( "crypto/sha256" "crypto/subtle" "encoding/hex" - "hash" - "io" - "net" "net/http" - "net/url" "regexp" "sort" "strconv" "strings" - "sync" - "sync/atomic" "time" "unicode/utf8" - "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" ) @@ -52,77 +46,99 @@ func (iam *IdentityAccessManagement) reqSignatureV4Verify(r *http.Request) (*Ide return nil, s3err.ErrAccessDenied } -// Streaming AWS Signature Version '4' constants. +// Constants specific to this file const ( - emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" - signV4ChunkedAlgorithm = "AWS4-HMAC-SHA256-PAYLOAD" - - // http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD" or "STREAMING-UNSIGNED-PAYLOAD-TRAILER" indicates that the - // client did not calculate sha256 of the payload. - unsignedPayload = "UNSIGNED-PAYLOAD" + emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" streamingUnsignedPayload = "STREAMING-UNSIGNED-PAYLOAD-TRAILER" + unsignedPayload = "UNSIGNED-PAYLOAD" ) -// AWS S3 authentication headers that should be skipped when signing the request -// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html -var awsS3AuthHeaders = map[string]struct{}{ - "x-amz-content-sha256": {}, - "x-amz-security-token": {}, - "x-amz-algorithm": {}, - "x-amz-date": {}, - "x-amz-expires": {}, - "x-amz-signedheaders": {}, - "x-amz-credential": {}, - "x-amz-signature": {}, -} - -// Returns SHA256 for calculating canonical-request. +// getContentSha256Cksum retrieves the "x-amz-content-sha256" header value. func getContentSha256Cksum(r *http.Request) string { - var ( - defaultSha256Cksum string - v []string - ok bool - ) + // If the client sends a SHA256 checksum of the object in this header, use it. + if v := r.Header.Get("X-Amz-Content-Sha256"); v != "" { + return v + } // For a presigned request we look at the query param for sha256. if isRequestPresignedSignatureV4(r) { - // X-Amz-Content-Sha256, if not set in presigned requests, checksum - // will default to 'UNSIGNED-PAYLOAD'. - defaultSha256Cksum = unsignedPayload - v, ok = r.URL.Query()["X-Amz-Content-Sha256"] - if !ok { - v, ok = r.Header["X-Amz-Content-Sha256"] - } - } else { - // X-Amz-Content-Sha256, if not set in signed requests, checksum - // will default to sha256([]byte("")). - defaultSha256Cksum = emptySHA256 - v, ok = r.Header["X-Amz-Content-Sha256"] + // X-Amz-Content-Sha256 header value is optional for presigned requests. + return unsignedPayload } - // We found 'X-Amz-Content-Sha256' return the captured value. - if ok { - return v[0] + // X-Amz-Content-Sha256 header value is required for all non-presigned requests. + return emptySHA256 +} + +// signValues data type represents structured form of AWS Signature V4 header. +type signValues struct { + Credential credentialHeader + SignedHeaders []string + Signature string +} + +// parseSignV4 parses the authorization header for signature v4. +func parseSignV4(v4Auth string) (sv signValues, aec s3err.ErrorCode) { + // Replace all spaced strings, some clients can send spaced + // parameters and some won't. So we pro-actively remove any spaces + // to make parsing easier. + v4Auth = strings.Replace(v4Auth, " ", "", -1) + if v4Auth == "" { + return sv, s3err.ErrAuthHeaderEmpty } - // We couldn't find 'X-Amz-Content-Sha256'. - return defaultSha256Cksum + // Verify if the header algorithm is supported or not. + if !strings.HasPrefix(v4Auth, signV4Algorithm) { + return sv, s3err.ErrSignatureVersionNotSupported + } + + // Strip off the Algorithm prefix. + v4Auth = strings.TrimPrefix(v4Auth, signV4Algorithm) + authFields := strings.Split(strings.TrimSpace(v4Auth), ",") + if len(authFields) != 3 { + return sv, s3err.ErrMissingFields + } + + // Initialize signature version '4' structured header. + signV4Values := signValues{} + + var err s3err.ErrorCode + // Save credential values. + signV4Values.Credential, err = parseCredentialHeader(authFields[0]) + if err != s3err.ErrNone { + return sv, err + } + + // Save signed headers. + signV4Values.SignedHeaders, err = parseSignedHeader(authFields[1]) + if err != s3err.ErrNone { + return sv, err + } + + // Save signature. + signV4Values.Signature, err = parseSignature(authFields[2]) + if err != s3err.ErrNone { + return sv, err + } + + // Return the structure here. + return signV4Values, s3err.ErrNone } -// Verify authorization header - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html +// Wrapper to verify if request came with a valid signature. func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r *http.Request) (*Identity, s3err.ErrorCode) { - // Copy request. + // Copy request req := *r // Save authorization header. v4Auth := req.Header.Get("Authorization") // Parse signature version '4' header. - signV4Values, err := parseSignV4(v4Auth) - if err != s3err.ErrNone { - return nil, err + signV4Values, errCode := parseSignV4(v4Auth) + if errCode != s3err.ErrNone { + return nil, errCode } // Extract all the signed headers along with its values. @@ -131,21 +147,26 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r return nil, errCode } - // Verify if the access key id matches. - identity, cred, found := iam.lookupByAccessKey(signV4Values.Credential.accessKey) + cred := signV4Values.Credential + identity, foundCred, found := iam.lookupByAccessKey(cred.accessKey) if !found { return nil, s3err.ErrInvalidAccessKeyID } + bucket, object := s3_constants.GetBucketAndObject(r) + if !identity.canDo(s3_constants.ACTION_WRITE, bucket, object) { + return nil, s3err.ErrAccessDenied + } + // Extract date, if not present throw error. - var date string - if date = req.Header.Get(http.CanonicalHeaderKey("X-Amz-Date")); date == "" { - if date = r.Header.Get("Date"); date == "" { + var dateStr string + if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" { + if dateStr = r.Header.Get("Date"); dateStr == "" { return nil, s3err.ErrMissingDateHeader } } // Parse date header. - t, e := time.Parse(iso8601Format, date) + t, e := time.Parse(iso8601Format, dateStr) if e != nil { return nil, s3err.ErrMalformedDate } @@ -153,62 +174,134 @@ func (iam *IdentityAccessManagement) doesSignatureMatch(hashedPayload string, r // Query string. queryStr := req.URL.Query().Encode() - // Get hashed Payload - if signV4Values.Credential.scope.service != "s3" && hashedPayload == emptySHA256 && r.Body != nil { - buf, _ := io.ReadAll(r.Body) - r.Body = io.NopCloser(bytes.NewBuffer(buf)) - b, _ := io.ReadAll(bytes.NewBuffer(buf)) - if len(b) != 0 { - bodyHash := sha256.Sum256(b) - hashedPayload = hex.EncodeToString(bodyHash[:]) - } + // Get canonical request. + canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method) + + // Get string to sign from canonical request. + stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope()) + + // Get hmac signing key. + signingKey := getSigningKey(foundCred.SecretKey, signV4Values.Credential.scope.date.Format(yyyymmdd), signV4Values.Credential.scope.region, "s3") + + // Calculate signature. + newSignature := getSignature(signingKey, stringToSign) + + // Verify if signature match. + if !compareSignatureV4(newSignature, signV4Values.Signature) { + return nil, s3err.ErrSignatureDoesNotMatch } - if forwardedPrefix := r.Header.Get("X-Forwarded-Prefix"); forwardedPrefix != "" { - // Handling usage of reverse proxy at prefix. - // Trying with prefix before main path. + // Return error none. + return identity, s3err.ErrNone +} - // Get canonical request. - glog.V(4).Infof("Forwarded Prefix: %s", forwardedPrefix) +// Simple implementation for presigned signature verification +func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload string, r *http.Request) (*Identity, s3err.ErrorCode) { + // Parse presigned signature values from query parameters + query := r.URL.Query() - canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, forwardedPrefix+req.URL.Path, req.Method) - errCode = iam.genAndCompareSignatureV4(canonicalRequest, cred.SecretKey, t, signV4Values) - if errCode == s3err.ErrNone { - return identity, errCode - } + // Check required parameters + algorithm := query.Get("X-Amz-Algorithm") + if algorithm != signV4Algorithm { + return nil, s3err.ErrSignatureVersionNotSupported } - // Get canonical request. - canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method) + credential := query.Get("X-Amz-Credential") + if credential == "" { + return nil, s3err.ErrMissingFields + } - errCode = iam.genAndCompareSignatureV4(canonicalRequest, cred.SecretKey, t, signV4Values) + signature := query.Get("X-Amz-Signature") + if signature == "" { + return nil, s3err.ErrMissingFields + } - if errCode == s3err.ErrNone { - return identity, errCode + signedHeadersStr := query.Get("X-Amz-SignedHeaders") + if signedHeadersStr == "" { + return nil, s3err.ErrMissingFields } - return nil, errCode -} -// Generate and compare signature for request. -func (iam *IdentityAccessManagement) genAndCompareSignatureV4(canonicalRequest, secretKey string, t time.Time, signV4Values signValues) s3err.ErrorCode { - // Get string to sign from canonical request. - stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope()) - glog.V(4).Infof("String to Sign:\n%s", stringToSign) - // Calculate signature. - newSignature := iam.getSignature( - secretKey, - signV4Values.Credential.scope.date, - signV4Values.Credential.scope.region, - signV4Values.Credential.scope.service, - stringToSign, - ) - glog.V(4).Infof("Signature:\n%s", newSignature) + dateStr := query.Get("X-Amz-Date") + if dateStr == "" { + return nil, s3err.ErrMissingDateHeader + } - // Verify if signature match. - if !compareSignatureV4(newSignature, signV4Values.Signature) { - return s3err.ErrSignatureDoesNotMatch + // Parse credential + credHeader, err := parseCredentialHeader("Credential=" + credential) + if err != s3err.ErrNone { + return nil, err } - return s3err.ErrNone + + // Look up identity by access key + identity, foundCred, found := iam.lookupByAccessKey(credHeader.accessKey) + if !found { + return nil, s3err.ErrInvalidAccessKeyID + } + + // Check permissions + bucket, object := s3_constants.GetBucketAndObject(r) + if !identity.canDo(s3_constants.ACTION_READ, bucket, object) { + return nil, s3err.ErrAccessDenied + } + + // Parse date + t, e := time.Parse(iso8601Format, dateStr) + if e != nil { + return nil, s3err.ErrMalformedDate + } + + // Check expiration + expiresStr := query.Get("X-Amz-Expires") + if expiresStr != "" { + expires, parseErr := strconv.ParseInt(expiresStr, 10, 64) + if parseErr != nil { + return nil, s3err.ErrMalformedDate + } + // Check if current time is after the expiration time + expirationTime := t.Add(time.Duration(expires) * time.Second) + if time.Now().UTC().After(expirationTime) { + return nil, s3err.ErrExpiredPresignRequest + } + } + + // Parse signed headers + signedHeaders := strings.Split(signedHeadersStr, ";") + + // Extract signed headers from request + extractedSignedHeaders := make(http.Header) + for _, header := range signedHeaders { + if header == "host" { + extractedSignedHeaders[header] = []string{r.Host} + continue + } + if values := r.Header[http.CanonicalHeaderKey(header)]; len(values) > 0 { + extractedSignedHeaders[http.CanonicalHeaderKey(header)] = values + } + } + + // Remove signature from query for canonical request calculation + queryForCanonical := r.URL.Query() + queryForCanonical.Del("X-Amz-Signature") + queryStr := strings.Replace(queryForCanonical.Encode(), "+", "%20", -1) + + // Get canonical request + canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, r.URL.Path, r.Method) + + // Get string to sign + stringToSign := getStringToSign(canonicalRequest, t, credHeader.getScope()) + + // Get signing key + signingKey := getSigningKey(foundCred.SecretKey, credHeader.scope.date.Format(yyyymmdd), credHeader.scope.region, "s3") + + // Calculate expected signature + expectedSignature := getSignature(signingKey, stringToSign) + + // Verify signature + if !compareSignatureV4(expectedSignature, signature) { + return nil, s3err.ErrSignatureDoesNotMatch + } + + return identity, s3err.ErrNone } // credentialHeader data type represents structured form of Credential @@ -223,14 +316,6 @@ type credentialHeader struct { } } -// signValues data type represents structured form of AWS Signature V4 header. -type signValues struct { - Credential credentialHeader - SignedHeaders []string - Signature string -} - -// Return scope string. func (c credentialHeader) getScope() string { return strings.Join([]string{ c.scope.date.Format(yyyymmdd), @@ -240,58 +325,9 @@ func (c credentialHeader) getScope() string { }, "/") } -// Authorization: algorithm Credential=accessKeyID/credScope, \ -// SignedHeaders=signedHeaders, Signature=signature -func parseSignV4(v4Auth string) (sv signValues, aec s3err.ErrorCode) { - // Replace all spaced strings, some clients can send spaced - // parameters and some won't. So we pro-actively remove any spaces - // to make parsing easier. - v4Auth = strings.Replace(v4Auth, " ", "", -1) - if v4Auth == "" { - return sv, s3err.ErrAuthHeaderEmpty - } - - // Verify if the header algorithm is supported or not. - if !strings.HasPrefix(v4Auth, signV4Algorithm) { - return sv, s3err.ErrSignatureVersionNotSupported - } - - // Strip off the Algorithm prefix. - v4Auth = strings.TrimPrefix(v4Auth, signV4Algorithm) - authFields := strings.Split(strings.TrimSpace(v4Auth), ",") - if len(authFields) != 3 { - return sv, s3err.ErrMissingFields - } - - // Initialize signature version '4' structured header. - signV4Values := signValues{} - - var err s3err.ErrorCode - // Save credential values. - signV4Values.Credential, err = parseCredentialHeader(authFields[0]) - if err != s3err.ErrNone { - return sv, err - } - - // Save signed headers. - signV4Values.SignedHeaders, err = parseSignedHeader(authFields[1]) - if err != s3err.ErrNone { - return sv, err - } - - // Save signature. - signV4Values.Signature, err = parseSignature(authFields[2]) - if err != s3err.ErrNone { - return sv, err - } - - // Return the structure here. - return signV4Values, s3err.ErrNone -} - // parse credentialHeader string into its structured form. func parseCredentialHeader(credElement string) (ch credentialHeader, aec s3err.ErrorCode) { - creds := strings.Split(strings.TrimSpace(credElement), "=") + creds := strings.SplitN(strings.TrimSpace(credElement), "=", 2) if len(creds) != 2 { return ch, s3err.ErrMissingFields } @@ -318,22 +354,6 @@ func parseCredentialHeader(credElement string) (ch credentialHeader, aec s3err.E return cred, s3err.ErrNone } -// Parse slice of signed headers from signed headers tag. -func parseSignedHeader(signedHdrElement string) ([]string, s3err.ErrorCode) { - signedHdrFields := strings.Split(strings.TrimSpace(signedHdrElement), "=") - if len(signedHdrFields) != 2 { - return nil, s3err.ErrMissingFields - } - if signedHdrFields[0] != "SignedHeaders" { - return nil, s3err.ErrMissingSignHeadersTag - } - if signedHdrFields[1] == "" { - return nil, s3err.ErrMissingFields - } - signedHeaders := strings.Split(signedHdrFields[1], ";") - return signedHeaders, s3err.ErrNone -} - // Parse signature from signature tag. func parseSignature(signElement string) (string, s3err.ErrorCode) { signFields := strings.Split(strings.TrimSpace(signElement), "=") @@ -350,437 +370,90 @@ func parseSignature(signElement string) (string, s3err.ErrorCode) { return signature, s3err.ErrNone } -// doesPolicySignatureV4Match - Verify query headers with post policy -// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html -// -// returns ErrNone if the signature matches. -func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.Header) s3err.ErrorCode { - - // Parse credential tag. - credHeader, err := parseCredentialHeader("Credential=" + formValues.Get("X-Amz-Credential")) - if err != s3err.ErrNone { - return s3err.ErrMissingFields +// Parse slice of signed headers from signed headers tag. +func parseSignedHeader(signedHdrElement string) ([]string, s3err.ErrorCode) { + signedHdrFields := strings.Split(strings.TrimSpace(signedHdrElement), "=") + if len(signedHdrFields) != 2 { + return nil, s3err.ErrMissingFields } - - _, cred, found := iam.lookupByAccessKey(credHeader.accessKey) - if !found { - return s3err.ErrInvalidAccessKeyID + if signedHdrFields[0] != "SignedHeaders" { + return nil, s3err.ErrMissingSignHeadersTag } - - // Get signature. - newSignature := iam.getSignature( - cred.SecretKey, - credHeader.scope.date, - credHeader.scope.region, - credHeader.scope.service, - formValues.Get("Policy"), - ) - - // Verify signature. - if !compareSignatureV4(newSignature, formValues.Get("X-Amz-Signature")) { - return s3err.ErrSignatureDoesNotMatch + if signedHdrFields[1] == "" { + return nil, s3err.ErrMissingFields } - - // Success. - return s3err.ErrNone + signedHeaders := strings.Split(signedHdrFields[1], ";") + return signedHeaders, s3err.ErrNone } -// check query headers with presigned signature -// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html -func (iam *IdentityAccessManagement) doesPresignedSignatureMatch(hashedPayload string, r *http.Request) (*Identity, s3err.ErrorCode) { - - // Copy request - req := *r +func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.Header) s3err.ErrorCode { - // Parse request query string. - pSignValues, err := parsePreSignV4(req.URL.Query()) + // Parse credential tag. + credHeader, err := parseCredentialHeader("Credential=" + formValues.Get("X-Amz-Credential")) if err != s3err.ErrNone { - return nil, err + return err } - // Verify if the access key id matches. - identity, cred, found := iam.lookupByAccessKey(pSignValues.Credential.accessKey) + identity, cred, found := iam.lookupByAccessKey(credHeader.accessKey) if !found { - return nil, s3err.ErrInvalidAccessKeyID - } - - // Extract all the signed headers along with its values. - extractedSignedHeaders, errCode := extractSignedHeaders(pSignValues.SignedHeaders, r) - if errCode != s3err.ErrNone { - return nil, errCode - } - // Construct new query. - query := make(url.Values) - if req.URL.Query().Get("X-Amz-Content-Sha256") != "" { - query.Set("X-Amz-Content-Sha256", hashedPayload) - } - - query.Set("X-Amz-Algorithm", signV4Algorithm) - - now := time.Now().UTC() - - // If the host which signed the request is slightly ahead in time (by less than globalMaxSkewTime) the - // request should still be allowed. - if pSignValues.Date.After(now.Add(15 * time.Minute)) { - return nil, s3err.ErrRequestNotReadyYet - } - - if now.Sub(pSignValues.Date) > pSignValues.Expires { - return nil, s3err.ErrExpiredPresignRequest - } - - // Save the date and expires. - t := pSignValues.Date - expireSeconds := int(pSignValues.Expires / time.Second) - - // Construct the query. - query.Set("X-Amz-Date", t.Format(iso8601Format)) - query.Set("X-Amz-Expires", strconv.Itoa(expireSeconds)) - query.Set("X-Amz-SignedHeaders", getSignedHeaders(extractedSignedHeaders)) - query.Set("X-Amz-Credential", cred.AccessKey+"/"+getScope(t, pSignValues.Credential.scope.region)) - - // Save other headers available in the request parameters. - for k, v := range req.URL.Query() { - // Skip AWS S3 authentication headers - if _, ok := awsS3AuthHeaders[strings.ToLower(k)]; ok { - continue - } - - query[k] = v + return s3err.ErrInvalidAccessKeyID } - // Get the encoded query. - encodedQuery := query.Encode() - - // Verify if date query is same. - if req.URL.Query().Get("X-Amz-Date") != query.Get("X-Amz-Date") { - return nil, s3err.ErrSignatureDoesNotMatch - } - // Verify if expires query is same. - if req.URL.Query().Get("X-Amz-Expires") != query.Get("X-Amz-Expires") { - return nil, s3err.ErrSignatureDoesNotMatch - } - // Verify if signed headers query is same. - if req.URL.Query().Get("X-Amz-SignedHeaders") != query.Get("X-Amz-SignedHeaders") { - return nil, s3err.ErrSignatureDoesNotMatch - } - // Verify if credential query is same. - if req.URL.Query().Get("X-Amz-Credential") != query.Get("X-Amz-Credential") { - return nil, s3err.ErrSignatureDoesNotMatch - } - // Verify if sha256 payload query is same. - if req.URL.Query().Get("X-Amz-Content-Sha256") != "" { - if req.URL.Query().Get("X-Amz-Content-Sha256") != query.Get("X-Amz-Content-Sha256") { - return nil, s3err.ErrContentSHA256Mismatch - } + bucket := formValues.Get("bucket") + if !identity.canDo(s3_constants.ACTION_WRITE, bucket, "") { + return s3err.ErrAccessDenied } - // / Verify finally if signature is same. - - // Get canonical request. - presignedCanonicalReq := getCanonicalRequest(extractedSignedHeaders, hashedPayload, encodedQuery, req.URL.Path, req.Method) - - // Get string to sign from canonical request. - presignedStringToSign := getStringToSign(presignedCanonicalReq, t, pSignValues.Credential.getScope()) + // Get signing key. + signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date.Format(yyyymmdd), credHeader.scope.region, "s3") - // Get new signature. - newSignature := iam.getSignature( - cred.SecretKey, - pSignValues.Credential.scope.date, - pSignValues.Credential.scope.region, - pSignValues.Credential.scope.service, - presignedStringToSign, - ) + // Get signature. + newSignature := getSignature(signingKey, formValues.Get("Policy")) // Verify signature. - if !compareSignatureV4(req.URL.Query().Get("X-Amz-Signature"), newSignature) { - return nil, s3err.ErrSignatureDoesNotMatch - } - return identity, s3err.ErrNone -} - -func (iam *IdentityAccessManagement) getSignature(secretKey string, t time.Time, region string, service string, stringToSign string) string { - pool := iam.getSignatureHashPool(secretKey, t, region, service) - h := pool.Get().(hash.Hash) - defer pool.Put(h) - - h.Reset() - h.Write([]byte(stringToSign)) - sig := hex.EncodeToString(h.Sum(nil)) - - return sig -} - -func (iam *IdentityAccessManagement) getSignatureHashPool(secretKey string, t time.Time, region string, service string) *sync.Pool { - // Build a caching key for the pool. - date := t.Format(yyyymmdd) - hashID := "AWS4" + secretKey + "/" + date + "/" + region + "/" + service + "/" + "aws4_request" - - // Try to find an existing pool and return it. - iam.hashMu.RLock() - pool, ok := iam.hashes[hashID] - iam.hashMu.RUnlock() - - if !ok { - iam.hashMu.Lock() - defer iam.hashMu.Unlock() - pool, ok = iam.hashes[hashID] - } - - if ok { - atomic.StoreInt32(iam.hashCounters[hashID], 1) - return pool - } - - // Create a pool that returns HMAC hashers for the requested parameters to avoid expensive re-initializing - // of new instances on every request. - iam.hashes[hashID] = &sync.Pool{ - New: func() any { - signingKey := getSigningKey(secretKey, date, region, service) - return hmac.New(sha256.New, signingKey) - }, - } - iam.hashCounters[hashID] = new(int32) - - // Clean up unused pools automatically after one hour of inactivity - ticker := time.NewTicker(time.Hour) - go func() { - for range ticker.C { - old := atomic.SwapInt32(iam.hashCounters[hashID], 0) - if old == 0 { - break - } - } - - ticker.Stop() - iam.hashMu.Lock() - delete(iam.hashes, hashID) - delete(iam.hashCounters, hashID) - iam.hashMu.Unlock() - }() - - return iam.hashes[hashID] -} - -func contains(list []string, elem string) bool { - for _, t := range list { - if t == elem { - return true - } - } - return false -} - -// preSignValues data type represents structured form of AWS Signature V4 query string. -type preSignValues struct { - signValues - Date time.Time - Expires time.Duration -} - -// Parses signature version '4' query string of the following form. -// -// querystring = X-Amz-Algorithm=algorithm -// querystring += &X-Amz-Credential= urlencode(accessKey + '/' + credential_scope) -// querystring += &X-Amz-Date=date -// querystring += &X-Amz-Expires=timeout interval -// querystring += &X-Amz-SignedHeaders=signed_headers -// querystring += &X-Amz-Signature=signature -// -// verifies if any of the necessary query params are missing in the presigned request. -func doesV4PresignParamsExist(query url.Values) s3err.ErrorCode { - v4PresignQueryParams := []string{"X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Signature", "X-Amz-Date", "X-Amz-SignedHeaders", "X-Amz-Expires"} - for _, v4PresignQueryParam := range v4PresignQueryParams { - if _, ok := query[v4PresignQueryParam]; !ok { - return s3err.ErrInvalidQueryParams - } + if !compareSignatureV4(newSignature, formValues.Get("X-Amz-Signature")) { + return s3err.ErrSignatureDoesNotMatch } return s3err.ErrNone } -// Parses all the presigned signature values into separate elements. -func parsePreSignV4(query url.Values) (psv preSignValues, aec s3err.ErrorCode) { - var err s3err.ErrorCode - // verify whether the required query params exist. - err = doesV4PresignParamsExist(query) - if err != s3err.ErrNone { - return psv, err - } - - // Verify if the query algorithm is supported or not. - if query.Get("X-Amz-Algorithm") != signV4Algorithm { - return psv, s3err.ErrInvalidQuerySignatureAlgo - } - - // Initialize signature version '4' structured header. - preSignV4Values := preSignValues{} - - // Save credential. - preSignV4Values.Credential, err = parseCredentialHeader("Credential=" + query.Get("X-Amz-Credential")) - if err != s3err.ErrNone { - return psv, err - } - - var e error - // Save date in native time.Time. - preSignV4Values.Date, e = time.Parse(iso8601Format, query.Get("X-Amz-Date")) - if e != nil { - return psv, s3err.ErrMalformedPresignedDate - } - - // Save expires in native time.Duration. - preSignV4Values.Expires, e = time.ParseDuration(query.Get("X-Amz-Expires") + "s") - if e != nil { - return psv, s3err.ErrMalformedExpires - } - - if preSignV4Values.Expires < 0 { - return psv, s3err.ErrNegativeExpires - } - - // Check if Expiry time is less than 7 days (value in seconds). - if preSignV4Values.Expires.Seconds() > 604800 { - return psv, s3err.ErrMaximumExpires - } - - // Save signed headers. - preSignV4Values.SignedHeaders, err = parseSignedHeader("SignedHeaders=" + query.Get("X-Amz-SignedHeaders")) - if err != s3err.ErrNone { - return psv, err - } - - // Save signature. - preSignV4Values.Signature, err = parseSignature("Signature=" + query.Get("X-Amz-Signature")) - if err != s3err.ErrNone { - return psv, err - } - - // Return structured form of signature query string. - return preSignV4Values, s3err.ErrNone -} - -// extractSignedHeaders extract signed headers from Authorization header +// Verify if extracted signed headers are not properly signed. func extractSignedHeaders(signedHeaders []string, r *http.Request) (http.Header, s3err.ErrorCode) { reqHeaders := r.Header - // find whether "host" is part of list of signed headers. - // if not return ErrUnsignedHeaders. "host" is mandatory. - if !contains(signedHeaders, "host") { - return nil, s3err.ErrUnsignedHeaders + // If no signed headers are provided, then return an error. + if len(signedHeaders) == 0 { + return nil, s3err.ErrMissingFields } extractedSignedHeaders := make(http.Header) for _, header := range signedHeaders { - // `host` will not be found in the headers, can be found in r.Host. - // but its alway necessary that the list of signed headers containing host in it. - val, ok := reqHeaders[http.CanonicalHeaderKey(header)] - if ok { - for _, enc := range val { - extractedSignedHeaders.Add(header, enc) - } + // `host` is not a case-sensitive header, unlike other headers such as `x-amz-date`. + if header == "host" { + // Get host value. + hostHeaderValue := extractHostHeader(r) + extractedSignedHeaders[header] = []string{hostHeaderValue} continue } - switch header { - case "expect": - // Set the default value of the Expect header for compatibility. - // - // In NGINX v1.1, the Expect header is removed when handling 100-continue requests. - // - // `aws-cli` sets this as part of signed headers. - // - // According to - // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.20 - // Expect header is always of form: - // - // Expect = "Expect" ":" 1#expectation - // expectation = "100-continue" | expectation-extension - // - // So it safe to assume that '100-continue' is what would - // be sent, for the time being keep this work around. - extractedSignedHeaders.Set(header, "100-continue") - case "host": - extractedHost := extractHostHeader(r) - extractedSignedHeaders.Set(header, extractedHost) - case "transfer-encoding": - for _, enc := range r.TransferEncoding { - extractedSignedHeaders.Add(header, enc) - } - case "content-length": - // Signature-V4 spec excludes Content-Length from signed headers list for signature calculation. - // But some clients deviate from this rule. Hence we consider Content-Length for signature - // calculation to be compatible with such clients. - extractedSignedHeaders.Set(header, strconv.FormatInt(r.ContentLength, 10)) - default: - return nil, s3err.ErrUnsignedHeaders + // For all other headers we need to find them in the HTTP headers and copy them over. + // We skip non-existent headers to be compatible with AWS signatures. + if values, ok := reqHeaders[http.CanonicalHeaderKey(header)]; ok { + extractedSignedHeaders[header] = values } } return extractedSignedHeaders, s3err.ErrNone } +// extractHostHeader returns the value of host header if available. func extractHostHeader(r *http.Request) string { - - forwardedHost := r.Header.Get("X-Forwarded-Host") - forwardedPort := r.Header.Get("X-Forwarded-Port") - forwardedProto := r.Header.Get("X-Forwarded-Proto") - - // If X-Forwarded-Host is set, use that as the host. - // If X-Forwarded-Port is set, use that too to form the host. - // If X-Forwarded-Proto is set, check if is it default to omit the port. - if forwardedHost != "" { - extractedHost := forwardedHost - host, port, err := net.SplitHostPort(extractedHost) - if err == nil { - extractedHost = host - if forwardedPort == "" { - forwardedPort = port - } - } - scheme := r.URL.Scheme - if forwardedProto != "" { - scheme = forwardedProto - } - if !isDefaultPort(scheme, forwardedPort) { - extractedHost = net.JoinHostPort(extractedHost, forwardedPort) - } - return extractedHost - } else { - // Go http server removes "host" from Request.Header - host := r.Host - if host == "" { - host = r.URL.Host - } - h, port, err := net.SplitHostPort(host) - if err != nil { - return host - } - if isDefaultPort(r.URL.Scheme, port) { - return h - } - return host + hostHeaderValue := r.Host + // For standard requests, this should be fine. + if r.Host != "" { + return hostHeaderValue } -} - -func isDefaultPort(scheme, port string) bool { - if port == "" { - return true - } - - switch port { - case "80": - return strings.EqualFold(scheme, "http") - case "443": - return strings.EqualFold(scheme, "https") - default: - return false - } -} - -// getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names -func getSignedHeaders(signedHeaders http.Header) string { - var headers []string - for k := range signedHeaders { - headers = append(headers, strings.ToLower(k)) + // If no host header is found, then check for host URL value. + if r.URL.Host != "" { + hostHeaderValue = r.URL.Host } - sort.Strings(headers) - return strings.Join(headers, ";") + return hostHeaderValue } // getScope generate a string of a specific date, an AWS region, and a service. @@ -815,8 +488,6 @@ func getCanonicalRequest(extractedSignedHeaders http.Header, payload, queryStr, getSignedHeaders(extractedSignedHeaders), payload, }, "\n") - - glog.V(4).Infof("Canonical Request:\n%s", canonicalRequest) return canonicalRequest } @@ -824,11 +495,16 @@ func getCanonicalRequest(extractedSignedHeaders http.Header, payload, queryStr, func getStringToSign(canonicalRequest string, t time.Time, scope string) string { stringToSign := signV4Algorithm + "\n" + t.Format(iso8601Format) + "\n" stringToSign = stringToSign + scope + "\n" - canonicalRequestBytes := sha256.Sum256([]byte(canonicalRequest)) - stringToSign = stringToSign + hex.EncodeToString(canonicalRequestBytes[:]) + stringToSign = stringToSign + getSHA256Hash([]byte(canonicalRequest)) return stringToSign } +// getSHA256Hash returns hex-encoded SHA256 hash of the input data. +func getSHA256Hash(data []byte) string { + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]) +} + // sumHMAC calculate hmac between two input byte array. func sumHMAC(key []byte, data []byte) []byte { hash := hmac.New(sha256.New, key) @@ -850,9 +526,11 @@ func getCanonicalHeaders(signedHeaders http.Header) string { var headers []string vals := make(http.Header) for k, vv := range signedHeaders { - headers = append(headers, strings.ToLower(k)) vals[strings.ToLower(k)] = vv } + for k := range vals { + headers = append(headers, k) + } sort.Strings(headers) var buf bytes.Buffer @@ -870,18 +548,28 @@ func getCanonicalHeaders(signedHeaders http.Header) string { return buf.String() } -// Trim leading and trailing spaces and replace sequential spaces with one space, following Trimall() -// in http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html +// signV4TrimAll trims leading and trailing spaces from each string in the slice, and trims sequential spaces. func signV4TrimAll(input string) string { // Compress adjacent spaces (a space is determined by - // unicode.IsSpace() internally here) to one space and return + // unicode.IsSpace() internally here) to a single space and trim + // leading and trailing spaces. return strings.Join(strings.Fields(input), " ") } +// getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names +func getSignedHeaders(signedHeaders http.Header) string { + var headers []string + for k := range signedHeaders { + headers = append(headers, strings.ToLower(k)) + } + sort.Strings(headers) + return strings.Join(headers, ";") +} + // if object matches reserved string, no need to encode them var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") -// EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences +// encodePath encodes the strings from UTF-8 byte representations to HTML hex escape sequences // // This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 // non english characters cannot be parsed due to the nature in which url.Encode() is written @@ -896,34 +584,38 @@ func encodePath(pathName string) string { for _, s := range pathName { if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) encodedPathname = encodedPathname + string(s) - continue - } - switch s { - case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) - encodedPathname = encodedPathname + string(s) - continue - default: - len := utf8.RuneLen(s) - if len < 0 { - // if utf8 cannot convert return the same string as is - return pathName - } - u := make([]byte, len) - utf8.EncodeRune(u, s) - for _, r := range u { - hex := hex.EncodeToString([]byte{r}) - encodedPathname = encodedPathname + "%" + strings.ToUpper(hex) + } else { + switch s { + case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) + encodedPathname = encodedPathname + string(s) + default: + runeLen := utf8.RuneLen(s) + if runeLen < 0 { + return pathName + } + u := make([]byte, runeLen) + utf8.EncodeRune(u, s) + for _, r := range u { + hex := hex.EncodeToString([]byte{r}) + encodedPathname = encodedPathname + "%" + strings.ToUpper(hex) + } } } } return encodedPathname } +// getSignature final signature in hexadecimal form. +func getSignature(signingKey []byte, stringToSign string) string { + return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) +} + // compareSignatureV4 returns true if and only if both signatures -// are equal. The signatures are expected to be HEX encoded strings +// are equal. The signatures are expected to be hex-encoded strings // according to the AWS S3 signature V4 spec. func compareSignatureV4(sig1, sig2 string) bool { - // The CTC using []byte(str) works because the hex encoding - // is unique for a sequence of bytes. See also compareSignatureV2. + // The CTC using []byte(str) works because the hex encoding doesn't use + // non-ASCII characters. Otherwise, we'd need to convert the strings to + // a []rune of UTF-8 characters. return subtle.ConstantTimeCompare([]byte(sig1), []byte(sig2)) == 1 } diff --git a/weed/s3api/auto_signature_v4_test.go b/weed/s3api/auto_signature_v4_test.go index 86fbbd19e..b8b817ab8 100644 --- a/weed/s3api/auto_signature_v4_test.go +++ b/weed/s3api/auto_signature_v4_test.go @@ -6,13 +6,10 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" - "errors" "fmt" "io" "net/http" - "net/url" "sort" - "strconv" "strings" "sync" "testing" @@ -76,7 +73,7 @@ func TestIsReqAuthenticated(t *testing.T) { SecretKey: "secret_key_1", }, }, - Actions: []string{}, + Actions: []string{"Read", "Write"}, }, }, }) @@ -149,7 +146,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) { SecretKey: "secret_key_1", }, }, - Actions: []string{}, + Actions: []string{"Admin", "Read", "Write"}, }, }, }) @@ -170,15 +167,12 @@ func TestCheckAdminRequestAuthType(t *testing.T) { func BenchmarkGetSignature(b *testing.B) { t := time.Now() - iam := IdentityAccessManagement{ - hashes: make(map[string]*sync.Pool), - hashCounters: make(map[string]*int32), - } b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - iam.getSignature("secret-key", t, "us-east-1", "s3", "random data") + signingKey := getSigningKey("secret-key", t.Format(yyyymmdd), "us-east-1", "s3") + getSignature(signingKey, "random data") } } @@ -213,6 +207,53 @@ func mustNewPresignedRequest(iam *IdentityAccessManagement, method string, urlSt return req } +// preSignV4 adds presigned URL parameters to the request +func preSignV4(iam *IdentityAccessManagement, req *http.Request, accessKey, secretKey string, expires int64) error { + // Create credential scope + now := time.Now().UTC() + dateStr := now.Format(iso8601Format) + + // Create credential header + scope := fmt.Sprintf("%s/%s/%s/%s", now.Format(yyyymmdd), "us-east-1", "s3", "aws4_request") + credential := fmt.Sprintf("%s/%s", accessKey, scope) + + // Get the query parameters + query := req.URL.Query() + query.Set("X-Amz-Algorithm", signV4Algorithm) + query.Set("X-Amz-Credential", credential) + query.Set("X-Amz-Date", dateStr) + query.Set("X-Amz-Expires", fmt.Sprintf("%d", expires)) + query.Set("X-Amz-SignedHeaders", "host") + + // Set the query on the URL (without signature yet) + req.URL.RawQuery = query.Encode() + + // Get the payload hash + hashedPayload := getContentSha256Cksum(req) + + // Extract signed headers + extractedSignedHeaders := make(http.Header) + extractedSignedHeaders["host"] = []string{req.Host} + + // Get canonical request + canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, req.URL.RawQuery, req.URL.Path, req.Method) + + // Get string to sign + stringToSign := getStringToSign(canonicalRequest, now, scope) + + // Get signing key + signingKey := getSigningKey(secretKey, now.Format(yyyymmdd), "us-east-1", "s3") + + // Calculate signature + signature := getSignature(signingKey, stringToSign) + + // Add signature to query + query.Set("X-Amz-Signature", signature) + req.URL.RawQuery = query.Encode() + + return nil +} + // Returns new HTTP request object. func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) { if method == "" { @@ -254,11 +295,6 @@ func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeek return req, nil } -// getSHA256Hash returns SHA-256 hash in hex encoding of given data. -func getSHA256Hash(data []byte) string { - return hex.EncodeToString(getSHA256Sum(data)) -} - // getMD5HashBase64 returns MD5 hash in base64 encoding of given data. func getMD5HashBase64(data []byte) string { return base64.StdEncoding.EncodeToString(getMD5Sum(data)) @@ -467,46 +503,6 @@ func signRequestV4(req *http.Request, accessKey, secretKey string) error { return nil } -// preSignV4 presign the request, in accordance with -// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html. -func preSignV4(iam *IdentityAccessManagement, req *http.Request, accessKeyID, secretAccessKey string, expires int64) error { - // Presign is not needed for anonymous credentials. - if accessKeyID == "" || secretAccessKey == "" { - return errors.New("Presign cannot be generated without access and secret keys") - } - - region := "us-east-1" - date := time.Now().UTC() - scope := getScope(date, region) - credential := fmt.Sprintf("%s/%s", accessKeyID, scope) - - // Set URL query. - query := req.URL.Query() - query.Set("X-Amz-Algorithm", signV4Algorithm) - query.Set("X-Amz-Date", date.Format(iso8601Format)) - query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10)) - query.Set("X-Amz-SignedHeaders", "host") - query.Set("X-Amz-Credential", credential) - query.Set("X-Amz-Content-Sha256", unsignedPayload) - - // "host" is the only header required to be signed for Presigned URLs. - extractedSignedHeaders := make(http.Header) - extractedSignedHeaders.Set("host", req.Host) - - queryStr := strings.Replace(query.Encode(), "+", "%20", -1) - canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayload, queryStr, req.URL.Path, req.Method) - stringToSign := getStringToSign(canonicalRequest, date, scope) - signature := iam.getSignature(secretAccessKey, date, region, "s3", stringToSign) - - req.URL.RawQuery = query.Encode() - - // Add signature header to RawQuery. - req.URL.RawQuery += "&X-Amz-Signature=" + url.QueryEscape(signature) - - // Construct the final presigned URL. - return nil -} - // EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences // // This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 @@ -529,12 +525,12 @@ func EncodePath(pathName string) string { encodedPathname = encodedPathname + string(s) continue default: - len := utf8.RuneLen(s) - if len < 0 { + runeLen := utf8.RuneLen(s) + if runeLen < 0 { // if utf8 cannot convert return the same string as is return pathName } - u := make([]byte, len) + u := make([]byte, runeLen) utf8.EncodeRune(u, s) for _, r := range u { hex := hex.EncodeToString([]byte{r}) diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go index 6ee46c82b..53ea8e768 100644 --- a/weed/s3api/chunked_reader_v4.go +++ b/weed/s3api/chunked_reader_v4.go @@ -102,13 +102,12 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr return nil, "", "", time.Time{}, s3err.ErrMissingDateHeader } } + // Parse date header. - var err error - date, err = time.Parse(iso8601Format, dateStr) + date, err := time.Parse(iso8601Format, dateStr) if err != nil { return nil, "", "", time.Time{}, s3err.ErrMalformedDate } - // Query string. queryStr := req.URL.Query().Encode() @@ -118,14 +117,11 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr // Get string to sign from canonical request. stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope()) + // Get hmac signing key. + signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date.Format(yyyymmdd), region, "s3") + // Calculate signature. - newSignature := iam.getSignature( - cred.SecretKey, - signV4Values.Credential.scope.date, - region, - "s3", - stringToSign, - ) + newSignature := getSignature(signingKey, stringToSign) // Verify if signature match. if !compareSignatureV4(newSignature, signV4Values.Signature) { @@ -469,58 +465,47 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) { // getChunkSignature - get chunk signature. func (cr *s3ChunkedReader) getChunkSignature(hashedChunk string) string { // Calculate string to sign. - stringToSign := signV4ChunkedAlgorithm + "\n" + + stringToSign := signV4Algorithm + "-PAYLOAD" + "\n" + cr.seedDate.Format(iso8601Format) + "\n" + getScope(cr.seedDate, cr.region) + "\n" + cr.seedSignature + "\n" + emptySHA256 + "\n" + hashedChunk - // Calculate signature. - return cr.iam.getSignature( - cr.cred.SecretKey, - cr.seedDate, - cr.region, - "s3", - stringToSign, - ) + // Get hmac signing key. + signingKey := getSigningKey(cr.cred.SecretKey, cr.seedDate.Format(yyyymmdd), cr.region, "s3") + + // Calculate and return signature. + return getSignature(signingKey, stringToSign) } -// readCRLF - check if reader only has '\r\n' CRLF character. -// returns malformed encoding if it doesn't. func readCRLF(reader *bufio.Reader) error { buf := make([]byte, 2) - _, err := reader.Read(buf) + _, err := io.ReadFull(reader, buf) if err != nil { return err } return checkCRLF(buf) } -// peekCRLF - peeks at the next two bytes to check for CRLF without consuming them. func peekCRLF(reader *bufio.Reader) error { - peeked, err := reader.Peek(2) + buf, err := reader.Peek(2) if err != nil { return err } - if err := checkCRLF(peeked); err != nil { + if err := checkCRLF(buf); err != nil { return err } return nil } -// checkCRLF - checks if the buffer contains '\r\n' CRLF character. func checkCRLF(buf []byte) error { - if buf[0] != '\r' || buf[1] != '\n' { + if len(buf) != 2 || buf[0] != '\r' || buf[1] != '\n' { return errMalformedEncoding } return nil } -// Read a line of bytes (up to \n) from b. -// Give up if the line exceeds maxLineLength. -// The returned bytes are owned by the bufio.Reader -// so they are only valid until the next bufio read. func readChunkLine(b *bufio.Reader) ([]byte, error) { buf, err := b.ReadSlice('\n') if err != nil { @@ -536,8 +521,7 @@ func readChunkLine(b *bufio.Reader) ([]byte, error) { if len(buf) >= maxLineLength { return nil, errLineTooLong } - - return buf, nil + return trimTrailingWhitespace(buf), nil } // trimTrailingWhitespace - trim trailing white space. @@ -608,13 +592,11 @@ func parseChunkChecksum(b *bufio.Reader) (ChecksumAlgorithm, []byte) { return extractedAlgorithm, checksumValue } -// parseChunkSignature - parse chunk signature. func parseChunkSignature(chunk []byte) []byte { - chunkSplits := bytes.SplitN(chunk, []byte(s3ChunkSignatureStr), 2) - return chunkSplits[1] + chunkSplits := bytes.SplitN(chunk, []byte("="), 2) + return chunkSplits[1] // Keep only the signature. } -// parse hex to uint64. func parseHexUint(v []byte) (n uint64, err error) { for i, b := range v { switch { @@ -636,6 +618,7 @@ func parseHexUint(v []byte) (n uint64, err error) { return } +// Checksum Algorithm represents the various checksum algorithms supported. type ChecksumAlgorithm int const ( @@ -649,18 +632,18 @@ const ( func (ca ChecksumAlgorithm) String() string { switch ca { + case ChecksumAlgorithmNone: + return "" case ChecksumAlgorithmCRC32: - return "CRC32" + return "x-amz-checksum-crc32" case ChecksumAlgorithmCRC32C: - return "CRC32C" + return "x-amz-checksum-crc32c" case ChecksumAlgorithmCRC64NVMe: - return "CRC64NVMe" + return "x-amz-checksum-crc64nvme" case ChecksumAlgorithmSHA1: - return "SHA1" + return "x-amz-checksum-sha1" case ChecksumAlgorithmSHA256: - return "SHA256" - case ChecksumAlgorithmNone: - return "" + return "x-amz-checksum-sha256" } return "" }