add v2 support

weed/command/s3.go

@@ -54,7 +54,7 @@ var cmdS3 = &Command{
       "credentials": [
         {
           "accessKey": "some_access_key1",
-          "secretKey": "some_secret_key2"
+          "secretKey": "some_secret_key1"
         }
       ],
       "actions": [
@@ -67,8 +67,8 @@ var cmdS3 = &Command{
       "name": "some_read_only_user",
       "credentials": [
         {
-          "accessKey": "some_access_key1",
-          "secretKey": "some_secret_key1"
+          "accessKey": "some_access_key2",
+          "secretKey": "some_secret_key2"
         }
       ],
       "actions": [
@@ -79,8 +79,8 @@ var cmdS3 = &Command{
       "name": "some_normal_user",
       "credentials": [
         {
-          "accessKey": "some_access_key2",
-          "secretKey": "some_secret_key2"
+          "accessKey": "some_access_key3",
+          "secretKey": "some_secret_key3"
         }
       ],
       "actions": [

weed/s3api/auth_credentials.go

@@ -26,6 +26,7 @@ type Iam interface {
 
 type IdentityAccessManagement struct {
 	identities []*Identity
+	domain     string
 }
 
 type Identity struct {
@@ -39,8 +40,10 @@ type Credential struct {
 	SecretKey string
 }
 
-func NewIdentityAccessManagement(fileName string) *IdentityAccessManagement {
-	iam := &IdentityAccessManagement{}
+func NewIdentityAccessManagement(fileName string, domain string) *IdentityAccessManagement {
+	iam := &IdentityAccessManagement{
+		domain: domain,
+	}
 	if fileName == "" {
 		return iam
 	}
@@ -119,16 +122,25 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, actions []Acti
 	var identity *Identity
 	var s3Err ErrorCode
 	switch getRequestAuthType(r) {
-	case authTypeUnknown, authTypeStreamingSigned:
+	case authTypeStreamingSigned:
+		return ErrNone
+	case authTypeUnknown:
+		glog.V(3).Infof("unknown auth type")
 		return ErrAccessDenied
 	case authTypePresignedV2, authTypeSignedV2:
-		return ErrNotImplemented
+		glog.V(3).Infof("v2 auth type")
+		identity, s3Err = iam.isReqAuthenticatedV2(r)
 	case authTypeSigned, authTypePresigned:
+		glog.V(3).Infof("v4 auth type")
 		identity, s3Err = iam.reqSignatureV4Verify(r)
+	}
+
+	glog.V(3).Infof("auth error: %v", s3Err)
 	if s3Err != ErrNone {
 		return s3Err
 	}
-	}
+
+	glog.V(3).Infof("user name: %v actions: %v", identity.Name, identity.Actions)
 
 	if !identity.canDo(actions) {
 		return ErrAccessDenied

weed/s3api/auth_signature_v2.go

@@ -0,0 +1,412 @@
+/*
+ * The following code tries to reverse engineer the Amazon S3 APIs,
+ * and is mostly copied from minio implementation.
+ */
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+// implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package s3api
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"crypto/subtle"
+	"encoding/base64"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Whitelist resource list that will be used in query string for signature-V2 calculation.
+// The list should be alphabetically sorted
+var resourceList = []string{
+	"acl",
+	"delete",
+	"lifecycle",
+	"location",
+	"logging",
+	"notification",
+	"partNumber",
+	"policy",
+	"requestPayment",
+	"response-cache-control",
+	"response-content-disposition",
+	"response-content-encoding",
+	"response-content-language",
+	"response-content-type",
+	"response-expires",
+	"torrent",
+	"uploadId",
+	"uploads",
+	"versionId",
+	"versioning",
+	"versions",
+	"website",
+}
+
+// Verify if request has valid AWS Signature Version '2'.
+func (iam *IdentityAccessManagement) isReqAuthenticatedV2(r *http.Request) (*Identity, ErrorCode) {
+	if isRequestSignatureV2(r) {
+		return iam.doesSignV2Match(r)
+	}
+	return iam.doesPresignV2SignatureMatch(r)
+}
+
+// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
+// Signature = Base64( HMAC-SHA1( YourSecretKey, UTF-8-Encoding-Of( StringToSign ) ) );
+//
+// StringToSign = HTTP-Verb + "\n" +
+//  	Content-Md5 + "\n" +
+//  	Content-Type + "\n" +
+//  	Date + "\n" +
+//  	CanonicalizedProtocolHeaders +
+//  	CanonicalizedResource;
+//
+// CanonicalizedResource = [ "/" + Bucket ] +
+//  	<HTTP-Request-URI, from the protocol name up to the query string> +
+//  	[ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
+//
+// CanonicalizedProtocolHeaders = <described below>
+
+// doesSignV2Match - Verify authorization header with calculated header in accordance with
+//     - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html
+// returns true if matches, false otherwise. if error is not nil then it is always false
+
+func validateV2AuthHeader(v2Auth string) (accessKey string, errCode ErrorCode) {
+	if v2Auth == "" {
+		return "", ErrAuthHeaderEmpty
+	}
+	// Verify if the header algorithm is supported or not.
+	if !strings.HasPrefix(v2Auth, signV2Algorithm) {
+		return "", ErrSignatureVersionNotSupported
+	}
+
+	// below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string).
+	// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature
+	authFields := strings.Split(v2Auth, " ")
+	if len(authFields) != 2 {
+		return "", ErrMissingFields
+	}
+
+	// Then will be splitting on ":", this will seprate `AWSAccessKeyId` and `Signature` string.
+	keySignFields := strings.Split(strings.TrimSpace(authFields[1]), ":")
+	if len(keySignFields) != 2 {
+		return "", ErrMissingFields
+	}
+
+	return keySignFields[0], ErrNone
+}
+
+func (iam *IdentityAccessManagement) doesSignV2Match(r *http.Request) (*Identity, ErrorCode) {
+	v2Auth := r.Header.Get("Authorization")
+
+	accessKey, apiError := validateV2AuthHeader(v2Auth)
+	if apiError != ErrNone {
+		return nil, apiError
+	}
+
+	// Access credentials.
+	// Validate if access key id same.
+	ident, cred, found := iam.lookupByAccessKey(accessKey)
+	if !found {
+		return nil, ErrInvalidAccessKeyID
+	}
+
+	// r.RequestURI will have raw encoded URI as sent by the client.
+	tokens := strings.SplitN(r.RequestURI, "?", 2)
+	encodedResource := tokens[0]
+	encodedQuery := ""
+	if len(tokens) == 2 {
+		encodedQuery = tokens[1]
+	}
+
+	unescapedQueries, err := unescapeQueries(encodedQuery)
+	if err != nil {
+		return nil, ErrInvalidQueryParams
+	}
+
+	encodedResource, err = getResource(encodedResource, r.Host, iam.domain)
+	if err != nil {
+		return nil, ErrInvalidRequest
+	}
+
+	prefix := fmt.Sprintf("%s %s:", signV2Algorithm, cred.AccessKey)
+	if !strings.HasPrefix(v2Auth, prefix) {
+		return nil, ErrSignatureDoesNotMatch
+	}
+	v2Auth = v2Auth[len(prefix):]
+	expectedAuth := signatureV2(cred, r.Method, encodedResource, strings.Join(unescapedQueries, "&"), r.Header)
+	if !compareSignatureV2(v2Auth, expectedAuth) {
+		return nil, ErrSignatureDoesNotMatch
+	}
+	return ident, ErrNone
+}
+
+// doesPresignV2SignatureMatch - Verify query headers with presigned signature
+//     - http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationQueryStringAuth
+// returns ErrNone if matches. S3 errors otherwise.
+func (iam *IdentityAccessManagement) doesPresignV2SignatureMatch(r *http.Request) (*Identity, ErrorCode) {
+
+	// r.RequestURI will have raw encoded URI as sent by the client.
+	tokens := strings.SplitN(r.RequestURI, "?", 2)
+	encodedResource := tokens[0]
+	encodedQuery := ""
+	if len(tokens) == 2 {
+		encodedQuery = tokens[1]
+	}
+
+	var (
+		filteredQueries []string
+		gotSignature    string
+		expires         string
+		accessKey       string
+		err             error
+	)
+
+	var unescapedQueries []string
+	unescapedQueries, err = unescapeQueries(encodedQuery)
+	if err != nil {
+		return nil, ErrInvalidQueryParams
+	}
+
+	// Extract the necessary values from presigned query, construct a list of new filtered queries.
+	for _, query := range unescapedQueries {
+		keyval := strings.SplitN(query, "=", 2)
+		if len(keyval) != 2 {
+			return nil, ErrInvalidQueryParams
+		}
+		switch keyval[0] {
+		case "AWSAccessKeyId":
+			accessKey = keyval[1]
+		case "Signature":
+			gotSignature = keyval[1]
+		case "Expires":
+			expires = keyval[1]
+		default:
+			filteredQueries = append(filteredQueries, query)
+		}
+	}
+
+	// Invalid values returns error.
+	if accessKey == "" || gotSignature == "" || expires == "" {
+		return nil, ErrInvalidQueryParams
+	}
+
+	// Validate if access key id same.
+	ident, cred, found := iam.lookupByAccessKey(accessKey)
+	if !found {
+		return nil, ErrInvalidAccessKeyID
+	}
+
+	// Make sure the request has not expired.
+	expiresInt, err := strconv.ParseInt(expires, 10, 64)
+	if err != nil {
+		return nil, ErrMalformedExpires
+	}
+
+	// Check if the presigned URL has expired.
+	if expiresInt < time.Now().UTC().Unix() {
+		return nil, ErrExpiredPresignRequest
+	}
+
+	encodedResource, err = getResource(encodedResource, r.Host, iam.domain)
+	if err != nil {
+		return nil, ErrInvalidRequest
+	}
+
+	expectedSignature := preSignatureV2(cred, r.Method, encodedResource, strings.Join(filteredQueries, "&"), r.Header, expires)
+	if !compareSignatureV2(gotSignature, expectedSignature) {
+		return nil, ErrSignatureDoesNotMatch
+	}
+
+	return ident, ErrNone
+}
+
+// Escape encodedQuery string into unescaped list of query params, returns error
+// if any while unescaping the values.
+func unescapeQueries(encodedQuery string) (unescapedQueries []string, err error) {
+	for _, query := range strings.Split(encodedQuery, "&") {
+		var unescapedQuery string
+		unescapedQuery, err = url.QueryUnescape(query)
+		if err != nil {
+			return nil, err
+		}
+		unescapedQueries = append(unescapedQueries, unescapedQuery)
+	}
+	return unescapedQueries, nil
+}
+
+// Returns "/bucketName/objectName" for path-style or virtual-host-style requests.
+func getResource(path string, host string, domain string) (string, error) {
+	if domain == "" {
+		return path, nil
+	}
+	// If virtual-host-style is enabled construct the "resource" properly.
+	if strings.Contains(host, ":") {
+		// In bucket.mydomain.com:9000, strip out :9000
+		var err error
+		if host, _, err = net.SplitHostPort(host); err != nil {
+			return "", err
+		}
+	}
+	if !strings.HasSuffix(host, "."+domain) {
+		return path, nil
+	}
+	bucket := strings.TrimSuffix(host, "."+domain)
+	return "/" + pathJoin(bucket, path), nil
+}
+
+// pathJoin - like path.Join() but retains trailing "/" of the last element
+func pathJoin(elem ...string) string {
+	trailingSlash := ""
+	if len(elem) > 0 {
+		if strings.HasSuffix(elem[len(elem)-1], "/") {
+			trailingSlash = "/"
+		}
+	}
+	return path.Join(elem...) + trailingSlash
+}
+
+// Return the signature v2 of a given request.
+func signatureV2(cred *Credential, method string, encodedResource string, encodedQuery string, headers http.Header) string {
+	stringToSign := getStringToSignV2(method, encodedResource, encodedQuery, headers, "")
+	signature := calculateSignatureV2(stringToSign, cred.SecretKey)
+	return signature
+}
+
+// Return string to sign under two different conditions.
+// - if expires string is set then string to sign includes date instead of the Date header.
+// - if expires string is empty then string to sign includes date header instead.
+func getStringToSignV2(method string, encodedResource, encodedQuery string, headers http.Header, expires string) string {
+	canonicalHeaders := canonicalizedAmzHeadersV2(headers)
+	if len(canonicalHeaders) > 0 {
+		canonicalHeaders += "\n"
+	}
+
+	date := expires // Date is set to expires date for presign operations.
+	if date == "" {
+		// If expires date is empty then request header Date is used.
+		date = headers.Get("Date")
+	}
+
+	// From the Amazon docs:
+	//
+	// StringToSign = HTTP-Verb + "\n" +
+	// 	 Content-Md5 + "\n" +
+	//	 Content-Type + "\n" +
+	//	 Date/Expires + "\n" +
+	//	 CanonicalizedProtocolHeaders +
+	//	 CanonicalizedResource;
+	stringToSign := strings.Join([]string{
+		method,
+		headers.Get("Content-MD5"),
+		headers.Get("Content-Type"),
+		date,
+		canonicalHeaders,
+	}, "\n")
+
+	return stringToSign + canonicalizedResourceV2(encodedResource, encodedQuery)
+}
+
+// Return canonical resource string.
+func canonicalizedResourceV2(encodedResource, encodedQuery string) string {
+	queries := strings.Split(encodedQuery, "&")
+	keyval := make(map[string]string)
+	for _, query := range queries {
+		key := query
+		val := ""
+		index := strings.Index(query, "=")
+		if index != -1 {
+			key = query[:index]
+			val = query[index+1:]
+		}
+		keyval[key] = val
+	}
+
+	var canonicalQueries []string
+	for _, key := range resourceList {
+		val, ok := keyval[key]
+		if !ok {
+			continue
+		}
+		if val == "" {
+			canonicalQueries = append(canonicalQueries, key)
+			continue
+		}
+		canonicalQueries = append(canonicalQueries, key+"="+val)
+	}
+
+	// The queries will be already sorted as resourceList is sorted, if canonicalQueries
+	// is empty strings.Join returns empty.
+	canonicalQuery := strings.Join(canonicalQueries, "&")
+	if canonicalQuery != "" {
+		return encodedResource + "?" + canonicalQuery
+	}
+	return encodedResource
+}
+
+// Return canonical headers.
+func canonicalizedAmzHeadersV2(headers http.Header) string {
+	var keys []string
+	keyval := make(map[string]string)
+	for key := range headers {
+		lkey := strings.ToLower(key)
+		if !strings.HasPrefix(lkey, "x-amz-") {
+			continue
+		}
+		keys = append(keys, lkey)
+		keyval[lkey] = strings.Join(headers[key], ",")
+	}
+	sort.Strings(keys)
+	var canonicalHeaders []string
+	for _, key := range keys {
+		canonicalHeaders = append(canonicalHeaders, key+":"+keyval[key])
+	}
+	return strings.Join(canonicalHeaders, "\n")
+}
+
+func calculateSignatureV2(stringToSign string, secret string) string {
+	hm := hmac.New(sha1.New, []byte(secret))
+	hm.Write([]byte(stringToSign))
+	return base64.StdEncoding.EncodeToString(hm.Sum(nil))
+}
+
+// compareSignatureV2 returns true if and only if both signatures
+// are equal. The signatures are expected to be base64 encoded strings
+// according to the AWS S3 signature V2 spec.
+func compareSignatureV2(sig1, sig2 string) bool {
+	// Decode signature string to binary byte-sequence representation is required
+	// as Base64 encoding of a value is not unique:
+	// For example "aGVsbG8=" and "aGVsbG8=\r" will result in the same byte slice.
+	signature1, err := base64.StdEncoding.DecodeString(sig1)
+	if err != nil {
+		return false
+	}
+	signature2, err := base64.StdEncoding.DecodeString(sig2)
+	if err != nil {
+		return false
+	}
+	return subtle.ConstantTimeCompare(signature1, signature2) == 1
+}
+
+// Return signature-v2 for the presigned request.
+func preSignatureV2(cred *Credential, method string, encodedResource string, encodedQuery string, headers http.Header, expires string) string {
+	stringToSign := getStringToSignV2(method, encodedResource, encodedQuery, headers, expires)
+	return calculateSignatureV2(stringToSign, cred.SecretKey)
+}

weed/s3api/auto_signature_v4_test.go

@@ -57,7 +57,7 @@ func TestIsRequestPresignedSignatureV4(t *testing.T) {
 
 // Tests is requested authenticated function, tests replies for s3 errors.
 func TestIsReqAuthenticated(t *testing.T) {
-	iam := NewIdentityAccessManagement("")
+	iam := NewIdentityAccessManagement("", "")
 	iam.identities = []*Identity{
 		{
 			Name: "someone",
@@ -92,7 +92,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 }
 
 func TestCheckAdminRequestAuthType(t *testing.T) {
-	iam := NewIdentityAccessManagement("")
+	iam := NewIdentityAccessManagement("", "")
 	iam.identities = []*Identity{
 		{
 			Name: "someone",

weed/s3api/s3api_errors.go

@@ -66,6 +66,7 @@ const (
 	ErrInvalidAccessKeyID
 	ErrRequestNotReadyYet
 	ErrMissingDateHeader
+	ErrInvalidRequest
 	ErrNotImplemented
 )
 
@@ -270,7 +271,11 @@ var errorCodeResponse = map[ErrorCode]APIError{
 		Description:    "AWS authentication requires a valid Date or x-amz-date header",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
+	ErrInvalidRequest: {
+		Code:           "InvalidRequest",
+		Description:    "Invalid Request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrNotImplemented: {
 		Code:           "NotImplemented",
 		Description:    "A header you provided implies functionality that is not implemented",

weed/s3api/s3api_server.go

@@ -24,7 +24,7 @@ type S3ApiServer struct {
 func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) {
 	s3ApiServer = &S3ApiServer{
 		option: option,
-		iam:    NewIdentityAccessManagement(option.Config),
+		iam:    NewIdentityAccessManagement(option.Config, option.DomainName),
 	}
 
 	s3ApiServer.registerRouter(router)