From b52ee30fddad1b19e8485aa962d86eae0828c818 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Wed, 12 Oct 2022 13:33:58 +0800 Subject: [PATCH 01/52] add acl helper functionalities Signed-off-by: changlin.shi --- weed/s3api/s3_constants/acp_grantee_group.go | 12 + weed/s3api/s3_constants/header.go | 8 + weed/s3api/s3acl/acl_helper.go | 495 +++++++++++++++++++ weed/s3api/s3acl/acl_helper_test.go | 220 +++++++++ 4 files changed, 735 insertions(+) create mode 100644 weed/s3api/s3acl/acl_helper.go create mode 100644 weed/s3api/s3acl/acl_helper_test.go diff --git a/weed/s3api/s3_constants/acp_grantee_group.go b/weed/s3api/s3_constants/acp_grantee_group.go index a315fb0f7..7058a4e9f 100644 --- a/weed/s3api/s3_constants/acp_grantee_group.go +++ b/weed/s3api/s3_constants/acp_grantee_group.go @@ -6,3 +6,15 @@ var ( GranteeGroupAuthenticatedUsers = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" GranteeGroupLogDelivery = "http://acs.amazonaws.com/groups/s3/LogDelivery" ) + +func ValidateGroup(group string) bool { + valid := true + switch group { + case GranteeGroupAllUsers: + case GranteeGroupLogDelivery: + case GranteeGroupAuthenticatedUsers: + default: + valid = false + } + return valid +} diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index 5e19d67be..5037f4691 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -38,6 +38,14 @@ const ( AmzTagCount = "x-amz-tagging-count" X_SeaweedFS_Header_Directory_Key = "x-seaweedfs-is-directory-key" + + // S3 ACL headers + AmzCannedAcl = "X-Amz-Acl" + AmzAclFullControl = "X-Amz-Grant-Full-Control" + AmzAclRead = "X-Amz-Grant-Read" + AmzAclWrite = "X-Amz-Grant-Write" + AmzAclReadAcp = "X-Amz-Grant-Read-Acp" + AmzAclWriteAcp = "X-Amz-Grant-Write-Acp" ) // Non-Standard S3 HTTP request constants diff --git a/weed/s3api/s3acl/acl_helper.go b/weed/s3api/s3acl/acl_helper.go new file mode 100644 index 000000000..3150e4daf --- /dev/null +++ b/weed/s3api/s3acl/acl_helper.go @@ -0,0 +1,495 @@ +package s3acl + +import ( + "encoding/json" + "encoding/xml" + "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/util" + "net/http" + "strings" +) + +// GetAccountId get AccountId from request headers, AccountAnonymousId will be return if not presen +func GetAccountId(r *http.Request) string { + id := r.Header.Get(s3_constants.AmzAccountId) + if len(id) == 0 { + return s3account.AccountAnonymous.Id + } else { + return id + } +} + +// ExtractAcl extracts the acl from the request body, or from the header if request body is empty +func ExtractAcl(r *http.Request, accountManager *s3account.AccountManager, ownership, bucketOwnerId, ownerId, accountId string) (grants []*s3.Grant, errCode s3err.ErrorCode) { + if r.Body != nil && r.Body != http.NoBody { + defer util.CloseRequest(r) + + var acp s3.AccessControlPolicy + err := xmlutil.UnmarshalXML(&acp, xml.NewDecoder(r.Body), "") + if err != nil { + return nil, s3err.ErrInvalidRequest + } + + //owner should present && owner is immutable + if acp.Owner == nil || acp.Owner.ID == nil || *acp.Owner.ID != ownerId { + glog.V(3).Infof("set acl denied! owner account is not consistent, request account id: %s, expect account id: %s", accountId, ownerId) + return nil, s3err.ErrAccessDenied + } + + return ValidateAndTransferGrants(accountManager, acp.Grants) + } else { + _, grants, errCode = ParseAndValidateAclHeadersOrElseDefault(r, accountManager, ownership, bucketOwnerId, accountId, true) + return grants, errCode + } +} + +// ParseAndValidateAclHeadersOrElseDefault will callParseAndValidateAclHeaders to get Grants, if empty, it will return Grant that grant `accountId` with `FullControl` permission +func ParseAndValidateAclHeadersOrElseDefault(r *http.Request, accountManager *s3account.AccountManager, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) { + ownerId, grants, errCode = ParseAndValidateAclHeaders(r, accountManager, ownership, bucketOwnerId, accountId, putAcl) + if errCode != s3err.ErrNone { + return + } + if len(grants) == 0 { + //if no acl(both customAcl and cannedAcl) specified, grant accountId(object writer) with full control permission + grants = append(grants, &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountId, + }, + Permission: &s3_constants.PermissionFullControl, + }) + } + return +} + +// ParseAndValidateAclHeaders parse and validate acl from header +func ParseAndValidateAclHeaders(r *http.Request, accountManager *s3account.AccountManager, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) { + ownerId, grants, errCode = ParseAclHeaders(r, ownership, bucketOwnerId, accountId, putAcl) + if errCode != s3err.ErrNone { + return + } + if len(grants) > 0 { + grants, errCode = ValidateAndTransferGrants(accountManager, grants) + } + return +} + +// ParseAclHeaders parse acl headers +// When `putAcl` is true, only `CannedAcl` is parsed, such as `PutBucketAcl` or `PutObjectAcl` +// is requested, `CustomAcl` is parsed from the request body not from headers, and only if the +// request body is empty, `CannedAcl` is parsed from the header, and will not parse `CustomAcl` from the header +// +// Since `CustomAcl` has higher priority, it will be parsed first; if `CustomAcl` does not exist, `CannedAcl` will be parsed +func ParseAclHeaders(r *http.Request, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) { + if !putAcl { + errCode = ParseCustomAclHeaders(r, &grants) + if errCode != s3err.ErrNone { + return "", nil, errCode + } + } + if len(grants) > 0 { + return accountId, grants, s3err.ErrNone + } + + cannedAcl := r.Header.Get(s3_constants.AmzCannedAcl) + if len(cannedAcl) == 0 { + return accountId, grants, s3err.ErrNone + } + + //if canned acl specified, parse cannedAcl (lower priority to custom acl) + ownerId, grants, errCode = ParseCannedAclHeader(ownership, bucketOwnerId, accountId, cannedAcl, putAcl) + if errCode != s3err.ErrNone { + return "", nil, errCode + } + return ownerId, grants, errCode +} + +func ParseCustomAclHeaders(r *http.Request, grants *[]*s3.Grant) s3err.ErrorCode { + customAclHeaders := []string{s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp} + var errCode s3err.ErrorCode + for _, customAclHeader := range customAclHeaders { + headerValue := r.Header.Get(customAclHeader) + switch customAclHeader { + case s3_constants.AmzAclRead: + errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionRead, grants) + case s3_constants.AmzAclWrite: + errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWrite, grants) + case s3_constants.AmzAclReadAcp: + errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionReadAcp, grants) + case s3_constants.AmzAclWriteAcp: + errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWriteAcp, grants) + case s3_constants.AmzAclFullControl: + errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionFullControl, grants) + } + if errCode != s3err.ErrNone { + return errCode + } + } + return s3err.ErrNone +} + +func ParseCustomAclHeader(headerValue, permission string, grants *[]*s3.Grant) s3err.ErrorCode { + if len(headerValue) > 0 { + split := strings.Split(headerValue, ", ") + for _, grantStr := range split { + kv := strings.Split(grantStr, "=") + if len(kv) != 2 { + return s3err.ErrInvalidRequest + } + + switch kv[0] { + case "id": + var accountId string + _ = json.Unmarshal([]byte(kv[1]), &accountId) + *grants = append(*grants, &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountId, + }, + Permission: &permission, + }) + case "emailAddress": + var emailAddress string + _ = json.Unmarshal([]byte(kv[1]), &emailAddress) + *grants = append(*grants, &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeAmazonCustomerByEmail, + EmailAddress: &emailAddress, + }, + Permission: &permission, + }) + case "uri": + var groupName string + _ = json.Unmarshal([]byte(kv[1]), &groupName) + *grants = append(*grants, &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &groupName, + }, + Permission: &permission, + }) + } + } + } + return s3err.ErrNone + +} + +func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl string, putAcl bool) (ownerId string, grants []*s3.Grant, err s3err.ErrorCode) { + err = s3err.ErrNone + ownerId = accountId + + //objectWrite automatically has full control on current object + objectWriterFullControl := &s3.Grant{ + Grantee: &s3.Grantee{ + ID: &accountId, + Type: &s3_constants.GrantTypeCanonicalUser, + }, + Permission: &s3_constants.PermissionFullControl, + } + + switch cannedAcl { + case s3_constants.CannedAclPrivate: + grants = append(grants, objectWriterFullControl) + case s3_constants.CannedAclPublicRead: + grants = append(grants, objectWriterFullControl) + grants = append(grants, s3_constants.PublicRead...) + case s3_constants.CannedAclPublicReadWrite: + grants = append(grants, objectWriterFullControl) + grants = append(grants, s3_constants.PublicReadWrite...) + case s3_constants.CannedAclAuthenticatedRead: + grants = append(grants, objectWriterFullControl) + grants = append(grants, s3_constants.AuthenticatedRead...) + case s3_constants.CannedAclLogDeliveryWrite: + grants = append(grants, objectWriterFullControl) + grants = append(grants, s3_constants.LogDeliveryWrite...) + case s3_constants.CannedAclBucketOwnerRead: + grants = append(grants, objectWriterFullControl) + if bucketOwnerId != "" && bucketOwnerId != accountId { + grants = append(grants, + &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionRead, + }) + } + case s3_constants.CannedAclBucketOwnerFullControl: + if bucketOwnerId != "" { + // if set ownership to 'BucketOwnerPreferred' when upload object, the bucket owner will be the object owner + if !putAcl && bucketOwnership == s3_constants.OwnershipBucketOwnerPreferred { + ownerId = bucketOwnerId + grants = append(grants, + &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }) + } else { + grants = append(grants, objectWriterFullControl) + if accountId != bucketOwnerId { + grants = append(grants, + &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }) + } + } + } + case s3_constants.CannedAclAwsExecRead: + err = s3err.ErrNotImplemented + default: + err = s3err.ErrNotImplemented + } + return +} + +// ValidateAndTransferGrants validate grant & transfer Email-Grant to Id-Grant +func ValidateAndTransferGrants(accountManager *s3account.AccountManager, grants []*s3.Grant) ([]*s3.Grant, s3err.ErrorCode) { + var result []*s3.Grant + for _, grant := range grants { + grantee := grant.Grantee + if grantee == nil || grantee.Type == nil { + glog.Warning("invalid grantee! grantee or granteeType is nil") + return nil, s3err.ErrInvalidRequest + } + + switch *grantee.Type { + case s3_constants.GrantTypeGroup: + if grantee.URI == nil { + glog.Warning("invalid group grantee! group URI is nil") + return nil, s3err.ErrInvalidRequest + } + ok := s3_constants.ValidateGroup(*grantee.URI) + if !ok { + glog.Warningf("invalid group grantee! group name[%s] is not valid", *grantee.URI) + return nil, s3err.ErrInvalidRequest + } + result = append(result, grant) + case s3_constants.GrantTypeCanonicalUser: + if grantee.ID == nil { + glog.Warning("invalid canonical grantee! account id is nil") + return nil, s3err.ErrInvalidRequest + } + _, ok := accountManager.IdNameMapping[*grantee.ID] + if !ok { + glog.Warningf("invalid canonical grantee! account id[%s] is not exists", *grantee.ID) + return nil, s3err.ErrInvalidRequest + } + result = append(result, grant) + case s3_constants.GrantTypeAmazonCustomerByEmail: + if grantee.EmailAddress == nil { + glog.Warning("invalid email grantee! email address is nil") + return nil, s3err.ErrInvalidRequest + } + accountId, ok := accountManager.EmailIdMapping[*grantee.EmailAddress] + if !ok { + glog.Warningf("invalid email grantee! email address[%s] is not exists", *grantee.EmailAddress) + return nil, s3err.ErrInvalidRequest + } + result = append(result, &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountId, + }, + Permission: grant.Permission, + }) + default: + return nil, s3err.ErrInvalidRequest + } + } + return result, s3err.ErrNone +} + +// DetermineReqGrants generates the grant set (Grants) according to accountId and reqPermission. +func DetermineReqGrants(accountId, aclAction string) (grants []*s3.Grant) { + // group grantee (AllUsers) + grants = append(grants, &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &aclAction, + }) + grants = append(grants, &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }) + + // canonical grantee (accountId) + grants = append(grants, &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountId, + }, + Permission: &aclAction, + }) + grants = append(grants, &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountId, + }, + Permission: &s3_constants.PermissionFullControl, + }) + + // group grantee (AuthenticateUsers) + if accountId != s3account.AccountAnonymous.Id { + grants = append(grants, &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAuthenticatedUsers, + }, + Permission: &aclAction, + }) + grants = append(grants, &s3.Grant{ + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAuthenticatedUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }) + } + return +} + +func SetAcpOwnerHeader(r *http.Request, acpOwnerId string) { + r.Header.Set(s3_constants.ExtAmzOwnerKey, acpOwnerId) +} + +func GetAcpOwner(entryExtended map[string][]byte, defaultOwner string) string { + ownerIdBytes, ok := entryExtended[s3_constants.ExtAmzOwnerKey] + if ok { + return string(ownerIdBytes) + } + return defaultOwner +} + +func SetAcpGrantsHeader(r *http.Request, acpGrants []*s3.Grant) { + if len(acpGrants) > 0 { + a, err := json.Marshal(acpGrants) + if err == nil { + r.Header.Set(s3_constants.ExtAmzAclKey, string(a)) + } else { + glog.Warning("Marshal acp grants err", err) + } + } +} + +// GetAcpGrants return grants parsed from entry +func GetAcpGrants(entryExtended map[string][]byte) []*s3.Grant { + acpBytes, ok := entryExtended[s3_constants.ExtAmzAclKey] + if ok { + var grants []*s3.Grant + err := json.Unmarshal(acpBytes, &grants) + if err == nil { + return grants + } + } + return nil +} + +// AssembleEntryWithAcp fill entry with owner and grants +func AssembleEntryWithAcp(objectEntry *filer_pb.Entry, objectOwner string, grants []*s3.Grant) s3err.ErrorCode { + objectEntry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(objectOwner) + grantsBytes, err := json.Marshal(grants) + if err != nil { + glog.Warning("assemble acp to entry:", err) + return s3err.ErrInvalidRequest + } + objectEntry.Extended[s3_constants.ExtAmzAclKey] = grantsBytes + return s3err.ErrNone +} + +// GrantEquals Compare whether two Grants are equal in meaning, not completely +// equal (compare Grantee.Type and the corresponding Value for equality, other +// fields of Grantee are ignored) +func GrantEquals(a, b *s3.Grant) bool { + // grant + if a == b { + return true + } + + if a == nil || b == nil { + return false + } + + // grant.Permission + if a.Permission != b.Permission { + if a.Permission == nil || b.Permission == nil { + return false + } + + if *a.Permission != *b.Permission { + return false + } + } + + // grant.Grantee + ag := a.Grantee + bg := b.Grantee + if ag != bg { + if ag == nil || bg == nil { + return false + } + // grantee.Type + if ag.Type != bg.Type { + if ag.Type == nil || bg.Type == nil { + return false + } + if *ag.Type != *bg.Type { + return false + } + } + // value corresponding to granteeType + if ag.Type != nil { + switch *ag.Type { + case s3_constants.GrantTypeGroup: + if ag.URI != bg.URI { + if ag.URI == nil || bg.URI == nil { + return false + } + + if *ag.URI != *bg.URI { + return false + } + } + case s3_constants.GrantTypeCanonicalUser: + if ag.ID != bg.ID { + if ag.ID == nil || bg.ID == nil { + return false + } + + if *ag.ID != *bg.ID { + return false + } + } + case s3_constants.GrantTypeAmazonCustomerByEmail: + if ag.EmailAddress != bg.EmailAddress { + if ag.EmailAddress == nil || bg.EmailAddress == nil { + return false + } + + if *ag.EmailAddress != *bg.EmailAddress { + return false + } + } + } + } + } + return true +} diff --git a/weed/s3api/s3acl/acl_helper_test.go b/weed/s3api/s3acl/acl_helper_test.go new file mode 100644 index 000000000..b58fcc171 --- /dev/null +++ b/weed/s3api/s3acl/acl_helper_test.go @@ -0,0 +1,220 @@ +package s3acl + +import ( + "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "testing" +) + +func TestParseAclHeaders(t *testing.T) { + accountManager := &s3account.AccountManager{ + IdNameMapping: map[string]string{ + s3account.AccountAdmin.Id: s3account.AccountAdmin.Name, + s3account.AccountAnonymous.Id: s3account.AccountAnonymous.Name, + }, + EmailIdMapping: map[string]string{ + s3account.AccountAdmin.EmailAddress: s3account.AccountAdmin.Id, + s3account.AccountAnonymous.EmailAddress: s3account.AccountAnonymous.Id, + }, + } + + //good value + grants := make([]*s3.Grant, 0) + validHeaderValue := `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example.com"` + errCode := ParseCustomAclHeader(validHeaderValue, s3_constants.PermissionFullControl, &grants) + if errCode != s3err.ErrNone { + t.Fatal(errCode) + } + _, errCode = ValidateAndTransferGrants(accountManager, grants) + if errCode != s3err.ErrNone { + t.Fatal(errCode) + } + + //bad case: acl header format error + grants = make([]*s3.Grant, 0) + formatErrCase := `uri, id="anonymous", emailAddress="admin@example.com"` + errCode = ParseCustomAclHeader(formatErrCase, s3_constants.PermissionFullControl, &grants) + if errCode != s3err.ErrInvalidRequest { + t.Fatal(errCode) + } + + //bad case: email not exists + grants = make([]*s3.Grant, 0) + badCaseOfEmail := `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example1.com"` + errCode = ParseCustomAclHeader(badCaseOfEmail, s3_constants.PermissionFullControl, &grants) + if errCode != s3err.ErrNone { + t.Fatal(errCode) + } + _, errCode = ValidateAndTransferGrants(accountManager, grants) + if errCode != s3err.ErrInvalidRequest { + t.Fatal(errCode) + } + + //bad case: account id not exists + grants = make([]*s3.Grant, 0) + badCaseOfAccountId := "uri=\"http://acs.amazonaws.com/groups/global/AllUsers\", id=\"xxxxxx\", emailAddress=\"admin@example.com\"" + errCode = ParseCustomAclHeader(badCaseOfAccountId, s3_constants.PermissionFullControl, &grants) + if errCode != s3err.ErrNone { + t.Fatal(errCode) + } + _, errCode = ValidateAndTransferGrants(accountManager, grants) + if errCode != s3err.ErrInvalidRequest { + t.Fatal(errCode) + } + + //bad case: group url not valid + grants = make([]*s3.Grant, 0) + badCaseOfURL := "uri=\"http://acs.amazonaws.com/groups/global/AllUsers111xxxx\", id=\"anonymous\", emailAddress=\"admin@example.com\"" + errCode = ParseCustomAclHeader(badCaseOfURL, s3_constants.PermissionFullControl, &grants) + if errCode != s3err.ErrNone { + t.Fatal(errCode) + } + _, errCode = ValidateAndTransferGrants(accountManager, grants) + if errCode != s3err.ErrInvalidRequest { + t.Fatal(errCode) + } +} + +func TestGrantEquals(t *testing.T) { + testCases := map[bool]bool{ + GrantEquals(nil, nil): true, + + GrantEquals(&s3.Grant{}, nil): false, + + GrantEquals(&s3.Grant{}, &s3.Grant{}): true, + + GrantEquals(&s3.Grant{ + Permission: &s3_constants.PermissionRead, + }, &s3.Grant{}): false, + + GrantEquals(&s3.Grant{ + Permission: &s3_constants.PermissionRead, + }, &s3.Grant{ + Permission: &s3_constants.PermissionRead, + }): true, + + GrantEquals(&s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{}, + }, &s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{}, + }): true, + + GrantEquals(&s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + }, + }, &s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{}, + }): false, + + //type not present, compare other fields of grant is meaningless + GrantEquals(&s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + ID: &s3account.AccountAdmin.Id, + EmailAddress: &s3account.AccountAdmin.EmailAddress, + }, + }, &s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + ID: &s3account.AccountAdmin.Id, + }, + }): true, + + GrantEquals(&s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + }, + }, &s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + }, + }): true, + + GrantEquals(&s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + }, &s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + }): true, + + GrantEquals(&s3.Grant{ + Permission: &s3_constants.PermissionWrite, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + }, &s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + }): false, + + GrantEquals(&s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + ID: &s3account.AccountAdmin.Id, + }, + }, &s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + ID: &s3account.AccountAdmin.Id, + }, + }): true, + + GrantEquals(&s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + ID: &s3account.AccountAdmin.Id, + URI: &s3_constants.GranteeGroupAllUsers, + }, + }, &s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + ID: &s3account.AccountAdmin.Id, + }, + }): false, + + GrantEquals(&s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + ID: &s3account.AccountAdmin.Id, + URI: &s3_constants.GranteeGroupAllUsers, + }, + }, &s3.Grant{ + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + }): true, + } + + for tc, expect := range testCases { + if tc != expect { + t.Fatal("TestGrantEquals not expect!") + } + } +} From 293113d1b8bc5ac94b4230e6c8d50dd18eaeed44 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Wed, 12 Oct 2022 21:16:24 +0800 Subject: [PATCH 02/52] add tests Signed-off-by: changlin.shi --- weed/s3api/s3acl/acl_helper.go | 32 +- weed/s3api/s3acl/acl_helper_test.go | 576 +++++++++++++++++++++++++--- 2 files changed, 553 insertions(+), 55 deletions(-) diff --git a/weed/s3api/s3acl/acl_helper.go b/weed/s3api/s3acl/acl_helper.go index 3150e4daf..350e703c4 100644 --- a/weed/s3api/s3acl/acl_helper.go +++ b/weed/s3api/s3acl/acl_helper.go @@ -32,12 +32,12 @@ func ExtractAcl(r *http.Request, accountManager *s3account.AccountManager, owner var acp s3.AccessControlPolicy err := xmlutil.UnmarshalXML(&acp, xml.NewDecoder(r.Body), "") - if err != nil { + if err != nil || acp.Owner == nil || acp.Owner.ID == nil { return nil, s3err.ErrInvalidRequest } //owner should present && owner is immutable - if acp.Owner == nil || acp.Owner.ID == nil || *acp.Owner.ID != ownerId { + if *acp.Owner.ID != ownerId { glog.V(3).Infof("set acl denied! owner account is not consistent, request account id: %s, expect account id: %s", accountId, ownerId) return nil, s3err.ErrAccessDenied } @@ -251,7 +251,7 @@ func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl s case s3_constants.CannedAclAwsExecRead: err = s3err.ErrNotImplemented default: - err = s3err.ErrNotImplemented + err = s3err.ErrInvalidRequest } return } @@ -373,7 +373,7 @@ func SetAcpOwnerHeader(r *http.Request, acpOwnerId string) { func GetAcpOwner(entryExtended map[string][]byte, defaultOwner string) string { ownerIdBytes, ok := entryExtended[s3_constants.ExtAmzOwnerKey] - if ok { + if ok && len(ownerIdBytes) > 0 { return string(ownerIdBytes) } return defaultOwner @@ -393,7 +393,7 @@ func SetAcpGrantsHeader(r *http.Request, acpGrants []*s3.Grant) { // GetAcpGrants return grants parsed from entry func GetAcpGrants(entryExtended map[string][]byte) []*s3.Grant { acpBytes, ok := entryExtended[s3_constants.ExtAmzAclKey] - if ok { + if ok && len(acpBytes) > 0 { var grants []*s3.Grant err := json.Unmarshal(acpBytes, &grants) if err == nil { @@ -405,13 +405,23 @@ func GetAcpGrants(entryExtended map[string][]byte) []*s3.Grant { // AssembleEntryWithAcp fill entry with owner and grants func AssembleEntryWithAcp(objectEntry *filer_pb.Entry, objectOwner string, grants []*s3.Grant) s3err.ErrorCode { - objectEntry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(objectOwner) - grantsBytes, err := json.Marshal(grants) - if err != nil { - glog.Warning("assemble acp to entry:", err) - return s3err.ErrInvalidRequest + if objectEntry.Extended == nil { + objectEntry.Extended = make(map[string][]byte, 0) + } + + if len(objectOwner) > 0 { + objectEntry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(objectOwner) + } + + if len(grants) > 0 { + grantsBytes, err := json.Marshal(grants) + if err != nil { + glog.Warning("assemble acp to entry:", err) + return s3err.ErrInvalidRequest + } + objectEntry.Extended[s3_constants.ExtAmzAclKey] = grantsBytes } - objectEntry.Extended[s3_constants.ExtAmzAclKey] = grantsBytes + return s3err.ErrNone } diff --git a/weed/s3api/s3acl/acl_helper_test.go b/weed/s3api/s3acl/acl_helper_test.go index b58fcc171..41f719235 100644 --- a/weed/s3api/s3acl/acl_helper_test.go +++ b/weed/s3api/s3acl/acl_helper_test.go @@ -1,80 +1,534 @@ package s3acl import ( + "bytes" + "encoding/json" "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "io" + "net/http" "testing" ) -func TestParseAclHeaders(t *testing.T) { - accountManager := &s3account.AccountManager{ +var ( + accountManager = &s3account.AccountManager{ IdNameMapping: map[string]string{ s3account.AccountAdmin.Id: s3account.AccountAdmin.Name, s3account.AccountAnonymous.Id: s3account.AccountAnonymous.Name, + "accountA": "accountA", + "accountB": "accountB", }, EmailIdMapping: map[string]string{ s3account.AccountAdmin.EmailAddress: s3account.AccountAdmin.Id, s3account.AccountAnonymous.EmailAddress: s3account.AccountAnonymous.Id, + "accountA@example.com": "accountA", + "accountBexample.com": "accountB", }, } +) + +func TestGetAccountId(t *testing.T) { + req := &http.Request{ + Header: make(map[string][]string, 0), + } + //case1 + //accountId: "admin" + req.Header.Set(s3_constants.AmzAccountId, s3account.AccountAdmin.Id) + if GetAccountId(req) != s3account.AccountAdmin.Id { + t.Fatal("expect accountId: admin") + } + + //case2 + //accountId: "anoymous" + req.Header.Set(s3_constants.AmzAccountId, s3account.AccountAnonymous.Id) + if GetAccountId(req) != s3account.AccountAnonymous.Id { + t.Fatal("expect accountId: anonymous") + } + + //case3 + //accountId is nil => "anonymous" + req.Header.Del(s3_constants.AmzAccountId) + if GetAccountId(req) != s3account.AccountAnonymous.Id { + t.Fatal("expect accountId: anonymous") + } +} + +func TestExtractAcl(t *testing.T) { + type Case struct { + id int + resultErrCode, expectErrCode s3err.ErrorCode + resultGrants, expectGrants []*s3.Grant + } + testCases := make([]*Case, 0) + accountAdminId := "admin" + + { + //case1 (good case) + //parse acp from request body + req := &http.Request{ + Header: make(map[string][]string, 0), + } + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + objectWriter := "accountA" + grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter) + testCases = append(testCases, &Case{ + 1, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountAdminId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + + { + //case2 (good case) + //parse acp from header (cannedAcl) + req := &http.Request{ + Header: make(map[string][]string, 0), + } + req.Body = nil + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate) + objectWriter := "accountA" + grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter) + testCases = append(testCases, &Case{ + 2, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &objectWriter, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + + { + //case3 (bad case) + //parse acp from request body (content is invalid) + req := &http.Request{ + Header: make(map[string][]string, 0), + } + req.Body = io.NopCloser(bytes.NewReader([]byte("zdfsaf"))) + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate) + objectWriter := "accountA" + _, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter) + testCases = append(testCases, &Case{ + id: 3, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + //case4 (bad case) + //parse acp from header (cannedAcl is invalid) + req := &http.Request{ + Header: make(map[string][]string, 0), + } + req.Body = nil + req.Header.Set(s3_constants.AmzCannedAcl, "dfaksjfk") + objectWriter := "accountA" + _, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, "", objectWriter) + testCases = append(testCases, &Case{ + id: 4, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + + { + //case5 (bad case) + //parse acp from request body: owner is inconsistent + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + objectWriter = "accountA" + _, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, objectWriter, objectWriter) + testCases = append(testCases, &Case{ + id: 5, + resultErrCode: errCode, expectErrCode: s3err.ErrAccessDenied, + }) + } - //good value - grants := make([]*s3.Grant, 0) - validHeaderValue := `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example.com"` - errCode := ParseCustomAclHeader(validHeaderValue, s3_constants.PermissionFullControl, &grants) - if errCode != s3err.ErrNone { - t.Fatal(errCode) + for _, tc := range testCases { + if tc.resultErrCode != tc.expectErrCode { + t.Fatalf("case[%d]: errorCode not expect", tc.id) + } + if !grantsEquals(tc.resultGrants, tc.expectGrants) { + t.Fatalf("case[%d]: grants not expect", tc.id) + } } - _, errCode = ValidateAndTransferGrants(accountManager, grants) - if errCode != s3err.ErrNone { - t.Fatal(errCode) +} + +func TestParseAndValidateAclHeaders(t *testing.T) { + type Case struct { + id int + resultOwner, expectOwner string + resultErrCode, expectErrCode s3err.ErrorCode + resultGrants, expectGrants []*s3.Grant } + testCases := make([]*Case, 0) + bucketOwner := "admin" - //bad case: acl header format error - grants = make([]*s3.Grant, 0) - formatErrCase := `uri, id="anonymous", emailAddress="admin@example.com"` - errCode = ParseCustomAclHeader(formatErrCase, s3_constants.PermissionFullControl, &grants) - if errCode != s3err.ErrInvalidRequest { - t.Fatal(errCode) + { + //case1 (good case) + //parse custom acl + req := &http.Request{ + Header: make(map[string][]string, 0), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example.com"`) + ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + 1, + ownerId, objectWriter, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &s3account.AccountAnonymous.Id, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &s3account.AccountAdmin.Id, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //case2 (good case) + //parse canned acl (ownership=ObjectWriter) + req := &http.Request{ + Header: make(map[string][]string, 0), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + 2, + ownerId, objectWriter, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &objectWriter, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwner, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //case3 (good case) + //parse canned acl (ownership=OwnershipBucketOwnerPreferred) + req := &http.Request{ + Header: make(map[string][]string, 0), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + 3, + ownerId, bucketOwner, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwner, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //case4 (bad case) + //parse custom acl (grantee id not exists) + req := &http.Request{ + Header: make(map[string][]string, 0), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="notExistsAccount", emailAddress="admin@example.com"`) + _, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + id: 4, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + { + //case5 (bad case) + //parse custom acl (invalid format) + req := &http.Request{ + Header: make(map[string][]string, 0), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzAclFullControl, `uri="http:sfasf"`) + _, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + id: 5, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + { + //case6 (bad case) + //parse canned acl (invalid value) + req := &http.Request{ + Header: make(map[string][]string, 0), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzCannedAcl, `uri="http:sfasf"`) + _, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + id: 5, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + for _, tc := range testCases { + if tc.expectErrCode != tc.resultErrCode { + t.Errorf("case[%d]: errCode unexpect", tc.id) + } + if tc.resultOwner != tc.expectOwner { + t.Errorf("case[%d]: ownerId unexpect", tc.id) + } + if !grantsEquals(tc.resultGrants, tc.expectGrants) { + t.Fatalf("case[%d]: grants not expect", tc.id) + } } +} - //bad case: email not exists - grants = make([]*s3.Grant, 0) - badCaseOfEmail := `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example1.com"` - errCode = ParseCustomAclHeader(badCaseOfEmail, s3_constants.PermissionFullControl, &grants) - if errCode != s3err.ErrNone { - t.Fatal(errCode) +func grantsEquals(a, b []*s3.Grant) bool { + if len(a) != len(b) { + return false } - _, errCode = ValidateAndTransferGrants(accountManager, grants) - if errCode != s3err.ErrInvalidRequest { - t.Fatal(errCode) + for i, grant := range a { + if !GrantEquals(grant, b[i]) { + return false + } } + return true +} - //bad case: account id not exists - grants = make([]*s3.Grant, 0) - badCaseOfAccountId := "uri=\"http://acs.amazonaws.com/groups/global/AllUsers\", id=\"xxxxxx\", emailAddress=\"admin@example.com\"" - errCode = ParseCustomAclHeader(badCaseOfAccountId, s3_constants.PermissionFullControl, &grants) - if errCode != s3err.ErrNone { - t.Fatal(errCode) +func TestDetermineReqGrants(t *testing.T) { + { + //case1: request account is anonymous + accountId := s3account.AccountAnonymous.Id + reqPermission := s3_constants.PermissionRead + + resultGrants := DetermineReqGrants(accountId, reqPermission) + expectGrants := []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &reqPermission, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountId, + }, + Permission: &reqPermission, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + } + if !grantsEquals(resultGrants, expectGrants) { + t.Fatalf("grants not expect") + } } - _, errCode = ValidateAndTransferGrants(accountManager, grants) - if errCode != s3err.ErrInvalidRequest { - t.Fatal(errCode) + { + //case2: request account is not anonymous (Iam authed) + accountId := "accountX" + reqPermission := s3_constants.PermissionRead + + resultGrants := DetermineReqGrants(accountId, reqPermission) + expectGrants := []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &reqPermission, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountId, + }, + Permission: &reqPermission, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAuthenticatedUsers, + }, + Permission: &reqPermission, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAuthenticatedUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }, + } + if !grantsEquals(resultGrants, expectGrants) { + t.Fatalf("grants not expect") + } } +} + +func TestAssembleEntryWithAcp(t *testing.T) { + defaultOwner := "admin" + { + //case1 + expectOwner := "accountS" + expectGrants := []*s3.Grant{ + { + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + ID: &s3account.AccountAdmin.Id, + URI: &s3_constants.GranteeGroupAllUsers, + }, + }, + } + entry := &filer_pb.Entry{} + AssembleEntryWithAcp(entry, expectOwner, expectGrants) + + resultOwner := GetAcpOwner(entry.Extended, defaultOwner) + if resultOwner != expectOwner { + t.Fatalf("owner not expect") + } - //bad case: group url not valid - grants = make([]*s3.Grant, 0) - badCaseOfURL := "uri=\"http://acs.amazonaws.com/groups/global/AllUsers111xxxx\", id=\"anonymous\", emailAddress=\"admin@example.com\"" - errCode = ParseCustomAclHeader(badCaseOfURL, s3_constants.PermissionFullControl, &grants) - if errCode != s3err.ErrNone { - t.Fatal(errCode) + resultGrants := GetAcpGrants(entry.Extended) + if !grantsEquals(resultGrants, expectGrants) { + t.Fatal("grants not expect") + } } - _, errCode = ValidateAndTransferGrants(accountManager, grants) - if errCode != s3err.ErrInvalidRequest { - t.Fatal(errCode) + { + //case2 + entry := &filer_pb.Entry{} + AssembleEntryWithAcp(entry, "", nil) + + resultOwner := GetAcpOwner(entry.Extended, defaultOwner) + if resultOwner != defaultOwner { + t.Fatalf("owner not expect") + } + + resultGrants := GetAcpGrants(entry.Extended) + if len(resultGrants) != 0 { + t.Fatal("grants not expect") + } } + } func TestGrantEquals(t *testing.T) { @@ -218,3 +672,37 @@ func TestGrantEquals(t *testing.T) { } } } + +func TestSetAcpOwnerHeader(t *testing.T) { + ownerId := "accountZ" + req := &http.Request{ + Header: make(map[string][]string, 0), + } + SetAcpOwnerHeader(req, ownerId) + + if req.Header.Get(s3_constants.ExtAmzOwnerKey) != ownerId { + t.Fatalf("owner unexpect") + } +} + +func TestSetAcpGrantsHeader(t *testing.T) { + req := &http.Request{ + Header: make(map[string][]string, 0), + } + grants := []*s3.Grant{ + { + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + ID: &s3account.AccountAdmin.Id, + URI: &s3_constants.GranteeGroupAllUsers, + }, + }, + } + SetAcpGrantsHeader(req, grants) + + grantsJson, _ := json.Marshal(grants) + if req.Header.Get(s3_constants.ExtAmzAclKey) != string(grantsJson) { + t.Fatalf("owner unexpect") + } +} From f1c05eb2fbe2f513d273124145bf7033ed05e79f Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 13 Oct 2022 10:17:07 +0800 Subject: [PATCH 03/52] remove 0 when create map Signed-off-by: changlin.shi --- weed/s3api/s3acl/acl_helper.go | 2 +- weed/s3api/s3acl/acl_helper_test.go | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/weed/s3api/s3acl/acl_helper.go b/weed/s3api/s3acl/acl_helper.go index 350e703c4..e54e67556 100644 --- a/weed/s3api/s3acl/acl_helper.go +++ b/weed/s3api/s3acl/acl_helper.go @@ -406,7 +406,7 @@ func GetAcpGrants(entryExtended map[string][]byte) []*s3.Grant { // AssembleEntryWithAcp fill entry with owner and grants func AssembleEntryWithAcp(objectEntry *filer_pb.Entry, objectOwner string, grants []*s3.Grant) s3err.ErrorCode { if objectEntry.Extended == nil { - objectEntry.Extended = make(map[string][]byte, 0) + objectEntry.Extended = make(map[string][]byte) } if len(objectOwner) > 0 { diff --git a/weed/s3api/s3acl/acl_helper_test.go b/weed/s3api/s3acl/acl_helper_test.go index 41f719235..efc137989 100644 --- a/weed/s3api/s3acl/acl_helper_test.go +++ b/weed/s3api/s3acl/acl_helper_test.go @@ -32,7 +32,7 @@ var ( func TestGetAccountId(t *testing.T) { req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } //case1 //accountId: "admin" @@ -69,7 +69,7 @@ func TestExtractAcl(t *testing.T) { //case1 (good case) //parse acp from request body req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } req.Body = io.NopCloser(bytes.NewReader([]byte(` @@ -121,7 +121,7 @@ func TestExtractAcl(t *testing.T) { //case2 (good case) //parse acp from header (cannedAcl) req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } req.Body = nil req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate) @@ -146,7 +146,7 @@ func TestExtractAcl(t *testing.T) { //case3 (bad case) //parse acp from request body (content is invalid) req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } req.Body = io.NopCloser(bytes.NewReader([]byte("zdfsaf"))) req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate) @@ -161,7 +161,7 @@ func TestExtractAcl(t *testing.T) { //case4 (bad case) //parse acp from header (cannedAcl is invalid) req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } req.Body = nil req.Header.Set(s3_constants.AmzCannedAcl, "dfaksjfk") @@ -229,7 +229,7 @@ func TestParseAndValidateAclHeaders(t *testing.T) { //case1 (good case) //parse custom acl req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } objectWriter := "accountA" req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example.com"`) @@ -267,7 +267,7 @@ func TestParseAndValidateAclHeaders(t *testing.T) { //case2 (good case) //parse canned acl (ownership=ObjectWriter) req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } objectWriter := "accountA" req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) @@ -298,7 +298,7 @@ func TestParseAndValidateAclHeaders(t *testing.T) { //case3 (good case) //parse canned acl (ownership=OwnershipBucketOwnerPreferred) req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } objectWriter := "accountA" req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) @@ -322,7 +322,7 @@ func TestParseAndValidateAclHeaders(t *testing.T) { //case4 (bad case) //parse custom acl (grantee id not exists) req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } objectWriter := "accountA" req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="notExistsAccount", emailAddress="admin@example.com"`) @@ -337,7 +337,7 @@ func TestParseAndValidateAclHeaders(t *testing.T) { //case5 (bad case) //parse custom acl (invalid format) req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } objectWriter := "accountA" req.Header.Set(s3_constants.AmzAclFullControl, `uri="http:sfasf"`) @@ -352,7 +352,7 @@ func TestParseAndValidateAclHeaders(t *testing.T) { //case6 (bad case) //parse canned acl (invalid value) req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } objectWriter := "accountA" req.Header.Set(s3_constants.AmzCannedAcl, `uri="http:sfasf"`) @@ -676,7 +676,7 @@ func TestGrantEquals(t *testing.T) { func TestSetAcpOwnerHeader(t *testing.T) { ownerId := "accountZ" req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } SetAcpOwnerHeader(req, ownerId) @@ -687,7 +687,7 @@ func TestSetAcpOwnerHeader(t *testing.T) { func TestSetAcpGrantsHeader(t *testing.T) { req := &http.Request{ - Header: make(map[string][]string, 0), + Header: make(map[string][]string), } grants := []*s3.Grant{ { From daf5b4d59b1445e7555d478faff2331a976401a6 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 13 Oct 2022 17:39:37 +0800 Subject: [PATCH 04/52] add acl validation for s3 GetObjectHandler --- weed/s3api/s3_constants/header.go | 1 + weed/s3api/s3acl/acl_helper.go | 36 +++++++++++++++++++++++ weed/s3api/s3api_acp.go | 20 +++++++++++++ weed/s3api/s3api_object_handlers.go | 17 ++++++++--- weed/server/filer_server_handlers_read.go | 10 +++++++ 5 files changed, 80 insertions(+), 4 deletions(-) diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index 5037f4691..e4581bff6 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -38,6 +38,7 @@ const ( AmzTagCount = "x-amz-tagging-count" X_SeaweedFS_Header_Directory_Key = "x-seaweedfs-is-directory-key" + XSeaweedFSHeaderAmzBucketOwnerId = "x-seaweedfs-amz-bucket-owner-id" // S3 ACL headers AmzCannedAcl = "X-Amz-Acl" diff --git a/weed/s3api/s3acl/acl_helper.go b/weed/s3api/s3acl/acl_helper.go index e54e67556..28796451a 100644 --- a/weed/s3api/s3acl/acl_helper.go +++ b/weed/s3api/s3acl/acl_helper.go @@ -5,6 +5,7 @@ import ( "encoding/xml" "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" @@ -503,3 +504,38 @@ func GrantEquals(a, b *s3.Grant) bool { } return true } + +func CheckObjectAccessForReadObject(r *http.Request, w http.ResponseWriter, entry *filer.Entry, bucketOwnerId string) (statusCode int, ok bool) { + if entry.IsDirectory() { + w.Header().Set(s3_constants.X_SeaweedFS_Header_Directory_Key, "true") + return http.StatusMethodNotAllowed, false + } + + accountId := GetAccountId(r) + if len(accountId) == 0 { + glog.Warning("#checkObjectAccessForReadObject header[accountId] not exists!") + return http.StatusForbidden, false + } + + //owner access + objectOwner := GetAcpOwner(entry.Extended, bucketOwnerId) + if accountId == objectOwner { + return http.StatusOK, true + } + + //find in Grants + acpGrants := GetAcpGrants(entry.Extended) + if acpGrants != nil { + reqGrants := DetermineReqGrants(accountId, s3_constants.PermissionRead) + for _, requiredGrant := range reqGrants { + for _, grant := range acpGrants { + if GrantEquals(requiredGrant, grant) { + return http.StatusOK, true + } + } + } + } + + glog.V(3).Infof("acl denied! request account id: %s", accountId) + return http.StatusForbidden, false +} diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index 7a76c2a67..185a0cc7e 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -27,3 +27,23 @@ func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s } return s3err.ErrAccessDenied } + +// Check Object-Read related access +// includes: +// - GetObjectHandler +// +// offload object access validation to Filer layer +// - s3acl.CheckObjectAccessForReadObject +func (s3a *S3ApiServer) checkBucketAccessForReadObject(r *http.Request, bucket string) s3err.ErrorCode { + bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) + if errCode != s3err.ErrNone { + return errCode + } + + if bucketMetadata.ObjectOwnership != s3_constants.OwnershipBucketOwnerEnforced { + //offload object acl validation to filer layer + r.Header.Set(s3_constants.XSeaweedFSHeaderAmzBucketOwnerId, *bucketMetadata.Owner.ID) + } + + return s3err.ErrNone +} diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 2fc0111a4..d3aef57c3 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -164,6 +164,12 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) bucket, object := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("GetObjectHandler %s %s", bucket, object) + errCode := s3a.checkBucketAccessForReadObject(r, bucket) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + if strings.HasSuffix(r.URL.Path, "/") { s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) return @@ -371,14 +377,17 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des } defer util.CloseResponse(resp) - if resp.StatusCode == http.StatusPreconditionFailed { + switch resp.StatusCode { + case http.StatusPreconditionFailed: s3err.WriteErrorResponse(w, r, s3err.ErrPreconditionFailed) return - } - - if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable { + case http.StatusRequestedRangeNotSatisfiable: s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRange) return + case http.StatusForbidden: + s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied) + return + default: } if r.Method == "DELETE" { diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 645a3fb44..b36a64f1f 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/util/mem" "io" "math" @@ -107,6 +108,15 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) return } + //s3 acl offload to filer + offloadHeaderBucketOwner := r.Header.Get(s3_constants.XSeaweedFSHeaderAmzBucketOwnerId) + if len(offloadHeaderBucketOwner) > 0 { + if statusCode, ok := s3acl.CheckObjectAccessForReadObject(r, w, entry, offloadHeaderBucketOwner); !ok { + w.WriteHeader(statusCode) + return + } + } + query := r.URL.Query() if entry.IsDirectory() { From bbecba266d585044c6407a8a44dbf8296f777def Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 13 Oct 2022 16:23:58 +0800 Subject: [PATCH 05/52] extract and save acl when create bucket --- weed/s3api/s3api_acp.go | 21 +++++++++++++++++++++ weed/s3api/s3api_bucket_handlers.go | 8 ++++++++ 2 files changed, 29 insertions(+) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index 7a76c2a67..7aa19babd 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -1,8 +1,10 @@ package s3api import ( + "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "net/http" ) @@ -27,3 +29,22 @@ func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s } return s3err.ErrAccessDenied } + +func (s3a *S3ApiServer) ExtractBucketAcp(r *http.Request) (owner string, grants []*s3.Grant, errCode s3err.ErrorCode) { + accountId := s3acl.GetAccountId(r) + + ownership := s3_constants.DefaultOwnershipForCreate + if ownership == s3_constants.OwnershipBucketOwnerEnforced { + return accountId, []*s3.Grant{ + { + Permission: &s3_constants.PermissionFullControl, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountId, + }, + }, + }, s3err.ErrNone + } else { + return s3acl.ParseAndValidateAclHeadersOrElseDefault(r, s3a.accountManager, ownership, accountId, accountId, false) + } +} diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 9e215db9e..efe069237 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/util" "math" "net/http" @@ -121,6 +122,12 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) } } + acpOwner, acpGrants, errCode := s3a.ExtractBucketAcp(r) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + fn := func(entry *filer_pb.Entry) { if identityId := r.Header.Get(s3_constants.AmzIdentityId); identityId != "" { if entry.Extended == nil { @@ -128,6 +135,7 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) } entry.Extended[s3_constants.AmzIdentityId] = []byte(identityId) } + s3acl.AssembleEntryWithAcp(entry, acpOwner, acpGrants) } // create the folder for bucket, but lazily create actual collection From 558ba1094aeb78ea9284d2dedf5c2e833785e1d2 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 13 Oct 2022 17:03:31 +0800 Subject: [PATCH 06/52] implement `PutBucketAclHandler` Signed-off-by: changlin.shi --- weed/s3api/s3api_acp.go | 36 +++++++++++++++++++++ weed/s3api/s3api_bucket_handlers.go | 41 ++++++++++++++++++++++++ weed/s3api/s3api_bucket_skip_handlers.go | 6 ---- weed/s3api/s3err/s3api_errors.go | 7 +++- 4 files changed, 83 insertions(+), 7 deletions(-) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index 7a76c2a67..6d2a62965 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -1,8 +1,11 @@ package s3api import ( + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "net/http" ) @@ -27,3 +30,36 @@ func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s } return s3err.ErrAccessDenied } + +//Check access for PutBucketAclHandler +func (s3a *S3ApiServer) checkAccessForPutBucketAcl(accountId, bucket string) (*BucketMetaData, s3err.ErrorCode) { + bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) + if errCode != s3err.ErrNone { + return nil, errCode + } + + if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced { + return nil, s3err.AccessControlListNotSupported + } + + if accountId == s3account.AccountAdmin.Id || accountId == *bucketMetadata.Owner.ID { + return bucketMetadata, s3err.ErrNone + } + + if len(bucketMetadata.Acl) > 0 { + reqGrants := s3acl.DetermineReqGrants(accountId, s3_constants.PermissionWriteAcp) + for _, bucketGrant := range bucketMetadata.Acl { + for _, reqGrant := range reqGrants { + if s3acl.GrantEquals(bucketGrant, reqGrant) { + return bucketMetadata, s3err.ErrNone + } + } + } + } + glog.V(3).Infof("acl denied! request account id: %s", accountId) + return nil, s3err.ErrAccessDenied +} + +func updateBucketEntry(s3a *S3ApiServer, entry *filer_pb.Entry) error { + return s3a.updateEntry(s3a.option.BucketsPath, entry) +} diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 9e215db9e..7aa5ce16f 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/util" "math" "net/http" @@ -238,6 +239,46 @@ func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool { return true } +// PutBucketAclHandler Put bucket ACL +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html +func (s3a *S3ApiServer) PutBucketAclHandler(w http.ResponseWriter, r *http.Request) { + bucket, _ := s3_constants.GetBucketAndObject(r) + glog.V(3).Infof("PutBucketAclHandler %s", bucket) + + accountId := s3acl.GetAccountId(r) + bucketMetadata, errorCode := s3a.checkAccessForPutBucketAcl(accountId, bucket) + if errorCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errorCode) + return + } + + grants, errCode := s3acl.ExtractAcl(r, s3a.accountManager, bucketMetadata.ObjectOwnership, "", *bucketMetadata.Owner.ID, accountId) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + + bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket) + if err != nil { + glog.Warning(err) + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + + errCode = s3acl.AssembleEntryWithAcp(bucketEntry, *bucketMetadata.Owner.ID, grants) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + + err = updateBucketEntry(s3a, bucketEntry) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + s3err.WriteEmptyResponse(w, r, http.StatusOK) +} + // GetBucketAclHandler Get Bucket ACL // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html func (s3a *S3ApiServer) GetBucketAclHandler(w http.ResponseWriter, r *http.Request) { diff --git a/weed/s3api/s3api_bucket_skip_handlers.go b/weed/s3api/s3api_bucket_skip_handlers.go index 70fd38424..62d5b8ce7 100644 --- a/weed/s3api/s3api_bucket_skip_handlers.go +++ b/weed/s3api/s3api_bucket_skip_handlers.go @@ -41,9 +41,3 @@ func (s3a *S3ApiServer) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Re func (s3a *S3ApiServer) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { s3err.WriteErrorResponse(w, r, http.StatusNoContent) } - -// PutBucketAclHandler Put bucket ACL -// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html -func (s3a *S3ApiServer) PutBucketAclHandler(w http.ResponseWriter, r *http.Request) { - s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) -} diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index 0348d4ddc..cdda16e87 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -109,6 +109,7 @@ const ( ErrRequestBytesExceed OwnershipControlsNotFoundError + AccessControlListNotSupported ) // error code to APIError structure, these fields carry respective @@ -416,12 +417,16 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Simultaneous request bytes exceed limitations", HTTPStatusCode: http.StatusTooManyRequests, }, - OwnershipControlsNotFoundError: { Code: "OwnershipControlsNotFoundError", Description: "The bucket ownership controls were not found", HTTPStatusCode: http.StatusNotFound, }, + AccessControlListNotSupported: { + Code: "AccessControlListNotSupported", + Description: "The bucket does not allow ACLs", + HTTPStatusCode: http.StatusBadRequest, + }, } // GetAPIError provides API Error for input API error code. From e76f66a0d0878dd9647cdef2250a205206d2b98b Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 13 Oct 2022 17:12:06 +0800 Subject: [PATCH 07/52] add acl validation for s3 bucket read api --- weed/s3api/s3api_acp.go | 38 +++++++++++++++++++++ weed/s3api/s3api_bucket_handlers.go | 41 ++++++++--------------- weed/s3api/s3api_objects_list_handlers.go | 12 +++++++ 3 files changed, 64 insertions(+), 27 deletions(-) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index 7a76c2a67..065457ef9 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -1,8 +1,10 @@ package s3api import ( + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "net/http" ) @@ -27,3 +29,39 @@ func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s } return s3err.ErrAccessDenied } + +// Check Bucket/BucketAcl Read related access +// includes: +// - HeadBucketHandler +// - GetBucketAclHandler +// - ListObjectsV1Handler +// - ListObjectsV2Handler +func (s3a *S3ApiServer) checkAccessForReadBucket(r *http.Request, bucket, aclAction string) (*BucketMetaData, s3err.ErrorCode) { + bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) + if errCode != s3err.ErrNone { + return nil, errCode + } + + if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced { + return bucketMetadata, s3err.ErrNone + } + + accountId := s3acl.GetAccountId(r) + if accountId == s3account.AccountAdmin.Id || accountId == *bucketMetadata.Owner.ID { + return bucketMetadata, s3err.ErrNone + } + + if len(bucketMetadata.Acl) > 0 { + reqGrants := s3acl.DetermineReqGrants(accountId, aclAction) + for _, bucketGrant := range bucketMetadata.Acl { + for _, reqGrant := range reqGrants { + if s3acl.GrantEquals(bucketGrant, reqGrant) { + return bucketMetadata, s3err.ErrNone + } + } + } + } + + glog.V(3).Infof("acl denied! request account id: %s", accountId) + return nil, s3err.ErrAccessDenied +} diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 9e215db9e..6f94fd027 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -200,6 +200,11 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request bucket, _ := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("HeadBucketHandler %s", bucket) + _, errorCode := s3a.checkAccessForReadBucket(r, bucket, s3_constants.PermissionRead) + if errorCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errorCode) + return + } if entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket); entry == nil || err == filer_pb.ErrNotFound { s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket) return @@ -245,37 +250,19 @@ func (s3a *S3ApiServer) GetBucketAclHandler(w http.ResponseWriter, r *http.Reque bucket, _ := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("GetBucketAclHandler %s", bucket) - if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone { - s3err.WriteErrorResponse(w, r, err) + bucketMetadata, errorCode := s3a.checkAccessForReadBucket(r, bucket, s3_constants.PermissionReadAcp) + if s3err.ErrNone != errorCode { + s3err.WriteErrorResponse(w, r, errorCode) return } - response := AccessControlPolicy{} - for _, ident := range s3a.iam.identities { - if len(ident.Credentials) == 0 { - continue - } - for _, action := range ident.Actions { - if !action.overBucket(bucket) || action.getPermission() == "" { - continue - } - id := ident.Credentials[0].AccessKey - if response.Owner.DisplayName == "" && action.isOwner(bucket) && len(ident.Credentials) > 0 { - response.Owner.DisplayName = ident.Name - response.Owner.ID = id - } - response.AccessControlList.Grant = append(response.AccessControlList.Grant, Grant{ - Grantee: Grantee{ - ID: id, - DisplayName: ident.Name, - Type: "CanonicalUser", - XMLXSI: "CanonicalUser", - XMLNS: "http://www.w3.org/2001/XMLSchema-instance"}, - Permission: action.getPermission(), - }) - } + acp := &s3.PutBucketAclInput{ + AccessControlPolicy: &s3.AccessControlPolicy{ + Grants: bucketMetadata.Acl, + Owner: bucketMetadata.Owner, + }, } - writeSuccessResponseXML(w, r, response) + s3err.WriteAwsXMLResponse(w, r, http.StatusOK, acp) } // GetBucketLifecycleConfigurationHandler Get Bucket Lifecycle configuration diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 620969fd6..97ba83d19 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -41,6 +41,12 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ bucket, _ := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("ListObjectsV2Handler %s", bucket) + _, errCode := s3a.checkAccessForReadBucket(r, bucket, s3_constants.PermissionRead) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query()) if maxKeys < 0 { @@ -97,6 +103,12 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ bucket, _ := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("ListObjectsV1Handler %s", bucket) + _, errCode := s3a.checkAccessForReadBucket(r, bucket, s3_constants.PermissionRead) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query()) if maxKeys < 0 { From af093721a2632cde4ad68b548095cac318cfbc4d Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 13 Oct 2022 17:20:06 +0800 Subject: [PATCH 08/52] add acl support for `PutObject` and `PutObjectPart` --- weed/s3api/chunked_reader_v4.go | 28 ++--- weed/s3api/s3api_acp.go | 105 ++++++++++++++++++ weed/s3api/s3api_object_handlers.go | 19 +++- weed/s3api/s3api_object_multipart_handlers.go | 14 ++- weed/s3api/s3err/s3api_errors.go | 7 +- 5 files changed, 152 insertions(+), 21 deletions(-) diff --git a/weed/s3api/chunked_reader_v4.go b/weed/s3api/chunked_reader_v4.go index 8ba1bc479..f0aa30d54 100644 --- a/weed/s3api/chunked_reader_v4.go +++ b/weed/s3api/chunked_reader_v4.go @@ -59,7 +59,7 @@ func getChunkSignature(secretKey string, seedSignature string, region string, da // // returns signature, error otherwise if the signature mismatches or any other // error while parsing and validating. -func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cred *Credential, signature string, region string, date time.Time, errCode s3err.ErrorCode) { +func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (idnt *Identity, cred *Credential, signature string, region string, date time.Time, errCode s3err.ErrorCode) { // Copy request. req := *r @@ -70,7 +70,7 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr // Parse signature version '4' header. signV4Values, errCode := parseSignV4(v4Auth) if errCode != s3err.ErrNone { - return nil, "", "", time.Time{}, errCode + return nil, nil, "", "", time.Time{}, errCode } // Payload streaming. @@ -78,18 +78,18 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr // Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' if payload != req.Header.Get("X-Amz-Content-Sha256") { - return nil, "", "", time.Time{}, s3err.ErrContentSHA256Mismatch + return nil, nil, "", "", time.Time{}, s3err.ErrContentSHA256Mismatch } // Extract all the signed headers along with its values. extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r) if errCode != s3err.ErrNone { - return nil, "", "", time.Time{}, errCode + return nil, nil, "", "", time.Time{}, errCode } // Verify if the access key id matches. identity, cred, found := iam.lookupByAccessKey(signV4Values.Credential.accessKey) if !found { - return nil, "", "", time.Time{}, s3err.ErrInvalidAccessKeyID + return nil, nil, "", "", time.Time{}, s3err.ErrInvalidAccessKeyID } bucket, object := s3_constants.GetBucketAndObject(r) @@ -105,14 +105,14 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr var dateStr string if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" { if dateStr = r.Header.Get("Date"); dateStr == "" { - return nil, "", "", time.Time{}, s3err.ErrMissingDateHeader + return nil, nil, "", "", time.Time{}, s3err.ErrMissingDateHeader } } // Parse date header. var err error date, err = time.Parse(iso8601Format, dateStr) if err != nil { - return nil, "", "", time.Time{}, s3err.ErrMalformedDate + return nil, nil, "", "", time.Time{}, s3err.ErrMalformedDate } // Query string. @@ -132,11 +132,11 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr // Verify if signature match. if !compareSignatureV4(newSignature, signV4Values.Signature) { - return nil, "", "", time.Time{}, s3err.ErrSignatureDoesNotMatch + return nil, nil, "", "", time.Time{}, s3err.ErrSignatureDoesNotMatch } // Return calculated signature. - return cred, newSignature, region, date, s3err.ErrNone + return identity, cred, newSignature, region, date, s3err.ErrNone } const maxLineLength = 4 * humanize.KiByte // assumed <= bufio.defaultBufSize 4KiB @@ -150,20 +150,20 @@ var errMalformedEncoding = errors.New("malformed chunked encoding") // newSignV4ChunkedReader returns a new s3ChunkedReader that translates the data read from r // out of HTTP "chunked" format before returning it. // The s3ChunkedReader returns io.EOF when the final 0-length chunk is read. -func (iam *IdentityAccessManagement) newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, s3err.ErrorCode) { - ident, seedSignature, region, seedDate, errCode := iam.calculateSeedSignature(req) +func (iam *IdentityAccessManagement) newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, *Identity, s3err.ErrorCode) { + ident, cred, seedSignature, region, seedDate, errCode := iam.calculateSeedSignature(req) if errCode != s3err.ErrNone { - return nil, errCode + return nil, nil, errCode } return &s3ChunkedReader{ - cred: ident, + cred: cred, reader: bufio.NewReader(req.Body), seedSignature: seedSignature, seedDate: seedDate, region: region, chunkSHA256Writer: sha256.New(), state: readChunkHeader, - }, s3err.ErrNone + }, ident, s3err.ErrNone } // Represents the overall state that is required for decoding a diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index 7a76c2a67..fbbf2d080 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -1,9 +1,14 @@ package s3api import ( + "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/util" "net/http" ) @@ -27,3 +32,103 @@ func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s } return s3err.ErrAccessDenied } + +// Check Object-Write related access +// includes: +// - PutObjectHandler +// - PutObjectPartHandler +func (s3a *S3ApiServer) checkAccessForWriteObject(r *http.Request, bucket, object string) s3err.ErrorCode { + bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) + if errCode != s3err.ErrNone { + return errCode + } + + accountId := s3acl.GetAccountId(r) + if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced { + // validate grants (only bucketOwnerFullControl acl is allowed) + _, grants, errCode := s3acl.ParseAndValidateAclHeaders(r, s3a.accountManager, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, accountId, false) + if errCode != s3err.ErrNone { + return errCode + } + if len(grants) > 1 { + return s3err.AccessControlListNotSupported + } + bucketOwnerFullControlGrant := &s3.Grant{ + Permission: &s3_constants.PermissionFullControl, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: bucketMetadata.Owner.ID, + }, + } + if len(grants) == 0 { + // set default grants + s3acl.SetAcpOwnerHeader(r, accountId) + s3acl.SetAcpGrantsHeader(r, []*s3.Grant{bucketOwnerFullControlGrant}) + return s3err.ErrNone + } + + if !s3acl.GrantEquals(bucketOwnerFullControlGrant, grants[0]) { + return s3err.AccessControlListNotSupported + } + + s3acl.SetAcpOwnerHeader(r, accountId) + s3acl.SetAcpGrantsHeader(r, []*s3.Grant{bucketOwnerFullControlGrant}) + return s3err.ErrNone + } + + //bucket access allowed + bucketAclAllowed := false + if accountId == *bucketMetadata.Owner.ID { + bucketAclAllowed = true + } else { + if len(bucketMetadata.Acl) > 0 { + reqGrants := s3acl.DetermineReqGrants(accountId, s3_constants.PermissionWrite) + bucketLoop: + for _, bucketGrant := range bucketMetadata.Acl { + for _, requiredGrant := range reqGrants { + if s3acl.GrantEquals(bucketGrant, requiredGrant) { + bucketAclAllowed = true + break bucketLoop + } + } + } + } + } + if !bucketAclAllowed { + glog.V(3).Infof("acl denied! request account id: %s", accountId) + return s3err.ErrAccessDenied + } + + //object access allowed + entry, err := getObjectEntry(s3a, bucket, object) + if err != nil { + if err != filer_pb.ErrNotFound { + return s3err.ErrInternalError + } + } else { + if entry.IsDirectory { + return s3err.ErrExistingObjectIsDirectory + } + + //Only the owner of the bucket and the owner of the object can overwrite the object + objectOwner := s3acl.GetAcpOwner(entry.Extended, *bucketMetadata.Owner.ID) + if accountId != objectOwner && accountId != *bucketMetadata.Owner.ID { + glog.V(3).Infof("acl denied! request account id: %s, expect account id: %s", accountId, *bucketMetadata.Owner.ID) + return s3err.ErrAccessDenied + } + } + + ownerId, grants, errCode := s3acl.ParseAndValidateAclHeadersOrElseDefault(r, s3a.accountManager, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, accountId, false) + if errCode != s3err.ErrNone { + return errCode + } + + s3acl.SetAcpOwnerHeader(r, ownerId) + s3acl.SetAcpGrantsHeader(r, grants) + + return s3err.ErrNone +} + +func getObjectEntry(s3a *S3ApiServer, bucket, object string) (*filer_pb.Entry, error) { + return s3a.getEntry(util.Join(s3a.option.BucketsPath, bucket), object) +} diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 2fc0111a4..9885303a6 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -7,6 +7,7 @@ import ( "encoding/xml" "fmt" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util/mem" "golang.org/x/exp/slices" @@ -72,18 +73,24 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) rAuthType := getRequestAuthType(r) if s3a.iam.isEnabled() { var s3ErrCode s3err.ErrorCode + var identity *Identity switch rAuthType { case authTypeStreamingSigned: - dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r) + dataReader, identity, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r) case authTypeSignedV2, authTypePresignedV2: - _, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r) + identity, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r) case authTypePresigned, authTypeSigned: - _, s3ErrCode = s3a.iam.reqSignatureV4Verify(r) + identity, s3ErrCode = s3a.iam.reqSignatureV4Verify(r) + case authTypeAnonymous: + identity = IdentityAnonymous } if s3ErrCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, s3ErrCode) return } + if identity.AccountId != s3account.AccountAnonymous.Id { + r.Header.Set(s3_constants.AmzAccountId, identity.AccountId) + } } else { if authTypeStreamingSigned == rAuthType { s3err.WriteErrorResponse(w, r, s3err.ErrAuthNotSetup) @@ -92,6 +99,12 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) } defer dataReader.Close() + errCode := s3a.checkAccessForWriteObject(r, bucket, object) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + objectContentType := r.Header.Get("Content-Type") if strings.HasSuffix(object, "/") && r.ContentLength == 0 { if err := s3a.mkdir( diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 96000310c..38e167f6b 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -230,23 +230,31 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ if s3a.iam.isEnabled() { rAuthType := getRequestAuthType(r) var s3ErrCode s3err.ErrorCode + var identity *Identity switch rAuthType { case authTypeStreamingSigned: - dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r) + dataReader, identity, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r) case authTypeSignedV2, authTypePresignedV2: - _, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r) + identity, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r) case authTypePresigned, authTypeSigned: - _, s3ErrCode = s3a.iam.reqSignatureV4Verify(r) + identity, s3ErrCode = s3a.iam.reqSignatureV4Verify(r) } if s3ErrCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, s3ErrCode) return } + r.Header.Set(s3_constants.AmzAccountId, identity.AccountId) } defer dataReader.Close() glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID) + s3ErrCode := s3a.checkAccessForWriteObject(r, bucket, object) + if s3ErrCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, s3ErrCode) + return + } + uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part", s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID) diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index 0348d4ddc..cdda16e87 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -109,6 +109,7 @@ const ( ErrRequestBytesExceed OwnershipControlsNotFoundError + AccessControlListNotSupported ) // error code to APIError structure, these fields carry respective @@ -416,12 +417,16 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Simultaneous request bytes exceed limitations", HTTPStatusCode: http.StatusTooManyRequests, }, - OwnershipControlsNotFoundError: { Code: "OwnershipControlsNotFoundError", Description: "The bucket ownership controls were not found", HTTPStatusCode: http.StatusNotFound, }, + AccessControlListNotSupported: { + Code: "AccessControlListNotSupported", + Description: "The bucket does not allow ACLs", + HTTPStatusCode: http.StatusBadRequest, + }, } // GetAPIError provides API Error for input API error code. From f79a3a73ee2f4f59eef9d19815edd9c30fcd5b30 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 13 Oct 2022 17:33:10 +0800 Subject: [PATCH 09/52] add acl support for `PutObjectAcl` --- weed/s3api/s3api_acp.go | 78 ++++++++++++++++++++++++ weed/s3api/s3api_object_handlers.go | 32 ++++++++++ weed/s3api/s3api_object_skip_handlers.go | 8 --- weed/s3api/s3err/s3api_errors.go | 7 ++- 4 files changed, 116 insertions(+), 9 deletions(-) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index 7a76c2a67..bafe19e88 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -1,9 +1,13 @@ package s3api import ( + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/util" "net/http" ) @@ -27,3 +31,77 @@ func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s } return s3err.ErrAccessDenied } + +// Check ObjectAcl-Write related access +// includes: +// - PutObjectAclHandler +func (s3a *S3ApiServer) checkAccessForWriteObjectAcl(accountId, bucket, object string) (bucketMetadata *BucketMetaData, objectEntry *filer_pb.Entry, objectOwner string, errCode s3err.ErrorCode) { + bucketMetadata, errCode = s3a.bucketRegistry.GetBucketMetadata(bucket) + if errCode != s3err.ErrNone { + return nil, nil, "", errCode + } + + if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced { + return nil, nil, "", s3err.AccessControlListNotSupported + } + + //bucket acl + bucketAclAllowed := false + reqGrants := s3acl.DetermineReqGrants(accountId, s3_constants.PermissionWrite) + if accountId == *bucketMetadata.Owner.ID { + bucketAclAllowed = true + } else if bucketMetadata.Acl != nil { + bucketLoop: + for _, bucketGrant := range bucketMetadata.Acl { + for _, requiredGrant := range reqGrants { + if s3acl.GrantEquals(bucketGrant, requiredGrant) { + bucketAclAllowed = true + break bucketLoop + } + } + } + } + if !bucketAclAllowed { + return nil, nil, "", s3err.ErrAccessDenied + } + + //object acl + objectEntry, err := getObjectEntry(s3a, bucket, object) + if err != nil { + if err == filer_pb.ErrNotFound { + return nil, nil, "", s3err.ErrNoSuchKey + } + return nil, nil, "", s3err.ErrInternalError + } + + if objectEntry.IsDirectory { + return nil, nil, "", s3err.ErrExistingObjectIsDirectory + } + + objectOwner = s3acl.GetAcpOwner(objectEntry.Extended, *bucketMetadata.Owner.ID) + if accountId == objectOwner { + return bucketMetadata, objectEntry, objectOwner, s3err.ErrNone + } + + objectGrants := s3acl.GetAcpGrants(objectEntry.Extended) + if objectGrants != nil { + for _, objectGrant := range objectGrants { + for _, requiredGrant := range reqGrants { + if s3acl.GrantEquals(objectGrant, requiredGrant) { + return bucketMetadata, objectEntry, objectOwner, s3err.ErrNone + } + } + } + } + + glog.V(3).Infof("acl denied! request account id: %s", accountId) + return nil, nil, "", s3err.ErrAccessDenied +} + +func getObjectEntry(s3a *S3ApiServer, bucket, object string) (*filer_pb.Entry, error) { + return s3a.getEntry(util.Join(s3a.option.BucketsPath, bucket), object) +} + +func updateObjectEntry(s3a *S3ApiServer, bucket string, entry *filer_pb.Entry) error { + return s3a.updateEntry(util.Join(s3a.option.BucketsPath, bucket), entry) +} diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 2fc0111a4..8ee11c195 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -7,6 +7,7 @@ import ( "encoding/xml" "fmt" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util/mem" "golang.org/x/exp/slices" @@ -525,3 +526,34 @@ func (s3a *S3ApiServer) maybeGetFilerJwtAuthorizationToken(isWrite bool) string } return string(encodedJwt) } + +// PutObjectAclHandler Put object ACL +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjecthtml +func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) { + bucket, object := s3_constants.GetBucketAndObject(r) + + accountId := s3acl.GetAccountId(r) + bucketMetadata, objectEntry, objectOwner, errCode := s3a.checkAccessForWriteObjectAcl(accountId, bucket, object) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + + grants, errCode := s3acl.ExtractAcl(r, s3a.accountManager, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, objectOwner, accountId) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + + errCode = s3acl.AssembleEntryWithAcp(objectEntry, objectOwner, grants) + if errCode != s3err.ErrNone { + return + } + + err := updateObjectEntry(s3a, bucket, objectEntry) + if err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + w.WriteHeader(http.StatusOK) +} diff --git a/weed/s3api/s3api_object_skip_handlers.go b/weed/s3api/s3api_object_skip_handlers.go index 160d02475..197a2d9b7 100644 --- a/weed/s3api/s3api_object_skip_handlers.go +++ b/weed/s3api/s3api_object_skip_handlers.go @@ -12,14 +12,6 @@ func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Reque } -// PutObjectAclHandler Put object ACL -// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html -func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) { - - w.WriteHeader(http.StatusNoContent) - -} - // PutObjectRetentionHandler Put object Retention // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html func (s3a *S3ApiServer) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) { diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index 0348d4ddc..cdda16e87 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -109,6 +109,7 @@ const ( ErrRequestBytesExceed OwnershipControlsNotFoundError + AccessControlListNotSupported ) // error code to APIError structure, these fields carry respective @@ -416,12 +417,16 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Simultaneous request bytes exceed limitations", HTTPStatusCode: http.StatusTooManyRequests, }, - OwnershipControlsNotFoundError: { Code: "OwnershipControlsNotFoundError", Description: "The bucket ownership controls were not found", HTTPStatusCode: http.StatusNotFound, }, + AccessControlListNotSupported: { + Code: "AccessControlListNotSupported", + Description: "The bucket does not allow ACLs", + HTTPStatusCode: http.StatusBadRequest, + }, } // GetAPIError provides API Error for input API error code. From 4b36721323fac10f532d5e25ca0ec09b5497a953 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 13 Oct 2022 18:18:46 +0800 Subject: [PATCH 10/52] implement `GetObjectAclHandler` --- weed/s3api/s3api_acp.go | 76 ++++++++++++++++++++++++ weed/s3api/s3api_object_handlers.go | 16 +++++ weed/s3api/s3api_object_skip_handlers.go | 8 --- 3 files changed, 92 insertions(+), 8 deletions(-) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index 7a76c2a67..2894f0f6e 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -1,9 +1,14 @@ package s3api import ( + "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/util" "net/http" ) @@ -27,3 +32,74 @@ func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s } return s3err.ErrAccessDenied } + +//Check ObjectAcl-Read related access +// includes: +// - GetObjectAclHandler +func (s3a *S3ApiServer) checkAccessForReadObjectAcl(r *http.Request, bucket, object string) (acp *s3.AccessControlPolicy, errCode s3err.ErrorCode) { + bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) + if errCode != s3err.ErrNone { + return nil, errCode + } + + getAcpFunc := func() (*s3.AccessControlPolicy, s3err.ErrorCode) { + entry, err := getObjectEntry(s3a, bucket, object) + if err != nil { + if err == filer_pb.ErrNotFound { + return nil, s3err.ErrNoSuchKey + } else { + return nil, s3err.ErrInternalError + } + } + + if entry.IsDirectory { + return nil, s3err.ErrExistingObjectIsDirectory + } + + acpOwnerId := s3acl.GetAcpOwner(entry.Extended, *bucketMetadata.Owner.ID) + acpOwnerName := s3a.accountManager.IdNameMapping[acpOwnerId] + acpGrants := s3acl.GetAcpGrants(entry.Extended) + acp = &s3.AccessControlPolicy{ + Owner: &s3.Owner{ + ID: &acpOwnerId, + DisplayName: &acpOwnerName, + }, + Grants: acpGrants, + } + return acp, s3err.ErrNone + } + + if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced { + return getAcpFunc() + } else { + accountId := s3acl.GetAccountId(r) + + acp, errCode := getAcpFunc() + if errCode != s3err.ErrNone { + return nil, errCode + } + + if accountId == *acp.Owner.ID { + return acp, s3err.ErrNone + } + + //find in Grants + if acp.Grants != nil { + reqGrants := s3acl.DetermineReqGrants(accountId, s3_constants.PermissionReadAcp) + for _, requiredGrant := range reqGrants { + for _, grant := range acp.Grants { + if s3acl.GrantEquals(requiredGrant, grant) { + return acp, s3err.ErrNone + } + } + } + } + + glog.V(3).Infof("acl denied! request account id: %s", accountId) + return nil, s3err.ErrAccessDenied + } +} + +func getObjectEntry(s3a *S3ApiServer, bucket, object string) (*filer_pb.Entry, error) { + return s3a.getEntry(util.Join(s3a.option.BucketsPath, bucket), object) +} diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 2fc0111a4..d14cc2896 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -6,6 +6,7 @@ import ( "encoding/json" "encoding/xml" "fmt" + "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/security" "github.com/seaweedfs/seaweedfs/weed/util/mem" @@ -174,6 +175,21 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request) s3a.proxyToFiler(w, r, destUrl, false, passThroughResponse) } +// GetObjectAclHandler Put object ACL +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjecthtml +func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Request) { + bucket, object := s3_constants.GetBucketAndObject(r) + acp, errCode := s3a.checkAccessForReadObjectAcl(r, bucket, object) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + result := &s3.PutBucketAclInput{ + AccessControlPolicy: acp, + } + s3err.WriteAwsXMLResponse(w, r, http.StatusOK, &result) +} + func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { bucket, object := s3_constants.GetBucketAndObject(r) diff --git a/weed/s3api/s3api_object_skip_handlers.go b/weed/s3api/s3api_object_skip_handlers.go index 160d02475..935787fbb 100644 --- a/weed/s3api/s3api_object_skip_handlers.go +++ b/weed/s3api/s3api_object_skip_handlers.go @@ -4,14 +4,6 @@ import ( "net/http" ) -// GetObjectAclHandler Put object ACL -// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAcl.html -func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Request) { - - w.WriteHeader(http.StatusNoContent) - -} - // PutObjectAclHandler Put object ACL // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) { From e576e4cee8515c910e642be748aa0cc1cf14cf14 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 13 Oct 2022 18:20:07 +0800 Subject: [PATCH 11/52] enable anonymous access when bucket acl is enabled Signed-off-by: changlin.shi --- weed/s3api/auth_credentials.go | 37 ++++++++++++++++++++++++----- weed/s3api/s3api_circuit_breaker.go | 2 +- weed/s3api/s3api_server.go | 31 ++++++++++++++---------- 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 82ac3688c..4021dbbb4 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -210,13 +210,26 @@ func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, foun } func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) http.HandlerFunc { + return Auth(iam, nil, f, action, false) +} + +func (s3a *S3ApiServer) Auth(f http.HandlerFunc, action Action, supportAcl bool) http.HandlerFunc { + return Auth(s3a.iam, s3a.bucketRegistry, f, action, supportAcl) +} + +func Auth(iam *IdentityAccessManagement, br *BucketRegistry, f http.HandlerFunc, action Action, supportAcl bool) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + //unset predefined headers + delete(r.Header, s3_constants.AmzAccountId) + delete(r.Header, s3_constants.ExtAmzOwnerKey) + delete(r.Header, s3_constants.ExtAmzAclKey) + if !iam.isEnabled() { f(w, r) return } - identity, errCode := iam.authRequest(r, action) + identity, errCode := authRequest(iam, br, r, action, supportAcl) if errCode == s3err.ErrNone { if identity != nil && identity.Name != "" { r.Header.Set(s3_constants.AmzIdentityId, identity.Name) @@ -233,11 +246,13 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt } } -// check whether the request has valid access keys func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) { + return authRequest(iam, nil, r, action, false) +} + +func authRequest(iam *IdentityAccessManagement, br *BucketRegistry, r *http.Request, action Action, supportAcl bool) (*Identity, s3err.ErrorCode) { var identity *Identity var s3Err s3err.ErrorCode - var found bool var authType string switch getRequestAuthType(r) { case authTypeStreamingSigned: @@ -263,9 +278,19 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) r.Header.Set(s3_constants.AmzAuthType, "Jwt") return identity, s3err.ErrNotImplemented case authTypeAnonymous: + if supportAcl && br != nil { + bucket, _ := s3_constants.GetBucketAndObject(r) + bucketMetadata, errorCode := br.GetBucketMetadata(bucket) + if errorCode != s3err.ErrNone { + return nil, errorCode + } + if bucketMetadata.ObjectOwnership != s3_constants.OwnershipBucketOwnerEnforced { + return IdentityAnonymous, s3err.ErrNone + } + } authType = "Anonymous" - identity, found = iam.lookupAnonymous() - if !found { + identity = IdentityAnonymous + if len(identity.Actions) == 0 { r.Header.Set(s3_constants.AmzAuthType, authType) return identity, s3err.ErrAccessDenied } @@ -280,7 +305,7 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) return identity, s3Err } - glog.V(3).Infof("user name: %v actions: %v, action: %v", identity.Name, identity.Actions, action) + glog.V(3).Infof("user name: %v account id: %v actions: %v, action: %v", identity.Name, identity.AccountId, identity.Actions, action) bucket, object := s3_constants.GetBucketAndObject(r) diff --git a/weed/s3api/s3api_circuit_breaker.go b/weed/s3api/s3api_circuit_breaker.go index 870c65d2b..3f5cab589 100644 --- a/weed/s3api/s3api_circuit_breaker.go +++ b/weed/s3api/s3api_circuit_breaker.go @@ -82,7 +82,7 @@ func (cb *CircuitBreaker) loadCircuitBreakerConfig(cfg *s3_pb.S3CircuitBreakerCo return nil } -func (cb *CircuitBreaker) Limit(f func(w http.ResponseWriter, r *http.Request), action string) (http.HandlerFunc, Action) { +func (cb *CircuitBreaker) Limit(f http.HandlerFunc, action string) (http.HandlerFunc, Action) { return func(w http.ResponseWriter, r *http.Request) { if !cb.Enabled { f(w, r) diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index a8816424d..a0f7c1baa 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -126,7 +126,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // CopyObjectPart bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", `.*?(\/|%2F).*?`).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") // PutObjectPart - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.PutObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") // CompleteMultipartUpload bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CompleteMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploadId", "{uploadId:.*}") // NewMultipartUpload @@ -146,7 +146,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectTaggingHandler, ACTION_TAGGING)), "DELETE")).Queries("tagging", "") // PutObjectACL - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectAclHandler, ACTION_WRITE)), "PUT")).Queries("acl", "") + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.PutObjectAclHandler, ACTION_WRITE)), "PUT")).Queries("acl", "") // PutObjectRetention bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectRetentionHandler, ACTION_WRITE)), "PUT")).Queries("retention", "") // PutObjectLegalHold @@ -155,22 +155,22 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectLockConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("object-lock", "") // GetObjectACL - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectAclHandler, ACTION_READ)), "GET")).Queries("acl", "") + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.GetObjectAclHandler, ACTION_READ)), "GET")).Queries("acl", "") // objects with query // raw objects // HeadObject - bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.HeadObjectHandler, ACTION_READ)), "GET")) + bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.HeadObjectHandler, ACTION_READ)), "GET")) // GetObject, but directory listing is not supported - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectHandler, ACTION_READ)), "GET")) + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.GetObjectHandler, ACTION_READ)), "GET")) // CopyObject bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectHandler, ACTION_WRITE)), "COPY")) // PutObject - bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectHandler, ACTION_WRITE)), "PUT")) + bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.PutObjectHandler, ACTION_WRITE)), "PUT")) // DeleteObject bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectHandler, ACTION_WRITE)), "DELETE")) @@ -182,9 +182,9 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { bucket.Methods("POST").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE)), "DELETE")).Queries("delete", "") // GetBucketACL - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketAclHandler, ACTION_READ)), "GET")).Queries("acl", "") + bucket.Methods("GET").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.GetBucketAclHandler, ACTION_READ)), "GET")).Queries("acl", "") // PutBucketACL - bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketAclHandler, ACTION_WRITE)), "PUT")).Queries("acl", "") + bucket.Methods("PUT").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.PutBucketAclHandler, ACTION_WRITE)), "PUT")).Queries("acl", "") // GetBucketPolicy bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketPolicyHandler, ACTION_READ)), "GET")).Queries("policy", "") @@ -214,17 +214,17 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketRequestPaymentHandler, ACTION_READ)), "GET")).Queries("requestPayment", "") // ListObjectsV2 - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2") + bucket.Methods("GET").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2") // buckets with query // PutBucketOwnershipControls - bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN), "PUT")).Queries("ownershipControls", "") + bucket.Methods("PUT").HandlerFunc(track(s3a.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN, true), "PUT")).Queries("ownershipControls", "") //GetBucketOwnershipControls - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketOwnershipControls, ACTION_READ), "GET")).Queries("ownershipControls", "") + bucket.Methods("GET").HandlerFunc(track(s3a.Auth(s3a.GetBucketOwnershipControls, ACTION_READ, true), "GET")).Queries("ownershipControls", "") //DeleteBucketOwnershipControls - bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN), "DELETE")).Queries("ownershipControls", "") + bucket.Methods("DELETE").HandlerFunc(track(s3a.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN, true), "DELETE")).Queries("ownershipControls", "") // raw buckets @@ -240,7 +240,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_WRITE)), "DELETE")) // ListObjectsV1 (Legacy) - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST")) + bucket.Methods("GET").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST")) // raw buckets @@ -253,3 +253,8 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler) } + +func withAcl(limitFunc func(http.HandlerFunc, string) (http.HandlerFunc, Action), hf http.HandlerFunc, action string) (http.HandlerFunc, Action, bool) { + f, a := limitFunc(hf, action) + return f, a, true +} From 2227888acc1c44bfc8fdd0ac4910cf97bc34176f Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Fri, 14 Oct 2022 12:22:57 +0800 Subject: [PATCH 12/52] validate grants when setting ownership to `OwnershipBucketOwnerEnforced` Signed-off-by: changlin.shi --- weed/s3api/s3acl/acl_helper.go | 10 ++++++++++ weed/s3api/s3acl/acl_helper_test.go | 17 +++++++++++++++++ weed/s3api/s3api_bucket_handlers.go | 20 ++++++++++++++++++++ weed/s3api/s3err/s3api_errors.go | 6 ++++++ 4 files changed, 53 insertions(+) diff --git a/weed/s3api/s3acl/acl_helper.go b/weed/s3api/s3acl/acl_helper.go index e54e67556..4387e64ca 100644 --- a/weed/s3api/s3acl/acl_helper.go +++ b/weed/s3api/s3acl/acl_helper.go @@ -503,3 +503,13 @@ func GrantEquals(a, b *s3.Grant) bool { } return true } + +func GrantWithFullControl(accountId string) *s3.Grant { + return &s3.Grant{ + Permission: &s3_constants.PermissionFullControl, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountId, + }, + } +} diff --git a/weed/s3api/s3acl/acl_helper_test.go b/weed/s3api/s3acl/acl_helper_test.go index efc137989..3c5b09ff7 100644 --- a/weed/s3api/s3acl/acl_helper_test.go +++ b/weed/s3api/s3acl/acl_helper_test.go @@ -706,3 +706,20 @@ func TestSetAcpGrantsHeader(t *testing.T) { t.Fatalf("owner unexpect") } } + +func TestGrantWithFullControl(t *testing.T) { + accountId := "Accountaskdfj" + expect := &s3.Grant{ + Permission: &s3_constants.PermissionFullControl, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountId, + }, + } + + result := GrantWithFullControl(accountId) + + if !GrantEquals(result, expect) { + t.Fatal("GrantWithFullControl not expect") + } +} diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 9e215db9e..c4a84dcdd 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/util" "math" "net/http" @@ -400,6 +402,24 @@ func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *htt oldOwnership, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey] if !ok || string(oldOwnership) != ownership { + + // must reset bucket acl to default(bucket owner with full control permission) before setting ownership + // to `OwnershipBucketOwnerEnforced` (bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting) + if ownership == s3_constants.OwnershipBucketOwnerEnforced { + acpGrants := s3acl.GetAcpGrants(bucketEntry.Extended) + if len(acpGrants) != 1 { + s3err.WriteErrorResponse(w, r, s3err.InvalidBucketAclWithObjectOwnership) + return + } + + bucketOwner := s3acl.GetAcpOwner(bucketEntry.Extended, s3account.AccountAdmin.Id) + expectGrant := s3acl.GrantWithFullControl(bucketOwner) + if s3acl.GrantEquals(acpGrants[0], expectGrant) { + s3err.WriteErrorResponse(w, r, s3err.InvalidBucketAclWithObjectOwnership) + return + } + } + if bucketEntry.Extended == nil { bucketEntry.Extended = make(map[string][]byte) } diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index 0348d4ddc..24d432c42 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -109,6 +109,7 @@ const ( ErrRequestBytesExceed OwnershipControlsNotFoundError + InvalidBucketAclWithObjectOwnership ) // error code to APIError structure, these fields carry respective @@ -422,6 +423,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "The bucket ownership controls were not found", HTTPStatusCode: http.StatusNotFound, }, + InvalidBucketAclWithObjectOwnership: { + Code: "InvalidBucketAclWithObjectOwnership", + Description: "Bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting", + HTTPStatusCode: http.StatusNotFound, + }, } // GetAPIError provides API Error for input API error code. From b8f7e75c3d88578d3c85bfb209a5a07ddea57320 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Fri, 14 Oct 2022 12:45:54 +0800 Subject: [PATCH 13/52] delete when empty at `AssembleEntryWithAcp` `PutBucketAcl/PutObjectAcl` allow request with empty grants, `AssembleEntryWithAcp` shouldn't skip empty value Signed-off-by: changlin.shi --- weed/s3api/s3acl/acl_helper.go | 4 ++ weed/s3api/s3acl/acl_helper_test.go | 66 ++++++++++++++--------------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/weed/s3api/s3acl/acl_helper.go b/weed/s3api/s3acl/acl_helper.go index e54e67556..bb956368e 100644 --- a/weed/s3api/s3acl/acl_helper.go +++ b/weed/s3api/s3acl/acl_helper.go @@ -411,6 +411,8 @@ func AssembleEntryWithAcp(objectEntry *filer_pb.Entry, objectOwner string, grant if len(objectOwner) > 0 { objectEntry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(objectOwner) + } else { + delete(objectEntry.Extended, s3_constants.ExtAmzOwnerKey) } if len(grants) > 0 { @@ -420,6 +422,8 @@ func AssembleEntryWithAcp(objectEntry *filer_pb.Entry, objectOwner string, grant return s3err.ErrInvalidRequest } objectEntry.Extended[s3_constants.ExtAmzAclKey] = grantsBytes + } else { + delete(objectEntry.Extended, s3_constants.ExtAmzAclKey) } return s3err.ErrNone diff --git a/weed/s3api/s3acl/acl_helper_test.go b/weed/s3api/s3acl/acl_helper_test.go index efc137989..ce177595b 100644 --- a/weed/s3api/s3acl/acl_helper_test.go +++ b/weed/s3api/s3acl/acl_helper_test.go @@ -487,46 +487,44 @@ func TestDetermineReqGrants(t *testing.T) { func TestAssembleEntryWithAcp(t *testing.T) { defaultOwner := "admin" - { - //case1 - expectOwner := "accountS" - expectGrants := []*s3.Grant{ - { - Permission: &s3_constants.PermissionRead, - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, - URI: &s3_constants.GranteeGroupAllUsers, - }, + + //case1 + //assemble with non-empty grants + expectOwner := "accountS" + expectGrants := []*s3.Grant{ + { + Permission: &s3_constants.PermissionRead, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + ID: &s3account.AccountAdmin.Id, + URI: &s3_constants.GranteeGroupAllUsers, }, - } - entry := &filer_pb.Entry{} - AssembleEntryWithAcp(entry, expectOwner, expectGrants) + }, + } + entry := &filer_pb.Entry{} + AssembleEntryWithAcp(entry, expectOwner, expectGrants) - resultOwner := GetAcpOwner(entry.Extended, defaultOwner) - if resultOwner != expectOwner { - t.Fatalf("owner not expect") - } + resultOwner := GetAcpOwner(entry.Extended, defaultOwner) + if resultOwner != expectOwner { + t.Fatalf("owner not expect") + } - resultGrants := GetAcpGrants(entry.Extended) - if !grantsEquals(resultGrants, expectGrants) { - t.Fatal("grants not expect") - } + resultGrants := GetAcpGrants(entry.Extended) + if !grantsEquals(resultGrants, expectGrants) { + t.Fatal("grants not expect") } - { - //case2 - entry := &filer_pb.Entry{} - AssembleEntryWithAcp(entry, "", nil) - resultOwner := GetAcpOwner(entry.Extended, defaultOwner) - if resultOwner != defaultOwner { - t.Fatalf("owner not expect") - } + //case2 + //assemble with empty grants (override) + AssembleEntryWithAcp(entry, "", nil) + resultOwner = GetAcpOwner(entry.Extended, defaultOwner) + if resultOwner != defaultOwner { + t.Fatalf("owner not expect") + } - resultGrants := GetAcpGrants(entry.Extended) - if len(resultGrants) != 0 { - t.Fatal("grants not expect") - } + resultGrants = GetAcpGrants(entry.Extended) + if len(resultGrants) != 0 { + t.Fatal("grants not expect") } } From 1f2883f7717dabd488fe84dc849221e9726c15c1 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Mon, 17 Oct 2022 11:47:35 +0800 Subject: [PATCH 14/52] merge Signed-off-by: changlin.shi --- weed/s3api/s3api_object_skip_handlers.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/weed/s3api/s3api_object_skip_handlers.go b/weed/s3api/s3api_object_skip_handlers.go index 935787fbb..bf7908aad 100644 --- a/weed/s3api/s3api_object_skip_handlers.go +++ b/weed/s3api/s3api_object_skip_handlers.go @@ -4,14 +4,6 @@ import ( "net/http" ) -// PutObjectAclHandler Put object ACL -// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html -func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) { - - w.WriteHeader(http.StatusNoContent) - -} - // PutObjectRetentionHandler Put object Retention // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html func (s3a *S3ApiServer) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) { From 900feb62b698e662475401a6ef824f8430a83db4 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Wed, 26 Oct 2022 16:02:25 +0800 Subject: [PATCH 15/52] fix parent path Signed-off-by: changlin.shi --- weed/s3api/s3api_acp.go | 4 ++-- weed/s3api/s3api_object_handlers.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index bafe19e88..00a4bb96a 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -102,6 +102,6 @@ func getObjectEntry(s3a *S3ApiServer, bucket, object string) (*filer_pb.Entry, e return s3a.getEntry(util.Join(s3a.option.BucketsPath, bucket), object) } -func updateObjectEntry(s3a *S3ApiServer, bucket string, entry *filer_pb.Entry) error { - return s3a.updateEntry(util.Join(s3a.option.BucketsPath, bucket), entry) +func updateObjectEntry(s3a *S3ApiServer, bucket, object string, entry *filer_pb.Entry) error { + return s3a.updateEntry(util.Join(s3a.option.BucketsPath, bucket, object), entry) } diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 8ee11c195..59a4a9b91 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -550,7 +550,7 @@ func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Reque return } - err := updateObjectEntry(s3a, bucket, objectEntry) + err := updateObjectEntry(s3a, bucket, object, objectEntry) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return From ebe1e39e7dd6e02bb9d4ee8926acbe0611748c5c Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Wed, 26 Oct 2022 16:02:25 +0800 Subject: [PATCH 16/52] fix parent path Signed-off-by: changlin.shi --- weed/s3api/s3api_acp.go | 6 ++++-- weed/s3api/s3api_object_handlers.go | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index bafe19e88..00abc9155 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -9,6 +9,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/util" "net/http" + "path/filepath" ) func getAccountId(r *http.Request) string { @@ -102,6 +103,7 @@ func getObjectEntry(s3a *S3ApiServer, bucket, object string) (*filer_pb.Entry, e return s3a.getEntry(util.Join(s3a.option.BucketsPath, bucket), object) } -func updateObjectEntry(s3a *S3ApiServer, bucket string, entry *filer_pb.Entry) error { - return s3a.updateEntry(util.Join(s3a.option.BucketsPath, bucket), entry) +func updateObjectEntry(s3a *S3ApiServer, bucket, object string, entry *filer_pb.Entry) error { + dir, _ := filepath.Split(object) + return s3a.updateEntry(util.Join(s3a.option.BucketsPath, bucket, dir), entry) } diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 8ee11c195..59a4a9b91 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -550,7 +550,7 @@ func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Reque return } - err := updateObjectEntry(s3a, bucket, objectEntry) + err := updateObjectEntry(s3a, bucket, object, objectEntry) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return From 4364f4dfe32447201a05f872fd5595dd82a69792 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Fri, 11 Nov 2022 10:02:39 +0800 Subject: [PATCH 17/52] don't set acp when ownership is BucketOwnerEnfored Signed-off-by: changlin.shi --- weed/s3api/s3api_acp.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index fbbf2d080..ca79cb6d2 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -61,19 +61,14 @@ func (s3a *S3ApiServer) checkAccessForWriteObject(r *http.Request, bucket, objec }, } if len(grants) == 0 { - // set default grants - s3acl.SetAcpOwnerHeader(r, accountId) - s3acl.SetAcpGrantsHeader(r, []*s3.Grant{bucketOwnerFullControlGrant}) return s3err.ErrNone } - if !s3acl.GrantEquals(bucketOwnerFullControlGrant, grants[0]) { - return s3err.AccessControlListNotSupported + if s3acl.GrantEquals(bucketOwnerFullControlGrant, grants[0]) { + return s3err.ErrNone } - s3acl.SetAcpOwnerHeader(r, accountId) - s3acl.SetAcpGrantsHeader(r, []*s3.Grant{bucketOwnerFullControlGrant}) - return s3err.ErrNone + return s3err.AccessControlListNotSupported } //bucket access allowed From f0c53bfa363ace80787c67ab277a2452fa635b12 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Tue, 22 Nov 2022 13:24:12 +0800 Subject: [PATCH 18/52] fix content-encoding Signed-off-by: changlin.shi --- weed/server/filer_server_handlers_write_autochunk.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 7064cac02..4a6219c00 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -381,6 +381,10 @@ func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool metadata[s3_constants.ExtAmzOwnerKey] = []byte(acpOwner) } + if ce := r.Header.Get("Content-Encoding"); ce != "" { + metadata["Content-Encoding"] = []byte(ce) + } + //acp-grants acpGrants := r.Header.Get(s3_constants.ExtAmzAclKey) if len(acpOwner) > 0 { From 381b496132a48cdd3c9fe8504bca210e96b9ad59 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Tue, 22 Nov 2022 16:18:02 +0800 Subject: [PATCH 19/52] acl for Initializing multipart upload Signed-off-by: changlin.shi --- weed/s3api/s3api_acp.go | 22 +++++++++++++++---- weed/s3api/s3api_object_handlers.go | 2 +- weed/s3api/s3api_object_multipart_handlers.go | 22 +++++++++---------- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index ca79cb6d2..fb7309ab4 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -33,17 +33,31 @@ func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s return s3err.ErrAccessDenied } -// Check Object-Write related access +// CheckAccessForPutObject Check ACL for PutObject API // includes: // - PutObjectHandler -// - PutObjectPartHandler -func (s3a *S3ApiServer) checkAccessForWriteObject(r *http.Request, bucket, object string) s3err.ErrorCode { +func (s3a *S3ApiServer) CheckAccessForPutObject(r *http.Request, bucket, object string) s3err.ErrorCode { + accountId := s3acl.GetAccountId(r) + return s3a.checkAccessForWriteObject(r, bucket, object, accountId) +} + +// CheckAccessForNewMultipartUpload Check Acl for InitiateMultipartUploadResult API +// includes: +// - NewMultipartUploadHandler +func (s3a *S3ApiServer) CheckAccessForNewMultipartUpload(r *http.Request, bucket, object string) s3err.ErrorCode { + accountId := s3acl.GetAccountId(r) + if accountId == IdentityAnonymous.AccountId { + return s3err.ErrAccessDenied + } + return s3a.checkAccessForWriteObject(r, bucket, object, accountId) +} + +func (s3a *S3ApiServer) checkAccessForWriteObject(r *http.Request, bucket, object, accountId string) s3err.ErrorCode { bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) if errCode != s3err.ErrNone { return errCode } - accountId := s3acl.GetAccountId(r) if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced { // validate grants (only bucketOwnerFullControl acl is allowed) _, grants, errCode := s3acl.ParseAndValidateAclHeaders(r, s3a.accountManager, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, accountId, false) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 9885303a6..203e63b3d 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -99,7 +99,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) } defer dataReader.Close() - errCode := s3a.checkAccessForWriteObject(r, bucket, object) + errCode := s3a.CheckAccessForPutObject(r, bucket, object) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 38e167f6b..7efb7b3dc 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -30,6 +30,13 @@ const ( func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { bucket, object := s3_constants.GetBucketAndObject(r) + //acl + errCode := s3a.CheckAccessForNewMultipartUpload(r, bucket, object) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + createMultipartUploadInput := &s3.CreateMultipartUploadInput{ Bucket: aws.String(bucket), Key: objectKey(aws.String(object)), @@ -230,31 +237,22 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ if s3a.iam.isEnabled() { rAuthType := getRequestAuthType(r) var s3ErrCode s3err.ErrorCode - var identity *Identity switch rAuthType { case authTypeStreamingSigned: - dataReader, identity, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r) + dataReader, _, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r) case authTypeSignedV2, authTypePresignedV2: - identity, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r) + _, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r) case authTypePresigned, authTypeSigned: - identity, s3ErrCode = s3a.iam.reqSignatureV4Verify(r) + _, s3ErrCode = s3a.iam.reqSignatureV4Verify(r) } if s3ErrCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, s3ErrCode) return } - r.Header.Set(s3_constants.AmzAccountId, identity.AccountId) } defer dataReader.Close() glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID) - - s3ErrCode := s3a.checkAccessForWriteObject(r, bucket, object) - if s3ErrCode != s3err.ErrNone { - s3err.WriteErrorResponse(w, r, s3ErrCode) - return - } - uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part", s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID) From d1b1f5e790b704a4863a230e0df54c0227313eda Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Tue, 22 Nov 2022 16:25:35 +0800 Subject: [PATCH 20/52] fix http code for dir Signed-off-by: changlin.shi --- weed/s3api/s3acl/acl_helper.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/weed/s3api/s3acl/acl_helper.go b/weed/s3api/s3acl/acl_helper.go index 28796451a..6a2b826e4 100644 --- a/weed/s3api/s3acl/acl_helper.go +++ b/weed/s3api/s3acl/acl_helper.go @@ -507,8 +507,7 @@ func GrantEquals(a, b *s3.Grant) bool { func CheckObjectAccessForReadObject(r *http.Request, w http.ResponseWriter, entry *filer.Entry, bucketOwnerId string) (statusCode int, ok bool) { if entry.IsDirectory() { - w.Header().Set(s3_constants.X_SeaweedFS_Header_Directory_Key, "true") - return http.StatusMethodNotAllowed, false + return http.StatusOK, true } accountId := GetAccountId(r) From f1d63024007246125deb055071938e73af685803 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Tue, 22 Nov 2022 16:47:28 +0800 Subject: [PATCH 21/52] acl for complete multipart uplod Signed-off-by: changlin.shi --- weed/s3api/s3api_acp.go | 29 +++++++++++++++++++ weed/s3api/s3api_object_multipart_handlers.go | 1 + 2 files changed, 30 insertions(+) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index fb7309ab4..e53fda373 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -52,6 +52,35 @@ func (s3a *S3ApiServer) CheckAccessForNewMultipartUpload(r *http.Request, bucket return s3a.checkAccessForWriteObject(r, bucket, object, accountId) } +// CheckAccessForCompleteMultipartUpload Check Acl for CompleteMultipartUpload API +// includes: +// - CompleteMultipartUploadHandler +func (s3a *S3ApiServer) CheckAccessForCompleteMultipartUpload(r *http.Request, bucket, object string) s3err.ErrorCode { + bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) + if errCode != s3err.ErrNone { + return errCode + } + + //bucket access allowed + accountId := s3acl.GetAccountId(r) + if accountId == *bucketMetadata.Owner.ID { + return s3err.ErrNone + } else { + if len(bucketMetadata.Acl) > 0 { + reqGrants := s3acl.DetermineReqGrants(accountId, s3_constants.PermissionWrite) + for _, bucketGrant := range bucketMetadata.Acl { + for _, requiredGrant := range reqGrants { + if s3acl.GrantEquals(bucketGrant, requiredGrant) { + return s3err.ErrNone + } + } + } + } + } + glog.V(3).Infof("acl denied! request account id: %s", accountId) + return s3err.ErrAccessDenied +} + func (s3a *S3ApiServer) checkAccessForWriteObject(r *http.Request, bucket, object, accountId string) s3err.ErrorCode { bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) if errCode != s3err.ErrNone { diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 7efb7b3dc..c4b65a7d5 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -70,6 +70,7 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html bucket, object := s3_constants.GetBucketAndObject(r) + s3a.CheckAccessForCompleteMultipartUpload(r, bucket, object) parts := &CompleteMultipartUpload{} if err := xmlDecoder(r.Body, parts, r.ContentLength); err != nil { From f47aafa9b9699fff764c133b6e8b15235f01202d Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Wed, 23 Nov 2022 10:46:57 +0800 Subject: [PATCH 22/52] Revert "fix content-encoding" This reverts commit f0c53bfa363ace80787c67ab277a2452fa635b12. --- weed/server/filer_server_handlers_write_autochunk.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 4a6219c00..7064cac02 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -381,10 +381,6 @@ func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool metadata[s3_constants.ExtAmzOwnerKey] = []byte(acpOwner) } - if ce := r.Header.Get("Content-Encoding"); ce != "" { - metadata["Content-Encoding"] = []byte(ce) - } - //acp-grants acpGrants := r.Header.Get(s3_constants.ExtAmzAclKey) if len(acpOwner) > 0 { From 41204715a084da82e7b854cd2bf18aae68ab97a1 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Wed, 23 Nov 2022 10:48:49 +0800 Subject: [PATCH 23/52] save content-encoding header Signed-off-by: changlin.shi --- weed/server/filer_server_handlers_write_autochunk.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 7064cac02..05b29a2e8 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -375,6 +375,10 @@ func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool } } + if ce := r.Header.Get("Content-Encoding"); ce != "" { + metadata["Content-Encoding"] = []byte(ce) + } + //acp-owner acpOwner := r.Header.Get(s3_constants.ExtAmzOwnerKey) if len(acpOwner) > 0 { From 930fa4769979b1234d3d96ca33dc882885f1264c Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Wed, 23 Nov 2022 10:54:38 +0800 Subject: [PATCH 24/52] log leftover with url if present Signed-off-by: changlin.shi --- weed/util/http_util.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/weed/util/http_util.go b/weed/util/http_util.go index bb1a32ede..fdebd2cdf 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -415,7 +415,11 @@ func CloseResponse(resp *http.Response) { io.Copy(io.Discard, reader) resp.Body.Close() if reader.BytesRead > 0 { - glog.V(1).Infof("response leftover %d bytes", reader.BytesRead) + if resp.Request != nil && resp.Request.URL != nil { + glog.V(1).Infof("response leftover %d bytes, url: %s", resp.Request.URL.RequestURI()) + } else { + glog.V(1).Infof("response leftover %d bytes", reader.BytesRead) + } } } @@ -424,7 +428,12 @@ func CloseRequest(req *http.Request) { io.Copy(io.Discard, reader) req.Body.Close() if reader.BytesRead > 0 { - glog.V(1).Infof("request leftover %d bytes", reader.BytesRead) + if req.URL != nil { + glog.V(1).Infof("request leftover %d bytes, url: %s", reader.BytesRead, req.URL.RequestURI()) + } else { + glog.V(1).Infof("request leftover %d bytes", reader.BytesRead) + } + } } From 300a53005141b7befb2f4cb7b5618121081cf912 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Wed, 23 Nov 2022 16:01:10 +0800 Subject: [PATCH 25/52] correct log Signed-off-by: changlin.shi --- weed/util/http_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/util/http_util.go b/weed/util/http_util.go index fdebd2cdf..384da6df8 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -416,7 +416,7 @@ func CloseResponse(resp *http.Response) { resp.Body.Close() if reader.BytesRead > 0 { if resp.Request != nil && resp.Request.URL != nil { - glog.V(1).Infof("response leftover %d bytes, url: %s", resp.Request.URL.RequestURI()) + glog.V(1).Infof("response leftover %d bytes, url: %s", reader.BytesRead, resp.Request.URL.RequestURI()) } else { glog.V(1).Infof("response leftover %d bytes", reader.BytesRead) } From 3b69ce6c3d0c83c7bb3b8343d5725571c641fae1 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 24 Nov 2022 14:52:42 +0800 Subject: [PATCH 26/52] set accept-encoding if empty Signed-off-by: changlin.shi --- weed/s3api/s3api_object_handlers.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index d63a68e4b..2a6a4ce2c 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -376,6 +376,13 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des s3a.maybeAddFilerJwtAuthorization(proxyReq, isWrite) resp, postErr := s3a.client.Do(proxyReq) + if resp.Uncompressed && r.Header.Get("Accept-Encoding") == "" { + r.Header.Set("Accept-Encoding", "gzip") + util.CloseResponse(resp) + s3a.proxyToFiler(w, r, destUrl, false, passThroughResponse) + return + } + if postErr != nil { glog.Errorf("post to filer: %v", postErr) s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) From 2769a96f132b66a9ed9b7f71ce5f4ee35953c1eb Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Fri, 2 Dec 2022 11:05:26 +0800 Subject: [PATCH 27/52] dont't set destination when copy object Signed-off-by: changlin.shi --- weed/s3api/s3api_object_copy_handlers.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go index 83120ef07..5e97f5230 100644 --- a/weed/s3api/s3api_object_copy_handlers.go +++ b/weed/s3api/s3api_object_copy_handlers.go @@ -98,8 +98,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request return } glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl) - destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject) - etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body, destination) + etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body, "") if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) From 89942cf01509d096ae65b06e276623bf71491654 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Fri, 2 Dec 2022 11:25:59 +0800 Subject: [PATCH 28/52] fix nil pointer when no identity config init Signed-off-by: changlin.shi --- weed/s3api/auth_credentials.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 82ac3688c..0f3ab0b06 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -17,7 +17,10 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" ) -var IdentityAnonymous *Identity +var IdentityAnonymous = &Identity{ + Name: s3account.AccountAnonymous.Name, + AccountId: s3account.AccountAnonymous.Id, +} type Action string @@ -162,12 +165,6 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api identities = append(identities, t) } - if IdentityAnonymous == nil { - IdentityAnonymous = &Identity{ - Name: s3account.AccountAnonymous.Name, - AccountId: s3account.AccountAnonymous.Id, - } - } iam.m.Lock() // atomically switch iam.identities = identities From c2c4158238e135ef87a245a83cf9765106dd8d0f Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Fri, 2 Dec 2022 13:05:14 +0800 Subject: [PATCH 29/52] create bucket with ownership Signed-off-by: changlin.shi --- weed/s3api/s3api_acp.go | 7 +++---- weed/s3api/s3api_bucket_handlers.go | 7 ++++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index 7aa19babd..535bb3489 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -30,11 +30,10 @@ func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s return s3err.ErrAccessDenied } -func (s3a *S3ApiServer) ExtractBucketAcp(r *http.Request) (owner string, grants []*s3.Grant, errCode s3err.ErrorCode) { +func (s3a *S3ApiServer) ExtractBucketAcp(r *http.Request, objectOwnership string) (owner string, grants []*s3.Grant, errCode s3err.ErrorCode) { accountId := s3acl.GetAccountId(r) - ownership := s3_constants.DefaultOwnershipForCreate - if ownership == s3_constants.OwnershipBucketOwnerEnforced { + if objectOwnership == s3_constants.OwnershipBucketOwnerEnforced { return accountId, []*s3.Grant{ { Permission: &s3_constants.PermissionFullControl, @@ -45,6 +44,6 @@ func (s3a *S3ApiServer) ExtractBucketAcp(r *http.Request) (owner string, grants }, }, s3err.ErrNone } else { - return s3acl.ParseAndValidateAclHeadersOrElseDefault(r, s3a.accountManager, ownership, accountId, accountId, false) + return s3acl.ParseAndValidateAclHeadersOrElseDefault(r, s3a.accountManager, objectOwnership, accountId, accountId, false) } } diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index efe069237..f1a747797 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -122,7 +122,11 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) } } - acpOwner, acpGrants, errCode := s3a.ExtractBucketAcp(r) + objectOwnership := r.Header.Get("ObjectOwnership") + if objectOwnership == "" { + objectOwnership = s3_constants.DefaultOwnershipForCreate + } + acpOwner, acpGrants, errCode := s3a.ExtractBucketAcp(r, objectOwnership) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return @@ -134,6 +138,7 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) entry.Extended = make(map[string][]byte) } entry.Extended[s3_constants.AmzIdentityId] = []byte(identityId) + entry.Extended[s3_constants.ExtOwnershipKey] = []byte(objectOwnership) } s3acl.AssembleEntryWithAcp(entry, acpOwner, acpGrants) } From 5d582ab8480d676e11434d5e504484fec1f67242 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Fri, 2 Dec 2022 16:36:39 +0800 Subject: [PATCH 30/52] set bucket ownership if not empty Signed-off-by: changlin.shi --- weed/s3api/s3api_acp.go | 2 +- weed/s3api/s3api_bucket_handlers.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index 535bb3489..e69597ed0 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -33,7 +33,7 @@ func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s func (s3a *S3ApiServer) ExtractBucketAcp(r *http.Request, objectOwnership string) (owner string, grants []*s3.Grant, errCode s3err.ErrorCode) { accountId := s3acl.GetAccountId(r) - if objectOwnership == s3_constants.OwnershipBucketOwnerEnforced { + if objectOwnership == "" || objectOwnership == s3_constants.OwnershipBucketOwnerEnforced { return accountId, []*s3.Grant{ { Permission: &s3_constants.PermissionFullControl, diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index f1a747797..c1a4888e4 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -123,9 +123,6 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) } objectOwnership := r.Header.Get("ObjectOwnership") - if objectOwnership == "" { - objectOwnership = s3_constants.DefaultOwnershipForCreate - } acpOwner, acpGrants, errCode := s3a.ExtractBucketAcp(r, objectOwnership) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) @@ -138,6 +135,8 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) entry.Extended = make(map[string][]byte) } entry.Extended[s3_constants.AmzIdentityId] = []byte(identityId) + } + if objectOwnership != "" { entry.Extended[s3_constants.ExtOwnershipKey] = []byte(objectOwnership) } s3acl.AssembleEntryWithAcp(entry, acpOwner, acpGrants) From 0c3009911e8f944c2bee2df8f69b048e64ad2ed8 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Sun, 11 Dec 2022 18:25:59 +0800 Subject: [PATCH 31/52] optimize readability Signed-off-by: changlin.shi --- weed/s3api/auth_credentials_subscribe.go | 4 +- weed/s3api/bucket_metadata.go | 65 +- weed/s3api/bucket_metadata_test.go | 10 +- weed/s3api/filer_multipart.go | 24 +- weed/s3api/s3_constants/acp_ownership.go | 4 +- weed/s3api/s3_constants/extend_key.go | 7 +- weed/s3api/s3_constants/header.go | 5 +- weed/s3api/s3acl/acl_helper.go | 373 +++-- weed/s3api/s3acl/acl_helper_test.go | 1391 +++++++++++++---- weed/s3api/s3api_acp.go | 332 ++-- weed/s3api/s3api_bucket_handlers.go | 51 +- weed/s3api/s3api_object_handlers.go | 14 +- weed/s3api/s3api_object_multipart_handlers.go | 63 +- weed/s3api/s3api_server.go | 10 +- weed/s3api/s3err/s3api_errors.go | 14 +- weed/server/filer_server_handlers_read.go | 17 +- 16 files changed, 1703 insertions(+), 681 deletions(-) diff --git a/weed/s3api/auth_credentials_subscribe.go b/weed/s3api/auth_credentials_subscribe.go index 377cf2728..c9dd1b086 100644 --- a/weed/s3api/auth_credentials_subscribe.go +++ b/weed/s3api/auth_credentials_subscribe.go @@ -70,10 +70,10 @@ func (s3a *S3ApiServer) onBucketMetadataChange(dir string, oldEntry *filer_pb.En if dir == s3a.option.BucketsPath { if newEntry != nil { s3a.bucketRegistry.LoadBucketMetadata(newEntry) - glog.V(0).Infof("updated bucketMetadata %s/%s", dir, newEntry) + glog.V(1).Infof("updated bucketMetadata %s/%s", dir, newEntry) } else { s3a.bucketRegistry.RemoveBucketMetadata(oldEntry) - glog.V(0).Infof("remove bucketMetadata %s/%s", dir, newEntry) + glog.V(1).Infof("remove bucketMetadata %s/%s", dir, newEntry) } } return nil diff --git a/weed/s3api/bucket_metadata.go b/weed/s3api/bucket_metadata.go index f4088e6b3..412076da3 100644 --- a/weed/s3api/bucket_metadata.go +++ b/weed/s3api/bucket_metadata.go @@ -107,7 +107,6 @@ func buildBucketMetadata(accountManager *s3account.AccountManager, entry *filer_ } } - //access control policy //owner acpOwnerBytes, ok := entry.Extended[s3_constants.ExtAmzOwnerKey] if ok && len(acpOwnerBytes) > 0 { @@ -122,17 +121,31 @@ func buildBucketMetadata(accountManager *s3account.AccountManager, entry *filer_ } } } + //grants acpGrantsBytes, ok := entry.Extended[s3_constants.ExtAmzAclKey] - if ok && len(acpGrantsBytes) > 0 { - var grants []*s3.Grant - err := json.Unmarshal(acpGrantsBytes, &grants) - if err == nil { - bucketMetadata.Acl = grants - } else { - glog.Warningf("Unmarshal ACP grants: %s(%v), bucket: %s", string(acpGrantsBytes), err, bucketMetadata.Name) + if ok { + if len(acpGrantsBytes) > 0 { + var grants []*s3.Grant + err := json.Unmarshal(acpGrantsBytes, &grants) + if err == nil { + bucketMetadata.Acl = grants + } else { + glog.Warningf("Unmarshal ACP grants: %s(%v), bucket: %s", string(acpGrantsBytes), err, bucketMetadata.Name) + } + } + } else { + bucketMetadata.Acl = []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: bucketMetadata.Owner.ID, + }, + Permission: &s3_constants.PermissionFullControl, + }, } } + } return bucketMetadata } @@ -143,17 +156,12 @@ func (r *BucketRegistry) RemoveBucketMetadata(entry *filer_pb.Entry) { } func (r *BucketRegistry) GetBucketMetadata(bucketName string) (*BucketMetaData, s3err.ErrorCode) { - r.metadataCacheLock.RLock() - bucketMetadata, ok := r.metadataCache[bucketName] - r.metadataCacheLock.RUnlock() - if ok { + bucketMetadata := r.getMetadataCache(bucketName) + if bucketMetadata != nil { return bucketMetadata, s3err.ErrNone } - r.notFoundLock.RLock() - _, ok = r.notFound[bucketName] - r.notFoundLock.RUnlock() - if ok { + if r.isNotFound(bucketName) { return nil, s3err.ErrNoSuchBucket } @@ -172,10 +180,8 @@ func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*Bucket defer r.notFoundLock.Unlock() //check if already exists - r.metadataCacheLock.RLock() - bucketMetaData, ok := r.metadataCache[bucketName] - r.metadataCacheLock.RUnlock() - if ok { + bucketMetaData := r.getMetadataCache(bucketName) + if bucketMetaData != nil { return bucketMetaData, s3err.ErrNone } @@ -184,6 +190,7 @@ func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*Bucket if err != nil { if err == filer_pb.ErrNotFound { // The bucket doesn't actually exist and should no longer loaded from the filer + glog.Warning("bucket not found in filer: ", bucketName) r.notFound[bucketName] = struct{}{} return nil, s3err.ErrNoSuchBucket } @@ -192,6 +199,15 @@ func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*Bucket return bucketMetadata, s3err.ErrNone } +func (r *BucketRegistry) getMetadataCache(bucket string) *BucketMetaData { + r.metadataCacheLock.RLock() + defer r.metadataCacheLock.RUnlock() + if cache, ok := r.metadataCache[bucket]; ok { + return cache + } + return nil +} + func (r *BucketRegistry) setMetadataCache(metadata *BucketMetaData) { r.metadataCacheLock.Lock() defer r.metadataCacheLock.Unlock() @@ -204,10 +220,11 @@ func (r *BucketRegistry) removeMetadataCache(bucket string) { delete(r.metadataCache, bucket) } -func (r *BucketRegistry) markNotFound(bucket string) { - r.notFoundLock.Lock() - defer r.notFoundLock.Unlock() - r.notFound[bucket] = struct{}{} +func (r *BucketRegistry) isNotFound(bucket string) bool { + r.notFoundLock.RLock() + defer r.notFoundLock.RUnlock() + _, ok := r.notFound[bucket] + return ok } func (r *BucketRegistry) unMarkNotFound(bucket string) { diff --git a/weed/s3api/bucket_metadata_test.go b/weed/s3api/bucket_metadata_test.go index f852a272a..cd6aef78f 100644 --- a/weed/s3api/bucket_metadata_test.go +++ b/weed/s3api/bucket_metadata_test.go @@ -86,7 +86,7 @@ var tcs = []*BucketMetadataTestCase{ { badEntry, &BucketMetaData{ Name: badEntry.Name, - ObjectOwnership: s3_constants.DefaultOwnershipForExists, + ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ DisplayName: &s3account.AccountAdmin.Name, ID: &s3account.AccountAdmin.Id, @@ -108,7 +108,7 @@ var tcs = []*BucketMetadataTestCase{ { ownershipEmptyStr, &BucketMetaData{ Name: ownershipEmptyStr.Name, - ObjectOwnership: s3_constants.DefaultOwnershipForExists, + ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ DisplayName: &s3account.AccountAdmin.Name, ID: &s3account.AccountAdmin.Id, @@ -130,7 +130,7 @@ var tcs = []*BucketMetadataTestCase{ { acpEmptyStr, &BucketMetaData{ Name: acpEmptyStr.Name, - ObjectOwnership: s3_constants.DefaultOwnershipForExists, + ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ DisplayName: &s3account.AccountAdmin.Name, ID: &s3account.AccountAdmin.Id, @@ -141,7 +141,7 @@ var tcs = []*BucketMetadataTestCase{ { acpEmptyObject, &BucketMetaData{ Name: acpEmptyObject.Name, - ObjectOwnership: s3_constants.DefaultOwnershipForExists, + ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ DisplayName: &s3account.AccountAdmin.Name, ID: &s3account.AccountAdmin.Id, @@ -152,7 +152,7 @@ var tcs = []*BucketMetadataTestCase{ { acpOwnerNil, &BucketMetaData{ Name: acpOwnerNil.Name, - ObjectOwnership: s3_constants.DefaultOwnershipForExists, + ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ DisplayName: &s3account.AccountAdmin.Name, ID: &s3account.AccountAdmin.Id, diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 414ba4bb2..fefddf4b2 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -5,6 +5,7 @@ import ( "encoding/xml" "fmt" "github.com/google/uuid" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "golang.org/x/exp/slices" "math" @@ -27,7 +28,7 @@ type InitiateMultipartUploadResult struct { s3.CreateMultipartUploadOutput } -func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code s3err.ErrorCode) { +func (s3a *S3ApiServer) createMultipartUpload(initiatorId string, input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code s3err.ErrorCode) { glog.V(2).Infof("createMultipartUpload input %v", input) @@ -46,6 +47,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp if input.ContentType != nil { entry.Attributes.Mime = *input.ContentType } + entry.Extended[s3_constants.ExtAmzMultipartInitiator] = []byte(initiatorId) }); err != nil { glog.Errorf("NewMultipartUpload error: %v", err) return nil, s3err.ErrInternalError @@ -236,7 +238,7 @@ type ListMultipartUploadsResult struct { Upload []*s3.MultipartUpload `locationName:"Upload" type:"list" flattened:"true"` } -func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code s3err.ErrorCode) { +func (s3a *S3ApiServer) listMultipartUploads(bucketMetaData *BucketMetaData, input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code s3err.ErrorCode) { // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html glog.V(2).Infof("listMultipartUploads input %v", input) @@ -267,9 +269,27 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput if *input.Prefix != "" && !strings.HasPrefix(key, *input.Prefix) { continue } + initiatorId := string(entry.Extended[s3_constants.ExtAmzMultipartInitiator]) + if initiatorId == "" { + initiatorId = *bucketMetaData.Owner.ID + } + initiatorDisplayName := s3a.accountManager.IdNameMapping[initiatorId] + ownerId := string(entry.Extended[s3_constants.ExtAmzOwnerKey]) + if ownerId == "" { + ownerId = *bucketMetaData.Owner.ID + } + ownerDisplayName := s3a.accountManager.IdNameMapping[ownerId] output.Upload = append(output.Upload, &s3.MultipartUpload{ Key: objectKey(aws.String(key)), UploadId: aws.String(entry.Name), + Owner: &s3.Owner{ + ID: &initiatorId, + DisplayName: &ownerDisplayName, + }, + Initiator: &s3.Initiator{ + ID: &initiatorId, + DisplayName: &initiatorDisplayName, + }, }) uploadsCount += 1 } diff --git a/weed/s3api/s3_constants/acp_ownership.go b/weed/s3api/s3_constants/acp_ownership.go index e11e95935..7a9f508a9 100644 --- a/weed/s3api/s3_constants/acp_ownership.go +++ b/weed/s3api/s3_constants/acp_ownership.go @@ -4,9 +4,7 @@ var ( OwnershipBucketOwnerPreferred = "BucketOwnerPreferred" OwnershipObjectWriter = "ObjectWriter" OwnershipBucketOwnerEnforced = "BucketOwnerEnforced" - - DefaultOwnershipForCreate = OwnershipObjectWriter - DefaultOwnershipForExists = OwnershipBucketOwnerEnforced + DefaultObjectOwnership = OwnershipBucketOwnerEnforced ) func ValidateOwnership(ownership string) bool { diff --git a/weed/s3api/s3_constants/extend_key.go b/weed/s3api/s3_constants/extend_key.go index f78983a99..d69af2698 100644 --- a/weed/s3api/s3_constants/extend_key.go +++ b/weed/s3api/s3_constants/extend_key.go @@ -1,7 +1,8 @@ package s3_constants const ( - ExtAmzOwnerKey = "Seaweed-X-Amz-Owner" - ExtAmzAclKey = "Seaweed-X-Amz-Acl" - ExtOwnershipKey = "Seaweed-X-Amz-Ownership" + ExtAmzOwnerKey = "Seaweed-X-Amz-Owner" + ExtAmzMultipartInitiator = "Seaweed-X-Amz-Multipart-Initiator" + ExtAmzAclKey = "Seaweed-X-Amz-Acl" + ExtOwnershipKey = "Seaweed-X-Amz-Ownership" ) diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index e4581bff6..9d22b8e4f 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -37,8 +37,9 @@ const ( AmzObjectTaggingDirective = "X-Amz-Tagging-Directive" AmzTagCount = "x-amz-tagging-count" - X_SeaweedFS_Header_Directory_Key = "x-seaweedfs-is-directory-key" - XSeaweedFSHeaderAmzBucketOwnerId = "x-seaweedfs-amz-bucket-owner-id" + X_SeaweedFS_Header_Directory_Key = "x-seaweedfs-is-directory-key" + XSeaweedFSHeaderAmzBucketOwnerId = "x-seaweedfs-amz-bucket-owner-id" + XSeaweedFSHeaderAmzBucketAccessDenied = "x-seaweedfs-amz-bucket-access-denied" // S3 ACL headers AmzCannedAcl = "X-Amz-Acl" diff --git a/weed/s3api/s3acl/acl_helper.go b/weed/s3api/s3acl/acl_helper.go index ef56da313..942b55797 100644 --- a/weed/s3api/s3acl/acl_helper.go +++ b/weed/s3api/s3acl/acl_helper.go @@ -3,6 +3,7 @@ package s3acl import ( "encoding/json" "encoding/xml" + "fmt" "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -16,6 +17,8 @@ import ( "strings" ) +var customAclHeaders = []string{s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp} + // GetAccountId get AccountId from request headers, AccountAnonymousId will be return if not presen func GetAccountId(r *http.Request) string { id := r.Header.Get(s3_constants.AmzAccountId) @@ -26,11 +29,36 @@ func GetAccountId(r *http.Request) string { } } -// ExtractAcl extracts the acl from the request body, or from the header if request body is empty -func ExtractAcl(r *http.Request, accountManager *s3account.AccountManager, ownership, bucketOwnerId, ownerId, accountId string) (grants []*s3.Grant, errCode s3err.ErrorCode) { - if r.Body != nil && r.Body != http.NoBody { - defer util.CloseRequest(r) +// ValidateAccount validate weather request account id is allowed to access +func ValidateAccount(requestAccountId string, allowedAccounts ...string) bool { + for _, allowedAccount := range allowedAccounts { + if requestAccountId == allowedAccount { + return true + } + } + return false +} + +// ExtractBucketAcl extracts the acl from the request body, or from the header if request body is empty +func ExtractBucketAcl(r *http.Request, accountManager *s3account.AccountManager, objectOwnership, bucketOwnerId, requestAccountId string, createBucket bool) (grants []*s3.Grant, errCode s3err.ErrorCode) { + cannedAclPresent := false + if r.Header.Get(s3_constants.AmzCannedAcl) != "" { + cannedAclPresent = true + } + customAclPresent := false + for _, customAclHeader := range customAclHeaders { + if r.Header.Get(customAclHeader) != "" { + customAclPresent = true + break + } + } + // AccessControlList body is not support when create object/bucket + if !createBucket && r.Body != nil && r.Body != http.NoBody { + defer util.CloseRequest(r) + if cannedAclPresent || customAclPresent { + return nil, s3err.ErrUnexpectedContent + } var acp s3.AccessControlPolicy err := xmlutil.UnmarshalXML(&acp, xml.NewDecoder(r.Body), "") if err != nil || acp.Owner == nil || acp.Owner.ID == nil { @@ -38,116 +66,127 @@ func ExtractAcl(r *http.Request, accountManager *s3account.AccountManager, owner } //owner should present && owner is immutable - if *acp.Owner.ID != ownerId { - glog.V(3).Infof("set acl denied! owner account is not consistent, request account id: %s, expect account id: %s", accountId, ownerId) + if *acp.Owner.ID == "" || *acp.Owner.ID != bucketOwnerId { + glog.V(3).Infof("set acl denied! owner account is not consistent, request account id: %s, expect account id: %s", *acp.Owner.ID, bucketOwnerId) return nil, s3err.ErrAccessDenied } - - return ValidateAndTransferGrants(accountManager, acp.Grants) + grants = acp.Grants } else { - _, grants, errCode = ParseAndValidateAclHeadersOrElseDefault(r, accountManager, ownership, bucketOwnerId, accountId, true) - return grants, errCode + if cannedAclPresent && customAclPresent { + return nil, s3err.ErrInvalidRequest + } + if cannedAclPresent { + grants, errCode = ExtractBucketCannedAcl(r, requestAccountId) + } else if customAclPresent { + grants, errCode = ExtractCustomAcl(r) + } + if errCode != s3err.ErrNone { + return nil, errCode + } } -} - -// ParseAndValidateAclHeadersOrElseDefault will callParseAndValidateAclHeaders to get Grants, if empty, it will return Grant that grant `accountId` with `FullControl` permission -func ParseAndValidateAclHeadersOrElseDefault(r *http.Request, accountManager *s3account.AccountManager, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) { - ownerId, grants, errCode = ParseAndValidateAclHeaders(r, accountManager, ownership, bucketOwnerId, accountId, putAcl) + errCode = ValidateObjectOwnershipAndGrants(objectOwnership, bucketOwnerId, grants) if errCode != s3err.ErrNone { - return + return nil, errCode } - if len(grants) == 0 { - //if no acl(both customAcl and cannedAcl) specified, grant accountId(object writer) with full control permission - grants = append(grants, &s3.Grant{ - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeCanonicalUser, - ID: &accountId, - }, - Permission: &s3_constants.PermissionFullControl, - }) + grants, errCode = ValidateAndTransferGrants(accountManager, grants) + if errCode != s3err.ErrNone { + return nil, errCode } - return + return grants, s3err.ErrNone } -// ParseAndValidateAclHeaders parse and validate acl from header -func ParseAndValidateAclHeaders(r *http.Request, accountManager *s3account.AccountManager, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) { - ownerId, grants, errCode = ParseAclHeaders(r, ownership, bucketOwnerId, accountId, putAcl) - if errCode != s3err.ErrNone { - return +// ExtractObjectAcl extracts the acl from the request body, or from the header if request body is empty +func ExtractObjectAcl(r *http.Request, accountManager *s3account.AccountManager, objectOwnership, bucketOwnerId, requestAccountId string, createObject bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) { + cannedAclPresent := false + if r.Header.Get(s3_constants.AmzCannedAcl) != "" { + cannedAclPresent = true } - if len(grants) > 0 { - grants, errCode = ValidateAndTransferGrants(accountManager, grants) + customAclPresent := false + for _, customAclHeader := range customAclHeaders { + if r.Header.Get(customAclHeader) != "" { + customAclPresent = true + break + } } - return -} -// ParseAclHeaders parse acl headers -// When `putAcl` is true, only `CannedAcl` is parsed, such as `PutBucketAcl` or `PutObjectAcl` -// is requested, `CustomAcl` is parsed from the request body not from headers, and only if the -// request body is empty, `CannedAcl` is parsed from the header, and will not parse `CustomAcl` from the header -// -// Since `CustomAcl` has higher priority, it will be parsed first; if `CustomAcl` does not exist, `CannedAcl` will be parsed -func ParseAclHeaders(r *http.Request, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) { - if !putAcl { - errCode = ParseCustomAclHeaders(r, &grants) + // AccessControlList body is not support when create object/bucket + if !createObject && r.Body != nil && r.Body != http.NoBody { + defer util.CloseRequest(r) + if cannedAclPresent || customAclPresent { + return "", nil, s3err.ErrUnexpectedContent + } + var acp s3.AccessControlPolicy + err := xmlutil.UnmarshalXML(&acp, xml.NewDecoder(r.Body), "") + if err != nil || acp.Owner == nil || acp.Owner.ID == nil { + return "", nil, s3err.ErrInvalidRequest + } + + //owner should present && owner is immutable + if *acp.Owner.ID == "" { + glog.V(1).Infof("Access denied! The owner id is required when specifying grants using AccessControlList") + return "", nil, s3err.ErrAccessDenied + } + ownerId = *acp.Owner.ID + grants = acp.Grants + } else { + if cannedAclPresent && customAclPresent { + return "", nil, s3err.ErrInvalidRequest + } + if cannedAclPresent { + ownerId, grants, errCode = ExtractObjectCannedAcl(r, objectOwnership, bucketOwnerId, requestAccountId, createObject) + } else { + grants, errCode = ExtractCustomAcl(r) + } if errCode != s3err.ErrNone { return "", nil, errCode } } - if len(grants) > 0 { - return accountId, grants, s3err.ErrNone - } - - cannedAcl := r.Header.Get(s3_constants.AmzCannedAcl) - if len(cannedAcl) == 0 { - return accountId, grants, s3err.ErrNone - } - - //if canned acl specified, parse cannedAcl (lower priority to custom acl) - ownerId, grants, errCode = ParseCannedAclHeader(ownership, bucketOwnerId, accountId, cannedAcl, putAcl) + errCode = ValidateObjectOwnershipAndGrants(objectOwnership, bucketOwnerId, grants) if errCode != s3err.ErrNone { return "", nil, errCode } + grants, errCode = ValidateAndTransferGrants(accountManager, grants) return ownerId, grants, errCode } -func ParseCustomAclHeaders(r *http.Request, grants *[]*s3.Grant) s3err.ErrorCode { - customAclHeaders := []string{s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp} +func ExtractCustomAcl(r *http.Request) ([]*s3.Grant, s3err.ErrorCode) { var errCode s3err.ErrorCode + var grants []*s3.Grant for _, customAclHeader := range customAclHeaders { headerValue := r.Header.Get(customAclHeader) switch customAclHeader { case s3_constants.AmzAclRead: - errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionRead, grants) + errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionRead, &grants) case s3_constants.AmzAclWrite: - errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWrite, grants) + errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWrite, &grants) case s3_constants.AmzAclReadAcp: - errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionReadAcp, grants) + errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionReadAcp, &grants) case s3_constants.AmzAclWriteAcp: - errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWriteAcp, grants) + errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWriteAcp, &grants) case s3_constants.AmzAclFullControl: - errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionFullControl, grants) + errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionFullControl, &grants) + default: + errCode = s3err.ErrInvalidAclArgument } if errCode != s3err.ErrNone { - return errCode + return nil, errCode } } - return s3err.ErrNone + return grants, s3err.ErrNone } func ParseCustomAclHeader(headerValue, permission string, grants *[]*s3.Grant) s3err.ErrorCode { if len(headerValue) > 0 { - split := strings.Split(headerValue, ", ") + split := strings.Split(headerValue, ",") for _, grantStr := range split { kv := strings.Split(grantStr, "=") if len(kv) != 2 { return s3err.ErrInvalidRequest } - switch kv[0] { + switch strings.TrimSpace(kv[0]) { case "id": - var accountId string - _ = json.Unmarshal([]byte(kv[1]), &accountId) + accountId := decodeGranteeValue(kv[1]) *grants = append(*grants, &s3.Grant{ Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeCanonicalUser, @@ -156,8 +195,7 @@ func ParseCustomAclHeader(headerValue, permission string, grants *[]*s3.Grant) s Permission: &permission, }) case "emailAddress": - var emailAddress string - _ = json.Unmarshal([]byte(kv[1]), &emailAddress) + emailAddress := decodeGranteeValue(kv[1]) *grants = append(*grants, &s3.Grant{ Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeAmazonCustomerByEmail, @@ -167,7 +205,7 @@ func ParseCustomAclHeader(headerValue, permission string, grants *[]*s3.Grant) s }) case "uri": var groupName string - _ = json.Unmarshal([]byte(kv[1]), &groupName) + groupName = decodeGranteeValue(kv[1]) *grants = append(*grants, &s3.Grant{ Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, @@ -179,17 +217,66 @@ func ParseCustomAclHeader(headerValue, permission string, grants *[]*s3.Grant) s } } return s3err.ErrNone +} +func decodeGranteeValue(value string) (result string) { + if !strings.HasPrefix(value, "\"") { + return value + } + _ = json.Unmarshal([]byte(value), &result) + if result == "" { + result = value + } + return result } -func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl string, putAcl bool) (ownerId string, grants []*s3.Grant, err s3err.ErrorCode) { +// ExtractBucketCannedAcl parse bucket canned acl, includes: 'private'|'public-read'|'public-read-write'|'authenticated-read' +func ExtractBucketCannedAcl(request *http.Request, requestAccountId string) (grants []*s3.Grant, err s3err.ErrorCode) { + cannedAcl := request.Header.Get(s3_constants.AmzCannedAcl) + if cannedAcl == "" { + return grants, s3err.ErrNone + } err = s3err.ErrNone - ownerId = accountId + objectWriterFullControl := &s3.Grant{ + Grantee: &s3.Grantee{ + ID: &requestAccountId, + Type: &s3_constants.GrantTypeCanonicalUser, + }, + Permission: &s3_constants.PermissionFullControl, + } + switch cannedAcl { + case s3_constants.CannedAclPrivate: + grants = append(grants, objectWriterFullControl) + case s3_constants.CannedAclPublicRead: + grants = append(grants, objectWriterFullControl) + grants = append(grants, s3_constants.PublicRead...) + case s3_constants.CannedAclPublicReadWrite: + grants = append(grants, objectWriterFullControl) + grants = append(grants, s3_constants.PublicReadWrite...) + case s3_constants.CannedAclAuthenticatedRead: + grants = append(grants, objectWriterFullControl) + grants = append(grants, s3_constants.AuthenticatedRead...) + default: + err = s3err.ErrInvalidAclArgument + } + return +} - //objectWrite automatically has full control on current object +// ExtractObjectCannedAcl parse object canned acl, includes: 'private'|'public-read'|'public-read-write'|'authenticated-read'|'aws-exec-read'|'bucket-owner-read'|'bucket-owner-full-control' +func ExtractObjectCannedAcl(request *http.Request, objectOwnership, bucketOwnerId, requestAccountId string, createObject bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) { + if createObject { + ownerId = requestAccountId + } + + cannedAcl := request.Header.Get(s3_constants.AmzCannedAcl) + if cannedAcl == "" { + return ownerId, grants, s3err.ErrNone + } + + errCode = s3err.ErrNone objectWriterFullControl := &s3.Grant{ Grantee: &s3.Grantee{ - ID: &accountId, + ID: &requestAccountId, Type: &s3_constants.GrantTypeCanonicalUser, }, Permission: &s3_constants.PermissionFullControl, @@ -212,7 +299,7 @@ func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl s grants = append(grants, s3_constants.LogDeliveryWrite...) case s3_constants.CannedAclBucketOwnerRead: grants = append(grants, objectWriterFullControl) - if bucketOwnerId != "" && bucketOwnerId != accountId { + if requestAccountId != bucketOwnerId { grants = append(grants, &s3.Grant{ Grantee: &s3.Grantee{ @@ -225,7 +312,7 @@ func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl s case s3_constants.CannedAclBucketOwnerFullControl: if bucketOwnerId != "" { // if set ownership to 'BucketOwnerPreferred' when upload object, the bucket owner will be the object owner - if !putAcl && bucketOwnership == s3_constants.OwnershipBucketOwnerPreferred { + if createObject && objectOwnership == s3_constants.OwnershipBucketOwnerPreferred { ownerId = bucketOwnerId grants = append(grants, &s3.Grant{ @@ -237,7 +324,7 @@ func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl s }) } else { grants = append(grants, objectWriterFullControl) - if accountId != bucketOwnerId { + if requestAccountId != bucketOwnerId { grants = append(grants, &s3.Grant{ Grantee: &s3.Grantee{ @@ -249,15 +336,13 @@ func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl s } } } - case s3_constants.CannedAclAwsExecRead: - err = s3err.ErrNotImplemented default: - err = s3err.ErrInvalidRequest + errCode = s3err.ErrInvalidAclArgument } return } -// ValidateAndTransferGrants validate grant & transfer Email-Grant to Id-Grant +// ValidateAndTransferGrants validate grant entity exists and transfer Email-Grant to Id-Grant func ValidateAndTransferGrants(accountManager *s3account.AccountManager, grants []*s3.Grant) ([]*s3.Grant, s3err.ErrorCode) { var result []*s3.Grant for _, grant := range grants { @@ -314,15 +399,43 @@ func ValidateAndTransferGrants(accountManager *s3account.AccountManager, grants return result, s3err.ErrNone } -// DetermineReqGrants generates the grant set (Grants) according to accountId and reqPermission. -func DetermineReqGrants(accountId, aclAction string) (grants []*s3.Grant) { +// ValidateObjectOwnershipAndGrants validate if grants equals OwnerFullControl when 'ObjectOwnership' is 'BucketOwnerEnforced' +func ValidateObjectOwnershipAndGrants(objectOwnership, bucketOwnerId string, grants []*s3.Grant) s3err.ErrorCode { + if len(grants) == 0 { + return s3err.ErrNone + } + if objectOwnership == "" { + objectOwnership = s3_constants.DefaultObjectOwnership + } + if objectOwnership != s3_constants.OwnershipBucketOwnerEnforced { + return s3err.ErrNone + } + if len(grants) > 1 { + return s3err.AccessControlListNotSupported + } + + bucketOwnerFullControlGrant := &s3.Grant{ + Permission: &s3_constants.PermissionFullControl, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + } + if GrantEquals(bucketOwnerFullControlGrant, grants[0]) { + return s3err.ErrNone + } + return s3err.AccessControlListNotSupported +} + +// DetermineRequiredGrants generates the grant set (Grants) according to accountId and reqPermission. +func DetermineRequiredGrants(accountId, permission string) (grants []*s3.Grant) { // group grantee (AllUsers) grants = append(grants, &s3.Grant{ Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, URI: &s3_constants.GranteeGroupAllUsers, }, - Permission: &aclAction, + Permission: &permission, }) grants = append(grants, &s3.Grant{ Grantee: &s3.Grantee{ @@ -338,7 +451,7 @@ func DetermineReqGrants(accountId, aclAction string) (grants []*s3.Grant) { Type: &s3_constants.GrantTypeCanonicalUser, ID: &accountId, }, - Permission: &aclAction, + Permission: &permission, }) grants = append(grants, &s3.Grant{ Grantee: &s3.Grantee{ @@ -355,7 +468,7 @@ func DetermineReqGrants(accountId, aclAction string) (grants []*s3.Grant) { Type: &s3_constants.GrantTypeGroup, URI: &s3_constants.GranteeGroupAuthenticatedUsers, }, - Permission: &aclAction, + Permission: &permission, }) grants = append(grants, &s3.Grant{ Grantee: &s3.Grantee{ @@ -392,7 +505,7 @@ func SetAcpGrantsHeader(r *http.Request, acpGrants []*s3.Grant) { } // GetAcpGrants return grants parsed from entry -func GetAcpGrants(entryExtended map[string][]byte) []*s3.Grant { +func GetAcpGrants(ownerId *string, entryExtended map[string][]byte) []*s3.Grant { acpBytes, ok := entryExtended[s3_constants.ExtAmzAclKey] if ok && len(acpBytes) > 0 { var grants []*s3.Grant @@ -400,31 +513,43 @@ func GetAcpGrants(entryExtended map[string][]byte) []*s3.Grant { if err == nil { return grants } + glog.Warning("grants Unmarshal error", err) + } + if ownerId == nil { + return nil + } + return []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: ownerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, } - return nil } // AssembleEntryWithAcp fill entry with owner and grants -func AssembleEntryWithAcp(objectEntry *filer_pb.Entry, objectOwner string, grants []*s3.Grant) s3err.ErrorCode { - if objectEntry.Extended == nil { - objectEntry.Extended = make(map[string][]byte) +func AssembleEntryWithAcp(filerEntry *filer_pb.Entry, ownerId string, grants []*s3.Grant) s3err.ErrorCode { + if filerEntry.Extended == nil { + filerEntry.Extended = make(map[string][]byte) } - if len(objectOwner) > 0 { - objectEntry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(objectOwner) + if len(ownerId) > 0 { + filerEntry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(ownerId) } else { - delete(objectEntry.Extended, s3_constants.ExtAmzOwnerKey) + delete(filerEntry.Extended, s3_constants.ExtAmzOwnerKey) } - if len(grants) > 0 { + if grants != nil { grantsBytes, err := json.Marshal(grants) if err != nil { glog.Warning("assemble acp to entry:", err) return s3err.ErrInvalidRequest } - objectEntry.Extended[s3_constants.ExtAmzAclKey] = grantsBytes + filerEntry.Extended[s3_constants.ExtAmzAclKey] = grantsBytes } else { - delete(objectEntry.Extended, s3_constants.ExtAmzAclKey) + delete(filerEntry.Extended, s3_constants.ExtAmzAclKey) } return s3err.ErrNone @@ -509,6 +634,46 @@ func GrantEquals(a, b *s3.Grant) bool { return true } +func MarshalGrantsToJson(grants []*s3.Grant) ([]byte, error) { + if len(grants) == 0 { + return nil, nil + } + var GrantsToMap []map[string]any + for _, grant := range grants { + grantee := grant.Grantee + switch *grantee.Type { + case s3_constants.GrantTypeGroup: + GrantsToMap = append(GrantsToMap, map[string]any{ + "Permission": grant.Permission, + "Grantee": map[string]any{ + "Type": grantee.Type, + "URI": grantee.URI, + }, + }) + case s3_constants.GrantTypeCanonicalUser: + GrantsToMap = append(GrantsToMap, map[string]any{ + "Permission": grant.Permission, + "Grantee": map[string]any{ + "Type": grantee.Type, + "ID": grantee.ID, + }, + }) + case s3_constants.GrantTypeAmazonCustomerByEmail: + GrantsToMap = append(GrantsToMap, map[string]any{ + "Permission": grant.Permission, + "Grantee": map[string]any{ + "Type": grantee.Type, + "EmailAddress": grantee.EmailAddress, + }, + }) + default: + return nil, fmt.Errorf("grantee type[%s] is not valid", *grantee.Type) + } + } + + return json.Marshal(GrantsToMap) +} + func GrantWithFullControl(accountId string) *s3.Grant { return &s3.Grant{ Permission: &s3_constants.PermissionFullControl, @@ -524,22 +689,22 @@ func CheckObjectAccessForReadObject(r *http.Request, w http.ResponseWriter, entr return http.StatusOK, true } - accountId := GetAccountId(r) - if len(accountId) == 0 { + requestAccountId := GetAccountId(r) + if len(requestAccountId) == 0 { glog.Warning("#checkObjectAccessForReadObject header[accountId] not exists!") return http.StatusForbidden, false } //owner access objectOwner := GetAcpOwner(entry.Extended, bucketOwnerId) - if accountId == objectOwner { + if ValidateAccount(requestAccountId, objectOwner) { return http.StatusOK, true } //find in Grants - acpGrants := GetAcpGrants(entry.Extended) + acpGrants := GetAcpGrants(nil, entry.Extended) if acpGrants != nil { - reqGrants := DetermineReqGrants(accountId, s3_constants.PermissionRead) + reqGrants := DetermineRequiredGrants(requestAccountId, s3_constants.PermissionRead) for _, requiredGrant := range reqGrants { for _, grant := range acpGrants { if GrantEquals(requiredGrant, grant) { @@ -549,6 +714,6 @@ func CheckObjectAccessForReadObject(r *http.Request, w http.ResponseWriter, entr } } - glog.V(3).Infof("acl denied! request account id: %s", accountId) + glog.V(3).Infof("acl denied! request account id: %s", requestAccountId) return http.StatusForbidden, false } diff --git a/weed/s3api/s3acl/acl_helper_test.go b/weed/s3api/s3acl/acl_helper_test.go index 713e56a26..ebf253851 100644 --- a/weed/s3api/s3acl/acl_helper_test.go +++ b/weed/s3api/s3acl/acl_helper_test.go @@ -56,326 +56,6 @@ func TestGetAccountId(t *testing.T) { } } -func TestExtractAcl(t *testing.T) { - type Case struct { - id int - resultErrCode, expectErrCode s3err.ErrorCode - resultGrants, expectGrants []*s3.Grant - } - testCases := make([]*Case, 0) - accountAdminId := "admin" - - { - //case1 (good case) - //parse acp from request body - req := &http.Request{ - Header: make(map[string][]string), - } - req.Body = io.NopCloser(bytes.NewReader([]byte(` - - - admin - admin - - - - - admin - - FULL_CONTROL - - - - http://acs.amazonaws.com/groups/global/AllUsers - - FULL_CONTROL - - - - `))) - objectWriter := "accountA" - grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter) - testCases = append(testCases, &Case{ - 1, - errCode, s3err.ErrNone, - grants, []*s3.Grant{ - { - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeCanonicalUser, - ID: &accountAdminId, - }, - Permission: &s3_constants.PermissionFullControl, - }, - { - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeGroup, - URI: &s3_constants.GranteeGroupAllUsers, - }, - Permission: &s3_constants.PermissionFullControl, - }, - }, - }) - } - - { - //case2 (good case) - //parse acp from header (cannedAcl) - req := &http.Request{ - Header: make(map[string][]string), - } - req.Body = nil - req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate) - objectWriter := "accountA" - grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter) - testCases = append(testCases, &Case{ - 2, - errCode, s3err.ErrNone, - grants, []*s3.Grant{ - { - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeCanonicalUser, - ID: &objectWriter, - }, - Permission: &s3_constants.PermissionFullControl, - }, - }, - }) - } - - { - //case3 (bad case) - //parse acp from request body (content is invalid) - req := &http.Request{ - Header: make(map[string][]string), - } - req.Body = io.NopCloser(bytes.NewReader([]byte("zdfsaf"))) - req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate) - objectWriter := "accountA" - _, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter) - testCases = append(testCases, &Case{ - id: 3, - resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, - }) - } - - //case4 (bad case) - //parse acp from header (cannedAcl is invalid) - req := &http.Request{ - Header: make(map[string][]string), - } - req.Body = nil - req.Header.Set(s3_constants.AmzCannedAcl, "dfaksjfk") - objectWriter := "accountA" - _, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, "", objectWriter) - testCases = append(testCases, &Case{ - id: 4, - resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, - }) - - { - //case5 (bad case) - //parse acp from request body: owner is inconsistent - req.Body = io.NopCloser(bytes.NewReader([]byte(` - - - admin - admin - - - - - admin - - FULL_CONTROL - - - - http://acs.amazonaws.com/groups/global/AllUsers - - FULL_CONTROL - - - - `))) - objectWriter = "accountA" - _, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, objectWriter, objectWriter) - testCases = append(testCases, &Case{ - id: 5, - resultErrCode: errCode, expectErrCode: s3err.ErrAccessDenied, - }) - } - - for _, tc := range testCases { - if tc.resultErrCode != tc.expectErrCode { - t.Fatalf("case[%d]: errorCode not expect", tc.id) - } - if !grantsEquals(tc.resultGrants, tc.expectGrants) { - t.Fatalf("case[%d]: grants not expect", tc.id) - } - } -} - -func TestParseAndValidateAclHeaders(t *testing.T) { - type Case struct { - id int - resultOwner, expectOwner string - resultErrCode, expectErrCode s3err.ErrorCode - resultGrants, expectGrants []*s3.Grant - } - testCases := make([]*Case, 0) - bucketOwner := "admin" - - { - //case1 (good case) - //parse custom acl - req := &http.Request{ - Header: make(map[string][]string), - } - objectWriter := "accountA" - req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example.com"`) - ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) - testCases = append(testCases, &Case{ - 1, - ownerId, objectWriter, - errCode, s3err.ErrNone, - grants, []*s3.Grant{ - { - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeGroup, - URI: &s3_constants.GranteeGroupAllUsers, - }, - Permission: &s3_constants.PermissionFullControl, - }, - { - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeCanonicalUser, - ID: &s3account.AccountAnonymous.Id, - }, - Permission: &s3_constants.PermissionFullControl, - }, - { - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeCanonicalUser, - ID: &s3account.AccountAdmin.Id, - }, - Permission: &s3_constants.PermissionFullControl, - }, - }, - }) - } - { - //case2 (good case) - //parse canned acl (ownership=ObjectWriter) - req := &http.Request{ - Header: make(map[string][]string), - } - objectWriter := "accountA" - req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) - ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) - testCases = append(testCases, &Case{ - 2, - ownerId, objectWriter, - errCode, s3err.ErrNone, - grants, []*s3.Grant{ - { - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeCanonicalUser, - ID: &objectWriter, - }, - Permission: &s3_constants.PermissionFullControl, - }, - { - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeCanonicalUser, - ID: &bucketOwner, - }, - Permission: &s3_constants.PermissionFullControl, - }, - }, - }) - } - { - //case3 (good case) - //parse canned acl (ownership=OwnershipBucketOwnerPreferred) - req := &http.Request{ - Header: make(map[string][]string), - } - objectWriter := "accountA" - req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) - ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwner, objectWriter, false) - testCases = append(testCases, &Case{ - 3, - ownerId, bucketOwner, - errCode, s3err.ErrNone, - grants, []*s3.Grant{ - { - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeCanonicalUser, - ID: &bucketOwner, - }, - Permission: &s3_constants.PermissionFullControl, - }, - }, - }) - } - { - //case4 (bad case) - //parse custom acl (grantee id not exists) - req := &http.Request{ - Header: make(map[string][]string), - } - objectWriter := "accountA" - req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="notExistsAccount", emailAddress="admin@example.com"`) - _, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) - testCases = append(testCases, &Case{ - id: 4, - resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, - }) - } - - { - //case5 (bad case) - //parse custom acl (invalid format) - req := &http.Request{ - Header: make(map[string][]string), - } - objectWriter := "accountA" - req.Header.Set(s3_constants.AmzAclFullControl, `uri="http:sfasf"`) - _, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) - testCases = append(testCases, &Case{ - id: 5, - resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, - }) - } - - { - //case6 (bad case) - //parse canned acl (invalid value) - req := &http.Request{ - Header: make(map[string][]string), - } - objectWriter := "accountA" - req.Header.Set(s3_constants.AmzCannedAcl, `uri="http:sfasf"`) - _, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) - testCases = append(testCases, &Case{ - id: 5, - resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, - }) - } - - for _, tc := range testCases { - if tc.expectErrCode != tc.resultErrCode { - t.Errorf("case[%d]: errCode unexpect", tc.id) - } - if tc.resultOwner != tc.expectOwner { - t.Errorf("case[%d]: ownerId unexpect", tc.id) - } - if !grantsEquals(tc.resultGrants, tc.expectGrants) { - t.Fatalf("case[%d]: grants not expect", tc.id) - } - } -} - func grantsEquals(a, b []*s3.Grant) bool { if len(a) != len(b) { return false @@ -394,7 +74,7 @@ func TestDetermineReqGrants(t *testing.T) { accountId := s3account.AccountAnonymous.Id reqPermission := s3_constants.PermissionRead - resultGrants := DetermineReqGrants(accountId, reqPermission) + resultGrants := DetermineRequiredGrants(accountId, reqPermission) expectGrants := []*s3.Grant{ { Grantee: &s3.Grantee{ @@ -434,7 +114,7 @@ func TestDetermineReqGrants(t *testing.T) { accountId := "accountX" reqPermission := s3_constants.PermissionRead - resultGrants := DetermineReqGrants(accountId, reqPermission) + resultGrants := DetermineRequiredGrants(accountId, reqPermission) expectGrants := []*s3.Grant{ { Grantee: &s3.Grantee{ @@ -509,7 +189,7 @@ func TestAssembleEntryWithAcp(t *testing.T) { t.Fatalf("owner not expect") } - resultGrants := GetAcpGrants(entry.Extended) + resultGrants := GetAcpGrants(nil, entry.Extended) if !grantsEquals(resultGrants, expectGrants) { t.Fatal("grants not expect") } @@ -522,7 +202,7 @@ func TestAssembleEntryWithAcp(t *testing.T) { t.Fatalf("owner not expect") } - resultGrants = GetAcpGrants(entry.Extended) + resultGrants = GetAcpGrants(nil, entry.Extended) if len(resultGrants) != 0 { t.Fatal("grants not expect") } @@ -721,3 +401,1066 @@ func TestGrantWithFullControl(t *testing.T) { t.Fatal("GrantWithFullControl not expect") } } + +func TestExtractObjectAcl(t *testing.T) { + type Case struct { + id string + resultErrCode, expectErrCode s3err.ErrorCode + resultGrants, expectGrants []*s3.Grant + resultOwnerId, expectOwnerId string + } + testCases := make([]*Case, 0) + accountAdminId := "admin" + + //Request body to specify AccessControlList + { + //ownership: ObjectWriter + //s3:PutObjectAcl('createObject' is set to false), config acl through request body + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + requestAccountId := "accountA" + ownerId, grants, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, requestAccountId, false) + testCases = append(testCases, &Case{ + "TestExtractObjectAcl: ownership-ObjectWriter, createObject-false, acl-requestBody", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountAdminId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + ownerId, accountAdminId, + }) + } + { + //ownership: BucketOwnerEnforced (extra acl is not allowed) + //s3:PutObjectAcl('createObject' is set to false), config acl through request body + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + requestAccountId := "accountA" + _, _, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipBucketOwnerEnforced, accountAdminId, requestAccountId, false) + testCases = append(testCases, &Case{ + id: "TestExtractObjectAcl: ownership-BucketOwnerEnforced, createObject-false, acl-requestBody", + resultErrCode: errCode, expectErrCode: s3err.AccessControlListNotSupported, + }) + } + { + //ownership: ObjectWriter + //s3:PutObject('createObject' is set to false), request body will be ignored when parse acl + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + requestAccountId := "accountA" + ownerId, grants, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, requestAccountId, true) + testCases = append(testCases, &Case{ + "TestExtractObjectAcl: ownership-ObjectWriter, createObject-true, acl-requestBody", + errCode, s3err.ErrNone, + grants, []*s3.Grant{}, + ownerId, "", + }) + } + { + //ownership: BucketOwnerEnforced (extra acl is not allowed) + //s3:PutObject('createObject' is set to true), request body will be ignored when parse acl + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + requestAccountId := "accountA" + ownerId, grants, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipBucketOwnerEnforced, accountAdminId, requestAccountId, true) + testCases = append(testCases, &Case{ + "TestExtractObjectAcl: ownership-BucketOwnerEnforced, createObject-true, acl-requestBody", + errCode, s3err.ErrNone, + grants, []*s3.Grant{}, + ownerId, "", + }) + } + + //CannedAcl Header to specify ACL + //cannedAcl, putObjectACL + { + //ownership: ObjectWriter + //s3:PutObjectACL('createObject' is set to false), parse cannedACL header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + bucketOwnerId := "admin" + requestAccountId := "accountA" + ownerId, grants, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + "TestExtractObjectAcl: ownership-ObjectWriter, createObject-false, acl-CannedAcl", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &requestAccountId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + ownerId, "", + }) + } + { + //ownership: BucketOwnerPreferred + //s3:PutObjectACL('createObject' is set to false), parse cannedACL header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + bucketOwnerId := "admin" + requestAccountId := "accountA" + ownerId, grants, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + "TestExtractObjectAcl: ownership-BucketOwnerPreferred, createObject-false, acl-CannedAcl", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &requestAccountId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + ownerId, "", + }) + } + { + //ownership: BucketOwnerEnforced + //s3:PutObjectACL('createObject' is set to false), parse cannedACL header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, _, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipBucketOwnerEnforced, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + id: "TestExtractObjectAcl: ownership-BucketOwnerEnforced, createObject-false, acl-CannedAcl", + resultErrCode: errCode, expectErrCode: s3err.AccessControlListNotSupported, + }) + } + //cannedACL, putObject + { + //ownership: ObjectWriter + //s3:PutObject('createObject' is set to true), parse cannedACL header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + bucketOwnerId := "admin" + requestAccountId := "accountA" + ownerId, grants, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwnerId, requestAccountId, true) + testCases = append(testCases, &Case{ + "TestExtractObjectAcl: ownership-ObjectWriter, createObject-true, acl-CannedAcl", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &requestAccountId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + ownerId, "", + }) + } + { + //ownership: BucketOwnerPreferred + //s3:PutObject('createObject' is set to true), parse cannedACL header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + bucketOwnerId := "admin" + requestAccountId := "accountA" + ownerId, grants, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwnerId, requestAccountId, true) + testCases = append(testCases, &Case{ + "TestExtractObjectAcl: ownership-BucketOwnerPreferred, createObject-true, acl-CannedAcl", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + ownerId, "", + }) + } + { + //ownership: BucketOwnerEnforced + //s3:PutObject('createObject' is set to true), parse cannedACL header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, _, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipBucketOwnerEnforced, bucketOwnerId, requestAccountId, true) + testCases = append(testCases, &Case{ + id: "TestExtractObjectAcl: ownership-BucketOwnerEnforced, createObject-true, acl-CannedAcl", + resultErrCode: errCode, expectErrCode: s3err.AccessControlListNotSupported, + }) + } + + //cannedAcl, putObjectACL + { + //ownership: ObjectWriter + //s3:PutObjectACL('createObject' is set to false), parse customAcl header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=accountA,id=\"admin\"") + bucketOwnerId := "admin" + requestAccountId := "accountA" + ownerId, grants, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + "TestExtractObjectAcl: ownership-ObjectWriter, createObject-false, acl-customAcl", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &requestAccountId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + ownerId, "", + }) + } + { + //ownership: BucketOwnerPreferred + //s3:PutObjectACL('createObject' is set to false), parse customAcl header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=accountA,id=\"admin\"") + bucketOwnerId := "admin" + requestAccountId := "accountA" + ownerId, grants, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + "TestExtractObjectAcl: ownership-BucketOwnerPreferred, createObject-false, acl-customAcl", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &requestAccountId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + ownerId, "", + }) + } + { + //ownership: BucketOwnerEnforced + //s3:PutObjectACL('createObject' is set to false), parse customAcl header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=accountA,id=\"admin\"") + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, _, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipBucketOwnerEnforced, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + id: "TestExtractObjectAcl: ownership-BucketOwnerEnforced, createObject-false, acl-customAcl", + resultErrCode: errCode, expectErrCode: s3err.AccessControlListNotSupported, + }) + } + //customAcl, putObject + { + //ownership: ObjectWriter + //s3:PutObject('createObject' is set to true), parse customAcl header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=accountA,id=\"admin\"") + bucketOwnerId := "admin" + requestAccountId := "accountA" + ownerId, grants, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwnerId, requestAccountId, true) + testCases = append(testCases, &Case{ + "TestExtractObjectAcl: ownership-ObjectWriter, createObject-true, acl-customAcl", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &requestAccountId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + ownerId, "", + }) + } + { + //ownership: BucketOwnerPreferred + //s3:PutObject('createObject' is set to true), parse customAcl header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=\"admin\"") + bucketOwnerId := "admin" + requestAccountId := "accountA" + ownerId, grants, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwnerId, requestAccountId, true) + testCases = append(testCases, &Case{ + "TestExtractObjectAcl: ownership-BucketOwnerPreferred, createObject-true, acl-customAcl", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + ownerId, "", + }) + } + { + //ownership: BucketOwnerEnforced + //s3:PutObject('createObject' is set to true), parse customAcl header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=accountA,id=\"admin\"") + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, _, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipBucketOwnerEnforced, bucketOwnerId, requestAccountId, true) + testCases = append(testCases, &Case{ + id: "TestExtractObjectAcl: ownership-BucketOwnerEnforced, createObject-true, acl-customAcl", + resultErrCode: errCode, expectErrCode: s3err.AccessControlListNotSupported, + }) + } + + { + //parse acp from request header: both canned acl and custom acl not allowed + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=admin, id=\"accountA\"") + req.Header.Set(s3_constants.AmzCannedAcl, "private") + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, _, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + id: "Only one of cannedAcl, customAcl is allowed", + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + { + //Acl can only be specified in one of requestBody, cannedAcl, customAcl, simultaneous use is not allowed + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=admin, id=\"accountA\"") + req.Header.Set(s3_constants.AmzCannedAcl, "private") + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + requestAccountId := "accountA" + _, _, errCode := ExtractObjectAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, requestAccountId, false) + testCases = append(testCases, &Case{ + id: "Only one of requestBody, cannedAcl, customAcl is allowed", + resultErrCode: errCode, expectErrCode: s3err.ErrUnexpectedContent, + }) + } + for _, tc := range testCases { + if tc.resultErrCode != tc.expectErrCode { + t.Fatalf("case[%s]: errorCode[%v] not expect[%v]", tc.id, s3err.GetAPIError(tc.resultErrCode).Code, s3err.GetAPIError(tc.expectErrCode).Code) + } + if !grantsEquals(tc.resultGrants, tc.expectGrants) { + t.Fatalf("case[%s]: grants not expect", tc.id) + } + } +} + +func TestBucketObjectAcl(t *testing.T) { + type Case struct { + id string + resultErrCode, expectErrCode s3err.ErrorCode + resultGrants, expectGrants []*s3.Grant + } + testCases := make([]*Case, 0) + accountAdminId := "admin" + + //Request body to specify AccessControlList + { + //ownership: ObjectWriter + //s3:PutBucketAcl('createObject' is set to false), config acl through request body + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + bucketOwnerId := "admin" + requestAccountId := "accountA" + grants, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + "TestExtractBucketAcl: ownership-ObjectWriter, createObject-false, acl-requestBody", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountAdminId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //ownership: BucketOwnerEnforced (extra acl is not allowed) + //s3:PutBucketAcl('createObject' is set to false), config acl through request body + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + requestAccountId := "accountA" + _, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipBucketOwnerEnforced, accountAdminId, requestAccountId, false) + testCases = append(testCases, &Case{ + id: "TestExtractBucketAcl: ownership-BucketOwnerEnforced, createObject-false, acl-requestBody", + resultErrCode: errCode, expectErrCode: s3err.AccessControlListNotSupported, + }) + } + { + //ownership: ObjectWriter + //s3:PutObject('createObject' is set to false), request body will be ignored when parse acl + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + requestAccountId := "accountA" + grants, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, requestAccountId, true) + testCases = append(testCases, &Case{ + "TestExtractBucketAcl: ownership-ObjectWriter, createObject-true, acl-requestBody", + errCode, s3err.ErrNone, + grants, []*s3.Grant{}, + }) + } + { + //ownership: BucketOwnerEnforced (extra acl is not allowed) + //s3:PutObject('createObject' is set to true), request body will be ignored when parse acl + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + requestAccountId := "accountA" + grants, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipBucketOwnerEnforced, accountAdminId, requestAccountId, true) + testCases = append(testCases, &Case{ + "TestExtractBucketAcl: ownership-BucketOwnerEnforced, createObject-true, acl-requestBody", + errCode, s3err.ErrNone, + grants, []*s3.Grant{}, + }) + } + + //CannedAcl Header to specify ACL + //cannedAcl, PutBucketAcl + { + //ownership: ObjectWriter + //s3:PutBucketAcl('createObject' is set to false), parse cannedACL header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + id: "TestExtractBucketAcl: ownership-ObjectWriter, createObject-false, acl-CannedAcl", + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidAclArgument, + }) + } + { + //ownership: BucketOwnerPreferred + //s3:PutBucketAcl('createObject' is set to false), parse cannedACL header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + id: "TestExtractBucketAcl: ownership-BucketOwnerPreferred, createObject-false, acl-CannedAcl", + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidAclArgument, + }) + } + { + //ownership: BucketOwnerEnforced + //s3:PutBucketAcl('createObject' is set to false), parse cannedACL header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipBucketOwnerEnforced, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + id: "TestExtractBucketAcl: ownership-BucketOwnerEnforced, createObject-false, acl-CannedAcl", + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidAclArgument, + }) + } + //cannedACL, createBucket + { + //ownership: ObjectWriter + //s3:PutObject('createObject' is set to true), parse cannedACL header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwnerId, requestAccountId, true) + testCases = append(testCases, &Case{ + id: "TestExtractBucketAcl: ownership-BucketOwnerEnforced, createObject-true, acl-CannedAcl", + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidAclArgument, + }) + } + { + //ownership: BucketOwnerPreferred + //s3:PutObject('createObject' is set to true), parse cannedACL header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwnerId, requestAccountId, true) + testCases = append(testCases, &Case{ + id: "TestExtractBucketAcl: ownership-BucketOwnerPreferred, createObject-true, acl-CannedAcl", + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidAclArgument, + }) + } + { + //ownership: BucketOwnerEnforced + //s3:PutObject('createObject' is set to true), parse cannedACL header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipBucketOwnerEnforced, bucketOwnerId, requestAccountId, true) + testCases = append(testCases, &Case{ + id: "TestExtractBucketAcl: ownership-BucketOwnerEnforced, createObject-true, acl-CannedAcl", + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidAclArgument, + }) + } + + //customAcl, PutBucketAcl + { + //ownership: ObjectWriter + //s3:PutBucketAcl('createObject' is set to false), parse customAcl header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=accountA,id=\"admin\"") + bucketOwnerId := "admin" + requestAccountId := "accountA" + grants, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + "TestExtractBucketAcl: ownership-ObjectWriter, createObject-false, acl-customAcl", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &requestAccountId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //ownership: BucketOwnerPreferred + //s3:PutBucketAcl('createObject' is set to false), parse customAcl header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=accountA,id=\"admin\"") + bucketOwnerId := "admin" + requestAccountId := "accountA" + grants, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + "TestExtractBucketAcl: ownership-BucketOwnerPreferred, createObject-false, acl-customAcl", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &requestAccountId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //ownership: BucketOwnerEnforced + //s3:PutBucketAcl('createObject' is set to false), parse customAcl header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=accountA,id=\"admin\"") + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipBucketOwnerEnforced, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + id: "TestExtractBucketAcl: ownership-BucketOwnerEnforced, createObject-false, acl-customAcl", + resultErrCode: errCode, expectErrCode: s3err.AccessControlListNotSupported, + }) + } + //customAcl, putObject + { + //ownership: ObjectWriter + //s3:PutObject('createObject' is set to true), parse customAcl header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=accountA,id=\"admin\"") + bucketOwnerId := "admin" + requestAccountId := "accountA" + grants, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwnerId, requestAccountId, true) + testCases = append(testCases, &Case{ + "TestExtractBucketAcl: ownership-ObjectWriter, createObject-true, acl-customAcl", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &requestAccountId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //ownership: BucketOwnerPreferred + //s3:PutObject('createObject' is set to true), parse customAcl header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=\"admin\"") + bucketOwnerId := "admin" + requestAccountId := "accountA" + grants, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwnerId, requestAccountId, true) + testCases = append(testCases, &Case{ + "TestExtractBucketAcl: ownership-BucketOwnerPreferred, createObject-true, acl-customAcl", + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //ownership: BucketOwnerEnforced + //s3:PutObject('createObject' is set to true), parse customAcl header + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=accountA,id=\"admin\"") + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipBucketOwnerEnforced, bucketOwnerId, requestAccountId, true) + testCases = append(testCases, &Case{ + id: "TestExtractBucketAcl: ownership-BucketOwnerEnforced, createObject-true, acl-customAcl", + resultErrCode: errCode, expectErrCode: s3err.AccessControlListNotSupported, + }) + } + + { + //parse acp from request header: both canned acl and custom acl not allowed + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=admin, id=\"accountA\"") + req.Header.Set(s3_constants.AmzCannedAcl, "private") + bucketOwnerId := "admin" + requestAccountId := "accountA" + _, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwnerId, requestAccountId, false) + testCases = append(testCases, &Case{ + id: "Only one of cannedAcl, customAcl is allowed", + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + { + //Acl can only be specified in one of requestBody, cannedAcl, customAcl, simultaneous use is not allowed + req := &http.Request{ + Header: make(map[string][]string), + } + req.Header.Set(s3_constants.AmzAclFullControl, "id=admin, id=\"accountA\"") + req.Header.Set(s3_constants.AmzCannedAcl, "private") + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + requestAccountId := "accountA" + _, errCode := ExtractBucketAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, requestAccountId, false) + testCases = append(testCases, &Case{ + id: "Only one of requestBody, cannedAcl, customAcl is allowed", + resultErrCode: errCode, expectErrCode: s3err.ErrUnexpectedContent, + }) + } + for _, tc := range testCases { + if tc.resultErrCode != tc.expectErrCode { + t.Fatalf("case[%s]: errorCode[%v] not expect[%v]", tc.id, s3err.GetAPIError(tc.resultErrCode).Code, s3err.GetAPIError(tc.expectErrCode).Code) + } + if !grantsEquals(tc.resultGrants, tc.expectGrants) { + t.Fatalf("case[%s]: grants not expect", tc.id) + } + } +} + +func TestMarshalGrantsToJson(t *testing.T) { + //ok + bucketOwnerId := "admin" + requestAccountId := "accountA" + grants := []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &requestAccountId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwnerId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + } + result, err := MarshalGrantsToJson(grants) + if err != nil { + t.Error(err) + } + + var grants2 []*s3.Grant + err = json.Unmarshal(result, &grants2) + if err != nil { + t.Error(err) + } + + print(string(result)) + if !grantsEquals(grants, grants2) { + t.Fatal("grants not equal", grants, grants2) + } + + //ok + result, err = MarshalGrantsToJson(nil) + if result != nil && err != nil { + t.Fatal("error: result, err = MarshalGrantsToJson(nil)") + } +} diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index 5fdec0397..528bf7e1d 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -23,19 +23,19 @@ func getAccountId(r *http.Request) string { } func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s3err.ErrorCode { - metadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) + bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) if errCode != s3err.ErrNone { return errCode } - accountId := getAccountId(r) - if accountId == s3account.AccountAdmin.Id || accountId == *metadata.Owner.ID { + requestAccountId := getAccountId(r) + if s3acl.ValidateAccount(requestAccountId, *bucketMetadata.Owner.ID) { return s3err.ErrNone } return s3err.ErrAccessDenied } //Check access for PutBucketAclHandler -func (s3a *S3ApiServer) checkAccessForPutBucketAcl(accountId, bucket string) (*BucketMetaData, s3err.ErrorCode) { +func (s3a *S3ApiServer) checkAccessForPutBucketAcl(requestAccountId, bucket string) (*BucketMetaData, s3err.ErrorCode) { bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) if errCode != s3err.ErrNone { return nil, errCode @@ -45,12 +45,12 @@ func (s3a *S3ApiServer) checkAccessForPutBucketAcl(accountId, bucket string) (*B return nil, s3err.AccessControlListNotSupported } - if accountId == s3account.AccountAdmin.Id || accountId == *bucketMetadata.Owner.ID { + if s3acl.ValidateAccount(requestAccountId, *bucketMetadata.Owner.ID) { return bucketMetadata, s3err.ErrNone } if len(bucketMetadata.Acl) > 0 { - reqGrants := s3acl.DetermineReqGrants(accountId, s3_constants.PermissionWriteAcp) + reqGrants := s3acl.DetermineRequiredGrants(requestAccountId, s3_constants.PermissionWriteAcp) for _, bucketGrant := range bucketMetadata.Acl { for _, reqGrant := range reqGrants { if s3acl.GrantEquals(bucketGrant, reqGrant) { @@ -59,7 +59,7 @@ func (s3a *S3ApiServer) checkAccessForPutBucketAcl(accountId, bucket string) (*B } } } - glog.V(3).Infof("acl denied! request account id: %s", accountId) + glog.V(3).Infof("acl denied! request account id: %s", requestAccountId) return nil, s3err.ErrAccessDenied } @@ -73,6 +73,7 @@ func updateBucketEntry(s3a *S3ApiServer, entry *filer_pb.Entry) error { // - GetBucketAclHandler // - ListObjectsV1Handler // - ListObjectsV2Handler +// - ListMultipartUploadsHandler func (s3a *S3ApiServer) checkAccessForReadBucket(r *http.Request, bucket, aclAction string) (*BucketMetaData, s3err.ErrorCode) { bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) if errCode != s3err.ErrNone { @@ -83,13 +84,13 @@ func (s3a *S3ApiServer) checkAccessForReadBucket(r *http.Request, bucket, aclAct return bucketMetadata, s3err.ErrNone } - accountId := s3acl.GetAccountId(r) - if accountId == s3account.AccountAdmin.Id || accountId == *bucketMetadata.Owner.ID { + requestAccountId := s3acl.GetAccountId(r) + if s3acl.ValidateAccount(requestAccountId, *bucketMetadata.Owner.ID) { return bucketMetadata, s3err.ErrNone } if len(bucketMetadata.Acl) > 0 { - reqGrants := s3acl.DetermineReqGrants(accountId, aclAction) + reqGrants := s3acl.DetermineRequiredGrants(requestAccountId, aclAction) for _, bucketGrant := range bucketMetadata.Acl { for _, reqGrant := range reqGrants { if s3acl.GrantEquals(bucketGrant, reqGrant) { @@ -99,7 +100,7 @@ func (s3a *S3ApiServer) checkAccessForReadBucket(r *http.Request, bucket, aclAct } } - glog.V(3).Infof("acl denied! request account id: %s", accountId) + glog.V(3).Infof("acl denied! request account id: %s", requestAccountId) return nil, s3err.ErrAccessDenied } @@ -121,14 +122,12 @@ func (s3a *S3ApiServer) checkAccessForReadObjectAcl(r *http.Request, bucket, obj return nil, s3err.ErrInternalError } } - if entry.IsDirectory { return nil, s3err.ErrExistingObjectIsDirectory } - acpOwnerId := s3acl.GetAcpOwner(entry.Extended, *bucketMetadata.Owner.ID) acpOwnerName := s3a.accountManager.IdNameMapping[acpOwnerId] - acpGrants := s3acl.GetAcpGrants(entry.Extended) + acpGrants := s3acl.GetAcpGrants(&acpOwnerId, entry.Extended) acp = &s3.AccessControlPolicy{ Owner: &s3.Owner{ ID: &acpOwnerId, @@ -138,36 +137,29 @@ func (s3a *S3ApiServer) checkAccessForReadObjectAcl(r *http.Request, bucket, obj } return acp, s3err.ErrNone } - if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced { return getAcpFunc() - } else { - accountId := s3acl.GetAccountId(r) - - acp, errCode := getAcpFunc() - if errCode != s3err.ErrNone { - return nil, errCode - } - - if accountId == *acp.Owner.ID { - return acp, s3err.ErrNone - } - - //find in Grants - if acp.Grants != nil { - reqGrants := s3acl.DetermineReqGrants(accountId, s3_constants.PermissionReadAcp) - for _, requiredGrant := range reqGrants { - for _, grant := range acp.Grants { - if s3acl.GrantEquals(requiredGrant, grant) { - return acp, s3err.ErrNone - } + } + requestAccountId := s3acl.GetAccountId(r) + acp, errCode = getAcpFunc() + if errCode != s3err.ErrNone { + return nil, errCode + } + if s3acl.ValidateAccount(requestAccountId, *acp.Owner.ID) { + return acp, s3err.ErrNone + } + if acp.Grants != nil { + reqGrants := s3acl.DetermineRequiredGrants(requestAccountId, s3_constants.PermissionReadAcp) + for _, requiredGrant := range reqGrants { + for _, grant := range acp.Grants { + if s3acl.GrantEquals(requiredGrant, grant) { + return acp, s3err.ErrNone } } } - - glog.V(3).Infof("acl denied! request account id: %s", accountId) - return nil, s3err.ErrAccessDenied } + glog.V(3).Infof("CheckAccessForReadObjectAcl denied! request account id: %s", requestAccountId) + return nil, s3err.ErrAccessDenied } // Check Object-Read related access @@ -184,6 +176,10 @@ func (s3a *S3ApiServer) checkBucketAccessForReadObject(r *http.Request, bucket s if bucketMetadata.ObjectOwnership != s3_constants.OwnershipBucketOwnerEnforced { //offload object acl validation to filer layer + _, defaultErrorCode := s3a.checkAccessForReadBucket(r, bucket, s3_constants.PermissionRead) + if defaultErrorCode != s3err.ErrNone { + r.Header.Set(s3_constants.XSeaweedFSHeaderAmzBucketAccessDenied, "true") + } r.Header.Set(s3_constants.XSeaweedFSHeaderAmzBucketOwnerId, *bucketMetadata.Owner.ID) } @@ -193,67 +189,58 @@ func (s3a *S3ApiServer) checkBucketAccessForReadObject(r *http.Request, bucket s // Check ObjectAcl-Write related access // includes: // - PutObjectAclHandler -func (s3a *S3ApiServer) checkAccessForWriteObjectAcl(accountId, bucket, object string) (bucketMetadata *BucketMetaData, objectEntry *filer_pb.Entry, objectOwner string, errCode s3err.ErrorCode) { - bucketMetadata, errCode = s3a.bucketRegistry.GetBucketMetadata(bucket) +func (s3a *S3ApiServer) checkAccessForWriteObjectAcl(r *http.Request, bucket, object string) (*filer_pb.Entry, string, []*s3.Grant, s3err.ErrorCode) { + bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) if errCode != s3err.ErrNone { - return nil, nil, "", errCode + return nil, "", nil, errCode } - if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced { - return nil, nil, "", s3err.AccessControlListNotSupported + requestAccountId := s3acl.GetAccountId(r) + reqOwnerId, grants, errCode := s3acl.ExtractObjectAcl(r, s3a.accountManager, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, requestAccountId, false) + if errCode != s3err.ErrNone { + return nil, "", nil, errCode } - //bucket acl - bucketAclAllowed := false - reqGrants := s3acl.DetermineReqGrants(accountId, s3_constants.PermissionWrite) - if accountId == *bucketMetadata.Owner.ID { - bucketAclAllowed = true - } else if bucketMetadata.Acl != nil { - bucketLoop: - for _, bucketGrant := range bucketMetadata.Acl { - for _, requiredGrant := range reqGrants { - if s3acl.GrantEquals(bucketGrant, requiredGrant) { - bucketAclAllowed = true - break bucketLoop - } - } - } - } - if !bucketAclAllowed { - return nil, nil, "", s3err.ErrAccessDenied + if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced { + return nil, "", nil, s3err.AccessControlListNotSupported } //object acl objectEntry, err := getObjectEntry(s3a, bucket, object) if err != nil { if err == filer_pb.ErrNotFound { - return nil, nil, "", s3err.ErrNoSuchKey + return nil, "", nil, s3err.ErrNoSuchKey } - return nil, nil, "", s3err.ErrInternalError + return nil, "", nil, s3err.ErrInternalError } if objectEntry.IsDirectory { - return nil, nil, "", s3err.ErrExistingObjectIsDirectory + return nil, "", nil, s3err.ErrExistingObjectIsDirectory } - objectOwner = s3acl.GetAcpOwner(objectEntry.Extended, *bucketMetadata.Owner.ID) - if accountId == objectOwner { - return bucketMetadata, objectEntry, objectOwner, s3err.ErrNone + objectOwner := s3acl.GetAcpOwner(objectEntry.Extended, *bucketMetadata.Owner.ID) + //object owner is immutable + if reqOwnerId != "" && reqOwnerId != objectOwner { + return nil, "", nil, s3err.ErrAccessDenied + } + if s3acl.ValidateAccount(requestAccountId, objectOwner) { + return objectEntry, objectOwner, grants, s3err.ErrNone } - objectGrants := s3acl.GetAcpGrants(objectEntry.Extended) + objectGrants := s3acl.GetAcpGrants(nil, objectEntry.Extended) if objectGrants != nil { + requiredGrants := s3acl.DetermineRequiredGrants(requestAccountId, s3_constants.PermissionWriteAcp) for _, objectGrant := range objectGrants { - for _, requiredGrant := range reqGrants { + for _, requiredGrant := range requiredGrants { if s3acl.GrantEquals(objectGrant, requiredGrant) { - return bucketMetadata, objectEntry, objectOwner, s3err.ErrNone + return objectEntry, objectOwner, grants, s3err.ErrNone } } } } - glog.V(3).Infof("acl denied! request account id: %s", accountId) - return nil, nil, "", s3err.ErrAccessDenied + glog.V(3).Infof("checkAccessForWriteObjectAcl denied! request account id: %s", requestAccountId) + return nil, "", nil, s3err.ErrAccessDenied } func updateObjectEntry(s3a *S3ApiServer, bucket, object string, entry *filer_pb.Entry) error { @@ -269,150 +256,159 @@ func (s3a *S3ApiServer) CheckAccessForPutObject(r *http.Request, bucket, object return s3a.checkAccessForWriteObject(r, bucket, object, accountId) } -// CheckAccessForNewMultipartUpload Check Acl for InitiateMultipartUploadResult API +// CheckAccessForPutObjectPartHandler Check Acl for Upload object part // includes: -// - NewMultipartUploadHandler -func (s3a *S3ApiServer) CheckAccessForNewMultipartUpload(r *http.Request, bucket, object string) s3err.ErrorCode { +// - PutObjectPartHandler +func (s3a *S3ApiServer) CheckAccessForPutObjectPartHandler(r *http.Request, bucket string) s3err.ErrorCode { + bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) + if errCode != s3err.ErrNone { + return errCode + } + if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced { + return s3err.ErrNone + } accountId := s3acl.GetAccountId(r) - if accountId == IdentityAnonymous.AccountId { + if !CheckBucketAccess(accountId, bucketMetadata, s3_constants.PermissionWrite) { return s3err.ErrAccessDenied } - return s3a.checkAccessForWriteObject(r, bucket, object, accountId) + return s3err.ErrNone } -// CheckAccessForCompleteMultipartUpload Check Acl for CompleteMultipartUpload API +// CheckAccessForNewMultipartUpload Check Acl for API // includes: -// - CompleteMultipartUploadHandler +// - NewMultipartUploadHandler +func (s3a *S3ApiServer) CheckAccessForNewMultipartUpload(r *http.Request, bucket, object string) (s3err.ErrorCode, string) { + accountId := s3acl.GetAccountId(r) + if accountId == IdentityAnonymous.AccountId { + return s3err.ErrAccessDenied, "" + } + errCode := s3a.checkAccessForWriteObject(r, bucket, object, accountId) + return errCode, accountId +} + +func (s3a *S3ApiServer) CheckAccessForAbortMultipartUpload(r *http.Request, bucket, object string) s3err.ErrorCode { + return s3a.CheckAccessWithBucketOwnerAndInitiator(r, bucket, object) +} + func (s3a *S3ApiServer) CheckAccessForCompleteMultipartUpload(r *http.Request, bucket, object string) s3err.ErrorCode { bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) if errCode != s3err.ErrNone { return errCode } + if bucketMetadata.ObjectOwnership != s3_constants.OwnershipBucketOwnerEnforced { + accountId := getAccountId(r) + if !CheckBucketAccess(accountId, bucketMetadata, s3_constants.PermissionWrite) { + return s3err.ErrAccessDenied + } + } + return s3err.ErrNone +} + +func (s3a *S3ApiServer) CheckAccessForListMultipartUploadParts(r *http.Request, bucket, object string) s3err.ErrorCode { + return s3a.CheckAccessWithBucketOwnerAndInitiator(r, bucket, object) +} + +// CheckAccessWithBucketOwnerAndInitiator Check Access Permission with 'bucketOwner' and 'multipartUpload initiator' +func (s3a *S3ApiServer) CheckAccessWithBucketOwnerAndInitiator(r *http.Request, bucket, object string) s3err.ErrorCode { + bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) + if errCode != s3err.ErrNone { + return errCode + } + //bucket access allowed accountId := s3acl.GetAccountId(r) - if accountId == *bucketMetadata.Owner.ID { + if s3acl.ValidateAccount(*bucketMetadata.Owner.ID, accountId) { return s3err.ErrNone + } + + //multipart initiator allowed + entry, err := getMultipartUpload(s3a, bucket, object) + if err != nil { + if err != filer_pb.ErrNotFound { + return s3err.ErrInternalError + } } else { - if len(bucketMetadata.Acl) > 0 { - reqGrants := s3acl.DetermineReqGrants(accountId, s3_constants.PermissionWrite) - for _, bucketGrant := range bucketMetadata.Acl { - for _, requiredGrant := range reqGrants { - if s3acl.GrantEquals(bucketGrant, requiredGrant) { - return s3err.ErrNone - } - } - } + uploadInitiator, ok := entry.Extended[s3_constants.ExtAmzMultipartInitiator] + if !ok || accountId == string(uploadInitiator) { + return s3err.ErrNone } } - glog.V(3).Infof("acl denied! request account id: %s", accountId) + glog.V(3).Infof("CheckAccessWithBucketOwnerAndInitiator denied! request account id: %s", accountId) return s3err.ErrAccessDenied } -func (s3a *S3ApiServer) checkAccessForWriteObject(r *http.Request, bucket, object, accountId string) s3err.ErrorCode { +func (s3a *S3ApiServer) checkAccessForWriteObject(r *http.Request, bucket, object, requestAccountId string) s3err.ErrorCode { bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) if errCode != s3err.ErrNone { return errCode } + requestOwnerId, grants, errCode := s3acl.ExtractObjectAcl(r, s3a.accountManager, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, requestAccountId, true) + if errCode != s3err.ErrNone { + return errCode + } if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced { - // validate grants (only bucketOwnerFullControl acl is allowed) - _, grants, errCode := s3acl.ParseAndValidateAclHeaders(r, s3a.accountManager, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, accountId, false) - if errCode != s3err.ErrNone { - return errCode - } - if len(grants) > 1 { - return s3err.AccessControlListNotSupported - } - bucketOwnerFullControlGrant := &s3.Grant{ - Permission: &s3_constants.PermissionFullControl, - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeCanonicalUser, - ID: bucketMetadata.Owner.ID, - }, - } - if len(grants) == 0 { - return s3err.ErrNone - } + return s3err.ErrNone + } - if s3acl.GrantEquals(bucketOwnerFullControlGrant, grants[0]) { + if !CheckBucketAccess(requestAccountId, bucketMetadata, s3_constants.PermissionWrite) { + return s3err.ErrAccessDenied + } + + if requestOwnerId == "" { + requestOwnerId = requestAccountId + } + entry, err := getObjectEntry(s3a, bucket, object) + if err != nil { + if err == filer_pb.ErrNotFound { + s3acl.SetAcpOwnerHeader(r, requestOwnerId) + s3acl.SetAcpGrantsHeader(r, grants) return s3err.ErrNone } + return s3err.ErrInternalError + } - return s3err.AccessControlListNotSupported + objectOwnerId := s3acl.GetAcpOwner(entry.Extended, *bucketMetadata.Owner.ID) + //object owner is immutable + if requestOwnerId != "" && objectOwnerId != requestOwnerId { + return s3err.ErrAccessDenied } - //bucket access allowed - bucketAclAllowed := false - if accountId == *bucketMetadata.Owner.ID { - bucketAclAllowed = true + //Only the owner of the bucket and the owner of the object can overwrite the object + if s3acl.ValidateAccount(requestOwnerId, objectOwnerId, *bucketMetadata.Owner.ID) { + glog.V(3).Infof("checkAccessForWriteObject denied! request account id: %s, expect account id: %s", requestAccountId, *bucketMetadata.Owner.ID) + return s3err.ErrAccessDenied + } + + s3acl.SetAcpOwnerHeader(r, objectOwnerId) + s3acl.SetAcpGrantsHeader(r, grants) + return s3err.ErrNone +} + +func CheckBucketAccess(requestAccountId string, bucketMetadata *BucketMetaData, permission string) bool { + if s3acl.ValidateAccount(requestAccountId, *bucketMetadata.Owner.ID) { + return true } else { if len(bucketMetadata.Acl) > 0 { - reqGrants := s3acl.DetermineReqGrants(accountId, s3_constants.PermissionWrite) - bucketLoop: + reqGrants := s3acl.DetermineRequiredGrants(requestAccountId, permission) for _, bucketGrant := range bucketMetadata.Acl { for _, requiredGrant := range reqGrants { if s3acl.GrantEquals(bucketGrant, requiredGrant) { - bucketAclAllowed = true - break bucketLoop + return true } } } } } - if !bucketAclAllowed { - glog.V(3).Infof("acl denied! request account id: %s", accountId) - return s3err.ErrAccessDenied - } - - //object access allowed - entry, err := getObjectEntry(s3a, bucket, object) - if err != nil { - if err != filer_pb.ErrNotFound { - return s3err.ErrInternalError - } - } else { - if entry.IsDirectory { - return s3err.ErrExistingObjectIsDirectory - } - - //Only the owner of the bucket and the owner of the object can overwrite the object - objectOwner := s3acl.GetAcpOwner(entry.Extended, *bucketMetadata.Owner.ID) - if accountId != objectOwner && accountId != *bucketMetadata.Owner.ID { - glog.V(3).Infof("acl denied! request account id: %s, expect account id: %s", accountId, *bucketMetadata.Owner.ID) - return s3err.ErrAccessDenied - } - } - - ownerId, grants, errCode := s3acl.ParseAndValidateAclHeadersOrElseDefault(r, s3a.accountManager, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, accountId, false) - if errCode != s3err.ErrNone { - return errCode - } - - s3acl.SetAcpOwnerHeader(r, ownerId) - s3acl.SetAcpGrantsHeader(r, grants) - - return s3err.ErrNone + glog.V(3).Infof("CheckBucketAccess denied! request account id: %s", requestAccountId) + return false } func getObjectEntry(s3a *S3ApiServer, bucket, object string) (*filer_pb.Entry, error) { return s3a.getEntry(util.Join(s3a.option.BucketsPath, bucket), object) } -func (s3a *S3ApiServer) ExtractBucketAcp(r *http.Request, objectOwnership string) (owner string, grants []*s3.Grant, errCode s3err.ErrorCode) { - accountId := s3acl.GetAccountId(r) - - if objectOwnership == "" || objectOwnership == s3_constants.OwnershipBucketOwnerEnforced { - return accountId, []*s3.Grant{ - { - Permission: &s3_constants.PermissionFullControl, - Grantee: &s3.Grantee{ - Type: &s3_constants.GrantTypeCanonicalUser, - ID: &accountId, - }, - }, - }, s3err.ErrNone - } else { - return s3acl.ParseAndValidateAclHeadersOrElseDefault(r, s3a.accountManager, objectOwnership, accountId, accountId, false) - } +func getMultipartUpload(s3a *S3ApiServer, bucket, object string) (*filer_pb.Entry, error) { + return s3a.getEntry(s3a.genUploadsFolder(bucket), object) } diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 159f5b7bc..515073142 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -123,8 +123,9 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) } } - objectOwnership := r.Header.Get("ObjectOwnership") - acpOwner, acpGrants, errCode := s3a.ExtractBucketAcp(r, objectOwnership) + objectOwnership := r.Header.Get("X-Amz-Object-Ownership") + requestAccountId := s3acl.GetAccountId(r) + grants, errCode := s3acl.ExtractBucketAcl(r, s3a.accountManager, objectOwnership, requestAccountId, requestAccountId, true) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return @@ -138,9 +139,12 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) entry.Extended[s3_constants.AmzIdentityId] = []byte(identityId) } if objectOwnership != "" { + if entry.Extended == nil { + entry.Extended = make(map[string][]byte) + } entry.Extended[s3_constants.ExtOwnershipKey] = []byte(objectOwnership) } - s3acl.AssembleEntryWithAcp(entry, acpOwner, acpGrants) + s3acl.AssembleEntryWithAcp(entry, requestAccountId, grants) } // create the folder for bucket, but lazily create actual collection @@ -269,7 +273,7 @@ func (s3a *S3ApiServer) PutBucketAclHandler(w http.ResponseWriter, r *http.Reque return } - grants, errCode := s3acl.ExtractAcl(r, s3a.accountManager, bucketMetadata.ObjectOwnership, "", *bucketMetadata.Owner.ID, accountId) + grants, errCode := s3acl.ExtractBucketAcl(r, s3a.accountManager, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, accountId, false) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return @@ -293,6 +297,8 @@ func (s3a *S3ApiServer) PutBucketAclHandler(w http.ResponseWriter, r *http.Reque s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } + //update local cache + bucketMetadata.Acl = grants s3err.WriteEmptyResponse(w, r, http.StatusOK) } @@ -402,9 +408,8 @@ func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *htt return } - var v s3.OwnershipControls defer util.CloseRequest(r) - + var v s3.OwnershipControls err := xmlutil.UnmarshalXML(&v, xml.NewDecoder(r.Body), "") if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) @@ -417,12 +422,11 @@ func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *htt } printOwnership := true - ownership := *v.Rules[0].ObjectOwnership - switch ownership { + newObjectOwnership := *v.Rules[0].ObjectOwnership + switch newObjectOwnership { case s3_constants.OwnershipObjectWriter: case s3_constants.OwnershipBucketOwnerPreferred: case s3_constants.OwnershipBucketOwnerEnforced: - printOwnership = false default: s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) return @@ -439,29 +443,34 @@ func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *htt } oldOwnership, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey] - if !ok || string(oldOwnership) != ownership { + if !ok || string(oldOwnership) != newObjectOwnership { // must reset bucket acl to default(bucket owner with full control permission) before setting ownership // to `OwnershipBucketOwnerEnforced` (bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting) - if ownership == s3_constants.OwnershipBucketOwnerEnforced { - acpGrants := s3acl.GetAcpGrants(bucketEntry.Extended) - if len(acpGrants) != 1 { - s3err.WriteErrorResponse(w, r, s3err.InvalidBucketAclWithObjectOwnership) - return - } - - bucketOwner := s3acl.GetAcpOwner(bucketEntry.Extended, s3account.AccountAdmin.Id) - expectGrant := s3acl.GrantWithFullControl(bucketOwner) - if s3acl.GrantEquals(acpGrants[0], expectGrant) { + if newObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced { + acpGrants := s3acl.GetAcpGrants(nil, bucketEntry.Extended) + if len(acpGrants) > 1 { s3err.WriteErrorResponse(w, r, s3err.InvalidBucketAclWithObjectOwnership) return + } else if len(acpGrants) == 1 { + bucketOwner := s3acl.GetAcpOwner(bucketEntry.Extended, s3account.AccountAdmin.Id) + expectGrant := s3acl.GrantWithFullControl(bucketOwner) + if !s3acl.GrantEquals(acpGrants[0], expectGrant) { + s3err.WriteErrorResponse(w, r, s3err.InvalidBucketAclWithObjectOwnership) + return + } } } if bucketEntry.Extended == nil { bucketEntry.Extended = make(map[string][]byte) } - bucketEntry.Extended[s3_constants.ExtOwnershipKey] = []byte(ownership) + //update local cache + bucketMetadata, eCode := s3a.bucketRegistry.GetBucketMetadata(bucket) + if eCode == s3err.ErrNone { + bucketMetadata.ObjectOwnership = newObjectOwnership + } + bucketEntry.Extended[s3_constants.ExtOwnershipKey] = []byte(newObjectOwnership) err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry) if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 2e0b64ea3..fb8974b30 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -219,12 +219,16 @@ func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Reque } func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { - bucket, object := s3_constants.GetBucketAndObject(r) glog.V(3).Infof("HeadObjectHandler %s %s", bucket, object) - destUrl := s3a.toFilerUrl(bucket, object) + errCode := s3a.checkBucketAccessForReadObject(r, bucket) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + destUrl := s3a.toFilerUrl(bucket, object) s3a.proxyToFiler(w, r, destUrl, false, passThroughResponse) } @@ -592,20 +596,18 @@ func (s3a *S3ApiServer) maybeGetFilerJwtAuthorizationToken(isWrite bool) string func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) { bucket, object := s3_constants.GetBucketAndObject(r) - accountId := s3acl.GetAccountId(r) - bucketMetadata, objectEntry, objectOwner, errCode := s3a.checkAccessForWriteObjectAcl(accountId, bucket, object) + objectEntry, ownerId, grants, errCode := s3a.checkAccessForWriteObjectAcl(r, bucket, object) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return } - grants, errCode := s3acl.ExtractAcl(r, s3a.accountManager, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, objectOwner, accountId) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return } - errCode = s3acl.AssembleEntryWithAcp(objectEntry, objectOwner, grants) + errCode = s3acl.AssembleEntryWithAcp(objectEntry, ownerId, grants) if errCode != s3err.ErrNone { return } diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 881bb81d2..2cfa3e51c 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -30,8 +30,7 @@ const ( func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { bucket, object := s3_constants.GetBucketAndObject(r) - //acl - errCode := s3a.CheckAccessForNewMultipartUpload(r, bucket, object) + errCode, initiatorId := s3a.CheckAccessForNewMultipartUpload(r, bucket, object) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return @@ -52,7 +51,7 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http if contentType != "" { createMultipartUploadInput.ContentType = &contentType } - response, errCode := s3a.createMultipartUpload(createMultipartUploadInput) + response, errCode := s3a.createMultipartUpload(initiatorId, createMultipartUploadInput) glog.V(2).Info("NewMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode) @@ -70,7 +69,6 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r // https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html bucket, object := s3_constants.GetBucketAndObject(r) - s3a.CheckAccessForCompleteMultipartUpload(r, bucket, object) parts := &CompleteMultipartUpload{} if err := xmlDecoder(r.Body, parts, r.ContentLength); err != nil { @@ -86,6 +84,12 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r return } + errCode := s3a.CheckAccessForCompleteMultipartUpload(r, bucket, uploadID) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{ Bucket: aws.String(bucket), Key: objectKey(aws.String(object)), @@ -115,6 +119,24 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht return } + errCode := s3a.CheckAccessForAbortMultipartUpload(r, bucket, uploadID) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + + exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true) + if err != nil { + glog.V(1).Infof("list parts error: %v, request url: %s", err, r.RequestURI) + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) + return + } + if !exists { + glog.V(1).Infof("list parts not found, request url: %s", r.RequestURI) + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) + return + } + response, errCode := s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{ Bucket: aws.String(bucket), Key: objectKey(aws.String(object)), @@ -151,7 +173,12 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht } } - response, errCode := s3a.listMultipartUploads(&s3.ListMultipartUploadsInput{ + bucketMetaData, errorCode := s3a.checkAccessForReadBucket(r, bucket, s3_constants.PermissionRead) + if errorCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errorCode) + return + } + response, errCode := s3a.listMultipartUploads(bucketMetaData, &s3.ListMultipartUploadsInput{ Bucket: aws.String(bucket), Delimiter: aws.String(delimiter), EncodingType: aws.String(encodingType), @@ -193,6 +220,23 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re return } + errCode := s3a.CheckAccessForListMultipartUploadParts(r, bucket, uploadID) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + + exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true) + if err != nil { + glog.V(1).Infof("list parts error: %v, request url: %s", err, r.RequestURI) + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) + return + } + if !exists { + glog.V(1).Infof("list parts not found, request url: %s", r.RequestURI) + s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload) + return + } response, errCode := s3a.listObjectParts(&s3.ListPartsInput{ Bucket: aws.String(bucket), Key: objectKey(aws.String(object)), @@ -253,6 +297,12 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ } defer dataReader.Close() + errorCode := s3a.CheckAccessForPutObjectPartHandler(r, bucket) + if errorCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errorCode) + return + } + glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID) uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part", s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID) @@ -260,9 +310,8 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ if partID == 1 && r.Header.Get("Content-Type") == "" { dataReader = mimeDetect(r, dataReader) } - destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, object) - etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader, destination) + etag, errCode := s3a.putToFiler(r, uploadUrl, dataReader, "") if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) return diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index a0f7c1baa..ac181eac2 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -128,15 +128,15 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // PutObjectPart bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.PutObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") // CompleteMultipartUpload - bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CompleteMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploadId", "{uploadId:.*}") + bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.CompleteMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploadId", "{uploadId:.*}") // NewMultipartUpload - bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.NewMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploads", "") + bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.NewMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploads", "") // AbortMultipartUpload - bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.AbortMultipartUploadHandler, ACTION_WRITE)), "DELETE")).Queries("uploadId", "{uploadId:.*}") + bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.AbortMultipartUploadHandler, ACTION_WRITE)), "DELETE")).Queries("uploadId", "{uploadId:.*}") // ListObjectParts - bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectPartsHandler, ACTION_READ)), "GET")).Queries("uploadId", "{uploadId:.*}") + bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.ListObjectPartsHandler, ACTION_READ)), "GET")).Queries("uploadId", "{uploadId:.*}") // ListMultipartUploads - bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListMultipartUploadsHandler, ACTION_READ)), "GET")).Queries("uploads", "") + bucket.Methods("GET").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.ListMultipartUploadsHandler, ACTION_READ)), "GET")).Queries("uploads", "") // GetObjectTagging bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectTaggingHandler, ACTION_READ)), "GET")).Queries("tagging", "") diff --git a/weed/s3api/s3err/s3api_errors.go b/weed/s3api/s3err/s3api_errors.go index 117631d1b..f3d44a31a 100644 --- a/weed/s3api/s3err/s3api_errors.go +++ b/weed/s3api/s3err/s3api_errors.go @@ -111,6 +111,8 @@ const ( OwnershipControlsNotFoundError InvalidBucketAclWithObjectOwnership AccessControlListNotSupported + ErrUnexpectedContent + ErrInvalidAclArgument ) // error code to APIError structure, these fields carry respective @@ -426,13 +428,23 @@ var errorCodeResponse = map[ErrorCode]APIError{ InvalidBucketAclWithObjectOwnership: { Code: "InvalidBucketAclWithObjectOwnership", Description: "Bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting", - HTTPStatusCode: http.StatusNotFound, + HTTPStatusCode: http.StatusBadRequest, }, AccessControlListNotSupported: { Code: "AccessControlListNotSupported", Description: "The bucket does not allow ACLs", HTTPStatusCode: http.StatusBadRequest, }, + ErrUnexpectedContent: { + Code: "UnexpectedContent", + Description: "This request does not support content", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidAclArgument: { + Code: "InvalidArgument", + Description: "ACL argument is invalid", + HTTPStatusCode: http.StatusBadRequest, + }, } // GetAPIError provides API Error for input API error code. diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 9e4ef61a1..335c3f1cb 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -12,7 +12,7 @@ import ( "mime" "net/http" "path/filepath" - "strconv" + strconv "strconv" "strings" "time" @@ -99,7 +99,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) if err == filer_pb.ErrNotFound { glog.V(2).Infof("Not found %s: %v", path, err) stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadNotFound).Inc() - w.WriteHeader(http.StatusNotFound) + if r.Header.Get(s3_constants.XSeaweedFSHeaderAmzBucketAccessDenied) == "true" { + w.WriteHeader(http.StatusForbidden) + } else { + w.WriteHeader(http.StatusNotFound) + } } else { glog.Errorf("Internal %s: %v", path, err) stats.FilerRequestCounter.WithLabelValues(stats.ErrorReadInternal).Inc() @@ -174,10 +178,15 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) // print out the header from extended properties for k, v := range entry.Extended { - if !strings.HasPrefix(k, "xattr-") { + if strings.HasPrefix(k, "xattr-") { // "xattr-" prefix is set in filesys.XATTR_PREFIX - w.Header().Set(k, string(v)) + continue + } + if strings.HasPrefix(k, "Seaweed-X-") { + // key with "Seaweed-X-" prefix is builtin and should not expose to user + continue } + w.Header().Set(k, string(v)) } //Seaweed custom header are not visible to Vue or javascript From d2792c5618355957d1fdbe704bb2c5f6f81f7879 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 15 Dec 2022 18:39:08 +0800 Subject: [PATCH 32/52] clear cache when put bucket Signed-off-by: changlin.shi --- weed/s3api/bucket_metadata.go | 5 +++++ weed/s3api/s3api_bucket_handlers.go | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/weed/s3api/bucket_metadata.go b/weed/s3api/bucket_metadata.go index 412076da3..f4dd4cfe6 100644 --- a/weed/s3api/bucket_metadata.go +++ b/weed/s3api/bucket_metadata.go @@ -232,3 +232,8 @@ func (r *BucketRegistry) unMarkNotFound(bucket string) { defer r.notFoundLock.Unlock() delete(r.notFound, bucket) } + +func (r *BucketRegistry) ClearCache(bucket string) { + r.removeMetadataCache(bucket) + r.unMarkNotFound(bucket) +} diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 515073142..610efcf78 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -153,6 +153,10 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) return } + + // clear cache + s3a.bucketRegistry.ClearCache(bucket) + w.Header().Set("Location", "/"+bucket) writeSuccessResponseEmpty(w, r) } From 7fec1a97e26dc1d93d01b6610c5d23042937f7e6 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Wed, 28 Dec 2022 12:35:40 +0800 Subject: [PATCH 33/52] fix tests Signed-off-by: changlin.shi --- weed/s3api/bucket_metadata_test.go | 40 +++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/weed/s3api/bucket_metadata_test.go b/weed/s3api/bucket_metadata_test.go index cd6aef78f..98c569dea 100644 --- a/weed/s3api/bucket_metadata_test.go +++ b/weed/s3api/bucket_metadata_test.go @@ -113,7 +113,15 @@ var tcs = []*BucketMetadataTestCase{ DisplayName: &s3account.AccountAdmin.Name, ID: &s3account.AccountAdmin.Id, }, - Acl: nil, + Acl: []*s3.Grant{ + { + Permission: &s3_constants.PermissionFullControl, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &s3account.AccountAdmin.Id, + }, + }, + }, }, }, { @@ -124,7 +132,15 @@ var tcs = []*BucketMetadataTestCase{ DisplayName: &s3account.AccountAdmin.Name, ID: &s3account.AccountAdmin.Id, }, - Acl: nil, + Acl: []*s3.Grant{ + { + Permission: &s3_constants.PermissionFullControl, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &s3account.AccountAdmin.Id, + }, + }, + }, }, }, { @@ -135,7 +151,15 @@ var tcs = []*BucketMetadataTestCase{ DisplayName: &s3account.AccountAdmin.Name, ID: &s3account.AccountAdmin.Id, }, - Acl: nil, + Acl: []*s3.Grant{ + { + Permission: &s3_constants.PermissionFullControl, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &s3account.AccountAdmin.Id, + }, + }, + }, }, }, { @@ -146,7 +170,15 @@ var tcs = []*BucketMetadataTestCase{ DisplayName: &s3account.AccountAdmin.Name, ID: &s3account.AccountAdmin.Id, }, - Acl: nil, + Acl: []*s3.Grant{ + { + Permission: &s3_constants.PermissionFullControl, + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &s3account.AccountAdmin.Id, + }, + }, + }, }, }, { From 6f40451e4e7e0dcfe3c684781895650591d6a4e5 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 5 Jan 2023 18:02:36 +0800 Subject: [PATCH 34/52] optimize marshal Signed-off-by: changlin.shi --- weed/s3api/s3acl/acl_helper.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/weed/s3api/s3acl/acl_helper.go b/weed/s3api/s3acl/acl_helper.go index 942b55797..1e8ee9524 100644 --- a/weed/s3api/s3acl/acl_helper.go +++ b/weed/s3api/s3acl/acl_helper.go @@ -493,9 +493,9 @@ func GetAcpOwner(entryExtended map[string][]byte, defaultOwner string) string { return defaultOwner } -func SetAcpGrantsHeader(r *http.Request, acpGrants []*s3.Grant) { - if len(acpGrants) > 0 { - a, err := json.Marshal(acpGrants) +func SetAcpGrantsHeader(r *http.Request, grants []*s3.Grant) { + if len(grants) > 0 { + a, err := MarshalGrantsToJson(grants) if err == nil { r.Header.Set(s3_constants.ExtAmzAclKey, string(a)) } else { @@ -542,7 +542,7 @@ func AssembleEntryWithAcp(filerEntry *filer_pb.Entry, ownerId string, grants []* } if grants != nil { - grantsBytes, err := json.Marshal(grants) + grantsBytes, err := MarshalGrantsToJson(grants) if err != nil { glog.Warning("assemble acp to entry:", err) return s3err.ErrInvalidRequest @@ -636,7 +636,7 @@ func GrantEquals(a, b *s3.Grant) bool { func MarshalGrantsToJson(grants []*s3.Grant) ([]byte, error) { if len(grants) == 0 { - return nil, nil + return []byte{}, nil } var GrantsToMap []map[string]any for _, grant := range grants { From d393937c90f1f21be53313f02e7a46e3078ebe76 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Sat, 11 Feb 2023 15:22:39 +0800 Subject: [PATCH 35/52] compatible with aws-sdk-go's cxpect header Signed-off-by: changlin.shi --- weed/s3api/auth_signature_v4.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index 02a6bd4e0..059be69a2 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -55,7 +55,8 @@ const ( // http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD" indicates that the // client did not calculate sha256 of the payload. - unsignedPayload = "UNSIGNED-PAYLOAD" + unsignedPayload = "UNSIGNED-PAYLOAD" + expect100Continue = "100-contine" ) // Returns SHA256 for calculating canonical-request. @@ -592,7 +593,12 @@ func extractSignedHeaders(signedHeaders []string, r *http.Request) (http.Header, // be sent, for the time being keep this work around. // Adding a *TODO* to remove this later when Golang server // doesn't filter out the 'Expect' header. - extractedSignedHeaders.Set(header, "100-continue") + expectHeaderValue := extractedSignedHeaders.Get(header) + + // here in order to be compatible with the aws go sdk v1 version, it sets the expect header to '100-Continue' + if !strings.EqualFold(expectHeaderValue, expect100Continue) { + extractedSignedHeaders.Set(header, expect100Continue) + } case "host": // Go http server removes "host" from Request.Header extractedSignedHeaders.Set(header, r.Host) From af40b7d7a21994a45a1a31c3a8a8a9734b1757d4 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Wed, 22 Feb 2023 12:03:47 +0800 Subject: [PATCH 36/52] Fix when the stored data is actually enough but s3api_object_list_handlers returns less than the specified limit Signed-off-by: changlin.shi --- weed/s3api/s3api_objects_list_handlers.go | 86 +++++++++++++---------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index bf4d1961c..298f2b84f 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -158,57 +158,65 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m // check filer err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { - - nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) { - if entry.IsDirectory { - // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html - if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter. - commonPrefixes = append(commonPrefixes, PrefixEntry{ - Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], - }) - //All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns. - cursor.maxKeys-- - } else if entry.IsDirectoryKeyObject() { + for { + empty := true + nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) { + empty = false + if entry.IsDirectory { + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter. + commonPrefixes = append(commonPrefixes, PrefixEntry{ + Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], + }) + //All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns. + cursor.maxKeys-- + } else if entry.IsDirectoryKeyObject() { + contents = append(contents, ListEntry{ + Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], + LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), + ETag: "\"" + filer.ETag(entry) + "\"", + Owner: CanonicalUser{ + ID: fmt.Sprintf("%x", entry.Attributes.Uid), + DisplayName: entry.Attributes.UserName, + }, + StorageClass: "STANDARD", + }) + cursor.maxKeys-- + } + } else { + storageClass := "STANDARD" + if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { + storageClass = string(v) + } contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):], + Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), ETag: "\"" + filer.ETag(entry) + "\"", + Size: int64(filer.FileSize(entry)), Owner: CanonicalUser{ ID: fmt.Sprintf("%x", entry.Attributes.Uid), DisplayName: entry.Attributes.UserName, }, - StorageClass: "STANDARD", + StorageClass: StorageClass(storageClass), }) cursor.maxKeys-- } - } else { - storageClass := "STANDARD" - if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { - storageClass = string(v) - } - contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], - LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), - ETag: "\"" + filer.ETag(entry) + "\"", - Size: int64(filer.FileSize(entry)), - Owner: CanonicalUser{ - ID: fmt.Sprintf("%x", entry.Attributes.Uid), - DisplayName: entry.Attributes.UserName, - }, - StorageClass: StorageClass(storageClass), - }) - cursor.maxKeys-- + }) + if doErr != nil { + return doErr } - }) - if doErr != nil { - return doErr - } - if !cursor.isTruncated { - nextMarker = "" - } else { - if requestDir != "" { - nextMarker = requestDir + "/" + nextMarker + if cursor.isTruncated { + if requestDir != "" { + nextMarker = requestDir + "/" + nextMarker + } + break + } else if empty { + nextMarker = "" + break + } else { + // start next loop + marker = nextMarker } } From d0bfc260e79c110ba1c37e2ec3e21856d877846e Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Tue, 28 Feb 2023 16:47:03 +0800 Subject: [PATCH 37/52] =?UTF-8?q?chore(fs.configure=20=E5=8F=AF=E4=BB=A5?= =?UTF-8?q?=E6=8C=87=E5=AE=9A=E4=BF=AE=E6=94=B9=20collection):?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: changlin.shi --- weed/shell/command_fs_configure.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go index 60edb79c2..29e12ee44 100644 --- a/weed/shell/command_fs_configure.go +++ b/weed/shell/command_fs_configure.go @@ -4,12 +4,10 @@ import ( "bytes" "flag" "fmt" - "io" - "strings" - "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "io" ) func init() { @@ -87,9 +85,9 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io } // check collection - if *collection != "" && strings.HasPrefix(*locationPrefix, "/buckets/") { - return fmt.Errorf("one s3 bucket goes to one collection and not customizable") - } + //if *collection != "" && strings.HasPrefix(*locationPrefix, "/buckets/") { + // return fmt.Errorf("one s3 bucket goes to one collection and not customizable") + //} // check replication if *replication != "" { From fa8698f6c349dc426fe6b832274d617b9500256f Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Sun, 23 Apr 2023 17:04:37 +0800 Subject: [PATCH 38/52] perf(fix): fix bugs Signed-off-by: changlin.shi --- weed/s3api/bucket_metadata.go | 2 +- weed/s3api/s3api_acp.go | 13 ++++--------- weed/s3api/s3api_bucket_handlers.go | 3 ++- weed/s3api/s3api_object_copy_handlers.go | 8 ++++++-- weed/s3api/s3api_server.go | 2 ++ .../server/filer_server_handlers_write_autochunk.go | 2 +- weed/util/http_util.go | 1 + 7 files changed, 17 insertions(+), 14 deletions(-) diff --git a/weed/s3api/bucket_metadata.go b/weed/s3api/bucket_metadata.go index f4dd4cfe6..d3e47810d 100644 --- a/weed/s3api/bucket_metadata.go +++ b/weed/s3api/bucket_metadata.go @@ -190,7 +190,7 @@ func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*Bucket if err != nil { if err == filer_pb.ErrNotFound { // The bucket doesn't actually exist and should no longer loaded from the filer - glog.Warning("bucket not found in filer: ", bucketName) + glog.V(4).Info("bucket not found in filer: ", bucketName) r.notFound[bucketName] = struct{}{} return nil, s3err.ErrNoSuchBucket } diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index 528bf7e1d..bed718055 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -34,7 +34,7 @@ func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s return s3err.ErrAccessDenied } -//Check access for PutBucketAclHandler +// Check access for PutBucketAclHandler func (s3a *S3ApiServer) checkAccessForPutBucketAcl(requestAccountId, bucket string) (*BucketMetaData, s3err.ErrorCode) { bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket) if errCode != s3err.ErrNone { @@ -104,7 +104,7 @@ func (s3a *S3ApiServer) checkAccessForReadBucket(r *http.Request, bucket, aclAct return nil, s3err.ErrAccessDenied } -//Check ObjectAcl-Read related access +// Check ObjectAcl-Read related access // includes: // - GetObjectAclHandler func (s3a *S3ApiServer) checkAccessForReadObjectAcl(r *http.Request, bucket, object string) (acp *s3.AccessControlPolicy, errCode s3err.ErrorCode) { @@ -370,14 +370,9 @@ func (s3a *S3ApiServer) checkAccessForWriteObject(r *http.Request, bucket, objec } objectOwnerId := s3acl.GetAcpOwner(entry.Extended, *bucketMetadata.Owner.ID) - //object owner is immutable - if requestOwnerId != "" && objectOwnerId != requestOwnerId { - return s3err.ErrAccessDenied - } - //Only the owner of the bucket and the owner of the object can overwrite the object - if s3acl.ValidateAccount(requestOwnerId, objectOwnerId, *bucketMetadata.Owner.ID) { - glog.V(3).Infof("checkAccessForWriteObject denied! request account id: %s, expect account id: %s", requestAccountId, *bucketMetadata.Owner.ID) + //object owner is immutable + if !s3acl.ValidateAccount(requestOwnerId, objectOwnerId, *bucketMetadata.Owner.ID) { return s3err.ErrAccessDenied } diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index ad9f6abac..7f07c0f9d 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -6,9 +6,9 @@ import ( "errors" "fmt" "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket" "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket" "github.com/seaweedfs/seaweedfs/weed/util" "math" "net/http" @@ -146,6 +146,7 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request) entry.Extended = make(map[string][]byte) } entry.Extended[s3_constants.AmzIdentityId] = []byte(identityId) + s3a.bucketRegistry.LoadBucketMetadata(entry) } if objectOwnership != "" { if entry.Extended == nil { diff --git a/weed/s3api/s3api_object_copy_handlers.go b/weed/s3api/s3api_object_copy_handlers.go index 83120ef07..9bdc1cce3 100644 --- a/weed/s3api/s3api_object_copy_handlers.go +++ b/weed/s3api/s3api_object_copy_handlers.go @@ -98,8 +98,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request return } glog.V(2).Infof("copy from %s to %s", srcUrl, dstUrl) - destination := fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject) - etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body, destination) + etag, errCode := s3a.putToFiler(r, dstUrl, resp.Body, "") if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) @@ -213,6 +212,11 @@ func processMetadata(reqHeader, existing http.Header, replaceMeta, replaceTaggin } } + // content-encoding + if contentEncoding, ok := existing["Content-Encoding"]; ok { + reqHeader["Content-Encoding"] = contentEncoding + } + if !replaceMeta { for header, _ := range reqHeader { if strings.HasPrefix(header, s3_constants.AmzUserMetaPrefix) { diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index ac181eac2..30e2e6bb6 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -68,6 +68,7 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer s3ApiServer.client = &http.Client{Transport: &http.Transport{ MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024, + DisableCompression: true, }} } else { s3ApiServer.client = &http.Client{ @@ -75,6 +76,7 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { return net.Dial("unix", option.LocalFilerSocket) }, + DisableCompression: true, }, } } diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 5400194ee..33f585e61 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -237,7 +237,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa for k, v := range r.Header { if len(v) > 0 && len(v[0]) > 0 { - if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" { + if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" || k == "Content-Encoding" || (k == "Content-Type" && v[0] != "application/octet-stream") { entry.Extended[k] = []byte(v[0]) } if k == "Response-Content-Disposition" { diff --git a/weed/util/http_util.go b/weed/util/http_util.go index 384da6df8..4cd9fd83e 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -23,6 +23,7 @@ func init() { Transport = &http.Transport{ MaxIdleConns: 1024, MaxIdleConnsPerHost: 1024, + DisableCompression: true, } client = &http.Client{ Transport: Transport, From 682f24a9acc439fa613a0661ae6ea9331ab511c4 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Mon, 25 Dec 2023 11:39:28 +0800 Subject: [PATCH 39/52] =?UTF-8?q?fs.conf=20=E5=8F=8D=E5=BA=8F=E5=88=97?= =?UTF-8?q?=E5=8C=96=E9=97=AE=E9=A2=98=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: changlin.shi --- weed/filer/filer_conf.go | 7 +++++-- weed/shell/command_fs_configure.go | 8 +------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index ccb1acb3c..a4ab7f249 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -92,8 +92,11 @@ func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*file func (fc *FilerConf) LoadFromBytes(data []byte) (err error) { conf := &filer_pb.FilerConf{} - - if err := jsonpb.Unmarshal(data, conf); err != nil { + options := &jsonpb.UnmarshalOptions{ + DiscardUnknown: true, + AllowPartial: true, + } + if err := options.Unmarshal(data, conf); err != nil { return err } diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go index 60edb79c2..901d7971e 100644 --- a/weed/shell/command_fs_configure.go +++ b/weed/shell/command_fs_configure.go @@ -5,7 +5,6 @@ import ( "flag" "fmt" "io" - "strings" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -51,7 +50,7 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io locationPrefix := fsConfigureCommand.String("locationPrefix", "", "path prefix, required to update the path-specific configuration") collection := fsConfigureCommand.String("collection", "", "assign writes to this collection") replication := fsConfigureCommand.String("replication", "", "assign writes with this replication") - ttl := fsConfigureCommand.String("ttl", "", "assign writes with this ttl") + ttl := fsConfigureCommand.String("ttl", "", "assign writes with this ttl (e.g., 1m, 1h, 1d, 1w, 1y)") diskType := fsConfigureCommand.String("disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes") isReadOnly := fsConfigureCommand.Bool("readOnly", false, "disable writes") @@ -86,11 +85,6 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io DataNode: *dataNode, } - // check collection - if *collection != "" && strings.HasPrefix(*locationPrefix, "/buckets/") { - return fmt.Errorf("one s3 bucket goes to one collection and not customizable") - } - // check replication if *replication != "" { rp, err := super_block.NewReplicaPlacementFromString(*replication) From 7f8ab63b7dc11d84e22f21e49933d10edfe27ed5 Mon Sep 17 00:00:00 2001 From: LHHDZ Date: Thu, 30 Mar 2023 02:29:44 +0800 Subject: [PATCH 40/52] fix key corrupt when fs.configure copy path trie (#4353) Signed-off-by: changlin.shi --- weed/filer/filer_conf.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index b12d54fc8..f3a25242d 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -125,7 +125,8 @@ func (fc *FilerConf) DeleteLocationConf(locationPrefix string) { if string(key) == locationPrefix { return true } - rules.Put(key, value) + key = bytes.Clone(key) + _ = rules.Put(key, value) return true }) fc.rules = rules From e0947cec67730e6d6dc7d1cddff84a38f0806ca6 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Tue, 11 Jul 2023 14:27:17 +0800 Subject: [PATCH 41/52] refresh max volume count after adjust Signed-off-by: changlin.shi --- weed/storage/store.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/weed/storage/store.go b/weed/storage/store.go index 40bb7bbe7..711ab623b 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -574,6 +574,7 @@ func (s *Store) MaybeAdjustVolumeMax() (hasChanges bool) { if volumeSizeLimit == 0 { return } + var newMaxVolumeCount int32 for _, diskLocation := range s.Locations { if diskLocation.OriginalMaxVolumeCount == 0 { currentMaxVolumeCount := atomic.LoadInt32(&diskLocation.MaxVolumeCount) @@ -585,11 +586,15 @@ func (s *Store) MaybeAdjustVolumeMax() (hasChanges bool) { if unclaimedSpaces > int64(volumeSizeLimit) { maxVolumeCount += int32(uint64(unclaimedSpaces)/volumeSizeLimit) - 1 } + newMaxVolumeCount = newMaxVolumeCount + maxVolumeCount atomic.StoreInt32(&diskLocation.MaxVolumeCount, maxVolumeCount) glog.V(4).Infof("disk %s max %d unclaimedSpace:%dMB, unused:%dMB volumeSizeLimit:%dMB", diskLocation.Directory, maxVolumeCount, unclaimedSpaces/1024/1024, unusedSpace/1024/1024, volumeSizeLimit/1024/1024) hasChanges = hasChanges || currentMaxVolumeCount != atomic.LoadInt32(&diskLocation.MaxVolumeCount) + } else { + newMaxVolumeCount = newMaxVolumeCount + diskLocation.OriginalMaxVolumeCount } } + stats.VolumeServerMaxVolumeCounter.Set(float64(newMaxVolumeCount)) return } From 9736534a5787ae96552fbe5f9e58332ddd279592 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Mon, 15 Jan 2024 17:34:40 +0800 Subject: [PATCH 42/52] =?UTF-8?q?volume.fix.replication=20=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=20replicaPlacement=20=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: changlin.shi --- weed/shell/command_volume_balance.go | 2 +- weed/shell/command_volume_balance_test.go | 2 +- weed/shell/command_volume_check_disk.go | 2 +- weed/shell/command_volume_fix_replication.go | 36 +++++++++++++++++--- weed/shell/command_volume_server_evacuate.go | 2 +- weed/shell/command_volume_tier_move.go | 2 +- 6 files changed, 36 insertions(+), 10 deletions(-) diff --git a/weed/shell/command_volume_balance.go b/weed/shell/command_volume_balance.go index 1c599b8a0..6d4227a3b 100644 --- a/weed/shell/command_volume_balance.go +++ b/weed/shell/command_volume_balance.go @@ -84,7 +84,7 @@ func (c *commandVolumeBalance) Do(args []string, commandEnv *CommandEnv, writer } volumeServers := collectVolumeServersByDc(topologyInfo, *dc) - volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo) + volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo, nil) diskTypes := collectVolumeDiskTypes(topologyInfo) if *collection == "EACH_COLLECTION" { diff --git a/weed/shell/command_volume_balance_test.go b/weed/shell/command_volume_balance_test.go index 5bd170e71..b0d857b38 100644 --- a/weed/shell/command_volume_balance_test.go +++ b/weed/shell/command_volume_balance_test.go @@ -175,7 +175,7 @@ func TestIsGoodMove(t *testing.T) { func TestBalance(t *testing.T) { topologyInfo := parseOutput(topoData) volumeServers := collectVolumeServersByDc(topologyInfo, "") - volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo) + volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo, nil) diskTypes := collectVolumeDiskTypes(topologyInfo) if err := balanceVolumeServers(nil, diskTypes, volumeReplicas, volumeServers, 30*1024*1024*1024, "ALL_COLLECTIONS", false); err != nil { diff --git a/weed/shell/command_volume_check_disk.go b/weed/shell/command_volume_check_disk.go index 6a1634f8a..3325cf256 100644 --- a/weed/shell/command_volume_check_disk.go +++ b/weed/shell/command_volume_check_disk.go @@ -69,7 +69,7 @@ func (c *commandVolumeCheckDisk) Do(args []string, commandEnv *CommandEnv, write if err != nil { return err } - volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo) + volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo, nil) // pick 1 pairs of volume replica fileCount := func(replica *VolumeReplica) uint64 { diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go index 783b2cce8..b0a9b5c27 100644 --- a/weed/shell/command_volume_fix_replication.go +++ b/weed/shell/command_volume_fix_replication.go @@ -60,6 +60,7 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv, noDelete := volFixReplicationCommand.Bool("noDelete", false, "Do not delete over-replicated volumes, only fix under-replication") retryCount := volFixReplicationCommand.Int("retry", 5, "how many times to retry") volumesPerStep := volFixReplicationCommand.Int("volumesPerStep", 0, "how many volumes to fix in one cycle") + replicaPlacement := volFixReplicationCommand.String("replicaPlacement", "", "override the default replicaPlacement of volume") if err = volFixReplicationCommand.Parse(args); err != nil { return nil @@ -72,6 +73,15 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv, takeAction := !*skipChange doDeletes := !*noDelete + var rp *super_block.ReplicaPlacement + if len(*replicaPlacement) > 0 { + rp, err = super_block.NewReplicaPlacementFromString(*replicaPlacement) + if err != nil { + return err + } + fmt.Fprintf(writer, "override replicaPlacement: %s", rp.String()) + } + underReplicatedVolumeIdsCount := 1 for underReplicatedVolumeIdsCount > 0 { fixedVolumeReplicas := map[string]int{} @@ -84,7 +94,7 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv, // find all volumes that needs replication // collect all data nodes - volumeReplicas, allLocations := collectVolumeReplicaLocations(topologyInfo) + volumeReplicas, allLocations := collectVolumeReplicaLocations(topologyInfo, rp) if len(allLocations) == 0 { return fmt.Errorf("no data nodes at all") @@ -94,9 +104,19 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv, var underReplicatedVolumeIds, overReplicatedVolumeIds, misplacedVolumeIds []uint32 for vid, replicas := range volumeReplicas { replica := replicas[0] + if len(*c.collectionPattern) > 0 { + matched, err := filepath.Match(*c.collectionPattern, replica.info.Collection) + if err != nil { + return fmt.Errorf("match pattern %s with collection %s: %v", *c.collectionPattern, replica.info.Collection, err) + } + if !matched { + continue + } + } replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement)) if replicaPlacement.GetCopyCount() > len(replicas) { underReplicatedVolumeIds = append(underReplicatedVolumeIds, vid) + fmt.Fprintf(writer, "volume %d replication %s, current only %+d replicas\n", replica.info.Id, replicaPlacement, len(replicas)) } else if replicaPlacement.GetCopyCount() < len(replicas) { overReplicatedVolumeIds = append(overReplicatedVolumeIds, vid) fmt.Fprintf(writer, "volume %d replication %s, but over replicated %+d\n", replica.info.Id, replicaPlacement, len(replicas)) @@ -150,7 +170,7 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv, volumeIdLocationCount := len(volumeIdLocation.Locations) i := 0 for fixedVolumeReplicas[volumeId] >= volumeIdLocationCount { - fmt.Fprintf(writer, "the number of locations for volume %s has not increased yet, let's wait\n", volumeId) + fmt.Fprintf(writer, "the number of locations(current:%d, before:%d) for volume %s has not increased yet, let's wait\n", fixedVolumeReplicas[volumeId], volumeIdLocationCount, volumeId) time.Sleep(time.Duration(i+1) * time.Second * 7) volumeLocIds, err := lookupVolumeIds(commandEnv, []string{volumeId}) if err != nil { @@ -168,13 +188,16 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv, return nil } -func collectVolumeReplicaLocations(topologyInfo *master_pb.TopologyInfo) (map[uint32][]*VolumeReplica, []location) { +func collectVolumeReplicaLocations(topologyInfo *master_pb.TopologyInfo, rp *super_block.ReplicaPlacement) (map[uint32][]*VolumeReplica, []location) { volumeReplicas := make(map[uint32][]*VolumeReplica) var allLocations []location eachDataNode(topologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) { loc := newLocation(dc, string(rack), dn) for _, diskInfo := range dn.DiskInfos { for _, v := range diskInfo.VolumeInfos { + if rp != nil { + v.ReplicaPlacement = uint32(rp.Byte()) + } volumeReplicas[v.Id] = append(volumeReplicas[v.Id], &VolumeReplica{ location: &loc, info: v, @@ -237,6 +260,7 @@ func (c *commandVolumeFixReplication) fixUnderReplicatedVolumes(commandEnv *Comm underReplicatedVolumeIds = underReplicatedVolumeIds[0:volumesPerStep] } for _, vid := range underReplicatedVolumeIds { + fmt.Fprintf(writer, "begin fix volume: %d\n", vid) for i := 0; i < retryCount+1; i++ { if err = c.fixOneUnderReplicatedVolume(commandEnv, writer, takeAction, volumeReplicas, vid, allLocations); err == nil { if takeAction { @@ -267,6 +291,7 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co return fmt.Errorf("match pattern %s with collection %s: %v", *c.collectionPattern, replica.info.Collection, err) } if !matched { + fmt.Fprintf(writer, "collection(%s) skipped for volume: %d, filer collection pattern:%s", replica.info.Collection, vid, *c.collectionPattern) hasSkippedCollection = true break } @@ -290,6 +315,7 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co if replicateErr != nil { return fmt.Errorf("copying from %s => %s : %v", replica.location.dataNode.Id, dst.dataNode.Id, replicateErr) } + var bytesCount int64 for { resp, recvErr := stream.Recv() if recvErr != nil { @@ -300,10 +326,10 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co } } if resp.ProcessedBytes > 0 { - fmt.Fprintf(writer, "volume %d processed %d bytes\n", replica.info.Id, resp.ProcessedBytes) + bytesCount += resp.ProcessedBytes } } - + fmt.Fprintf(writer, "volume %d processed completed! total %d bytes\n", replica.info.Id, bytesCount) return nil }) diff --git a/weed/shell/command_volume_server_evacuate.go b/weed/shell/command_volume_server_evacuate.go index 94c457689..a1c736ab7 100644 --- a/weed/shell/command_volume_server_evacuate.go +++ b/weed/shell/command_volume_server_evacuate.go @@ -124,7 +124,7 @@ func (c *commandVolumeServerEvacuate) evacuateNormalVolumes(commandEnv *CommandE } } } - volumeReplicas, _ := collectVolumeReplicaLocations(c.topologyInfo) + volumeReplicas, _ := collectVolumeReplicaLocations(c.topologyInfo, nil) for _, vol := range diskInfo.VolumeInfos { hasMoved, err := moveAwayOneNormalVolume(commandEnv, volumeReplicas, vol, thisNode, otherNodes, applyChange) if err != nil { diff --git a/weed/shell/command_volume_tier_move.go b/weed/shell/command_volume_tier_move.go index a59119a40..511b757d6 100644 --- a/weed/shell/command_volume_tier_move.go +++ b/weed/shell/command_volume_tier_move.go @@ -94,7 +94,7 @@ func (c *commandVolumeTierMove) Do(args []string, commandEnv *CommandEnv, writer } fmt.Printf("tier move volumes: %v\n", volumeIds) - _, allLocations := collectVolumeReplicaLocations(topologyInfo) + _, allLocations := collectVolumeReplicaLocations(topologyInfo, nil) allLocations = filterLocationsByDiskType(allLocations, toDiskType) keepDataNodesSorted(allLocations, toDiskType) From 4b144c19405b2fa0909b36c0ae0f28f9853943ab Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Thu, 25 Jan 2024 15:54:35 +0800 Subject: [PATCH 43/52] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20stream=20sign?= =?UTF-8?q?ed=20=E9=89=B4=E6=9D=83=E5=9C=BA=E6=99=AF=E7=BC=BA=E5=A4=B1acl?= =?UTF-8?q?=20account=20id=20=E6=B3=A8=E5=85=A5bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: changlin.shi --- weed/s3api/s3api_object_multipart_handlers.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 5d049280f..07b6497ae 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -4,6 +4,7 @@ import ( "crypto/sha1" "encoding/xml" "fmt" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "io" "net/http" "net/url" @@ -282,18 +283,22 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ if s3a.iam.isEnabled() { rAuthType := getRequestAuthType(r) var s3ErrCode s3err.ErrorCode + var identity *Identity switch rAuthType { case authTypeStreamingSigned: - dataReader, _, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r) + dataReader, identity, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r) case authTypeSignedV2, authTypePresignedV2: - _, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r) + identity, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r) case authTypePresigned, authTypeSigned: - _, s3ErrCode = s3a.iam.reqSignatureV4Verify(r) + identity, s3ErrCode = s3a.iam.reqSignatureV4Verify(r) } if s3ErrCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, s3ErrCode) return } + if identity.AccountId != s3account.AccountAnonymous.Id { + r.Header.Set(s3_constants.AmzAccountId, identity.AccountId) + } } defer dataReader.Close() From 42fdce102d4a5464ff347dcb64a7e73bff5c2648 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Fri, 26 Jan 2024 19:03:29 +0800 Subject: [PATCH 44/52] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=E6=B7=BB=E5=8A=A0content-type=E4=B8=BAapplication/oct?= =?UTF-8?q?et-stream=E5=AF=BC=E8=87=B4=E6=B5=8F=E8=A7=88=E5=99=A8=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E9=A2=84=E8=A7=88=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: changlin.shi --- weed/server/filer_server_handlers_read.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index b1238af55..20cdbe01f 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -176,8 +176,6 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } if mimeType != "" { w.Header().Set("Content-Type", mimeType) - } else { - w.Header().Set("Content-Type", "application/octet-stream") } // print out the header from extended properties From 503ae6ad64373a3b6cfa9ff81c595ce3304351e3 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Sun, 28 Jan 2024 20:05:07 +0800 Subject: [PATCH 45/52] =?UTF-8?q?fix:=20=E5=BD=93=E7=BC=BA=E5=B0=91content?= =?UTF-8?q?-type=E5=A4=B4=E5=92=8C=E6=96=87=E4=BB=B6=E5=90=8E=E7=BC=80?= =?UTF-8?q?=E6=97=B6=E8=87=AA=E5=8A=A8=E6=A3=80=E6=B5=8Bmime=20type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: changlin.shi --- weed/s3api/filer_multipart.go | 9 ++++++--- weed/s3api/s3api_object_handlers.go | 2 +- weed/s3api/s3api_object_multipart_handlers.go | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index fefddf4b2..cbae59fae 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -99,10 +99,13 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa for _, entry := range entries { if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { - partETag, found := findByPartNumber(entry.Name, completedParts) + partETag, isFirstPart, found := findByPartNumber(entry.Name, completedParts) if !found { continue } + if isFirstPart { + mime = entry.GetAttributes().Mime + } entryETag := hex.EncodeToString(entry.Attributes.GetMd5()) if partETag != "" && len(partETag) == 32 && entryETag != "" && entryETag != partETag { glog.Errorf("completeMultipartUpload %s ETag mismatch chunk: %s part: %s", entry.Name, entryETag, partETag) @@ -176,7 +179,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa return } -func findByPartNumber(fileName string, parts []CompletedPart) (etag string, found bool) { +func findByPartNumber(fileName string, parts []CompletedPart) (etag string, firstPart, found bool) { partNumber, formatErr := strconv.Atoi(fileName[:4]) if formatErr != nil { return @@ -198,7 +201,7 @@ func findByPartNumber(fileName string, parts []CompletedPart) (etag string, foun break } } - return parts[x+y].ETag, true + return parts[x+y].ETag, partNumber == 1, true } func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code s3err.ErrorCode) { diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 4d0e374b2..2ba63c784 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -122,7 +122,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) } } else { uploadUrl := s3a.toFilerUrl(bucket, object) - if objectContentType == "" { + if objectContentType == "" || objectContentType == "application/octet-stream" { dataReader = mimeDetect(r, dataReader) } diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 07b6497ae..2784208fe 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -312,7 +312,8 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part", s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID) - if partID == 1 && r.Header.Get("Content-Type") == "" { + contentType := r.Header.Get("Content-Type") + if partID == 1 && contentType == "" || contentType == "application/octet-stream" { dataReader = mimeDetect(r, dataReader) } From 618668dda8458a353e5a483b83722d8fcac8b786 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Mon, 19 Feb 2024 17:10:39 +0800 Subject: [PATCH 46/52] =?UTF-8?q?Revert=20"fix:=20=E5=BD=93=E7=BC=BA?= =?UTF-8?q?=E5=B0=91content-type=E5=A4=B4=E5=92=8C=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=90=8E=E7=BC=80=E6=97=B6=E8=87=AA=E5=8A=A8=E6=A3=80=E6=B5=8B?= =?UTF-8?q?mime=20type"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 503ae6ad64373a3b6cfa9ff81c595ce3304351e3. --- weed/s3api/filer_multipart.go | 9 +++------ weed/s3api/s3api_object_handlers.go | 2 +- weed/s3api/s3api_object_multipart_handlers.go | 3 +-- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index cbae59fae..fefddf4b2 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -99,13 +99,10 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa for _, entry := range entries { if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { - partETag, isFirstPart, found := findByPartNumber(entry.Name, completedParts) + partETag, found := findByPartNumber(entry.Name, completedParts) if !found { continue } - if isFirstPart { - mime = entry.GetAttributes().Mime - } entryETag := hex.EncodeToString(entry.Attributes.GetMd5()) if partETag != "" && len(partETag) == 32 && entryETag != "" && entryETag != partETag { glog.Errorf("completeMultipartUpload %s ETag mismatch chunk: %s part: %s", entry.Name, entryETag, partETag) @@ -179,7 +176,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa return } -func findByPartNumber(fileName string, parts []CompletedPart) (etag string, firstPart, found bool) { +func findByPartNumber(fileName string, parts []CompletedPart) (etag string, found bool) { partNumber, formatErr := strconv.Atoi(fileName[:4]) if formatErr != nil { return @@ -201,7 +198,7 @@ func findByPartNumber(fileName string, parts []CompletedPart) (etag string, firs break } } - return parts[x+y].ETag, partNumber == 1, true + return parts[x+y].ETag, true } func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code s3err.ErrorCode) { diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 2ba63c784..4d0e374b2 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -122,7 +122,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request) } } else { uploadUrl := s3a.toFilerUrl(bucket, object) - if objectContentType == "" || objectContentType == "application/octet-stream" { + if objectContentType == "" { dataReader = mimeDetect(r, dataReader) } diff --git a/weed/s3api/s3api_object_multipart_handlers.go b/weed/s3api/s3api_object_multipart_handlers.go index 2784208fe..07b6497ae 100644 --- a/weed/s3api/s3api_object_multipart_handlers.go +++ b/weed/s3api/s3api_object_multipart_handlers.go @@ -312,8 +312,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ uploadUrl := fmt.Sprintf("http://%s%s/%s/%04d.part", s3a.option.Filer.ToHttpAddress(), s3a.genUploadsFolder(bucket), uploadID, partID) - contentType := r.Header.Get("Content-Type") - if partID == 1 && contentType == "" || contentType == "application/octet-stream" { + if partID == 1 && r.Header.Get("Content-Type") == "" { dataReader = mimeDetect(r, dataReader) } From a2c5ad0faf975462791d28c9f508877623a38a6f Mon Sep 17 00:00:00 2001 From: 7y-9 <121850020+7y-9@users.noreply.github.com> Date: Mon, 19 Feb 2024 16:38:05 +0800 Subject: [PATCH 47/52] fix: only admin auth can delete S3 bucket (#5312) --- weed/s3api/s3api_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 30e2e6bb6..40783ea73 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -239,7 +239,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // PutBucket bucket.Methods("PUT").HandlerFunc(track(s3a.PutBucketHandler, "PUT")) // DeleteBucket - bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_WRITE)), "DELETE")) + bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_ADMIN)), "DELETE")) // ListObjectsV1 (Legacy) bucket.Methods("GET").HandlerFunc(track(s3a.Auth(withAcl(s3a.cb.Limit, s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST")) From 18d941ba25552fd10efc726eabcd7dfbd87bd831 Mon Sep 17 00:00:00 2001 From: "changlin.shi" Date: Fri, 15 Mar 2024 10:56:46 +0800 Subject: [PATCH 48/52] =?UTF-8?q?=E7=A6=81=E7=94=A8shell=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=91=BD=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: changlin.shi --- weed/shell/command_collection_delete.go | 2 +- weed/shell/command_fs_mv.go | 2 +- weed/shell/command_fs_rm.go | 2 +- weed/shell/command_s3_bucket_delete.go | 2 +- weed/shell/command_volume_delete.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/weed/shell/command_collection_delete.go b/weed/shell/command_collection_delete.go index 936f35b46..caf52aa9b 100644 --- a/weed/shell/command_collection_delete.go +++ b/weed/shell/command_collection_delete.go @@ -10,7 +10,7 @@ import ( ) func init() { - Commands = append(Commands, &commandCollectionDelete{}) + //Commands = append(Commands, &commandCollectionDelete{}) } type commandCollectionDelete struct { diff --git a/weed/shell/command_fs_mv.go b/weed/shell/command_fs_mv.go index 8e609edc9..73c792efa 100644 --- a/weed/shell/command_fs_mv.go +++ b/weed/shell/command_fs_mv.go @@ -10,7 +10,7 @@ import ( ) func init() { - Commands = append(Commands, &commandFsMv{}) + //Commands = append(Commands, &commandFsMv{}) } type commandFsMv struct { diff --git a/weed/shell/command_fs_rm.go b/weed/shell/command_fs_rm.go index b8445b7e9..2343e91fd 100644 --- a/weed/shell/command_fs_rm.go +++ b/weed/shell/command_fs_rm.go @@ -11,7 +11,7 @@ import ( ) func init() { - Commands = append(Commands, &commandFsRm{}) + //Commands = append(Commands, &commandFsRm{}) } type commandFsRm struct { diff --git a/weed/shell/command_s3_bucket_delete.go b/weed/shell/command_s3_bucket_delete.go index d0b4cb505..0fcbcc40e 100644 --- a/weed/shell/command_s3_bucket_delete.go +++ b/weed/shell/command_s3_bucket_delete.go @@ -11,7 +11,7 @@ import ( ) func init() { - Commands = append(Commands, &commandS3BucketDelete{}) + //Commands = append(Commands, &commandS3BucketDelete{}) } type commandS3BucketDelete struct { diff --git a/weed/shell/command_volume_delete.go b/weed/shell/command_volume_delete.go index eb4473565..e70877cca 100644 --- a/weed/shell/command_volume_delete.go +++ b/weed/shell/command_volume_delete.go @@ -9,7 +9,7 @@ import ( ) func init() { - Commands = append(Commands, &commandVolumeDelete{}) + //Commands = append(Commands, &commandVolumeDelete{}) } type commandVolumeDelete struct { From 8ebcd18b5692b18bfd971dc2908a2467e8ef2f62 Mon Sep 17 00:00:00 2001 From: "steve.wei" Date: Mon, 8 Apr 2024 22:30:04 +0800 Subject: [PATCH 49/52] fix(volume.fix.replication): adjust volume count, not free volume count (#5479) (cherry picked from commit 67ead9b18f0e7f23f690c7f04e521be9e4d88270) --- weed/shell/command_volume_fix_replication.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go index b0a9b5c27..6492e1a36 100644 --- a/weed/shell/command_volume_fix_replication.go +++ b/weed/shell/command_volume_fix_replication.go @@ -302,8 +302,8 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co fmt.Fprintf(writer, "replicating volume %d %s from %s to dataNode %s ...\n", replica.info.Id, replicaPlacement, replica.location.dataNode.Id, dst.dataNode.Id) if !takeAction { - // adjust free volume count - dst.dataNode.DiskInfos[replica.info.DiskType].FreeVolumeCount-- + // adjust volume count + dst.dataNode.DiskInfos[replica.info.DiskType].VolumeCount++ break } @@ -337,8 +337,8 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co return err } - // adjust free volume count - dst.dataNode.DiskInfos[replica.info.DiskType].FreeVolumeCount-- + // adjust volume count + dst.dataNode.DiskInfos[replica.info.DiskType].VolumeCount++ break } } From 7e85e1c5f942fe15a4f6a538b9b8e081e9c88f20 Mon Sep 17 00:00:00 2001 From: "chuanhai.wei" Date: Thu, 25 Apr 2024 10:50:49 +0800 Subject: [PATCH 50/52] fix(volume.fix.replication): processedBytes --- weed/shell/command_volume_fix_replication.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go index 6492e1a36..1dc111d6e 100644 --- a/weed/shell/command_volume_fix_replication.go +++ b/weed/shell/command_volume_fix_replication.go @@ -315,7 +315,7 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co if replicateErr != nil { return fmt.Errorf("copying from %s => %s : %v", replica.location.dataNode.Id, dst.dataNode.Id, replicateErr) } - var bytesCount int64 + var processedBytes int64 for { resp, recvErr := stream.Recv() if recvErr != nil { @@ -326,10 +326,10 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co } } if resp.ProcessedBytes > 0 { - bytesCount += resp.ProcessedBytes + processedBytes = resp.ProcessedBytes } } - fmt.Fprintf(writer, "volume %d processed completed! total %d bytes\n", replica.info.Id, bytesCount) + fmt.Fprintf(writer, "volume %d processed completed! total %d bytes\n", replica.info.Id, processedBytes) return nil }) From 3add86ef9e815ca8dd6398a0a7fe222330cb1522 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Thu, 21 Sep 2023 23:20:05 +0800 Subject: [PATCH 51/52] [s3acl] Step 0: Put bucket ACL only responds success if the ACL is private. (#4856) * Passing test: test_bucket_acl_default test_bucket_acl_canned_private_to_private https://github.com/seaweedfs/seaweedfs/issues/4519 * Update weed/s3api/s3api_bucket_handlers.go --------- Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co> Co-authored-by: Chris Lu Signed-off-by: LHHDZ --- docker/compose/s3tests.conf | 4 ++-- weed/s3api/s3api_bucket_handlers.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/compose/s3tests.conf b/docker/compose/s3tests.conf index 68d9ddeb7..2bffe20d4 100644 --- a/docker/compose/s3tests.conf +++ b/docker/compose/s3tests.conf @@ -18,10 +18,10 @@ bucket prefix = yournamehere-{random}- [s3 main] # main display_name set in vstart.sh -display_name = M. Tester +display_name = s3_tests # main user_idname set in vstart.sh -user_id = testid +user_id = s3_tests # main email set in vstart.sh email = tester@ceph.com diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 7f07c0f9d..1fd212d7f 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -424,6 +424,7 @@ func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *htt defer util.CloseRequest(r) var v s3.OwnershipControls + err := xmlutil.UnmarshalXML(&v, xml.NewDecoder(r.Body), "") if err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) From b253a20faf92df280299545477e1e5d9bbbbcf04 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Mon, 25 Sep 2023 23:34:12 +0800 Subject: [PATCH 52/52] [s3acl] Step1: move s3account.AccountManager into to iam.S3ApiConfiguration (#4859) * move s3account.AccountManager into to iam.S3ApiConfiguration and switch to Interface https://github.com/seaweedfs/seaweedfs/issues/4519 * fix: test bucket acl default and adjust the variable names * fix: s3 api config test --------- Co-authored-by: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.co> Co-authored-by: Chris Lu Signed-off-by: LHHDZ --- Makefile | 5 +- docker/compose/s3.json | 14 +- docker/compose/s3tests.conf | 4 +- weed/pb/iam.proto | 9 +- weed/pb/iam_pb/iam.pb.go | 170 ++++++-- weed/pb/iam_pb/iam_grpc.pb.go | 6 +- weed/s3api/auth_credentials.go | 185 +++++++-- weed/s3api/auth_credentials_test.go | 37 +- weed/s3api/bucket_metadata.go | 15 +- weed/s3api/bucket_metadata_test.go | 42 +- weed/s3api/s3_constants/s3_acp.go | 6 + weed/s3api/s3account/s3_account.go | 70 ---- .../acl_helper.go => s3api_acl_helper.go} | 28 +- ...elper_test.go => s3api_acl_helper_test.go} | 389 ++++++++++++++++-- weed/s3api/s3api_acp.go | 4 +- weed/s3api/s3api_bucket_handlers.go | 2 - weed/s3api/s3api_server.go | 3 - 17 files changed, 736 insertions(+), 253 deletions(-) create mode 100644 weed/s3api/s3_constants/s3_acp.go delete mode 100644 weed/s3api/s3account/s3_account.go rename weed/s3api/{s3acl/acl_helper.go => s3api_acl_helper.go} (95%) rename weed/s3api/{s3acl/acl_helper_test.go => s3api_acl_helper_test.go} (80%) diff --git a/Makefile b/Makefile index 315a62428..6ca6cae0d 100644 --- a/Makefile +++ b/Makefile @@ -10,5 +10,8 @@ install: full_install: cd weed; go install -tags "elastic gocdk sqlite ydb tikv" -tests: +server: install + weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json + +test: cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./... diff --git a/docker/compose/s3.json b/docker/compose/s3.json index 64dedb681..ce230863b 100644 --- a/docker/compose/s3.json +++ b/docker/compose/s3.json @@ -40,7 +40,10 @@ "List", "Tagging", "Write" - ] + ], + "account": { + "id": "testid" + } }, { "name": "s3_tests_alt", @@ -101,5 +104,12 @@ "Write" ] } - ] + ], + "accounts": [ + { + "id" : "testid", + "displayName": "M. Tester", + "emailAddress": "tester@ceph.com" + } + ] } \ No newline at end of file diff --git a/docker/compose/s3tests.conf b/docker/compose/s3tests.conf index 2bffe20d4..68d9ddeb7 100644 --- a/docker/compose/s3tests.conf +++ b/docker/compose/s3tests.conf @@ -18,10 +18,10 @@ bucket prefix = yournamehere-{random}- [s3 main] # main display_name set in vstart.sh -display_name = s3_tests +display_name = M. Tester # main user_idname set in vstart.sh -user_id = s3_tests +user_id = testid # main email set in vstart.sh email = tester@ceph.com diff --git a/weed/pb/iam.proto b/weed/pb/iam.proto index 1a6027292..99bb65ef2 100644 --- a/weed/pb/iam.proto +++ b/weed/pb/iam.proto @@ -16,13 +16,14 @@ service SeaweedIdentityAccessManagement { message S3ApiConfiguration { repeated Identity identities = 1; + repeated Account accounts = 2; } message Identity { string name = 1; repeated Credential credentials = 2; repeated string actions = 3; - string accountId = 4; + Account account = 4; } message Credential { @@ -32,6 +33,12 @@ message Credential { // bool is_disabled = 4; } +message Account { + string id = 1; + string display_name = 2; + string email_address = 3; +} + /* message Policy { repeated Statement statements = 1; diff --git a/weed/pb/iam_pb/iam.pb.go b/weed/pb/iam_pb/iam.pb.go index e94454e47..074e255e6 100644 --- a/weed/pb/iam_pb/iam.pb.go +++ b/weed/pb/iam_pb/iam.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 -// protoc v3.21.12 +// protoc-gen-go v1.30.0 +// protoc v4.23.2 // source: iam.proto package iam_pb @@ -26,6 +26,7 @@ type S3ApiConfiguration struct { unknownFields protoimpl.UnknownFields Identities []*Identity `protobuf:"bytes,1,rep,name=identities,proto3" json:"identities,omitempty"` + Accounts []*Account `protobuf:"bytes,2,rep,name=accounts,proto3" json:"accounts,omitempty"` } func (x *S3ApiConfiguration) Reset() { @@ -67,6 +68,13 @@ func (x *S3ApiConfiguration) GetIdentities() []*Identity { return nil } +func (x *S3ApiConfiguration) GetAccounts() []*Account { + if x != nil { + return x.Accounts + } + return nil +} + type Identity struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -75,7 +83,7 @@ type Identity struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Credentials []*Credential `protobuf:"bytes,2,rep,name=credentials,proto3" json:"credentials,omitempty"` Actions []string `protobuf:"bytes,3,rep,name=actions,proto3" json:"actions,omitempty"` - AccountId string `protobuf:"bytes,4,opt,name=accountId,proto3" json:"accountId,omitempty"` + Account *Account `protobuf:"bytes,4,opt,name=account,proto3" json:"account,omitempty"` } func (x *Identity) Reset() { @@ -131,11 +139,11 @@ func (x *Identity) GetActions() []string { return nil } -func (x *Identity) GetAccountId() string { +func (x *Identity) GetAccount() *Account { if x != nil { - return x.AccountId + return x.Account } - return "" + return nil } type Credential struct { @@ -193,36 +201,109 @@ func (x *Credential) GetSecretKey() string { return "" } +type Account struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + DisplayName string `protobuf:"bytes,2,opt,name=display_name,json=displayName,proto3" json:"display_name,omitempty"` + EmailAddress string `protobuf:"bytes,3,opt,name=email_address,json=emailAddress,proto3" json:"email_address,omitempty"` +} + +func (x *Account) Reset() { + *x = Account{} + if protoimpl.UnsafeEnabled { + mi := &file_iam_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Account) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Account) ProtoMessage() {} + +func (x *Account) ProtoReflect() protoreflect.Message { + mi := &file_iam_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Account.ProtoReflect.Descriptor instead. +func (*Account) Descriptor() ([]byte, []int) { + return file_iam_proto_rawDescGZIP(), []int{3} +} + +func (x *Account) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Account) GetDisplayName() string { + if x != nil { + return x.DisplayName + } + return "" +} + +func (x *Account) GetEmailAddress() string { + if x != nil { + return x.EmailAddress + } + return "" +} + var File_iam_proto protoreflect.FileDescriptor var file_iam_proto_rawDesc = []byte{ 0x0a, 0x09, 0x69, 0x61, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x69, 0x61, 0x6d, - 0x5f, 0x70, 0x62, 0x22, 0x46, 0x0a, 0x12, 0x53, 0x33, 0x41, 0x70, 0x69, 0x43, 0x6f, 0x6e, 0x66, + 0x5f, 0x70, 0x62, 0x22, 0x73, 0x0a, 0x12, 0x53, 0x33, 0x41, 0x70, 0x69, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, - 0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x8c, 0x01, 0x0a, 0x08, - 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b, - 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, - 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x4a, 0x0a, 0x0a, 0x43, 0x72, - 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, - 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, - 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65, - 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x63, - 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, - 0x64, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65, 0x61, - 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08, 0x49, - 0x61, 0x6d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, - 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, - 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x2b, 0x0a, 0x08, 0x61, + 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, + 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x08, + 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x22, 0x99, 0x01, 0x0a, 0x08, 0x49, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b, 0x63, 0x72, 0x65, + 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x12, + 0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x29, 0x0a, 0x07, 0x61, 0x63, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x69, 0x61, 0x6d, + 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x07, 0x61, 0x63, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x4a, 0x0a, 0x0a, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, + 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, + 0x22, 0x61, 0x0a, 0x07, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x64, + 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, + 0x0a, 0x0d, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x64, 0x64, 0x72, + 0x65, 0x73, 0x73, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x49, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, + 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08, 0x49, 0x61, 0x6d, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, + 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x69, 0x61, 0x6d, + 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -237,20 +318,23 @@ func file_iam_proto_rawDescGZIP() []byte { return file_iam_proto_rawDescData } -var file_iam_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_iam_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_iam_proto_goTypes = []interface{}{ (*S3ApiConfiguration)(nil), // 0: iam_pb.S3ApiConfiguration (*Identity)(nil), // 1: iam_pb.Identity (*Credential)(nil), // 2: iam_pb.Credential + (*Account)(nil), // 3: iam_pb.Account } var file_iam_proto_depIdxs = []int32{ 1, // 0: iam_pb.S3ApiConfiguration.identities:type_name -> iam_pb.Identity - 2, // 1: iam_pb.Identity.credentials:type_name -> iam_pb.Credential - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 3, // 1: iam_pb.S3ApiConfiguration.accounts:type_name -> iam_pb.Account + 2, // 2: iam_pb.Identity.credentials:type_name -> iam_pb.Credential + 3, // 3: iam_pb.Identity.account:type_name -> iam_pb.Account + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_iam_proto_init() } @@ -295,6 +379,18 @@ func file_iam_proto_init() { return nil } } + file_iam_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Account); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -302,7 +398,7 @@ func file_iam_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_iam_proto_rawDesc, NumEnums: 0, - NumMessages: 3, + NumMessages: 4, NumExtensions: 0, NumServices: 1, }, diff --git a/weed/pb/iam_pb/iam_grpc.pb.go b/weed/pb/iam_pb/iam_grpc.pb.go index ea4e1bb41..c4fe7becc 100644 --- a/weed/pb/iam_pb/iam_grpc.pb.go +++ b/weed/pb/iam_pb/iam_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.12 +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.23.2 // source: iam.proto package iam_pb @@ -15,6 +15,8 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const () + // SeaweedIdentityAccessManagementClient is the client API for SeaweedIdentityAccessManagement service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index f79448dcc..e7fff0dce 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -2,7 +2,6 @@ package s3api import ( "fmt" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "net/http" "os" "strings" @@ -17,11 +16,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" ) -var IdentityAnonymous = &Identity{ - Name: s3account.AccountAnonymous.Name, - AccountId: s3account.AccountAnonymous.Id, -} - type Action string type Iam interface { @@ -31,27 +25,64 @@ type Iam interface { type IdentityAccessManagement struct { m sync.RWMutex - identities []*Identity - isAuthEnabled bool - domain string + identities []*Identity + accessKeyIdent map[string]*Identity + accounts map[string]*Account + emailAccount map[string]*Account + hashes map[string]*sync.Pool + hashCounters map[string]*int32 + identityAnonymous *Identity + hashMu sync.RWMutex + domain string + isAuthEnabled bool } type Identity struct { Name string - AccountId string + Account *Account Credentials []*Credential Actions []Action } -func (i *Identity) isAnonymous() bool { - return i.Name == s3account.AccountAnonymous.Name +// Account represents a system user, a system user can +// configure multiple IAM-Users, IAM-Users can configure +// permissions respectively, and each IAM-User can +// configure multiple security credentials +type Account struct { + //Name is also used to display the "DisplayName" as the owner of the bucket or object + DisplayName string + EmailAddress string + + //Id is used to identify an Account when granting cross-account access(ACLs) to buckets and objects + Id string } +// Predefined Accounts +var ( + // AccountAdmin is used as the default account for IAM-Credentials access without Account configured + AccountAdmin = Account{ + DisplayName: "admin", + EmailAddress: "admin@example.com", + Id: s3_constants.AccountAdminId, + } + + // AccountAnonymous is used to represent the account for anonymous access + AccountAnonymous = Account{ + DisplayName: "anonymous", + EmailAddress: "anonymous@example.com", + Id: s3_constants.AccountAnonymousId, + } +) + type Credential struct { AccessKey string SecretKey string } +func (i *Identity) isAnonymous() bool { + return i.Account.Id == s3_constants.AccountAnonymousId +} + func (action Action) isAdmin() bool { return strings.HasPrefix(string(action), s3_constants.ACTION_ADMIN) } @@ -64,14 +95,19 @@ func (action Action) overBucket(bucket string) bool { return strings.HasSuffix(string(action), ":"+bucket) || strings.HasSuffix(string(action), ":*") } +// "Permission": "FULL_CONTROL"|"WRITE"|"WRITE_ACP"|"READ"|"READ_ACP" func (action Action) getPermission() Permission { switch act := strings.Split(string(action), ":")[0]; act { case s3_constants.ACTION_ADMIN: return Permission("FULL_CONTROL") case s3_constants.ACTION_WRITE: return Permission("WRITE") + case s3_constants.ACTION_WRITE_ACP: + return Permission("WRITE_ACP") case s3_constants.ACTION_READ: return Permission("READ") + case s3_constants.ACTION_READ_ACP: + return Permission("READ_ACP") default: return Permission("") } @@ -79,7 +115,9 @@ func (action Action) getPermission() Permission { func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement { iam := &IdentityAccessManagement{ - domain: option.DomainName, + domain: option.DomainName, + hashes: make(map[string]*sync.Pool), + hashCounters: make(map[string]*int32), } if option.Config != "" { if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil { @@ -133,26 +171,71 @@ func (iam *IdentityAccessManagement) LoadS3ApiConfigurationFromBytes(content []b func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error { var identities []*Identity + var identityAnonymous *Identity + accessKeyIdent := make(map[string]*Identity) + accounts := make(map[string]*Account) + emailAccount := make(map[string]*Account) + foundAccountAdmin := false + foundAccountAnonymous := false + + for _, account := range config.Accounts { + switch account.Id { + case AccountAdmin.Id: + AccountAdmin = Account{ + Id: account.Id, + DisplayName: account.DisplayName, + EmailAddress: account.EmailAddress, + } + accounts[account.Id] = &AccountAdmin + foundAccountAdmin = true + case AccountAnonymous.Id: + AccountAnonymous = Account{ + Id: account.Id, + DisplayName: account.DisplayName, + EmailAddress: account.EmailAddress, + } + accounts[account.Id] = &AccountAnonymous + foundAccountAnonymous = true + default: + t := Account{ + Id: account.Id, + DisplayName: account.DisplayName, + EmailAddress: account.EmailAddress, + } + accounts[account.Id] = &t + } + if account.EmailAddress != "" { + emailAccount[account.EmailAddress] = accounts[account.Id] + } + } + if !foundAccountAdmin { + accounts[AccountAdmin.Id] = &AccountAdmin + emailAccount[AccountAdmin.EmailAddress] = &AccountAdmin + } + if !foundAccountAnonymous { + accounts[AccountAnonymous.Id] = &AccountAnonymous + emailAccount[AccountAnonymous.EmailAddress] = &AccountAnonymous + } for _, ident := range config.Identities { t := &Identity{ Name: ident.Name, - AccountId: s3account.AccountAdmin.Id, Credentials: nil, Actions: nil, } - - if ident.Name == s3account.AccountAnonymous.Name { - if ident.AccountId != "" && ident.AccountId != s3account.AccountAnonymous.Id { - glog.Warningf("anonymous identity is associated with a non-anonymous account ID, the association is invalid") - } - t.AccountId = s3account.AccountAnonymous.Id - IdentityAnonymous = t - } else { - if len(ident.AccountId) > 0 { - t.AccountId = ident.AccountId + switch { + case ident.Name == AccountAnonymous.Id: + t.Account = &AccountAnonymous + identityAnonymous = t + case ident.Account == nil: + t.Account = &AccountAdmin + default: + if account, ok := accounts[ident.Account.Id]; ok { + t.Account = account + } else { + t.Account = &AccountAdmin + glog.Warningf("identity %s is associated with a non exist account ID, the association is invalid", ident.Name) } } - for _, action := range ident.Actions { t.Actions = append(t.Actions, Action(action)) } @@ -161,17 +244,22 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api AccessKey: cred.AccessKey, SecretKey: cred.SecretKey, }) + accessKeyIdent[cred.AccessKey] = t } identities = append(identities, t) } - iam.m.Lock() // atomically switch iam.identities = identities + iam.identityAnonymous = identityAnonymous + iam.accounts = accounts + iam.emailAccount = emailAccount + iam.accessKeyIdent = accessKeyIdent if !iam.isAuthEnabled { // one-directional, no toggling iam.isAuthEnabled = len(identities) > 0 } iam.m.Unlock() + return nil } @@ -180,14 +268,12 @@ func (iam *IdentityAccessManagement) isEnabled() bool { } func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) { - iam.m.RLock() defer iam.m.RUnlock() - for _, ident := range iam.identities { - for _, cred := range ident.Credentials { - // println("checking", ident.Name, cred.AccessKey) - if cred.AccessKey == accessKey { - return ident, cred, true + if ident, ok := iam.accessKeyIdent[accessKey]; ok { + for _, credential := range ident.Credentials { + if credential.AccessKey == accessKey { + return ident, credential, true } } } @@ -198,14 +284,30 @@ func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identi func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, found bool) { iam.m.RLock() defer iam.m.RUnlock() - for _, ident := range iam.identities { - if ident.isAnonymous() { - return ident, true - } + if iam.identityAnonymous != nil { + return iam.identityAnonymous, true } return nil, false } +func (iam *IdentityAccessManagement) GetAccountNameById(canonicalId string) string { + iam.m.RLock() + defer iam.m.RUnlock() + if account, ok := iam.accounts[canonicalId]; ok { + return account.DisplayName + } + return "" +} + +func (iam *IdentityAccessManagement) GetAccountIdByEmail(email string) string { + iam.m.RLock() + defer iam.m.RUnlock() + if account, ok := iam.emailAccount[email]; ok { + return account.Id + } + return "" +} + func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) http.HandlerFunc { return Auth(iam, nil, f, action, false) } @@ -243,6 +345,7 @@ func Auth(iam *IdentityAccessManagement, br *BucketRegistry, f http.HandlerFunc, } } +// check whether the request has valid access keys func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) { return authRequest(iam, nil, r, action, false) } @@ -286,8 +389,7 @@ func authRequest(iam *IdentityAccessManagement, br *BucketRegistry, r *http.Requ } } authType = "Anonymous" - identity = IdentityAnonymous - if len(identity.Actions) == 0 { + if identity, found = iam.lookupAnonymous(); !found || len(identity.Actions) == 0 { r.Header.Set(s3_constants.AmzAuthType, authType) return identity, s3err.ErrAccessDenied } @@ -302,7 +404,7 @@ func authRequest(iam *IdentityAccessManagement, br *BucketRegistry, r *http.Requ return identity, s3Err } - glog.V(3).Infof("user name: %v account id: %v actions: %v, action: %v", identity.Name, identity.AccountId, identity.Actions, action) + glog.V(3).Infof("user name: %v actions: %v, action: %v", identity.Name, identity.Actions, action) bucket, object := s3_constants.GetBucketAndObject(r) @@ -310,9 +412,8 @@ func authRequest(iam *IdentityAccessManagement, br *BucketRegistry, r *http.Requ return identity, s3err.ErrAccessDenied } - if !identity.isAnonymous() { - r.Header.Set(s3_constants.AmzAccountId, identity.AccountId) - } + r.Header.Set(s3_constants.AmzAccountId, identity.Account.Id) + return identity, s3err.ErrNone } diff --git a/weed/s3api/auth_credentials_test.go b/weed/s3api/auth_credentials_test.go index 1f0ffc1cc..5d1823537 100644 --- a/weed/s3api/auth_credentials_test.go +++ b/weed/s3api/auth_credentials_test.go @@ -2,14 +2,12 @@ package s3api import ( . "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/stretchr/testify/assert" "reflect" "testing" - jsonpb "google.golang.org/protobuf/encoding/protojson" - "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" + jsonpb "google.golang.org/protobuf/encoding/protojson" ) func TestIdentityListFileFormat(t *testing.T) { @@ -129,11 +127,22 @@ func TestCanDo(t *testing.T) { } type LoadS3ApiConfigurationTestCase struct { + pbAccount *iam_pb.Account pbIdent *iam_pb.Identity expectIdent *Identity } func TestLoadS3ApiConfiguration(t *testing.T) { + specifiedAccount := Account{ + Id: "specifiedAccountID", + DisplayName: "specifiedAccountName", + EmailAddress: "specifiedAccounEmail@example.com", + } + pbSpecifiedAccount := iam_pb.Account{ + Id: "specifiedAccountID", + DisplayName: "specifiedAccountName", + EmailAddress: "specifiedAccounEmail@example.com", + } testCases := map[string]*LoadS3ApiConfigurationTestCase{ "notSpecifyAccountId": { pbIdent: &iam_pb.Identity{ @@ -150,8 +159,8 @@ func TestLoadS3ApiConfiguration(t *testing.T) { }, }, expectIdent: &Identity{ - Name: "notSpecifyAccountId", - AccountId: s3account.AccountAdmin.Id, + Name: "notSpecifyAccountId", + Account: &AccountAdmin, Actions: []Action{ "Read", "Write", @@ -165,17 +174,18 @@ func TestLoadS3ApiConfiguration(t *testing.T) { }, }, "specifiedAccountID": { + pbAccount: &pbSpecifiedAccount, pbIdent: &iam_pb.Identity{ - Name: "specifiedAccountID", - AccountId: "specifiedAccountID", + Name: "specifiedAccountID", + Account: &pbSpecifiedAccount, Actions: []string{ "Read", "Write", }, }, expectIdent: &Identity{ - Name: "specifiedAccountID", - AccountId: "specifiedAccountID", + Name: "specifiedAccountID", + Account: &specifiedAccount, Actions: []Action{ "Read", "Write", @@ -191,8 +201,8 @@ func TestLoadS3ApiConfiguration(t *testing.T) { }, }, expectIdent: &Identity{ - Name: "anonymous", - AccountId: "anonymous", + Name: "anonymous", + Account: &AccountAnonymous, Actions: []Action{ "Read", "Write", @@ -206,6 +216,9 @@ func TestLoadS3ApiConfiguration(t *testing.T) { } for _, v := range testCases { config.Identities = append(config.Identities, v.pbIdent) + if v.pbAccount != nil { + config.Accounts = append(config.Accounts, v.pbAccount) + } } iam := IdentityAccessManagement{} @@ -217,7 +230,7 @@ func TestLoadS3ApiConfiguration(t *testing.T) { for _, ident := range iam.identities { tc := testCases[ident.Name] if !reflect.DeepEqual(ident, tc.expectIdent) { - t.Error("not expect") + t.Errorf("not expect for ident name %s", ident.Name) } } } diff --git a/weed/s3api/bucket_metadata.go b/weed/s3api/bucket_metadata.go index d3e47810d..fcf3d187d 100644 --- a/weed/s3api/bucket_metadata.go +++ b/weed/s3api/bucket_metadata.go @@ -6,7 +6,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/util" "math" @@ -19,7 +18,7 @@ var loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*B return nil, err } - return buildBucketMetadata(r.s3a.accountManager, entry), nil + return buildBucketMetadata(r.s3a.iam, entry), nil } type BucketMetaData struct { @@ -73,13 +72,13 @@ func (r *BucketRegistry) init() error { } func (r *BucketRegistry) LoadBucketMetadata(entry *filer_pb.Entry) { - bucketMetadata := buildBucketMetadata(r.s3a.accountManager, entry) + bucketMetadata := buildBucketMetadata(r.s3a.iam, entry) r.metadataCacheLock.Lock() defer r.metadataCacheLock.Unlock() r.metadataCache[entry.Name] = bucketMetadata } -func buildBucketMetadata(accountManager *s3account.AccountManager, entry *filer_pb.Entry) *BucketMetaData { +func buildBucketMetadata(accountManager AccountManager, entry *filer_pb.Entry) *BucketMetaData { entryJson, _ := json.Marshal(entry) glog.V(3).Infof("build bucket metadata,entry=%s", entryJson) bucketMetadata := &BucketMetaData{ @@ -90,8 +89,8 @@ func buildBucketMetadata(accountManager *s3account.AccountManager, entry *filer_ // Default owner: `AccountAdmin` Owner: &s3.Owner{ - ID: &s3account.AccountAdmin.Id, - DisplayName: &s3account.AccountAdmin.Name, + ID: &AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, }, } if entry.Extended != nil { @@ -111,8 +110,8 @@ func buildBucketMetadata(accountManager *s3account.AccountManager, entry *filer_ acpOwnerBytes, ok := entry.Extended[s3_constants.ExtAmzOwnerKey] if ok && len(acpOwnerBytes) > 0 { ownerAccountId := string(acpOwnerBytes) - ownerAccountName, exists := accountManager.IdNameMapping[ownerAccountId] - if !exists { + ownerAccountName := accountManager.GetAccountNameById(ownerAccountId) + if ownerAccountName == "" { glog.Warningf("owner[id=%s] is invalid, bucket: %s", ownerAccountId, bucketMetadata.Name) } else { bucketMetadata.Owner = &s3.Owner{ diff --git a/weed/s3api/bucket_metadata_test.go b/weed/s3api/bucket_metadata_test.go index 98c569dea..960f6d3ee 100644 --- a/weed/s3api/bucket_metadata_test.go +++ b/weed/s3api/bucket_metadata_test.go @@ -5,8 +5,8 @@ import ( "fmt" "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "reflect" "sync" @@ -31,7 +31,7 @@ var ( Name: "entryWithValidAcp", Extended: map[string][]byte{ s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced), - s3_constants.ExtAmzOwnerKey: []byte(s3account.AccountAdmin.Name), + s3_constants.ExtAmzOwnerKey: []byte(AccountAdmin.DisplayName), s3_constants.ExtAmzAclKey: goodEntryAcl, }, } @@ -88,8 +88,8 @@ var tcs = []*BucketMetadataTestCase{ Name: badEntry.Name, ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: nil, }, @@ -99,8 +99,8 @@ var tcs = []*BucketMetadataTestCase{ Name: goodEntry.Name, ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: s3_constants.PublicRead, }, @@ -110,8 +110,8 @@ var tcs = []*BucketMetadataTestCase{ Name: ownershipEmptyStr.Name, ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: []*s3.Grant{ { @@ -129,8 +129,8 @@ var tcs = []*BucketMetadataTestCase{ Name: ownershipValid.Name, ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: []*s3.Grant{ { @@ -148,8 +148,8 @@ var tcs = []*BucketMetadataTestCase{ Name: acpEmptyStr.Name, ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: []*s3.Grant{ { @@ -167,8 +167,8 @@ var tcs = []*BucketMetadataTestCase{ Name: acpEmptyObject.Name, ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: []*s3.Grant{ { @@ -186,8 +186,8 @@ var tcs = []*BucketMetadataTestCase{ Name: acpOwnerNil.Name, ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: make([]*s3.Grant, 0), }, @@ -195,14 +195,10 @@ var tcs = []*BucketMetadataTestCase{ } func TestBuildBucketMetadata(t *testing.T) { - accountManager := &s3account.AccountManager{ - IdNameMapping: map[string]string{ - s3account.AccountAdmin.Id: s3account.AccountAdmin.Name, - s3account.AccountAnonymous.Id: s3account.AccountAnonymous.Name, - }, - } + iam := &IdentityAccessManagement{} + _ = iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{}) for _, tc := range tcs { - resultBucketMetadata := buildBucketMetadata(accountManager, tc.filerEntry) + resultBucketMetadata := buildBucketMetadata(iam, tc.filerEntry) if !reflect.DeepEqual(resultBucketMetadata, tc.expectBucketMetadata) { t.Fatalf("result is unexpect: \nresult: %v, \nexpect: %v", resultBucketMetadata, tc.expectBucketMetadata) } diff --git a/weed/s3api/s3_constants/s3_acp.go b/weed/s3api/s3_constants/s3_acp.go new file mode 100644 index 000000000..d24e07e24 --- /dev/null +++ b/weed/s3api/s3_constants/s3_acp.go @@ -0,0 +1,6 @@ +package s3_constants + +const ( + AccountAnonymousId = "anonymous" + AccountAdminId = "admin" +) diff --git a/weed/s3api/s3account/s3_account.go b/weed/s3api/s3account/s3_account.go deleted file mode 100644 index 9b1b01123..000000000 --- a/weed/s3api/s3account/s3_account.go +++ /dev/null @@ -1,70 +0,0 @@ -package s3account - -import ( - "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" - "sync" -) - -//Predefined Accounts -var ( - // AccountAdmin is used as the default account for IAM-Credentials access without Account configured - AccountAdmin = Account{ - Name: "admin", - EmailAddress: "admin@example.com", - Id: "admin", - } - - // AccountAnonymous is used to represent the account for anonymous access - AccountAnonymous = Account{ - Name: "anonymous", - EmailAddress: "anonymous@example.com", - Id: "anonymous", - } -) - -//Account represents a system user, a system user can -//configure multiple IAM-Users, IAM-Users can configure -//permissions respectively, and each IAM-User can -//configure multiple security credentials -type Account struct { - //Name is also used to display the "DisplayName" as the owner of the bucket or object - Name string - EmailAddress string - - //Id is used to identify an Account when granting cross-account access(ACLs) to buckets and objects - Id string -} - -type AccountManager struct { - sync.Mutex - filerClient filer_pb.FilerClient - - IdNameMapping map[string]string - EmailIdMapping map[string]string -} - -func NewAccountManager(filerClient filer_pb.FilerClient) *AccountManager { - am := &AccountManager{ - filerClient: filerClient, - IdNameMapping: make(map[string]string), - EmailIdMapping: make(map[string]string), - } - am.initialize() - return am -} - -func (am *AccountManager) GetAccountNameById(canonicalId string) string { - return am.IdNameMapping[canonicalId] -} - -func (am *AccountManager) GetAccountIdByEmail(email string) string { - return am.EmailIdMapping[email] -} - -func (am *AccountManager) initialize() { - // load predefined Accounts - for _, account := range []Account{AccountAdmin, AccountAnonymous} { - am.IdNameMapping[account.Id] = account.Name - am.EmailIdMapping[account.EmailAddress] = account.Id - } -} diff --git a/weed/s3api/s3acl/acl_helper.go b/weed/s3api/s3api_acl_helper.go similarity index 95% rename from weed/s3api/s3acl/acl_helper.go rename to weed/s3api/s3api_acl_helper.go index 1e8ee9524..e68e49b78 100644 --- a/weed/s3api/s3acl/acl_helper.go +++ b/weed/s3api/s3api_acl_helper.go @@ -1,4 +1,4 @@ -package s3acl +package s3api import ( "encoding/json" @@ -10,7 +10,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/util" "net/http" @@ -19,11 +18,16 @@ import ( var customAclHeaders = []string{s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp} +type AccountManager interface { + GetAccountNameById(canonicalId string) string + GetAccountIdByEmail(email string) string +} + // GetAccountId get AccountId from request headers, AccountAnonymousId will be return if not presen func GetAccountId(r *http.Request) string { id := r.Header.Get(s3_constants.AmzAccountId) if len(id) == 0 { - return s3account.AccountAnonymous.Id + return s3_constants.AccountAnonymousId } else { return id } @@ -40,7 +44,7 @@ func ValidateAccount(requestAccountId string, allowedAccounts ...string) bool { } // ExtractBucketAcl extracts the acl from the request body, or from the header if request body is empty -func ExtractBucketAcl(r *http.Request, accountManager *s3account.AccountManager, objectOwnership, bucketOwnerId, requestAccountId string, createBucket bool) (grants []*s3.Grant, errCode s3err.ErrorCode) { +func ExtractBucketAcl(r *http.Request, accountManager AccountManager, objectOwnership, bucketOwnerId, requestAccountId string, createBucket bool) (grants []*s3.Grant, errCode s3err.ErrorCode) { cannedAclPresent := false if r.Header.Get(s3_constants.AmzCannedAcl) != "" { cannedAclPresent = true @@ -96,7 +100,7 @@ func ExtractBucketAcl(r *http.Request, accountManager *s3account.AccountManager, } // ExtractObjectAcl extracts the acl from the request body, or from the header if request body is empty -func ExtractObjectAcl(r *http.Request, accountManager *s3account.AccountManager, objectOwnership, bucketOwnerId, requestAccountId string, createObject bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) { +func ExtractObjectAcl(r *http.Request, accountManager AccountManager, objectOwnership, bucketOwnerId, requestAccountId string, createObject bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) { cannedAclPresent := false if r.Header.Get(s3_constants.AmzCannedAcl) != "" { cannedAclPresent = true @@ -342,8 +346,8 @@ func ExtractObjectCannedAcl(request *http.Request, objectOwnership, bucketOwnerI return } -// ValidateAndTransferGrants validate grant entity exists and transfer Email-Grant to Id-Grant -func ValidateAndTransferGrants(accountManager *s3account.AccountManager, grants []*s3.Grant) ([]*s3.Grant, s3err.ErrorCode) { +// ValidateAndTransferGrants validate grant & transfer Email-Grant to Id-Grant +func ValidateAndTransferGrants(accountManager AccountManager, grants []*s3.Grant) ([]*s3.Grant, s3err.ErrorCode) { var result []*s3.Grant for _, grant := range grants { grantee := grant.Grantee @@ -369,8 +373,8 @@ func ValidateAndTransferGrants(accountManager *s3account.AccountManager, grants glog.Warning("invalid canonical grantee! account id is nil") return nil, s3err.ErrInvalidRequest } - _, ok := accountManager.IdNameMapping[*grantee.ID] - if !ok { + name := accountManager.GetAccountNameById(*grantee.ID) + if len(name) == 0 { glog.Warningf("invalid canonical grantee! account id[%s] is not exists", *grantee.ID) return nil, s3err.ErrInvalidRequest } @@ -380,8 +384,8 @@ func ValidateAndTransferGrants(accountManager *s3account.AccountManager, grants glog.Warning("invalid email grantee! email address is nil") return nil, s3err.ErrInvalidRequest } - accountId, ok := accountManager.EmailIdMapping[*grantee.EmailAddress] - if !ok { + accountId := accountManager.GetAccountIdByEmail(*grantee.EmailAddress) + if len(accountId) == 0 { glog.Warningf("invalid email grantee! email address[%s] is not exists", *grantee.EmailAddress) return nil, s3err.ErrInvalidRequest } @@ -462,7 +466,7 @@ func DetermineRequiredGrants(accountId, permission string) (grants []*s3.Grant) }) // group grantee (AuthenticateUsers) - if accountId != s3account.AccountAnonymous.Id { + if accountId != s3_constants.AccountAnonymousId { grants = append(grants, &s3.Grant{ Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, diff --git a/weed/s3api/s3acl/acl_helper_test.go b/weed/s3api/s3api_acl_helper_test.go similarity index 80% rename from weed/s3api/s3acl/acl_helper_test.go rename to weed/s3api/s3api_acl_helper_test.go index ebf253851..58f5ac5d8 100644 --- a/weed/s3api/s3acl/acl_helper_test.go +++ b/weed/s3api/s3api_acl_helper_test.go @@ -1,34 +1,38 @@ -package s3acl +package s3api import ( "bytes" "encoding/json" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "io" "net/http" "testing" ) -var ( - accountManager = &s3account.AccountManager{ - IdNameMapping: map[string]string{ - s3account.AccountAdmin.Id: s3account.AccountAdmin.Name, - s3account.AccountAnonymous.Id: s3account.AccountAnonymous.Name, - "accountA": "accountA", - "accountB": "accountB", - }, - EmailIdMapping: map[string]string{ - s3account.AccountAdmin.EmailAddress: s3account.AccountAdmin.Id, - s3account.AccountAnonymous.EmailAddress: s3account.AccountAnonymous.Id, - "accountA@example.com": "accountA", - "accountBexample.com": "accountB", +var accountManager *IdentityAccessManagement + +func init() { + accountManager = &IdentityAccessManagement{} + _ = accountManager.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{ + Accounts: []*iam_pb.Account{ + { + Id: "accountA", + DisplayName: "accountAName", + EmailAddress: "accountA@example.com", + }, + { + Id: "accountB", + DisplayName: "accountBName", + EmailAddress: "accountB@example.com", + }, }, - } -) + }) +} func TestGetAccountId(t *testing.T) { req := &http.Request{ @@ -36,26 +40,345 @@ func TestGetAccountId(t *testing.T) { } //case1 //accountId: "admin" - req.Header.Set(s3_constants.AmzAccountId, s3account.AccountAdmin.Id) - if GetAccountId(req) != s3account.AccountAdmin.Id { + req.Header.Set(s3_constants.AmzAccountId, s3_constants.AccountAdminId) + if GetAccountId(req) != s3_constants.AccountAdminId { t.Fatal("expect accountId: admin") } //case2 //accountId: "anoymous" - req.Header.Set(s3_constants.AmzAccountId, s3account.AccountAnonymous.Id) - if GetAccountId(req) != s3account.AccountAnonymous.Id { + req.Header.Set(s3_constants.AmzAccountId, s3_constants.AccountAnonymousId) + if GetAccountId(req) != s3_constants.AccountAnonymousId { t.Fatal("expect accountId: anonymous") } //case3 //accountId is nil => "anonymous" req.Header.Del(s3_constants.AmzAccountId) - if GetAccountId(req) != s3account.AccountAnonymous.Id { + if GetAccountId(req) != s3_constants.AccountAnonymousId { t.Fatal("expect accountId: anonymous") } } +func TestExtractAcl(t *testing.T) { + type Case struct { + id int + resultErrCode, expectErrCode s3err.ErrorCode + resultGrants, expectGrants []*s3.Grant + } + testCases := make([]*Case, 0) + accountAdminId := "admin" + { + //case1 (good case) + //parse acp from request body + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + objectWriter := "accountA" + grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter) + testCases = append(testCases, &Case{ + 1, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountAdminId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + + { + //case2 (good case) + //parse acp from header (cannedAcl) + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = nil + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate) + objectWriter := "accountA" + grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter) + testCases = append(testCases, &Case{ + 2, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &objectWriter, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + + { + //case3 (bad case) + //parse acp from request body (content is invalid) + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = io.NopCloser(bytes.NewReader([]byte("zdfsaf"))) + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate) + objectWriter := "accountA" + _, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter) + testCases = append(testCases, &Case{ + id: 3, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + //case4 (bad case) + //parse acp from header (cannedAcl is invalid) + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = nil + req.Header.Set(s3_constants.AmzCannedAcl, "dfaksjfk") + objectWriter := "accountA" + _, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, "", objectWriter) + testCases = append(testCases, &Case{ + id: 4, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + + { + //case5 (bad case) + //parse acp from request body: owner is inconsistent + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + objectWriter = "accountA" + _, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, objectWriter, objectWriter) + testCases = append(testCases, &Case{ + id: 5, + resultErrCode: errCode, expectErrCode: s3err.ErrAccessDenied, + }) + } + + for _, tc := range testCases { + if tc.resultErrCode != tc.expectErrCode { + t.Fatalf("case[%d]: errorCode not expect", tc.id) + } + if !grantsEquals(tc.resultGrants, tc.expectGrants) { + t.Fatalf("case[%d]: grants not expect", tc.id) + } + } +} + +func TestParseAndValidateAclHeaders(t *testing.T) { + type Case struct { + id int + resultOwner, expectOwner string + resultErrCode, expectErrCode s3err.ErrorCode + resultGrants, expectGrants []*s3.Grant + } + testCases := make([]*Case, 0) + bucketOwner := "admin" + + { + //case1 (good case) + //parse custom acl + req := &http.Request{ + Header: make(map[string][]string), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example.com"`) + ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + 1, + ownerId, objectWriter, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: aws.String(s3_constants.AccountAnonymousId), + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: aws.String(s3_constants.AccountAdminId), + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //case2 (good case) + //parse canned acl (ownership=ObjectWriter) + req := &http.Request{ + Header: make(map[string][]string), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + 2, + ownerId, objectWriter, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &objectWriter, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwner, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //case3 (good case) + //parse canned acl (ownership=OwnershipBucketOwnerPreferred) + req := &http.Request{ + Header: make(map[string][]string), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + 3, + ownerId, bucketOwner, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwner, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //case4 (bad case) + //parse custom acl (grantee id not exists) + req := &http.Request{ + Header: make(map[string][]string), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="notExistsAccount", emailAddress="admin@example.com"`) + _, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + id: 4, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + { + //case5 (bad case) + //parse custom acl (invalid format) + req := &http.Request{ + Header: make(map[string][]string), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzAclFullControl, `uri="http:sfasf"`) + _, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + id: 5, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + { + //case6 (bad case) + //parse canned acl (invalid value) + req := &http.Request{ + Header: make(map[string][]string), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzCannedAcl, `uri="http:sfasf"`) + _, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + id: 5, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + for _, tc := range testCases { + if tc.expectErrCode != tc.resultErrCode { + t.Errorf("case[%d]: errCode unexpect", tc.id) + } + if tc.resultOwner != tc.expectOwner { + t.Errorf("case[%d]: ownerId unexpect", tc.id) + } + if !grantsEquals(tc.resultGrants, tc.expectGrants) { + t.Fatalf("case[%d]: grants not expect", tc.id) + } + } +} + func grantsEquals(a, b []*s3.Grant) bool { if len(a) != len(b) { return false @@ -71,7 +394,7 @@ func grantsEquals(a, b []*s3.Grant) bool { func TestDetermineReqGrants(t *testing.T) { { //case1: request account is anonymous - accountId := s3account.AccountAnonymous.Id + accountId := s3_constants.AccountAnonymousId reqPermission := s3_constants.PermissionRead resultGrants := DetermineRequiredGrants(accountId, reqPermission) @@ -176,7 +499,7 @@ func TestAssembleEntryWithAcp(t *testing.T) { Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), URI: &s3_constants.GranteeGroupAllUsers, }, }, @@ -249,13 +572,13 @@ func TestGrantEquals(t *testing.T) { GrantEquals(&s3.Grant{ Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ - ID: &s3account.AccountAdmin.Id, - EmailAddress: &s3account.AccountAdmin.EmailAddress, + ID: aws.String(s3_constants.AccountAdminId), + //EmailAddress: &s3account.AccountAdmin.EmailAddress, }, }, &s3.Grant{ Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), }, }): true, @@ -303,13 +626,13 @@ func TestGrantEquals(t *testing.T) { Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), }, }, &s3.Grant{ Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), }, }): true, @@ -317,14 +640,14 @@ func TestGrantEquals(t *testing.T) { Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), URI: &s3_constants.GranteeGroupAllUsers, }, }, &s3.Grant{ Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), }, }): false, @@ -332,7 +655,7 @@ func TestGrantEquals(t *testing.T) { Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), URI: &s3_constants.GranteeGroupAllUsers, }, }, &s3.Grant{ @@ -372,7 +695,7 @@ func TestSetAcpGrantsHeader(t *testing.T) { Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), URI: &s3_constants.GranteeGroupAllUsers, }, }, diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index bed718055..6fe2b624a 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -5,8 +5,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/util" "net/http" @@ -16,7 +14,7 @@ import ( func getAccountId(r *http.Request) string { id := r.Header.Get(s3_constants.AmzAccountId) if len(id) == 0 { - return s3account.AccountAnonymous.Id + return AccountAnonymous.Id } else { return id } diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 1fd212d7f..fd049fd11 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -6,8 +6,6 @@ import ( "errors" "fmt" "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket" "github.com/seaweedfs/seaweedfs/weed/util" "math" diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 40783ea73..fb99afd30 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "net" "net/http" "strings" @@ -41,7 +40,6 @@ type S3ApiServer struct { randomClientId int32 filerGuard *security.Guard client *http.Client - accountManager *s3account.AccountManager bucketRegistry *BucketRegistry } @@ -62,7 +60,6 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer filerGuard: security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec), cb: NewCircuitBreaker(option), } - s3ApiServer.accountManager = s3account.NewAccountManager(s3ApiServer) s3ApiServer.bucketRegistry = NewBucketRegistry(s3ApiServer) if option.LocalFilerSocket == "" { s3ApiServer.client = &http.Client{Transport: &http.Transport{