Browse Source

acl merge

pull/5936/head
LHHDZ 5 months ago
parent
commit
d7ddda04d7
  1. 9
      weed/filer/filer_conf.go
  2. 2
      weed/pb/iam.proto
  3. 68
      weed/pb/iam_pb/iam.pb.go
  4. 54
      weed/s3api/auth_credentials.go
  5. 4
      weed/s3api/auth_credentials_subscribe.go
  6. 9
      weed/s3api/auth_credentials_test.go
  7. 10
      weed/s3api/auth_signature_v4.go
  8. 6
      weed/s3api/auth_signature_v4_test.go
  9. 73
      weed/s3api/bucket_metadata.go
  10. 52
      weed/s3api/bucket_metadata_test.go
  11. 33
      weed/s3api/chunked_reader_v4.go
  12. 29
      weed/s3api/filer_multipart.go
  13. 4
      weed/s3api/s3_constants/acp_ownership.go
  14. 7
      weed/s3api/s3_constants/extend_key.go
  15. 3
      weed/s3api/s3_constants/header.go
  16. 416
      weed/s3api/s3acl/s3api_acl_helper.go
  17. 1477
      weed/s3api/s3acl/s3api_acl_helper_test.go
  18. 709
      weed/s3api/s3api_acl_helper_test.go
  19. 400
      weed/s3api/s3api_acp.go
  20. 3
      weed/s3api/s3api_auth.go
  21. 160
      weed/s3api/s3api_bucket_handlers.go
  22. 2
      weed/s3api/s3api_circuit_breaker.go
  23. 81
      weed/s3api/s3api_object_handlers.go
  24. 5
      weed/s3api/s3api_object_handlers_copy.go
  25. 14
      weed/s3api/s3api_object_handlers_list.go
  26. 72
      weed/s3api/s3api_object_handlers_multipart.go
  27. 36
      weed/s3api/s3api_object_handlers_put.go
  28. 16
      weed/s3api/s3api_object_handlers_skip.go
  29. 42
      weed/s3api/s3api_server.go
  30. 25
      weed/s3api/s3err/s3api_errors.go
  31. 32
      weed/server/filer_server_handlers_read.go
  32. 6
      weed/server/filer_server_handlers_write_autochunk.go
  33. 2
      weed/shell/command_collection_delete.go
  34. 2
      weed/shell/command_fs_mv.go
  35. 2
      weed/shell/command_fs_rm.go
  36. 2
      weed/shell/command_s3_bucket_delete.go
  37. 2
      weed/shell/command_volume_balance_test.go
  38. 2
      weed/shell/command_volume_delete.go
  39. 17
      weed/util/http/http_global_client_util.go

9
weed/filer/filer_conf.go

@ -92,8 +92,11 @@ func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*file
func (fc *FilerConf) LoadFromBytes(data []byte) (err error) {
conf := &filer_pb.FilerConf{}
if err := jsonpb.Unmarshal(data, conf); err != nil {
options := &jsonpb.UnmarshalOptions{
DiscardUnknown: true,
AllowPartial: true,
}
if err := options.Unmarshal(data, conf); err != nil {
return err
}
@ -111,7 +114,7 @@ func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) (err error) {
return nil
}
func (fc *FilerConf) GetLocationConf(locationPrefix string)(locConf *filer_pb.FilerConf_PathConf, found bool) {
func (fc *FilerConf) GetLocationConf(locationPrefix string) (locConf *filer_pb.FilerConf_PathConf, found bool) {
return fc.rules.Get([]byte(locationPrefix))
}

2
weed/pb/iam.proto

@ -23,7 +23,7 @@ message Identity {
string name = 1;
repeated Credential credentials = 2;
repeated string actions = 3;
Account account = 4;
string AccountId = 4;
}
message Credential {

68
weed/pb/iam_pb/iam.pb.go

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.32.0
// protoc v4.25.3
// protoc-gen-go v1.28.0
// protoc v3.21.1
// source: iam.proto
package iam_pb
@ -83,7 +83,7 @@ type Identity struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
Credentials []*Credential `protobuf:"bytes,2,rep,name=credentials,proto3" json:"credentials,omitempty"`
Actions []string `protobuf:"bytes,3,rep,name=actions,proto3" json:"actions,omitempty"`
Account *Account `protobuf:"bytes,4,opt,name=account,proto3" json:"account,omitempty"`
AccountId string `protobuf:"bytes,4,opt,name=AccountId,proto3" json:"AccountId,omitempty"`
}
func (x *Identity) Reset() {
@ -139,11 +139,11 @@ func (x *Identity) GetActions() []string {
return nil
}
func (x *Identity) GetAccount() *Account {
func (x *Identity) GetAccountId() string {
if x != nil {
return x.Account
return x.AccountId
}
return nil
return ""
}
type Credential struct {
@ -275,35 +275,34 @@ var file_iam_proto_rawDesc = []byte{
0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x2b, 0x0a, 0x08, 0x61,
0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e,
0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x08,
0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x22, 0x99, 0x01, 0x0a, 0x08, 0x49, 0x64, 0x65,
0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x22, 0x8c, 0x01, 0x0a, 0x08, 0x49, 0x64, 0x65,
0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b, 0x63, 0x72, 0x65,
0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12,
0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x12,
0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09,
0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x29, 0x0a, 0x07, 0x61, 0x63, 0x63,
0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x69, 0x61, 0x6d,
0x5f, 0x70, 0x62, 0x2e, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x07, 0x61, 0x63, 0x63,
0x6f, 0x75, 0x6e, 0x74, 0x22, 0x4a, 0x0a, 0x0a, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69,
0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65,
0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79,
0x22, 0x61, 0x0a, 0x07, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69,
0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x64,
0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
0x09, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23,
0x0a, 0x0d, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18,
0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x64, 0x64, 0x72,
0x65, 0x73, 0x73, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x49, 0x64,
0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4d, 0x61, 0x6e, 0x61,
0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65,
0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08, 0x49, 0x61, 0x6d, 0x50,
0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65,
0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x69, 0x61, 0x6d,
0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x41, 0x63, 0x63,
0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x41, 0x63,
0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x4a, 0x0a, 0x0a, 0x43, 0x72, 0x65, 0x64, 0x65,
0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f,
0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x65, 0x73,
0x73, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b,
0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74,
0x4b, 0x65, 0x79, 0x22, 0x61, 0x0a, 0x07, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x0e,
0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x21,
0x0a, 0x0c, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02,
0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d,
0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65,
0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x41,
0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65,
0x64, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4d,
0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65, 0x61,
0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08, 0x49,
0x61, 0x6d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65,
0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f,
0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@ -329,12 +328,11 @@ var file_iam_proto_depIdxs = []int32{
1, // 0: iam_pb.S3ApiConfiguration.identities:type_name -> iam_pb.Identity
3, // 1: iam_pb.S3ApiConfiguration.accounts:type_name -> iam_pb.Account
2, // 2: iam_pb.Identity.credentials:type_name -> iam_pb.Credential
3, // 3: iam_pb.Identity.account:type_name -> iam_pb.Account
4, // [4:4] is the sub-list for method output_type
4, // [4:4] is the sub-list for method input_type
4, // [4:4] is the sub-list for extension type_name
4, // [4:4] is the sub-list for extension extendee
0, // [0:4] is the sub-list for field type_name
3, // [3:3] is the sub-list for method output_type
3, // [3:3] is the sub-list for method input_type
3, // [3:3] is the sub-list for extension type_name
3, // [3:3] is the sub-list for extension extendee
0, // [0:3] is the sub-list for field type_name
}
func init() { file_iam_proto_init() }

54
weed/s3api/auth_credentials.go

@ -163,13 +163,13 @@ func (iam *IdentityAccessManagement) LoadS3ApiConfigurationFromBytes(content []b
return err
}
if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil {
if err := iam.LoadS3ApiConfiguration(s3ApiConfiguration); err != nil {
return err
}
return nil
}
func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error {
func (iam *IdentityAccessManagement) LoadS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error {
var identities []*Identity
var identityAnonymous *Identity
accessKeyIdent := make(map[string]*Identity)
@ -226,10 +226,12 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
case ident.Name == AccountAnonymous.Id:
t.Account = &AccountAnonymous
identityAnonymous = t
case ident.Account == nil:
case ident.AccountId == AccountAdmin.Id:
t.Account = &AccountAdmin
case ident.AccountId == "":
t.Account = &AccountAdmin
default:
if account, ok := accounts[ident.Account.Id]; ok {
if account, ok := accounts[ident.AccountId]; ok {
t.Account = account
} else {
t.Account = &AccountAdmin
@ -310,14 +312,30 @@ func (iam *IdentityAccessManagement) GetAccountIdByEmail(email string) string {
}
func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) http.HandlerFunc {
return Auth(iam, nil, f, action, false)
}
func (s3a *S3ApiServer) AuthWithAcl(f http.HandlerFunc, action Action) http.HandlerFunc {
return Auth(s3a.iam, s3a.bucketRegistry, f, action, true)
}
func (s3a *S3ApiServer) Auth(f http.HandlerFunc, action Action, supportAcl bool) http.HandlerFunc {
return Auth(s3a.iam, s3a.bucketRegistry, f, action, supportAcl)
}
func Auth(iam *IdentityAccessManagement, br *BucketRegistry, f http.HandlerFunc, action Action, supportAcl bool) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
//unset predefined headers
delete(r.Header, s3_constants.AmzAccountId)
delete(r.Header, s3_constants.ExtAmzOwnerKey)
delete(r.Header, s3_constants.ExtAmzAclKey)
if !iam.isEnabled() {
f(w, r)
return
}
identity, errCode := iam.authRequest(r, action)
glog.V(3).Infof("auth error: %v", errCode)
identity, errCode := authRequest(iam, br, r, action, supportAcl)
if errCode == s3err.ErrNone {
if identity != nil && identity.Name != "" {
r.Header.Set(s3_constants.AmzIdentityId, identity.Name)
@ -336,9 +354,12 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt
// check whether the request has valid access keys
func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) {
return authRequest(iam, nil, r, action, false)
}
func authRequest(iam *IdentityAccessManagement, br *BucketRegistry, r *http.Request, action Action, supportAcl bool) (*Identity, s3err.ErrorCode) {
var identity *Identity
var s3Err s3err.ErrorCode
var found bool
var authType string
switch getRequestAuthType(r) {
case authTypeStreamingSigned:
@ -364,14 +385,25 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
r.Header.Set(s3_constants.AmzAuthType, "Jwt")
return identity, s3err.ErrNotImplemented
case authTypeAnonymous:
authType = "Anonymous"
if identity, found = iam.lookupAnonymous(); !found {
r.Header.Set(s3_constants.AmzAuthType, authType)
return identity, s3err.ErrAccessDenied
if supportAcl && br != nil {
bucket, _ := s3_constants.GetBucketAndObject(r)
bucketMetadata, errorCode := br.GetBucketMetadata(bucket)
if errorCode != s3err.ErrNone {
return nil, errorCode
}
if bucketMetadata.ObjectOwnership != s3_constants.OwnershipBucketOwnerEnforced {
return iam.identityAnonymous, s3err.ErrNone
}
}
authType = "Anonymous"
identity = iam.identityAnonymous
default:
return identity, s3err.ErrNotImplemented
}
if identity == nil || len(identity.Actions) == 0 {
r.Header.Set(s3_constants.AmzAuthType, authType)
return identity, s3err.ErrAccessDenied
}
if len(authType) > 0 {
r.Header.Set(s3_constants.AmzAuthType, authType)

4
weed/s3api/auth_credentials_subscribe.go

@ -81,10 +81,10 @@ func (s3a *S3ApiServer) onBucketMetadataChange(dir string, oldEntry *filer_pb.En
if dir == s3a.option.BucketsPath {
if newEntry != nil {
s3a.bucketRegistry.LoadBucketMetadata(newEntry)
glog.V(0).Infof("updated bucketMetadata %s/%s", dir, newEntry)
glog.V(1).Infof("updated bucketMetadata %s/%s", dir, newEntry)
} else {
s3a.bucketRegistry.RemoveBucketMetadata(oldEntry)
glog.V(0).Infof("remove bucketMetadata %s/%s", dir, newEntry)
glog.V(1).Infof("remove bucketMetadata %s/%s", dir, newEntry)
}
}
return nil

9
weed/s3api/auth_credentials_test.go

@ -1,11 +1,10 @@
package s3api
import (
"reflect"
"testing"
. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/stretchr/testify/assert"
"reflect"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
jsonpb "google.golang.org/protobuf/encoding/protojson"
@ -205,7 +204,7 @@ func TestLoadS3ApiConfiguration(t *testing.T) {
pbAccount: &pbSpecifiedAccount,
pbIdent: &iam_pb.Identity{
Name: "specifiedAccountID",
Account: &pbSpecifiedAccount,
AccountId: pbSpecifiedAccount.Id,
Actions: []string{
"Read",
"Write",
@ -250,7 +249,7 @@ func TestLoadS3ApiConfiguration(t *testing.T) {
}
iam := IdentityAccessManagement{}
err := iam.loadS3ApiConfiguration(config)
err := iam.LoadS3ApiConfiguration(config)
if err != nil {
return
}

10
weed/s3api/auth_signature_v4.go

@ -58,7 +58,8 @@ const (
// http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD" indicates that the
// client did not calculate sha256 of the payload.
unsignedPayload = "UNSIGNED-PAYLOAD"
unsignedPayload = "UNSIGNED-PAYLOAD"
expect100Continue = "100-contine"
)
// Returns SHA256 for calculating canonical-request.
@ -661,7 +662,12 @@ func extractSignedHeaders(signedHeaders []string, r *http.Request) (http.Header,
// be sent, for the time being keep this work around.
// Adding a *TODO* to remove this later when Golang server
// doesn't filter out the 'Expect' header.
extractedSignedHeaders.Set(header, "100-continue")
expectHeaderValue := extractedSignedHeaders.Get(header)
// here in order to be compatible with the aws go sdk v1 version, it sets the expect header to '100-Continue'
if !strings.EqualFold(expectHeaderValue, expect100Continue) {
extractedSignedHeaders.Set(header, expect100Continue)
}
case "host":
// Go http server removes "host" from Request.Header
extractedSignedHeaders.Set(header, r.Host)

6
weed/s3api/auto_signature_v4_test.go → weed/s3api/auth_signature_v4_test.go

@ -64,7 +64,7 @@ func TestIsReqAuthenticated(t *testing.T) {
hashes: make(map[string]*sync.Pool),
hashCounters: make(map[string]*int32),
}
_ = iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
_ = iam.LoadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
Identities: []*iam_pb.Identity{
{
Name: "someone",
@ -104,7 +104,7 @@ func TestCheckaAnonymousRequestAuthType(t *testing.T) {
hashes: make(map[string]*sync.Pool),
hashCounters: make(map[string]*int32),
}
_ = iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
_ = iam.LoadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
Identities: []*iam_pb.Identity{
{
Name: "anonymous",
@ -137,7 +137,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
hashes: make(map[string]*sync.Pool),
hashCounters: make(map[string]*int32),
}
_ = iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
_ = iam.LoadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
Identities: []*iam_pb.Identity{
{
Name: "someone",

73
weed/s3api/bucket_metadata.go

@ -6,6 +6,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3acl"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/util"
"math"
@ -78,7 +79,7 @@ func (r *BucketRegistry) LoadBucketMetadata(entry *filer_pb.Entry) {
r.metadataCache[entry.Name] = bucketMetadata
}
func buildBucketMetadata(accountManager AccountManager, entry *filer_pb.Entry) *BucketMetaData {
func buildBucketMetadata(accountManager s3acl.AccountManager, entry *filer_pb.Entry) *BucketMetaData {
entryJson, _ := json.Marshal(entry)
glog.V(3).Infof("build bucket metadata,entry=%s", entryJson)
bucketMetadata := &BucketMetaData{
@ -106,7 +107,6 @@ func buildBucketMetadata(accountManager AccountManager, entry *filer_pb.Entry) *
}
}
//access control policy
//owner
acpOwnerBytes, ok := entry.Extended[s3_constants.ExtAmzOwnerKey]
if ok && len(acpOwnerBytes) > 0 {
@ -121,17 +121,31 @@ func buildBucketMetadata(accountManager AccountManager, entry *filer_pb.Entry) *
}
}
}
//grants
acpGrantsBytes, ok := entry.Extended[s3_constants.ExtAmzAclKey]
if ok && len(acpGrantsBytes) > 0 {
var grants []*s3.Grant
err := json.Unmarshal(acpGrantsBytes, &grants)
if err == nil {
bucketMetadata.Acl = grants
} else {
glog.Warningf("Unmarshal ACP grants: %s(%v), bucket: %s", string(acpGrantsBytes), err, bucketMetadata.Name)
if ok {
if len(acpGrantsBytes) > 0 {
var grants []*s3.Grant
err := json.Unmarshal(acpGrantsBytes, &grants)
if err == nil {
bucketMetadata.Acl = grants
} else {
glog.Warningf("Unmarshal ACP grants: %s(%v), bucket: %s", string(acpGrantsBytes), err, bucketMetadata.Name)
}
}
} else {
bucketMetadata.Acl = []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: bucketMetadata.Owner.ID,
},
Permission: &s3_constants.PermissionFullControl,
},
}
}
}
return bucketMetadata
}
@ -142,17 +156,12 @@ func (r *BucketRegistry) RemoveBucketMetadata(entry *filer_pb.Entry) {
}
func (r *BucketRegistry) GetBucketMetadata(bucketName string) (*BucketMetaData, s3err.ErrorCode) {
r.metadataCacheLock.RLock()
bucketMetadata, ok := r.metadataCache[bucketName]
r.metadataCacheLock.RUnlock()
if ok {
bucketMetadata := r.getMetadataCache(bucketName)
if bucketMetadata != nil {
return bucketMetadata, s3err.ErrNone
}
r.notFoundLock.RLock()
_, ok = r.notFound[bucketName]
r.notFoundLock.RUnlock()
if ok {
if r.isNotFound(bucketName) {
return nil, s3err.ErrNoSuchBucket
}
@ -171,10 +180,8 @@ func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*Bucket
defer r.notFoundLock.Unlock()
//check if already exists
r.metadataCacheLock.RLock()
bucketMetaData, ok := r.metadataCache[bucketName]
r.metadataCacheLock.RUnlock()
if ok {
bucketMetaData := r.getMetadataCache(bucketName)
if bucketMetaData != nil {
return bucketMetaData, s3err.ErrNone
}
@ -183,6 +190,7 @@ func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*Bucket
if err != nil {
if err == filer_pb.ErrNotFound {
// The bucket doesn't actually exist and should no longer loaded from the filer
glog.V(4).Info("bucket not found in filer: ", bucketName)
r.notFound[bucketName] = struct{}{}
return nil, s3err.ErrNoSuchBucket
}
@ -191,6 +199,15 @@ func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*Bucket
return bucketMetadata, s3err.ErrNone
}
func (r *BucketRegistry) getMetadataCache(bucket string) *BucketMetaData {
r.metadataCacheLock.RLock()
defer r.metadataCacheLock.RUnlock()
if cache, ok := r.metadataCache[bucket]; ok {
return cache
}
return nil
}
func (r *BucketRegistry) setMetadataCache(metadata *BucketMetaData) {
r.metadataCacheLock.Lock()
defer r.metadataCacheLock.Unlock()
@ -203,10 +220,11 @@ func (r *BucketRegistry) removeMetadataCache(bucket string) {
delete(r.metadataCache, bucket)
}
func (r *BucketRegistry) markNotFound(bucket string) {
r.notFoundLock.Lock()
defer r.notFoundLock.Unlock()
r.notFound[bucket] = struct{}{}
func (r *BucketRegistry) isNotFound(bucket string) bool {
r.notFoundLock.RLock()
defer r.notFoundLock.RUnlock()
_, ok := r.notFound[bucket]
return ok
}
func (r *BucketRegistry) unMarkNotFound(bucket string) {
@ -214,3 +232,8 @@ func (r *BucketRegistry) unMarkNotFound(bucket string) {
defer r.notFoundLock.Unlock()
delete(r.notFound, bucket)
}
func (r *BucketRegistry) ClearCache(bucket string) {
r.removeMetadataCache(bucket)
r.unMarkNotFound(bucket)
}

52
weed/s3api/bucket_metadata_test.go

@ -86,7 +86,7 @@ var tcs = []*BucketMetadataTestCase{
{
badEntry, &BucketMetaData{
Name: badEntry.Name,
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
ObjectOwnership: s3_constants.DefaultObjectOwnership,
Owner: &s3.Owner{
DisplayName: &AccountAdmin.DisplayName,
ID: &AccountAdmin.Id,
@ -108,12 +108,20 @@ var tcs = []*BucketMetadataTestCase{
{
ownershipEmptyStr, &BucketMetaData{
Name: ownershipEmptyStr.Name,
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
ObjectOwnership: s3_constants.DefaultObjectOwnership,
Owner: &s3.Owner{
DisplayName: &AccountAdmin.DisplayName,
ID: &AccountAdmin.Id,
},
Acl: nil,
Acl: []*s3.Grant{
{
Permission: &s3_constants.PermissionFullControl,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &AccountAdmin.Id,
},
},
},
},
},
{
@ -124,35 +132,59 @@ var tcs = []*BucketMetadataTestCase{
DisplayName: &AccountAdmin.DisplayName,
ID: &AccountAdmin.Id,
},
Acl: nil,
Acl: []*s3.Grant{
{
Permission: &s3_constants.PermissionFullControl,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &AccountAdmin.Id,
},
},
},
},
},
{
acpEmptyStr, &BucketMetaData{
Name: acpEmptyStr.Name,
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
ObjectOwnership: s3_constants.DefaultObjectOwnership,
Owner: &s3.Owner{
DisplayName: &AccountAdmin.DisplayName,
ID: &AccountAdmin.Id,
},
Acl: nil,
Acl: []*s3.Grant{
{
Permission: &s3_constants.PermissionFullControl,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &AccountAdmin.Id,
},
},
},
},
},
{
acpEmptyObject, &BucketMetaData{
Name: acpEmptyObject.Name,
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
ObjectOwnership: s3_constants.DefaultObjectOwnership,
Owner: &s3.Owner{
DisplayName: &AccountAdmin.DisplayName,
ID: &AccountAdmin.Id,
},
Acl: nil,
Acl: []*s3.Grant{
{
Permission: &s3_constants.PermissionFullControl,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &AccountAdmin.Id,
},
},
},
},
},
{
acpOwnerNil, &BucketMetaData{
Name: acpOwnerNil.Name,
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
ObjectOwnership: s3_constants.DefaultObjectOwnership,
Owner: &s3.Owner{
DisplayName: &AccountAdmin.DisplayName,
ID: &AccountAdmin.Id,
@ -164,7 +196,7 @@ var tcs = []*BucketMetadataTestCase{
func TestBuildBucketMetadata(t *testing.T) {
iam := &IdentityAccessManagement{}
_ = iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{})
_ = iam.LoadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{})
for _, tc := range tcs {
resultBucketMetadata := buildBucketMetadata(iam, tc.filerEntry)
if !reflect.DeepEqual(resultBucketMetadata, tc.expectBucketMetadata) {

33
weed/s3api/chunked_reader_v4.go

@ -24,14 +24,13 @@ import (
"crypto/sha256"
"encoding/hex"
"errors"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"hash"
"io"
"net/http"
"time"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/dustin/go-humanize"
)
@ -40,7 +39,7 @@ import (
//
// returns signature, error otherwise if the signature mismatches or any other
// error while parsing and validating.
func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cred *Credential, signature string, region string, date time.Time, errCode s3err.ErrorCode) {
func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (idnt *Identity, cred *Credential, signature string, region string, date time.Time, errCode s3err.ErrorCode) {
// Copy request.
req := *r
@ -51,7 +50,7 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
// Parse signature version '4' header.
signV4Values, errCode := parseSignV4(v4Auth)
if errCode != s3err.ErrNone {
return nil, "", "", time.Time{}, errCode
return nil, nil, "", "", time.Time{}, errCode
}
// Payload streaming.
@ -59,18 +58,18 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
if payload != req.Header.Get("X-Amz-Content-Sha256") {
return nil, "", "", time.Time{}, s3err.ErrContentSHA256Mismatch
return nil, nil, "", "", time.Time{}, s3err.ErrContentSHA256Mismatch
}
// Extract all the signed headers along with its values.
extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r)
if errCode != s3err.ErrNone {
return nil, "", "", time.Time{}, errCode
return nil, nil, "", "", time.Time{}, errCode
}
// Verify if the access key id matches.
identity, cred, found := iam.lookupByAccessKey(signV4Values.Credential.accessKey)
if !found {
return nil, "", "", time.Time{}, s3err.ErrInvalidAccessKeyID
return nil, nil, "", "", time.Time{}, s3err.ErrInvalidAccessKeyID
}
bucket, object := s3_constants.GetBucketAndObject(r)
@ -86,14 +85,14 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
var dateStr string
if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" {
if dateStr = r.Header.Get("Date"); dateStr == "" {
return nil, "", "", time.Time{}, s3err.ErrMissingDateHeader
return nil, nil, "", "", time.Time{}, s3err.ErrMissingDateHeader
}
}
// Parse date header.
var err error
date, err = time.Parse(iso8601Format, dateStr)
if err != nil {
return nil, "", "", time.Time{}, s3err.ErrMalformedDate
return nil, nil, "", "", time.Time{}, s3err.ErrMalformedDate
}
// Query string.
@ -116,11 +115,11 @@ func (iam *IdentityAccessManagement) calculateSeedSignature(r *http.Request) (cr
// Verify if signature match.
if !compareSignatureV4(newSignature, signV4Values.Signature) {
return nil, "", "", time.Time{}, s3err.ErrSignatureDoesNotMatch
return nil, nil, "", "", time.Time{}, s3err.ErrSignatureDoesNotMatch
}
// Return calculated signature.
return cred, newSignature, region, date, s3err.ErrNone
return identity, cred, newSignature, region, date, s3err.ErrNone
}
const maxLineLength = 4 * humanize.KiByte // assumed <= bufio.defaultBufSize 4KiB
@ -134,13 +133,13 @@ var errMalformedEncoding = errors.New("malformed chunked encoding")
// newSignV4ChunkedReader returns a new s3ChunkedReader that translates the data read from r
// out of HTTP "chunked" format before returning it.
// The s3ChunkedReader returns io.EOF when the final 0-length chunk is read.
func (iam *IdentityAccessManagement) newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, s3err.ErrorCode) {
ident, seedSignature, region, seedDate, errCode := iam.calculateSeedSignature(req)
func (iam *IdentityAccessManagement) newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, *Identity, s3err.ErrorCode) {
ident, cred, seedSignature, region, seedDate, errCode := iam.calculateSeedSignature(req)
if errCode != s3err.ErrNone {
return nil, errCode
return nil, nil, errCode
}
return &s3ChunkedReader{
cred: ident,
cred: cred,
reader: bufio.NewReader(req.Body),
seedSignature: seedSignature,
seedDate: seedDate,
@ -148,7 +147,7 @@ func (iam *IdentityAccessManagement) newSignV4ChunkedReader(req *http.Request) (
chunkSHA256Writer: sha256.New(),
state: readChunkHeader,
iam: iam,
}, s3err.ErrNone
}, ident, s3err.ErrNone
}
// Represents the overall state that is required for decoding a

29
weed/s3api/filer_multipart.go

@ -5,7 +5,9 @@ import (
"encoding/hex"
"encoding/xml"
"fmt"
"github.com/google/uuid"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/stats"
"golang.org/x/exp/slices"
"math"
@ -17,9 +19,6 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/google/uuid"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@ -35,7 +34,7 @@ type InitiateMultipartUploadResult struct {
s3.CreateMultipartUploadOutput
}
func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code s3err.ErrorCode) {
func (s3a *S3ApiServer) createMultipartUpload(initiatorId string, input *s3.CreateMultipartUploadInput) (output *InitiateMultipartUploadResult, code s3err.ErrorCode) {
glog.V(2).Infof("createMultipartUpload input %v", input)
@ -54,6 +53,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
if input.ContentType != nil {
entry.Attributes.Mime = *input.ContentType
}
entry.Extended[s3_constants.ExtAmzMultipartInitiator] = []byte(initiatorId)
}); err != nil {
glog.Errorf("NewMultipartUpload error: %v", err)
return nil, s3err.ErrInternalError
@ -93,6 +93,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
sort.Ints(completedPartNumbers)
uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId
entries, _, err := s3a.list(uploadDirectory, "", "", false, maxPartsList)
if err != nil {
glog.Errorf("completeMultipartUpload %s %s error: %v, entries:%d", *input.Bucket, *input.UploadId, err, len(entries))
@ -330,7 +331,7 @@ type ListMultipartUploadsResult struct {
Upload []*s3.MultipartUpload `locationName:"Upload" type:"list" flattened:"true"`
}
func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code s3err.ErrorCode) {
func (s3a *S3ApiServer) listMultipartUploads(bucketMetaData *BucketMetaData, input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code s3err.ErrorCode) {
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html
glog.V(2).Infof("listMultipartUploads input %v", input)
@ -361,9 +362,27 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput
if *input.Prefix != "" && !strings.HasPrefix(key, *input.Prefix) {
continue
}
initiatorId := string(entry.Extended[s3_constants.ExtAmzMultipartInitiator])
if initiatorId == "" {
initiatorId = *bucketMetaData.Owner.ID
}
initiatorDisplayName := s3a.iam.GetAccountNameById(initiatorId)
ownerId := string(entry.Extended[s3_constants.ExtAmzOwnerKey])
if ownerId == "" {
ownerId = *bucketMetaData.Owner.ID
}
ownerDisplayName := s3a.iam.GetAccountNameById(ownerId)
output.Upload = append(output.Upload, &s3.MultipartUpload{
Key: objectKey(aws.String(key)),
UploadId: aws.String(entry.Name),
Owner: &s3.Owner{
ID: &initiatorId,
DisplayName: &ownerDisplayName,
},
Initiator: &s3.Initiator{
ID: &initiatorId,
DisplayName: &initiatorDisplayName,
},
})
uploadsCount += 1
}

4
weed/s3api/s3_constants/acp_ownership.go

@ -4,9 +4,7 @@ var (
OwnershipBucketOwnerPreferred = "BucketOwnerPreferred"
OwnershipObjectWriter = "ObjectWriter"
OwnershipBucketOwnerEnforced = "BucketOwnerEnforced"
DefaultOwnershipForCreate = OwnershipObjectWriter
DefaultOwnershipForExists = OwnershipBucketOwnerEnforced
DefaultObjectOwnership = OwnershipBucketOwnerEnforced
)
func ValidateOwnership(ownership string) bool {

7
weed/s3api/s3_constants/extend_key.go

@ -1,7 +1,8 @@
package s3_constants
const (
ExtAmzOwnerKey = "Seaweed-X-Amz-Owner"
ExtAmzAclKey = "Seaweed-X-Amz-Acl"
ExtOwnershipKey = "Seaweed-X-Amz-Ownership"
ExtAmzOwnerKey = "Seaweed-X-Amz-Owner"
ExtAmzMultipartInitiator = "Seaweed-X-Amz-Multipart-Initiator"
ExtAmzAclKey = "Seaweed-X-Amz-Acl"
ExtOwnershipKey = "Seaweed-X-Amz-Ownership"
)

3
weed/s3api/s3_constants/header.go

@ -42,6 +42,9 @@ const (
SeaweedFSPartNumber = "X-Seaweedfs-Part-Number"
SeaweedFSUploadId = "X-Seaweedfs-Upload-Id"
XAmzBucketOwnerId = "x-seaweedfs-amz-bucket-owner-id"
XAmzBucketAccessDenied = "x-seaweedfs-amz-bucket-access-denied"
// S3 ACL headers
AmzCannedAcl = "X-Amz-Acl"
AmzAclFullControl = "X-Amz-Grant-Full-Control"

416
weed/s3api/s3api_acl_helper.go → weed/s3api/s3acl/s3api_acl_helper.go

@ -1,19 +1,23 @@
package s3api
package s3acl
import (
"encoding/json"
"encoding/xml"
"fmt"
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
"net/http"
"strings"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
var customAclHeaders = []string{s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp}
type AccountManager interface {
GetAccountNameById(canonicalId string) string
GetAccountIdByEmail(email string) string
@ -29,11 +33,36 @@ func GetAccountId(r *http.Request) string {
}
}
// ExtractAcl extracts the acl from the request body, or from the header if request body is empty
func ExtractAcl(r *http.Request, accountManager AccountManager, ownership, bucketOwnerId, ownerId, accountId string) (grants []*s3.Grant, errCode s3err.ErrorCode) {
if r.Body != nil && r.Body != http.NoBody {
defer util_http.CloseRequest(r)
// ValidateAccount validate weather request account id is allowed to access
func ValidateAccount(requestAccountId string, allowedAccounts ...string) bool {
for _, allowedAccount := range allowedAccounts {
if requestAccountId == allowedAccount {
return true
}
}
return false
}
// ExtractBucketAcl extracts the acl from the request body, or from the header if request body is empty
func ExtractBucketAcl(r *http.Request, accountManager AccountManager, objectOwnership, bucketOwnerId, requestAccountId string, createBucket bool) (grants []*s3.Grant, errCode s3err.ErrorCode) {
cannedAclPresent := false
if r.Header.Get(s3_constants.AmzCannedAcl) != "" {
cannedAclPresent = true
}
customAclPresent := false
for _, customAclHeader := range customAclHeaders {
if r.Header.Get(customAclHeader) != "" {
customAclPresent = true
break
}
}
// AccessControlList body is not support when create object/bucket
if !createBucket && r.Body != nil && r.Body != http.NoBody {
defer util_http.CloseRequest(r)
if cannedAclPresent || customAclPresent {
return nil, s3err.ErrUnexpectedContent
}
var acp s3.AccessControlPolicy
err := xmlutil.UnmarshalXML(&acp, xml.NewDecoder(r.Body), "")
if err != nil || acp.Owner == nil || acp.Owner.ID == nil {
@ -41,116 +70,127 @@ func ExtractAcl(r *http.Request, accountManager AccountManager, ownership, bucke
}
//owner should present && owner is immutable
if *acp.Owner.ID != ownerId {
glog.V(3).Infof("set acl denied! owner account is not consistent, request account id: %s, expect account id: %s", accountId, ownerId)
if *acp.Owner.ID == "" || *acp.Owner.ID != bucketOwnerId {
glog.V(3).Infof("set acl denied! owner account is not consistent, request account id: %s, expect account id: %s", *acp.Owner.ID, bucketOwnerId)
return nil, s3err.ErrAccessDenied
}
return ValidateAndTransferGrants(accountManager, acp.Grants)
grants = acp.Grants
} else {
_, grants, errCode = ParseAndValidateAclHeadersOrElseDefault(r, accountManager, ownership, bucketOwnerId, accountId, true)
return grants, errCode
if cannedAclPresent && customAclPresent {
return nil, s3err.ErrInvalidRequest
}
if cannedAclPresent {
grants, errCode = ExtractBucketCannedAcl(r, requestAccountId)
} else if customAclPresent {
grants, errCode = ExtractCustomAcl(r)
}
if errCode != s3err.ErrNone {
return nil, errCode
}
}
}
// ParseAndValidateAclHeadersOrElseDefault will callParseAndValidateAclHeaders to get Grants, if empty, it will return Grant that grant `accountId` with `FullControl` permission
func ParseAndValidateAclHeadersOrElseDefault(r *http.Request, accountManager AccountManager, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) {
ownerId, grants, errCode = ParseAndValidateAclHeaders(r, accountManager, ownership, bucketOwnerId, accountId, putAcl)
errCode = ValidateObjectOwnershipAndGrants(objectOwnership, bucketOwnerId, grants)
if errCode != s3err.ErrNone {
return
return nil, errCode
}
if len(grants) == 0 {
//if no acl(both customAcl and cannedAcl) specified, grant accountId(object writer) with full control permission
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &s3_constants.PermissionFullControl,
})
grants, errCode = ValidateAndTransferGrants(accountManager, grants)
if errCode != s3err.ErrNone {
return nil, errCode
}
return
return grants, s3err.ErrNone
}
// ParseAndValidateAclHeaders parse and validate acl from header
func ParseAndValidateAclHeaders(r *http.Request, accountManager AccountManager, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) {
ownerId, grants, errCode = ParseAclHeaders(r, ownership, bucketOwnerId, accountId, putAcl)
if errCode != s3err.ErrNone {
return
// ExtractObjectAcl extracts the acl from the request body, or from the header if request body is empty
func ExtractObjectAcl(r *http.Request, accountManager AccountManager, objectOwnership, bucketOwnerId, requestAccountId string, createObject bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) {
cannedAclPresent := false
if r.Header.Get(s3_constants.AmzCannedAcl) != "" {
cannedAclPresent = true
}
if len(grants) > 0 {
grants, errCode = ValidateAndTransferGrants(accountManager, grants)
customAclPresent := false
for _, customAclHeader := range customAclHeaders {
if r.Header.Get(customAclHeader) != "" {
customAclPresent = true
break
}
}
return
}
// ParseAclHeaders parse acl headers
// When `putAcl` is true, only `CannedAcl` is parsed, such as `PutBucketAcl` or `PutObjectAcl`
// is requested, `CustomAcl` is parsed from the request body not from headers, and only if the
// request body is empty, `CannedAcl` is parsed from the header, and will not parse `CustomAcl` from the header
//
// Since `CustomAcl` has higher priority, it will be parsed first; if `CustomAcl` does not exist, `CannedAcl` will be parsed
func ParseAclHeaders(r *http.Request, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) {
if !putAcl {
errCode = ParseCustomAclHeaders(r, &grants)
// AccessControlList body is not support when create object/bucket
if !createObject && r.Body != nil && r.Body != http.NoBody {
defer util_http.CloseRequest(r)
if cannedAclPresent || customAclPresent {
return "", nil, s3err.ErrUnexpectedContent
}
var acp s3.AccessControlPolicy
err := xmlutil.UnmarshalXML(&acp, xml.NewDecoder(r.Body), "")
if err != nil || acp.Owner == nil || acp.Owner.ID == nil {
return "", nil, s3err.ErrInvalidRequest
}
//owner should present && owner is immutable
if *acp.Owner.ID == "" {
glog.V(1).Infof("Access denied! The owner id is required when specifying grants using AccessControlList")
return "", nil, s3err.ErrAccessDenied
}
ownerId = *acp.Owner.ID
grants = acp.Grants
} else {
if cannedAclPresent && customAclPresent {
return "", nil, s3err.ErrInvalidRequest
}
if cannedAclPresent {
ownerId, grants, errCode = ExtractObjectCannedAcl(r, objectOwnership, bucketOwnerId, requestAccountId, createObject)
} else {
grants, errCode = ExtractCustomAcl(r)
}
if errCode != s3err.ErrNone {
return "", nil, errCode
}
}
if len(grants) > 0 {
return accountId, grants, s3err.ErrNone
}
cannedAcl := r.Header.Get(s3_constants.AmzCannedAcl)
if len(cannedAcl) == 0 {
return accountId, grants, s3err.ErrNone
}
//if canned acl specified, parse cannedAcl (lower priority to custom acl)
ownerId, grants, errCode = ParseCannedAclHeader(ownership, bucketOwnerId, accountId, cannedAcl, putAcl)
errCode = ValidateObjectOwnershipAndGrants(objectOwnership, bucketOwnerId, grants)
if errCode != s3err.ErrNone {
return "", nil, errCode
}
grants, errCode = ValidateAndTransferGrants(accountManager, grants)
return ownerId, grants, errCode
}
func ParseCustomAclHeaders(r *http.Request, grants *[]*s3.Grant) s3err.ErrorCode {
customAclHeaders := []string{s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp}
func ExtractCustomAcl(r *http.Request) ([]*s3.Grant, s3err.ErrorCode) {
var errCode s3err.ErrorCode
var grants []*s3.Grant
for _, customAclHeader := range customAclHeaders {
headerValue := r.Header.Get(customAclHeader)
switch customAclHeader {
case s3_constants.AmzAclRead:
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionRead, grants)
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionRead, &grants)
case s3_constants.AmzAclWrite:
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWrite, grants)
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWrite, &grants)
case s3_constants.AmzAclReadAcp:
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionReadAcp, grants)
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionReadAcp, &grants)
case s3_constants.AmzAclWriteAcp:
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWriteAcp, grants)
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWriteAcp, &grants)
case s3_constants.AmzAclFullControl:
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionFullControl, grants)
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionFullControl, &grants)
default:
errCode = s3err.ErrInvalidAclArgument
}
if errCode != s3err.ErrNone {
return errCode
return nil, errCode
}
}
return s3err.ErrNone
return grants, s3err.ErrNone
}
func ParseCustomAclHeader(headerValue, permission string, grants *[]*s3.Grant) s3err.ErrorCode {
if len(headerValue) > 0 {
split := strings.Split(headerValue, ", ")
split := strings.Split(headerValue, ",")
for _, grantStr := range split {
kv := strings.Split(grantStr, "=")
if len(kv) != 2 {
return s3err.ErrInvalidRequest
}
switch kv[0] {
switch strings.TrimSpace(kv[0]) {
case "id":
var accountId string
_ = json.Unmarshal([]byte(kv[1]), &accountId)
accountId := decodeGranteeValue(kv[1])
*grants = append(*grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
@ -159,8 +199,7 @@ func ParseCustomAclHeader(headerValue, permission string, grants *[]*s3.Grant) s
Permission: &permission,
})
case "emailAddress":
var emailAddress string
_ = json.Unmarshal([]byte(kv[1]), &emailAddress)
emailAddress := decodeGranteeValue(kv[1])
*grants = append(*grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeAmazonCustomerByEmail,
@ -170,7 +209,7 @@ func ParseCustomAclHeader(headerValue, permission string, grants *[]*s3.Grant) s
})
case "uri":
var groupName string
_ = json.Unmarshal([]byte(kv[1]), &groupName)
groupName = decodeGranteeValue(kv[1])
*grants = append(*grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
@ -182,17 +221,66 @@ func ParseCustomAclHeader(headerValue, permission string, grants *[]*s3.Grant) s
}
}
return s3err.ErrNone
}
func decodeGranteeValue(value string) (result string) {
if !strings.HasPrefix(value, "\"") {
return value
}
_ = json.Unmarshal([]byte(value), &result)
if result == "" {
result = value
}
return result
}
func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl string, putAcl bool) (ownerId string, grants []*s3.Grant, err s3err.ErrorCode) {
// ExtractBucketCannedAcl parse bucket canned acl, includes: 'private'|'public-read'|'public-read-write'|'authenticated-read'
func ExtractBucketCannedAcl(request *http.Request, requestAccountId string) (grants []*s3.Grant, err s3err.ErrorCode) {
cannedAcl := request.Header.Get(s3_constants.AmzCannedAcl)
if cannedAcl == "" {
return grants, s3err.ErrNone
}
err = s3err.ErrNone
ownerId = accountId
objectWriterFullControl := &s3.Grant{
Grantee: &s3.Grantee{
ID: &requestAccountId,
Type: &s3_constants.GrantTypeCanonicalUser,
},
Permission: &s3_constants.PermissionFullControl,
}
switch cannedAcl {
case s3_constants.CannedAclPrivate:
grants = append(grants, objectWriterFullControl)
case s3_constants.CannedAclPublicRead:
grants = append(grants, objectWriterFullControl)
grants = append(grants, s3_constants.PublicRead...)
case s3_constants.CannedAclPublicReadWrite:
grants = append(grants, objectWriterFullControl)
grants = append(grants, s3_constants.PublicReadWrite...)
case s3_constants.CannedAclAuthenticatedRead:
grants = append(grants, objectWriterFullControl)
grants = append(grants, s3_constants.AuthenticatedRead...)
default:
err = s3err.ErrInvalidAclArgument
}
return
}
//objectWrite automatically has full control on current object
// ExtractObjectCannedAcl parse object canned acl, includes: 'private'|'public-read'|'public-read-write'|'authenticated-read'|'aws-exec-read'|'bucket-owner-read'|'bucket-owner-full-control'
func ExtractObjectCannedAcl(request *http.Request, objectOwnership, bucketOwnerId, requestAccountId string, createObject bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) {
if createObject {
ownerId = requestAccountId
}
cannedAcl := request.Header.Get(s3_constants.AmzCannedAcl)
if cannedAcl == "" {
return ownerId, grants, s3err.ErrNone
}
errCode = s3err.ErrNone
objectWriterFullControl := &s3.Grant{
Grantee: &s3.Grantee{
ID: &accountId,
ID: &requestAccountId,
Type: &s3_constants.GrantTypeCanonicalUser,
},
Permission: &s3_constants.PermissionFullControl,
@ -215,7 +303,7 @@ func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl s
grants = append(grants, s3_constants.LogDeliveryWrite...)
case s3_constants.CannedAclBucketOwnerRead:
grants = append(grants, objectWriterFullControl)
if bucketOwnerId != "" && bucketOwnerId != accountId {
if requestAccountId != bucketOwnerId {
grants = append(grants,
&s3.Grant{
Grantee: &s3.Grantee{
@ -228,7 +316,7 @@ func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl s
case s3_constants.CannedAclBucketOwnerFullControl:
if bucketOwnerId != "" {
// if set ownership to 'BucketOwnerPreferred' when upload object, the bucket owner will be the object owner
if !putAcl && bucketOwnership == s3_constants.OwnershipBucketOwnerPreferred {
if createObject && objectOwnership == s3_constants.OwnershipBucketOwnerPreferred {
ownerId = bucketOwnerId
grants = append(grants,
&s3.Grant{
@ -240,7 +328,7 @@ func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl s
})
} else {
grants = append(grants, objectWriterFullControl)
if accountId != bucketOwnerId {
if requestAccountId != bucketOwnerId {
grants = append(grants,
&s3.Grant{
Grantee: &s3.Grantee{
@ -253,9 +341,9 @@ func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl s
}
}
case s3_constants.CannedAclAwsExecRead:
err = s3err.ErrNotImplemented
errCode = s3err.ErrNotImplemented
default:
err = s3err.ErrInvalidRequest
errCode = s3err.ErrInvalidAclArgument
}
return
}
@ -317,15 +405,43 @@ func ValidateAndTransferGrants(accountManager AccountManager, grants []*s3.Grant
return result, s3err.ErrNone
}
// DetermineReqGrants generates the grant set (Grants) according to accountId and reqPermission.
func DetermineReqGrants(accountId, aclAction string) (grants []*s3.Grant) {
// ValidateObjectOwnershipAndGrants validate if grants equals OwnerFullControl when 'ObjectOwnership' is 'BucketOwnerEnforced'
func ValidateObjectOwnershipAndGrants(objectOwnership, bucketOwnerId string, grants []*s3.Grant) s3err.ErrorCode {
if len(grants) == 0 {
return s3err.ErrNone
}
if objectOwnership == "" {
objectOwnership = s3_constants.DefaultObjectOwnership
}
if objectOwnership != s3_constants.OwnershipBucketOwnerEnforced {
return s3err.ErrNone
}
if len(grants) > 1 {
return s3err.AccessControlListNotSupported
}
bucketOwnerFullControlGrant := &s3.Grant{
Permission: &s3_constants.PermissionFullControl,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &bucketOwnerId,
},
}
if GrantEquals(bucketOwnerFullControlGrant, grants[0]) {
return s3err.ErrNone
}
return s3err.AccessControlListNotSupported
}
// DetermineRequiredGrants generates the grant set (Grants) according to accountId and reqPermission.
func DetermineRequiredGrants(accountId, permission string) (grants []*s3.Grant) {
// group grantee (AllUsers)
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &aclAction,
Permission: &permission,
})
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
@ -341,7 +457,7 @@ func DetermineReqGrants(accountId, aclAction string) (grants []*s3.Grant) {
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &aclAction,
Permission: &permission,
})
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
@ -358,7 +474,7 @@ func DetermineReqGrants(accountId, aclAction string) (grants []*s3.Grant) {
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
},
Permission: &aclAction,
Permission: &permission,
})
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
@ -383,9 +499,9 @@ func GetAcpOwner(entryExtended map[string][]byte, defaultOwner string) string {
return defaultOwner
}
func SetAcpGrantsHeader(r *http.Request, acpGrants []*s3.Grant) {
if len(acpGrants) > 0 {
a, err := json.Marshal(acpGrants)
func SetAcpGrantsHeader(r *http.Request, grants []*s3.Grant) {
if len(grants) > 0 {
a, err := MarshalGrantsToJson(grants)
if err == nil {
r.Header.Set(s3_constants.ExtAmzAclKey, string(a))
} else {
@ -395,7 +511,7 @@ func SetAcpGrantsHeader(r *http.Request, acpGrants []*s3.Grant) {
}
// GetAcpGrants return grants parsed from entry
func GetAcpGrants(entryExtended map[string][]byte) []*s3.Grant {
func GetAcpGrants(ownerId *string, entryExtended map[string][]byte) []*s3.Grant {
acpBytes, ok := entryExtended[s3_constants.ExtAmzAclKey]
if ok && len(acpBytes) > 0 {
var grants []*s3.Grant
@ -403,31 +519,43 @@ func GetAcpGrants(entryExtended map[string][]byte) []*s3.Grant {
if err == nil {
return grants
}
glog.Warning("grants Unmarshal error", err)
}
if ownerId == nil {
return nil
}
return []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: ownerId,
},
Permission: &s3_constants.PermissionFullControl,
},
}
return nil
}
// AssembleEntryWithAcp fill entry with owner and grants
func AssembleEntryWithAcp(objectEntry *filer_pb.Entry, objectOwner string, grants []*s3.Grant) s3err.ErrorCode {
if objectEntry.Extended == nil {
objectEntry.Extended = make(map[string][]byte)
func AssembleEntryWithAcp(filerEntry *filer_pb.Entry, ownerId string, grants []*s3.Grant) s3err.ErrorCode {
if filerEntry.Extended == nil {
filerEntry.Extended = make(map[string][]byte)
}
if len(objectOwner) > 0 {
objectEntry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(objectOwner)
if len(ownerId) > 0 {
filerEntry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(ownerId)
} else {
delete(objectEntry.Extended, s3_constants.ExtAmzOwnerKey)
delete(filerEntry.Extended, s3_constants.ExtAmzOwnerKey)
}
if len(grants) > 0 {
grantsBytes, err := json.Marshal(grants)
if grants != nil {
grantsBytes, err := MarshalGrantsToJson(grants)
if err != nil {
glog.Warning("assemble acp to entry:", err)
return s3err.ErrInvalidRequest
}
objectEntry.Extended[s3_constants.ExtAmzAclKey] = grantsBytes
filerEntry.Extended[s3_constants.ExtAmzAclKey] = grantsBytes
} else {
delete(objectEntry.Extended, s3_constants.ExtAmzAclKey)
delete(filerEntry.Extended, s3_constants.ExtAmzAclKey)
}
return s3err.ErrNone
@ -511,3 +639,87 @@ func GrantEquals(a, b *s3.Grant) bool {
}
return true
}
func MarshalGrantsToJson(grants []*s3.Grant) ([]byte, error) {
if len(grants) == 0 {
return []byte{}, nil
}
var GrantsToMap []map[string]any
for _, grant := range grants {
grantee := grant.Grantee
switch *grantee.Type {
case s3_constants.GrantTypeGroup:
GrantsToMap = append(GrantsToMap, map[string]any{
"Permission": grant.Permission,
"Grantee": map[string]any{
"Type": grantee.Type,
"URI": grantee.URI,
},
})
case s3_constants.GrantTypeCanonicalUser:
GrantsToMap = append(GrantsToMap, map[string]any{
"Permission": grant.Permission,
"Grantee": map[string]any{
"Type": grantee.Type,
"ID": grantee.ID,
},
})
case s3_constants.GrantTypeAmazonCustomerByEmail:
GrantsToMap = append(GrantsToMap, map[string]any{
"Permission": grant.Permission,
"Grantee": map[string]any{
"Type": grantee.Type,
"EmailAddress": grantee.EmailAddress,
},
})
default:
return nil, fmt.Errorf("grantee type[%s] is not valid", *grantee.Type)
}
}
return json.Marshal(GrantsToMap)
}
func GrantWithFullControl(accountId string) *s3.Grant {
return &s3.Grant{
Permission: &s3_constants.PermissionFullControl,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
}
}
func CheckObjectAccessForReadObject(r *http.Request, w http.ResponseWriter, entry *filer.Entry, bucketOwnerId string) (statusCode int, ok bool) {
if entry.IsDirectory() {
return http.StatusOK, true
}
requestAccountId := GetAccountId(r)
if len(requestAccountId) == 0 {
glog.Warning("#checkObjectAccessForReadObject header[accountId] not exists!")
return http.StatusForbidden, false
}
//owner access
objectOwner := GetAcpOwner(entry.Extended, bucketOwnerId)
if ValidateAccount(requestAccountId, objectOwner) {
return http.StatusOK, true
}
//find in Grants
acpGrants := GetAcpGrants(nil, entry.Extended)
if acpGrants != nil {
reqGrants := DetermineRequiredGrants(requestAccountId, s3_constants.PermissionRead)
for _, requiredGrant := range reqGrants {
for _, grant := range acpGrants {
if GrantEquals(requiredGrant, grant) {
return http.StatusOK, true
}
}
}
}
glog.V(3).Infof("acl denied! request account id: %s", requestAccountId)
return http.StatusForbidden, false
}

1477
weed/s3api/s3acl/s3api_acl_helper_test.go
File diff suppressed because it is too large
View File

709
weed/s3api/s3api_acl_helper_test.go

@ -1,709 +0,0 @@
package s3api
import (
"bytes"
"encoding/json"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"io"
"net/http"
"testing"
)
var accountManager *IdentityAccessManagement
func init() {
accountManager = &IdentityAccessManagement{}
_ = accountManager.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
Accounts: []*iam_pb.Account{
{
Id: "accountA",
DisplayName: "accountAName",
EmailAddress: "accountA@example.com",
},
{
Id: "accountB",
DisplayName: "accountBName",
EmailAddress: "accountB@example.com",
},
},
})
}
func TestGetAccountId(t *testing.T) {
req := &http.Request{
Header: make(map[string][]string),
}
//case1
//accountId: "admin"
req.Header.Set(s3_constants.AmzAccountId, s3_constants.AccountAdminId)
if GetAccountId(req) != s3_constants.AccountAdminId {
t.Fatal("expect accountId: admin")
}
//case2
//accountId: "anoymous"
req.Header.Set(s3_constants.AmzAccountId, s3_constants.AccountAnonymousId)
if GetAccountId(req) != s3_constants.AccountAnonymousId {
t.Fatal("expect accountId: anonymous")
}
//case3
//accountId is nil => "anonymous"
req.Header.Del(s3_constants.AmzAccountId)
if GetAccountId(req) != s3_constants.AccountAnonymousId {
t.Fatal("expect accountId: anonymous")
}
}
func TestExtractAcl(t *testing.T) {
type Case struct {
id int
resultErrCode, expectErrCode s3err.ErrorCode
resultGrants, expectGrants []*s3.Grant
}
testCases := make([]*Case, 0)
accountAdminId := "admin"
{
//case1 (good case)
//parse acp from request body
req := &http.Request{
Header: make(map[string][]string),
}
req.Body = io.NopCloser(bytes.NewReader([]byte(`
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Owner>
<ID>admin</ID>
<DisplayName>admin</DisplayName>
</Owner>
<AccessControlList>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">
<ID>admin</ID>
</Grantee>
<Permission>FULL_CONTROL</Permission>
</Grant>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group">
<URI>http://acs.amazonaws.com/groups/global/AllUsers</URI>
</Grantee>
<Permission>FULL_CONTROL</Permission>
</Grant>
</AccessControlList>
</AccessControlPolicy>
`)))
objectWriter := "accountA"
grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter)
testCases = append(testCases, &Case{
1,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountAdminId,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case2 (good case)
//parse acp from header (cannedAcl)
req := &http.Request{
Header: make(map[string][]string),
}
req.Body = nil
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate)
objectWriter := "accountA"
grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter)
testCases = append(testCases, &Case{
2,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &objectWriter,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case3 (bad case)
//parse acp from request body (content is invalid)
req := &http.Request{
Header: make(map[string][]string),
}
req.Body = io.NopCloser(bytes.NewReader([]byte("zdfsaf")))
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate)
objectWriter := "accountA"
_, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter)
testCases = append(testCases, &Case{
id: 3,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
}
//case4 (bad case)
//parse acp from header (cannedAcl is invalid)
req := &http.Request{
Header: make(map[string][]string),
}
req.Body = nil
req.Header.Set(s3_constants.AmzCannedAcl, "dfaksjfk")
objectWriter := "accountA"
_, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, "", objectWriter)
testCases = append(testCases, &Case{
id: 4,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
{
//case5 (bad case)
//parse acp from request body: owner is inconsistent
req.Body = io.NopCloser(bytes.NewReader([]byte(`
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Owner>
<ID>admin</ID>
<DisplayName>admin</DisplayName>
</Owner>
<AccessControlList>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">
<ID>admin</ID>
</Grantee>
<Permission>FULL_CONTROL</Permission>
</Grant>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group">
<URI>http://acs.amazonaws.com/groups/global/AllUsers</URI>
</Grantee>
<Permission>FULL_CONTROL</Permission>
</Grant>
</AccessControlList>
</AccessControlPolicy>
`)))
objectWriter = "accountA"
_, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, objectWriter, objectWriter)
testCases = append(testCases, &Case{
id: 5,
resultErrCode: errCode, expectErrCode: s3err.ErrAccessDenied,
})
}
for _, tc := range testCases {
if tc.resultErrCode != tc.expectErrCode {
t.Fatalf("case[%d]: errorCode not expect", tc.id)
}
if !grantsEquals(tc.resultGrants, tc.expectGrants) {
t.Fatalf("case[%d]: grants not expect", tc.id)
}
}
}
func TestParseAndValidateAclHeaders(t *testing.T) {
type Case struct {
id int
resultOwner, expectOwner string
resultErrCode, expectErrCode s3err.ErrorCode
resultGrants, expectGrants []*s3.Grant
}
testCases := make([]*Case, 0)
bucketOwner := "admin"
{
//case1 (good case)
//parse custom acl
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example.com"`)
ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
1,
ownerId, objectWriter,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: aws.String(s3_constants.AccountAnonymousId),
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: aws.String(s3_constants.AccountAdminId),
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case2 (good case)
//parse canned acl (ownership=ObjectWriter)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl)
ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
2,
ownerId, objectWriter,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &objectWriter,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &bucketOwner,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case3 (good case)
//parse canned acl (ownership=OwnershipBucketOwnerPreferred)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl)
ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
3,
ownerId, bucketOwner,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &bucketOwner,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case4 (bad case)
//parse custom acl (grantee id not exists)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="notExistsAccount", emailAddress="admin@example.com"`)
_, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
id: 4,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
}
{
//case5 (bad case)
//parse custom acl (invalid format)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzAclFullControl, `uri="http:sfasf"`)
_, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
id: 5,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
}
{
//case6 (bad case)
//parse canned acl (invalid value)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzCannedAcl, `uri="http:sfasf"`)
_, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
id: 5,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
}
for _, tc := range testCases {
if tc.expectErrCode != tc.resultErrCode {
t.Errorf("case[%d]: errCode unexpect", tc.id)
}
if tc.resultOwner != tc.expectOwner {
t.Errorf("case[%d]: ownerId unexpect", tc.id)
}
if !grantsEquals(tc.resultGrants, tc.expectGrants) {
t.Fatalf("case[%d]: grants not expect", tc.id)
}
}
}
func grantsEquals(a, b []*s3.Grant) bool {
if len(a) != len(b) {
return false
}
for i, grant := range a {
if !GrantEquals(grant, b[i]) {
return false
}
}
return true
}
func TestDetermineReqGrants(t *testing.T) {
{
//case1: request account is anonymous
accountId := s3_constants.AccountAnonymousId
reqPermission := s3_constants.PermissionRead
resultGrants := DetermineReqGrants(accountId, reqPermission)
expectGrants := []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &s3_constants.PermissionFullControl,
},
}
if !grantsEquals(resultGrants, expectGrants) {
t.Fatalf("grants not expect")
}
}
{
//case2: request account is not anonymous (Iam authed)
accountId := "accountX"
reqPermission := s3_constants.PermissionRead
resultGrants := DetermineReqGrants(accountId, reqPermission)
expectGrants := []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
}
if !grantsEquals(resultGrants, expectGrants) {
t.Fatalf("grants not expect")
}
}
}
func TestAssembleEntryWithAcp(t *testing.T) {
defaultOwner := "admin"
//case1
//assemble with non-empty grants
expectOwner := "accountS"
expectGrants := []*s3.Grant{
{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
URI: &s3_constants.GranteeGroupAllUsers,
},
},
}
entry := &filer_pb.Entry{}
AssembleEntryWithAcp(entry, expectOwner, expectGrants)
resultOwner := GetAcpOwner(entry.Extended, defaultOwner)
if resultOwner != expectOwner {
t.Fatalf("owner not expect")
}
resultGrants := GetAcpGrants(entry.Extended)
if !grantsEquals(resultGrants, expectGrants) {
t.Fatal("grants not expect")
}
//case2
//assemble with empty grants (override)
AssembleEntryWithAcp(entry, "", nil)
resultOwner = GetAcpOwner(entry.Extended, defaultOwner)
if resultOwner != defaultOwner {
t.Fatalf("owner not expect")
}
resultGrants = GetAcpGrants(entry.Extended)
if len(resultGrants) != 0 {
t.Fatal("grants not expect")
}
}
func TestGrantEquals(t *testing.T) {
testCases := map[bool]bool{
GrantEquals(nil, nil): true,
GrantEquals(&s3.Grant{}, nil): false,
GrantEquals(&s3.Grant{}, &s3.Grant{}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
}, &s3.Grant{}): false,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{},
}): false,
//type not present, compare other fields of grant is meaningless
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
ID: aws.String(s3_constants.AccountAdminId),
//EmailAddress: &s3account.AccountAdmin.EmailAddress,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
ID: aws.String(s3_constants.AccountAdminId),
},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionWrite,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}): false,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
URI: &s3_constants.GranteeGroupAllUsers,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
},
}): false,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
URI: &s3_constants.GranteeGroupAllUsers,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}): true,
}
for tc, expect := range testCases {
if tc != expect {
t.Fatal("TestGrantEquals not expect!")
}
}
}
func TestSetAcpOwnerHeader(t *testing.T) {
ownerId := "accountZ"
req := &http.Request{
Header: make(map[string][]string),
}
SetAcpOwnerHeader(req, ownerId)
if req.Header.Get(s3_constants.ExtAmzOwnerKey) != ownerId {
t.Fatalf("owner unexpect")
}
}
func TestSetAcpGrantsHeader(t *testing.T) {
req := &http.Request{
Header: make(map[string][]string),
}
grants := []*s3.Grant{
{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: aws.String(s3_constants.AccountAdminId),
URI: &s3_constants.GranteeGroupAllUsers,
},
},
}
SetAcpGrantsHeader(req, grants)
grantsJson, _ := json.Marshal(grants)
if req.Header.Get(s3_constants.ExtAmzAclKey) != string(grantsJson) {
t.Fatalf("owner unexpect")
}
}

400
weed/s3api/s3api_acp.go

@ -1,28 +1,406 @@
package s3api
import (
"net/http"
"path/filepath"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3acl"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"net/http"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func getAccountId(r *http.Request) string {
id := r.Header.Get(s3_constants.AmzAccountId)
if len(id) == 0 {
return AccountAnonymous.Id
} else {
return id
func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s3err.ErrorCode {
bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
if errCode != s3err.ErrNone {
return errCode
}
requestAccountId := s3acl.GetAccountId(r)
if s3acl.ValidateAccount(requestAccountId, *bucketMetadata.Owner.ID) {
return s3err.ErrNone
}
return s3err.ErrAccessDenied
}
func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s3err.ErrorCode {
metadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
// Check access for PutBucketAclHandler
func (s3a *S3ApiServer) checkAccessForPutBucketAcl(requestAccountId, bucket string) (*BucketMetaData, s3err.ErrorCode) {
bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
if errCode != s3err.ErrNone {
return nil, errCode
}
if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced {
return nil, s3err.AccessControlListNotSupported
}
if s3acl.ValidateAccount(requestAccountId, *bucketMetadata.Owner.ID) {
return bucketMetadata, s3err.ErrNone
}
if len(bucketMetadata.Acl) > 0 {
reqGrants := s3acl.DetermineRequiredGrants(requestAccountId, s3_constants.PermissionWriteAcp)
for _, bucketGrant := range bucketMetadata.Acl {
for _, reqGrant := range reqGrants {
if s3acl.GrantEquals(bucketGrant, reqGrant) {
return bucketMetadata, s3err.ErrNone
}
}
}
}
glog.V(3).Infof("acl denied! request account id: %s", requestAccountId)
return nil, s3err.ErrAccessDenied
}
func updateBucketEntry(s3a *S3ApiServer, entry *filer_pb.Entry) error {
return s3a.updateEntry(s3a.option.BucketsPath, entry)
}
// Check Bucket/BucketAcl Read related access
// includes:
// - HeadBucketHandler
// - GetBucketAclHandler
// - ListObjectsV1Handler
// - ListObjectsV2Handler
// - ListMultipartUploadsHandler
func (s3a *S3ApiServer) checkAccessForReadBucket(r *http.Request, bucket, aclAction string) (*BucketMetaData, s3err.ErrorCode) {
bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
if errCode != s3err.ErrNone {
return nil, errCode
}
if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced {
return bucketMetadata, s3err.ErrNone
}
requestAccountId := s3acl.GetAccountId(r)
if s3acl.ValidateAccount(requestAccountId, *bucketMetadata.Owner.ID) {
return bucketMetadata, s3err.ErrNone
}
if len(bucketMetadata.Acl) > 0 {
reqGrants := s3acl.DetermineRequiredGrants(requestAccountId, aclAction)
for _, bucketGrant := range bucketMetadata.Acl {
for _, reqGrant := range reqGrants {
if s3acl.GrantEquals(bucketGrant, reqGrant) {
return bucketMetadata, s3err.ErrNone
}
}
}
}
glog.V(3).Infof("acl denied! request account id: %s", requestAccountId)
return nil, s3err.ErrAccessDenied
}
// Check ObjectAcl-Read related access
// includes:
// - GetObjectAclHandler
func (s3a *S3ApiServer) checkAccessForReadObjectAcl(r *http.Request, bucket, object string) (acp *s3.AccessControlPolicy, errCode s3err.ErrorCode) {
bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
if errCode != s3err.ErrNone {
return nil, errCode
}
getAcpFunc := func() (*s3.AccessControlPolicy, s3err.ErrorCode) {
entry, err := getObjectEntry(s3a, bucket, object)
if err != nil {
if err == filer_pb.ErrNotFound {
return nil, s3err.ErrNoSuchKey
} else {
return nil, s3err.ErrInternalError
}
}
if entry.IsDirectory {
return nil, s3err.ErrExistingObjectIsDirectory
}
acpOwnerId := s3acl.GetAcpOwner(entry.Extended, *bucketMetadata.Owner.ID)
acpOwnerName := s3a.iam.GetAccountNameById(acpOwnerId)
acpGrants := s3acl.GetAcpGrants(&acpOwnerId, entry.Extended)
acp = &s3.AccessControlPolicy{
Owner: &s3.Owner{
ID: &acpOwnerId,
DisplayName: &acpOwnerName,
},
Grants: acpGrants,
}
return acp, s3err.ErrNone
}
if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced {
return getAcpFunc()
}
requestAccountId := s3acl.GetAccountId(r)
acp, errCode = getAcpFunc()
if errCode != s3err.ErrNone {
return nil, errCode
}
if s3acl.ValidateAccount(requestAccountId, *acp.Owner.ID) {
return acp, s3err.ErrNone
}
if acp.Grants != nil {
reqGrants := s3acl.DetermineRequiredGrants(requestAccountId, s3_constants.PermissionReadAcp)
for _, requiredGrant := range reqGrants {
for _, grant := range acp.Grants {
if s3acl.GrantEquals(requiredGrant, grant) {
return acp, s3err.ErrNone
}
}
}
}
glog.V(3).Infof("CheckAccessForReadObjectAcl denied! request account id: %s", requestAccountId)
return nil, s3err.ErrAccessDenied
}
// Check Object-Read related access
// includes:
// - GetObjectHandler
//
// offload object access validation to Filer layer
// - CheckObjectAccessForReadObject
func (s3a *S3ApiServer) checkBucketAccessForReadObject(r *http.Request, bucket string) s3err.ErrorCode {
bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
if errCode != s3err.ErrNone {
return errCode
}
accountId := getAccountId(r)
if accountId == AccountAdmin.Id || accountId == *metadata.Owner.ID {
if bucketMetadata.ObjectOwnership != s3_constants.OwnershipBucketOwnerEnforced {
//offload object acl validation to filer layer
_, defaultErrorCode := s3a.checkAccessForReadBucket(r, bucket, s3_constants.PermissionRead)
if defaultErrorCode != s3err.ErrNone {
r.Header.Set(s3_constants.XAmzBucketAccessDenied, "true")
}
r.Header.Set(s3_constants.XAmzBucketOwnerId, *bucketMetadata.Owner.ID)
}
return s3err.ErrNone
}
// Check ObjectAcl-Write related access
// includes:
// - PutObjectAclHandler
func (s3a *S3ApiServer) checkAccessForWriteObjectAcl(r *http.Request, bucket, object string) (*filer_pb.Entry, string, []*s3.Grant, s3err.ErrorCode) {
bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
if errCode != s3err.ErrNone {
return nil, "", nil, errCode
}
requestAccountId := s3acl.GetAccountId(r)
reqOwnerId, grants, errCode := s3acl.ExtractObjectAcl(r, s3a.iam, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, requestAccountId, false)
if errCode != s3err.ErrNone {
return nil, "", nil, errCode
}
if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced {
return nil, "", nil, s3err.AccessControlListNotSupported
}
//object acl
objectEntry, err := getObjectEntry(s3a, bucket, object)
if err != nil {
if err == filer_pb.ErrNotFound {
return nil, "", nil, s3err.ErrNoSuchKey
}
return nil, "", nil, s3err.ErrInternalError
}
if objectEntry.IsDirectory {
return nil, "", nil, s3err.ErrExistingObjectIsDirectory
}
objectOwner := s3acl.GetAcpOwner(objectEntry.Extended, *bucketMetadata.Owner.ID)
//object owner is immutable
if reqOwnerId != "" && reqOwnerId != objectOwner {
return nil, "", nil, s3err.ErrAccessDenied
}
if s3acl.ValidateAccount(requestAccountId, objectOwner) {
return objectEntry, objectOwner, grants, s3err.ErrNone
}
objectGrants := s3acl.GetAcpGrants(nil, objectEntry.Extended)
if objectGrants != nil {
requiredGrants := s3acl.DetermineRequiredGrants(requestAccountId, s3_constants.PermissionWriteAcp)
for _, objectGrant := range objectGrants {
for _, requiredGrant := range requiredGrants {
if s3acl.GrantEquals(objectGrant, requiredGrant) {
return objectEntry, objectOwner, grants, s3err.ErrNone
}
}
}
}
glog.V(3).Infof("checkAccessForWriteObjectAcl denied! request account id: %s", requestAccountId)
return nil, "", nil, s3err.ErrAccessDenied
}
func updateObjectEntry(s3a *S3ApiServer, bucket, object string, entry *filer_pb.Entry) error {
dir, _ := filepath.Split(object)
return s3a.updateEntry(util.Join(s3a.option.BucketsPath, bucket, dir), entry)
}
// CheckAccessForPutObject Check ACL for PutObject API
// includes:
// - PutObjectHandler
func (s3a *S3ApiServer) CheckAccessForPutObject(r *http.Request, bucket, object string) s3err.ErrorCode {
accountId := s3acl.GetAccountId(r)
return s3a.checkAccessForPutObject(r, bucket, object, accountId)
}
// CheckAccessForPutObjectPartHandler Check Acl for Upload object part
// includes:
// - PutObjectPartHandler
func (s3a *S3ApiServer) CheckAccessForPutObjectPartHandler(r *http.Request, bucket string) s3err.ErrorCode {
bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
if errCode != s3err.ErrNone {
return errCode
}
if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced {
return s3err.ErrNone
}
accountId := s3acl.GetAccountId(r)
if !CheckBucketAccess(accountId, bucketMetadata, s3_constants.PermissionWrite) {
return s3err.ErrAccessDenied
}
return s3err.ErrNone
}
// CheckAccessForNewMultipartUpload Check Acl for API
// includes:
// - NewMultipartUploadHandler
func (s3a *S3ApiServer) CheckAccessForNewMultipartUpload(r *http.Request, bucket, object string) (s3err.ErrorCode, string) {
accountId := s3acl.GetAccountId(r)
if accountId == AccountAnonymous.Id {
return s3err.ErrAccessDenied, ""
}
errCode := s3a.checkAccessForPutObject(r, bucket, object, accountId)
return errCode, accountId
}
func (s3a *S3ApiServer) CheckAccessForAbortMultipartUpload(r *http.Request, bucket, object string) s3err.ErrorCode {
return s3a.CheckAccessWithBucketOwnerAndInitiator(r, bucket, object)
}
func (s3a *S3ApiServer) CheckAccessForCompleteMultipartUpload(r *http.Request, bucket, object string) s3err.ErrorCode {
bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
if errCode != s3err.ErrNone {
return errCode
}
if bucketMetadata.ObjectOwnership != s3_constants.OwnershipBucketOwnerEnforced {
accountId := s3acl.GetAccountId(r)
if !CheckBucketAccess(accountId, bucketMetadata, s3_constants.PermissionWrite) {
return s3err.ErrAccessDenied
}
}
return s3err.ErrNone
}
func (s3a *S3ApiServer) CheckAccessForListMultipartUploadParts(r *http.Request, bucket, object string) s3err.ErrorCode {
return s3a.CheckAccessWithBucketOwnerAndInitiator(r, bucket, object)
}
// CheckAccessWithBucketOwnerAndInitiator Check Access Permission with 'bucketOwner' and 'multipartUpload initiator'
func (s3a *S3ApiServer) CheckAccessWithBucketOwnerAndInitiator(r *http.Request, bucket, object string) s3err.ErrorCode {
bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
if errCode != s3err.ErrNone {
return errCode
}
//bucket access allowed
accountId := s3acl.GetAccountId(r)
if s3acl.ValidateAccount(*bucketMetadata.Owner.ID, accountId) {
return s3err.ErrNone
}
//multipart initiator allowed
entry, err := getMultipartUpload(s3a, bucket, object)
if err != nil {
if err != filer_pb.ErrNotFound {
return s3err.ErrInternalError
}
} else {
uploadInitiator, ok := entry.Extended[s3_constants.ExtAmzMultipartInitiator]
if !ok || accountId == string(uploadInitiator) {
return s3err.ErrNone
}
}
glog.V(3).Infof("CheckAccessWithBucketOwnerAndInitiator denied! request account id: %s", accountId)
return s3err.ErrAccessDenied
}
func (s3a *S3ApiServer) checkAccessForPutObject(r *http.Request, bucket, object, requestAccountId string) s3err.ErrorCode {
bucketMetadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
if errCode != s3err.ErrNone {
return errCode
}
// if ownership is 'OwnershipBucketOwnerEnforced', acl is not supportedG
if bucketMetadata.ObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced {
_, _, errCode := s3acl.ExtractObjectAcl(r, s3a.iam, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, requestAccountId, true)
if errCode != s3err.ErrNone {
return errCode
}
return s3err.ErrNone
}
requestOwnerId, grants, errCode := s3acl.ExtractObjectAcl(r, s3a.iam, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, requestAccountId, true)
if errCode != s3err.ErrNone {
return errCode
}
if !CheckBucketAccess(requestAccountId, bucketMetadata, s3_constants.PermissionWrite) {
return s3err.ErrAccessDenied
}
if requestOwnerId == "" {
requestOwnerId = requestAccountId
}
entry, err := getObjectEntry(s3a, bucket, object)
if err != nil {
if err == filer_pb.ErrNotFound {
s3acl.SetAcpOwnerHeader(r, requestOwnerId)
s3acl.SetAcpGrantsHeader(r, grants)
return s3err.ErrNone
}
return s3err.ErrInternalError
}
objectOwnerId := s3acl.GetAcpOwner(entry.Extended, *bucketMetadata.Owner.ID)
//object owner is immutable
if !s3acl.ValidateAccount(requestOwnerId, objectOwnerId, *bucketMetadata.Owner.ID) {
return s3err.ErrAccessDenied
}
s3acl.SetAcpOwnerHeader(r, objectOwnerId)
s3acl.SetAcpGrantsHeader(r, grants)
return s3err.ErrNone
}
func CheckBucketAccess(requestAccountId string, bucketMetadata *BucketMetaData, permission string) bool {
if s3acl.ValidateAccount(requestAccountId, *bucketMetadata.Owner.ID) {
return true
} else {
if len(bucketMetadata.Acl) > 0 {
reqGrants := s3acl.DetermineRequiredGrants(requestAccountId, permission)
for _, bucketGrant := range bucketMetadata.Acl {
for _, requiredGrant := range reqGrants {
if s3acl.GrantEquals(bucketGrant, requiredGrant) {
return true
}
}
}
}
}
glog.V(3).Infof("CheckBucketAccess denied! request account id: %s", requestAccountId)
return false
}
func getObjectEntry(s3a *S3ApiServer, bucket, object string) (*filer_pb.Entry, error) {
return s3a.getEntry(util.Join(s3a.option.BucketsPath, bucket), object)
}
func getMultipartUpload(s3a *S3ApiServer, bucket, object string) (*filer_pb.Entry, error) {
return s3a.getEntry(s3a.genUploadsFolder(bucket), object)
}

3
weed/s3api/s3api_auth.go

@ -49,8 +49,7 @@ func isRequestPostPolicySignatureV4(r *http.Request) bool {
// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
func isRequestSignStreamingV4(r *http.Request) bool {
return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 &&
r.Method == http.MethodPut
return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 && r.Method == http.MethodPut
}
// Authorization type.

160
weed/s3api/s3api_bucket_handlers.go

@ -6,6 +6,7 @@ import (
"encoding/xml"
"errors"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3acl"
"math"
"net/http"
"strings"
@ -120,13 +121,29 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
return
}
objectOwnership := r.Header.Get("X-Amz-Object-Ownership")
requestAccountId := s3acl.GetAccountId(r)
grants, errCode := s3acl.ExtractBucketAcl(r, s3a.iam, objectOwnership, requestAccountId, requestAccountId, true)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
fn := func(entry *filer_pb.Entry) {
if identityId := r.Header.Get(s3_constants.AmzIdentityId); identityId != "" {
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.AmzIdentityId] = []byte(identityId)
s3a.bucketRegistry.LoadBucketMetadata(entry)
}
if objectOwnership != "" {
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.ExtOwnershipKey] = []byte(objectOwnership)
}
s3acl.AssembleEntryWithAcp(entry, requestAccountId, grants)
}
// create the folder for bucket, but lazily create actual collection
@ -135,6 +152,10 @@ func (s3a *S3ApiServer) PutBucketHandler(w http.ResponseWriter, r *http.Request)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
// clear cache
s3a.bucketRegistry.ClearCache(bucket)
w.Header().Set("Location", "/"+bucket)
writeSuccessResponseEmpty(w, r)
}
@ -190,7 +211,9 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
// clear cache
s3a.bucketRegistry.ClearCache(bucket)
s3err.WriteEmptyResponse(w, r, http.StatusNoContent)
}
@ -199,6 +222,11 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("HeadBucketHandler %s", bucket)
_, errorCode := s3a.checkAccessForReadBucket(r, bucket, s3_constants.PermissionRead)
if errorCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errorCode)
return
}
if entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket); entry == nil || err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
@ -242,67 +270,68 @@ func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool {
return true
}
// GetBucketAclHandler Get Bucket ACL
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html
func (s3a *S3ApiServer) GetBucketAclHandler(w http.ResponseWriter, r *http.Request) {
// collect parameters
// PutBucketAclHandler Put bucket ACL
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html
func (s3a *S3ApiServer) PutBucketAclHandler(w http.ResponseWriter, r *http.Request) {
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("GetBucketAclHandler %s", bucket)
glog.V(3).Infof("PutBucketAclHandler %s", bucket)
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, err)
accountId := s3acl.GetAccountId(r)
bucketMetadata, errorCode := s3a.checkAccessForPutBucketAcl(accountId, bucket)
if errorCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errorCode)
return
}
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
amzDisplayName := s3a.iam.GetAccountNameById(amzAccountId)
response := AccessControlPolicy{
Owner: CanonicalUser{
ID: amzAccountId,
DisplayName: amzDisplayName,
},
grants, errCode := s3acl.ExtractBucketAcl(r, s3a.iam, bucketMetadata.ObjectOwnership, *bucketMetadata.Owner.ID, accountId, false)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
response.AccessControlList.Grant = append(response.AccessControlList.Grant, Grant{
Grantee: Grantee{
ID: amzAccountId,
DisplayName: amzDisplayName,
Type: "CanonicalUser",
XMLXSI: "CanonicalUser",
XMLNS: "http://www.w3.org/2001/XMLSchema-instance"},
Permission: s3.PermissionFullControl,
})
writeSuccessResponseXML(w, r, response)
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
if err != nil {
glog.Warning(err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
errCode = s3acl.AssembleEntryWithAcp(bucketEntry, *bucketMetadata.Owner.ID, grants)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
err = updateBucketEntry(s3a, bucketEntry)
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
//update local cache
bucketMetadata.Acl = grants
s3err.WriteEmptyResponse(w, r, http.StatusOK)
}
// PutBucketAclHandler Put bucket ACL only responds success if the ACL is private.
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html //
func (s3a *S3ApiServer) PutBucketAclHandler(w http.ResponseWriter, r *http.Request) {
// GetBucketAclHandler Get Bucket ACL
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html
func (s3a *S3ApiServer) GetBucketAclHandler(w http.ResponseWriter, r *http.Request) {
// collect parameters
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("PutBucketAclHandler %s", bucket)
glog.V(3).Infof("GetBucketAclHandler %s", bucket)
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, err)
bucketMetadata, errorCode := s3a.checkAccessForReadBucket(r, bucket, s3_constants.PermissionReadAcp)
if s3err.ErrNone != errorCode {
s3err.WriteErrorResponse(w, r, errorCode)
return
}
cannedAcl := r.Header.Get(s3_constants.AmzCannedAcl)
switch {
case cannedAcl == "":
acl := &s3.AccessControlPolicy{}
if err := xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
glog.Errorf("PutBucketAclHandler: %s", err)
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
if len(acl.Grants) == 1 && acl.Grants[0].Permission != nil && *acl.Grants[0].Permission == s3_constants.PermissionFullControl {
writeSuccessResponseEmpty(w, r)
return
}
case cannedAcl == s3_constants.CannedAclPrivate:
writeSuccessResponseEmpty(w, r)
return
acp := &s3.PutBucketAclInput{
AccessControlPolicy: &s3.AccessControlPolicy{
Grants: bucketMetadata.Acl,
Owner: bucketMetadata.Owner,
},
}
s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, acp)
}
// GetBucketLifecycleConfigurationHandler Get Bucket Lifecycle configuration
@ -510,9 +539,8 @@ func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *htt
return
}
var v s3.OwnershipControls
defer util_http.CloseRequest(r)
var v s3.OwnershipControls
err := xmlutil.UnmarshalXML(&v, xml.NewDecoder(r.Body), "")
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
@ -525,12 +553,11 @@ func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *htt
}
printOwnership := true
ownership := *v.Rules[0].ObjectOwnership
switch ownership {
newObjectOwnership := *v.Rules[0].ObjectOwnership
switch newObjectOwnership {
case s3_constants.OwnershipObjectWriter:
case s3_constants.OwnershipBucketOwnerPreferred:
case s3_constants.OwnershipBucketOwnerEnforced:
printOwnership = false
default:
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
@ -547,11 +574,34 @@ func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *htt
}
oldOwnership, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
if !ok || string(oldOwnership) != ownership {
if !ok || string(oldOwnership) != newObjectOwnership {
// must reset bucket acl to default(bucket owner with full control permission) before setting ownership
// to `OwnershipBucketOwnerEnforced` (bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting)
if newObjectOwnership == s3_constants.OwnershipBucketOwnerEnforced {
acpGrants := s3acl.GetAcpGrants(nil, bucketEntry.Extended)
if len(acpGrants) > 1 {
s3err.WriteErrorResponse(w, r, s3err.InvalidBucketAclWithObjectOwnership)
return
} else if len(acpGrants) == 1 {
bucketOwner := s3acl.GetAcpOwner(bucketEntry.Extended, AccountAdmin.Id)
expectGrant := s3acl.GrantWithFullControl(bucketOwner)
if !s3acl.GrantEquals(acpGrants[0], expectGrant) {
s3err.WriteErrorResponse(w, r, s3err.InvalidBucketAclWithObjectOwnership)
return
}
}
}
if bucketEntry.Extended == nil {
bucketEntry.Extended = make(map[string][]byte)
}
bucketEntry.Extended[s3_constants.ExtOwnershipKey] = []byte(ownership)
//update local cache
bucketMetadata, eCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
if eCode == s3err.ErrNone {
bucketMetadata.ObjectOwnership = newObjectOwnership
}
bucketEntry.Extended[s3_constants.ExtOwnershipKey] = []byte(newObjectOwnership)
err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)

2
weed/s3api/s3api_circuit_breaker.go

@ -87,7 +87,7 @@ func (cb *CircuitBreaker) loadCircuitBreakerConfig(cfg *s3_pb.S3CircuitBreakerCo
return nil
}
func (cb *CircuitBreaker) Limit(f func(w http.ResponseWriter, r *http.Request), action string) (http.HandlerFunc, Action) {
func (cb *CircuitBreaker) Limit(f http.HandlerFunc, action string) (http.HandlerFunc, Action) {
return func(w http.ResponseWriter, r *http.Request) {
if !cb.Enabled {
f(w, r)

81
weed/s3api/s3api_object_handlers.go

@ -3,19 +3,21 @@ package s3api
import (
"bytes"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3acl"
"github.com/seaweedfs/seaweedfs/weed/util/mem"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/util/mem"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
@ -115,6 +117,12 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
bucket, object := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("GetObjectHandler %s %s", bucket, object)
errCode := s3a.checkBucketAccessForReadObject(r, bucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
if strings.HasSuffix(r.URL.Path, "/") {
s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
return
@ -125,13 +133,56 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
s3a.proxyToFiler(w, r, destUrl, false, passThroughResponse)
}
func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
// PutObjectAclHandler Put object ACL
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjecthtml
func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) {
bucket, object := s3_constants.GetBucketAndObject(r)
objectEntry, ownerId, grants, errCode := s3a.checkAccessForWriteObjectAcl(r, bucket, object)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
errCode = s3acl.AssembleEntryWithAcp(objectEntry, ownerId, grants)
if errCode != s3err.ErrNone {
return
}
err := updateObjectEntry(s3a, bucket, object, objectEntry)
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
w.WriteHeader(http.StatusOK)
}
// GetObjectAclHandler Put object ACL
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjecthtml
func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Request) {
bucket, object := s3_constants.GetBucketAndObject(r)
acp, errCode := s3a.checkAccessForReadObjectAcl(r, bucket, object)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
result := &s3.PutBucketAclInput{
AccessControlPolicy: acp,
}
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, &result)
}
func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
bucket, object := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("HeadObjectHandler %s %s", bucket, object)
destUrl := s3a.toFilerUrl(bucket, object)
errCode := s3a.checkBucketAccessForReadObject(r, bucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
destUrl := s3a.toFilerUrl(bucket, object)
s3a.proxyToFiler(w, r, destUrl, false, passThroughResponse)
}
@ -167,6 +218,13 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
s3a.maybeAddFilerJwtAuthorization(proxyReq, isWrite)
resp, postErr := s3a.client.Do(proxyReq)
if resp.Uncompressed && r.Header.Get("Accept-Encoding") == "" {
r.Header.Set("Accept-Encoding", "gzip")
util_http.CloseResponse(resp)
s3a.proxyToFiler(w, r, destUrl, false, passThroughResponse)
return
}
if postErr != nil {
glog.Errorf("post to filer: %v", postErr)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
@ -174,14 +232,17 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
}
defer util_http.CloseResponse(resp)
if resp.StatusCode == http.StatusPreconditionFailed {
switch resp.StatusCode {
case http.StatusPreconditionFailed:
s3err.WriteErrorResponse(w, r, s3err.ErrPreconditionFailed)
return
}
if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
case http.StatusRequestedRangeNotSatisfiable:
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRange)
return
case http.StatusForbidden:
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
default:
}
if r.Method == http.MethodDelete {

5
weed/s3api/s3api_object_handlers_copy.go

@ -215,6 +215,11 @@ func processMetadata(reqHeader, existing http.Header, replaceMeta, replaceTaggin
}
}
// content-encoding
if contentEncoding, ok := existing["Content-Encoding"]; ok {
reqHeader["Content-Encoding"] = contentEncoding
}
if !replaceMeta {
for header, _ := range reqHeader {
if strings.HasPrefix(header, s3_constants.AmzUserMetaPrefix) {

14
weed/s3api/s3api_object_handlers_list.go

@ -52,8 +52,13 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("ListObjectsV2Handler %s", bucket)
originalPrefix, startAfter, delimiter, continuationToken, encodingTypeUrl, fetchOwner, maxKeys := getListObjectsV2Args(r.URL.Query())
_, errCode := s3a.checkAccessForReadBucket(r, bucket, s3_constants.PermissionRead)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
originalPrefix, startAfter, delimiter, continuationToken, encodingTypeUrl, fetchOwner, maxKeys := getListObjectsV2Args(r.URL.Query())
if maxKeys < 0 {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
return
@ -106,8 +111,13 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("ListObjectsV1Handler %s", bucket)
originalPrefix, marker, delimiter, encodingTypeUrl, maxKeys := getListObjectsV1Args(r.URL.Query())
_, errCode := s3a.checkAccessForReadBucket(r, bucket, s3_constants.PermissionRead)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
originalPrefix, marker, delimiter, encodingTypeUrl, maxKeys := getListObjectsV1Args(r.URL.Query())
if maxKeys < 0 {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
return

72
weed/s3api/s3api_object_handlers_multipart.go

@ -31,6 +31,12 @@ const (
func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
bucket, object := s3_constants.GetBucketAndObject(r)
errCode, initiatorId := s3a.CheckAccessForNewMultipartUpload(r, bucket, object)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
createMultipartUploadInput := &s3.CreateMultipartUploadInput{
Bucket: aws.String(bucket),
Key: objectKey(aws.String(object)),
@ -46,7 +52,7 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
if contentType != "" {
createMultipartUploadInput.ContentType = &contentType
}
response, errCode := s3a.createMultipartUpload(createMultipartUploadInput)
response, errCode := s3a.createMultipartUpload(initiatorId, createMultipartUploadInput)
glog.V(2).Info("NewMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)
@ -79,6 +85,12 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
return
}
errCode := s3a.CheckAccessForCompleteMultipartUpload(r, bucket, uploadID)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucket),
Key: objectKey(aws.String(object)),
@ -108,6 +120,24 @@ func (s3a *S3ApiServer) AbortMultipartUploadHandler(w http.ResponseWriter, r *ht
return
}
errCode := s3a.CheckAccessForAbortMultipartUpload(r, bucket, uploadID)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true)
if err != nil {
glog.V(1).Infof("list parts error: %v, request url: %s", err, r.RequestURI)
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
return
}
if !exists {
glog.V(1).Infof("list parts not found, request url: %s", r.RequestURI)
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
return
}
response, errCode := s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{
Bucket: aws.String(bucket),
Key: objectKey(aws.String(object)),
@ -144,7 +174,12 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht
}
}
response, errCode := s3a.listMultipartUploads(&s3.ListMultipartUploadsInput{
bucketMetaData, errorCode := s3a.checkAccessForReadBucket(r, bucket, s3_constants.PermissionRead)
if errorCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errorCode)
return
}
response, errCode := s3a.listMultipartUploads(bucketMetaData, &s3.ListMultipartUploadsInput{
Bucket: aws.String(bucket),
Delimiter: aws.String(delimiter),
EncodingType: aws.String(encodingType),
@ -186,6 +221,23 @@ func (s3a *S3ApiServer) ListObjectPartsHandler(w http.ResponseWriter, r *http.Re
return
}
errCode := s3a.CheckAccessForListMultipartUploadParts(r, bucket, uploadID)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true)
if err != nil {
glog.V(1).Infof("list parts error: %v, request url: %s", err, r.RequestURI)
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
return
}
if !exists {
glog.V(1).Infof("list parts not found, request url: %s", r.RequestURI)
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
return
}
response, errCode := s3a.listObjectParts(&s3.ListPartsInput{
Bucket: aws.String(bucket),
Key: objectKey(aws.String(object)),
@ -231,21 +283,31 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
if s3a.iam.isEnabled() {
rAuthType := getRequestAuthType(r)
var s3ErrCode s3err.ErrorCode
var identity *Identity
switch rAuthType {
case authTypeStreamingSigned:
dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r)
dataReader, identity, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r)
case authTypeSignedV2, authTypePresignedV2:
_, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)
identity, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)
case authTypePresigned, authTypeSigned:
_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
identity, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
}
if s3ErrCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, s3ErrCode)
return
}
if identity != nil && identity.Account.Id != AccountAnonymous.Id {
r.Header.Set(s3_constants.AmzAccountId, identity.Account.Id)
}
}
defer dataReader.Close()
errorCode := s3a.CheckAccessForPutObjectPartHandler(r, bucket)
if errorCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errorCode)
return
}
glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID)
uploadUrl := s3a.genPartUploadUrl(bucket, uploadID, partID)

36
weed/s3api/s3api_object_handlers_put.go

@ -4,18 +4,17 @@ import (
"crypto/md5"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/pquerna/cachecontrol/cacheobject"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/security"
"io"
"net/http"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
)
@ -50,18 +49,24 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
rAuthType := getRequestAuthType(r)
if s3a.iam.isEnabled() {
var s3ErrCode s3err.ErrorCode
var identity *Identity
switch rAuthType {
case authTypeStreamingSigned:
dataReader, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r)
dataReader, identity, s3ErrCode = s3a.iam.newSignV4ChunkedReader(r)
case authTypeSignedV2, authTypePresignedV2:
_, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)
identity, s3ErrCode = s3a.iam.isReqAuthenticatedV2(r)
case authTypePresigned, authTypeSigned:
_, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
identity, s3ErrCode = s3a.iam.reqSignatureV4Verify(r)
case authTypeAnonymous:
identity = s3a.iam.identityAnonymous
}
if s3ErrCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, s3ErrCode)
return
}
if identity != nil && identity.Account.Id != AccountAnonymous.Id {
r.Header.Set(s3_constants.AmzAccountId, identity.Account.Id)
}
} else {
if authTypeStreamingSigned == rAuthType {
s3err.WriteErrorResponse(w, r, s3err.ErrAuthNotSetup)
@ -70,16 +75,19 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
}
defer dataReader.Close()
errCode := s3a.CheckAccessForPutObject(r, bucket, object)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
objectContentType := r.Header.Get("Content-Type")
if strings.HasSuffix(object, "/") && r.ContentLength <= 1024 {
if strings.HasSuffix(object, "/") && r.ContentLength == 0 {
if err := s3a.mkdir(
s3a.option.BucketsPath, bucket+strings.TrimSuffix(object, "/"),
func(entry *filer_pb.Entry) {
if objectContentType == "" {
objectContentType = s3_constants.FolderMimeType
}
if r.ContentLength > 0 {
entry.Content, _ = io.ReadAll(r.Body)
objectContentType = "httpd/unix-directory"
}
entry.Attributes.Mime = objectContentType
}); err != nil {

16
weed/s3api/s3api_object_handlers_skip.go

@ -4,22 +4,6 @@ import (
"net/http"
)
// GetObjectAclHandler Put object ACL
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAcl.html
func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
// PutObjectAclHandler Put object ACL
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html
func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
// PutObjectRetentionHandler Put object Retention
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html
func (s3a *S3ApiServer) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) {

42
weed/s3api/s3api_server.go

@ -19,9 +19,9 @@ import (
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
"google.golang.org/grpc"
)
type S3ApiServerOption struct {
@ -162,19 +162,19 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
// objects with query
// CopyObjectPart
bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", `.*?(\/|%2F).*?`).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", `.*?(\/|%2F).*?`).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
// PutObjectPart
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.PutObjectPartHandler, ACTION_WRITE)), "PUT")).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
// CompleteMultipartUpload
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CompleteMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploadId", "{uploadId:.*}")
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.CompleteMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploadId", "{uploadId:.*}")
// NewMultipartUpload
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.NewMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploads", "")
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.NewMultipartUploadHandler, ACTION_WRITE)), "POST")).Queries("uploads", "")
// AbortMultipartUpload
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.AbortMultipartUploadHandler, ACTION_WRITE)), "DELETE")).Queries("uploadId", "{uploadId:.*}")
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.AbortMultipartUploadHandler, ACTION_WRITE)), "DELETE")).Queries("uploadId", "{uploadId:.*}")
// ListObjectParts
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectPartsHandler, ACTION_READ)), "GET")).Queries("uploadId", "{uploadId:.*}")
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.ListObjectPartsHandler, ACTION_READ)), "GET")).Queries("uploadId", "{uploadId:.*}")
// ListMultipartUploads
bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListMultipartUploadsHandler, ACTION_READ)), "GET")).Queries("uploads", "")
bucket.Methods("GET").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.ListMultipartUploadsHandler, ACTION_READ)), "GET")).Queries("uploads", "")
// GetObjectTagging
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectTaggingHandler, ACTION_READ)), "GET")).Queries("tagging", "")
@ -184,7 +184,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectTaggingHandler, ACTION_TAGGING)), "DELETE")).Queries("tagging", "")
// PutObjectACL
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectAclHandler, ACTION_WRITE_ACP)), "PUT")).Queries("acl", "")
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.PutObjectAclHandler, ACTION_WRITE_ACP)), "PUT")).Queries("acl", "")
// PutObjectRetention
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectRetentionHandler, ACTION_WRITE)), "PUT")).Queries("retention", "")
// PutObjectLegalHold
@ -193,22 +193,22 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectLockConfigurationHandler, ACTION_WRITE)), "PUT")).Queries("object-lock", "")
// GetObjectACL
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectAclHandler, ACTION_READ_ACP)), "GET")).Queries("acl", "")
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.GetObjectAclHandler, ACTION_READ_ACP)), "GET")).Queries("acl", "")
// objects with query
// raw objects
// HeadObject
bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.HeadObjectHandler, ACTION_READ)), "GET"))
bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.HeadObjectHandler, ACTION_READ)), "GET"))
// GetObject, but directory listing is not supported
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetObjectHandler, ACTION_READ)), "GET"))
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.GetObjectHandler, ACTION_READ)), "GET"))
// CopyObject
bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.CopyObjectHandler, ACTION_WRITE)), "COPY"))
// PutObject
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutObjectHandler, ACTION_WRITE)), "PUT"))
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.PutObjectHandler, ACTION_WRITE)), "PUT"))
// DeleteObject
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteObjectHandler, ACTION_WRITE)), "DELETE"))
@ -220,9 +220,9 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
bucket.Methods(http.MethodPost).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteMultipleObjectsHandler, ACTION_WRITE)), "DELETE")).Queries("delete", "")
// GetBucketACL
bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketAclHandler, ACTION_READ_ACP)), "GET")).Queries("acl", "")
bucket.Methods("GET").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.GetBucketAclHandler, ACTION_READ_ACP)), "GET")).Queries("acl", "")
// PutBucketACL
bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketAclHandler, ACTION_WRITE_ACP)), "PUT")).Queries("acl", "")
bucket.Methods("PUT").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.PutBucketAclHandler, ACTION_WRITE_ACP)), "PUT")).Queries("acl", "")
// GetBucketPolicy
bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.GetBucketPolicyHandler, ACTION_READ)), "GET")).Queries("policy", "")
@ -256,17 +256,17 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketVersioningHandler, ACTION_WRITE)), "PUT")).Queries("versioning", "")
// ListObjectsV2
bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2")
bucket.Methods("GET").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2")
// buckets with query
// PutBucketOwnershipControls
bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN), "PUT")).Queries("ownershipControls", "")
bucket.Methods("PUT").HandlerFunc(track(s3a.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN, true), "PUT")).Queries("ownershipControls", "")
//GetBucketOwnershipControls
bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketOwnershipControls, ACTION_READ), "GET")).Queries("ownershipControls", "")
bucket.Methods("GET").HandlerFunc(track(s3a.Auth(s3a.GetBucketOwnershipControls, ACTION_READ, true), "GET")).Queries("ownershipControls", "")
//DeleteBucketOwnershipControls
bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN), "DELETE")).Queries("ownershipControls", "")
bucket.Methods("DELETE").HandlerFunc(track(s3a.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN, true), "DELETE")).Queries("ownershipControls", "")
// raw buckets
@ -280,10 +280,10 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
bucket.Methods(http.MethodPut).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketHandler, ACTION_ADMIN)), "PUT"))
// DeleteBucket
bucket.Methods(http.MethodDelete).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_DELETE_BUCKET)), "DELETE"))
bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_DELETE_BUCKET)), "DELETE"))
// ListObjectsV1 (Legacy)
bucket.Methods(http.MethodGet).HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST"))
bucket.Methods("GET").HandlerFunc(track(s3a.AuthWithAcl(s3a.cb.Limit(s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST"))
// raw buckets

25
weed/s3api/s3err/s3api_errors.go

@ -109,6 +109,10 @@ const (
ErrRequestBytesExceed
OwnershipControlsNotFoundError
InvalidBucketAclWithObjectOwnership
AccessControlListNotSupported
ErrUnexpectedContent
ErrInvalidAclArgument
)
// error code to APIError structure, these fields carry respective
@ -416,12 +420,31 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "Simultaneous request bytes exceed limitations",
HTTPStatusCode: http.StatusTooManyRequests,
},
OwnershipControlsNotFoundError: {
Code: "OwnershipControlsNotFoundError",
Description: "The bucket ownership controls were not found",
HTTPStatusCode: http.StatusNotFound,
},
InvalidBucketAclWithObjectOwnership: {
Code: "InvalidBucketAclWithObjectOwnership",
Description: "Bucket cannot have ACLs set with ObjectOwnership's BucketOwnerEnforced setting",
HTTPStatusCode: http.StatusBadRequest,
},
AccessControlListNotSupported: {
Code: "AccessControlListNotSupported",
Description: "The bucket does not allow ACLs",
HTTPStatusCode: http.StatusBadRequest,
},
ErrUnexpectedContent: {
Code: "UnexpectedContent",
Description: "This request does not support content",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidAclArgument: {
Code: "InvalidArgument",
Description: "ACL argument is invalid",
HTTPStatusCode: http.StatusBadRequest,
},
}
// GetAPIError provides API Error for input API error code.

32
weed/server/filer_server_handlers_read.go

@ -7,6 +7,9 @@ import (
"encoding/hex"
"errors"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3acl"
"github.com/seaweedfs/seaweedfs/weed/util/mem"
"io"
"math"
"mime"
@ -16,14 +19,11 @@ import (
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util/mem"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/images"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
)
@ -105,7 +105,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
if err == filer_pb.ErrNotFound {
glog.V(2).Infof("Not found %s: %v", path, err)
stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadNotFound).Inc()
w.WriteHeader(http.StatusNotFound)
if r.Header.Get(s3_constants.XAmzBucketAccessDenied) == "true" {
w.WriteHeader(http.StatusForbidden)
} else {
w.WriteHeader(http.StatusNotFound)
}
} else {
glog.Errorf("Internal %s: %v", path, err)
stats.FilerHandlerCounter.WithLabelValues(stats.ErrorReadInternal).Inc()
@ -114,6 +118,15 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
return
}
//s3 acl offload to filer
offloadHeaderBucketOwner := r.Header.Get(s3_constants.XAmzBucketOwnerId)
if len(offloadHeaderBucketOwner) > 0 {
if statusCode, ok := s3acl.CheckObjectAccessForReadObject(r, w, entry, offloadHeaderBucketOwner); !ok {
w.WriteHeader(statusCode)
return
}
}
query := r.URL.Query()
if entry.IsDirectory() {
@ -193,10 +206,15 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
// print out the header from extended properties
for k, v := range entry.Extended {
if !strings.HasPrefix(k, "xattr-") {
if strings.HasPrefix(k, "xattr-") {
// "xattr-" prefix is set in filesys.XATTR_PREFIX
w.Header().Set(k, string(v))
continue
}
if strings.HasPrefix(k, "Seaweed-X-") {
// key with "Seaweed-X-" prefix is builtin and should not expose to user
continue
}
w.Header().Set(k, string(v))
}
//Seaweed custom header are not visible to Vue or javascript

6
weed/server/filer_server_handlers_write_autochunk.go

@ -261,7 +261,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
for k, v := range r.Header {
if len(v) > 0 && len(v[0]) > 0 {
if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" {
if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" || k == "Content-Encoding" || (k == "Content-Type" && v[0] != "application/octet-stream") {
entry.Extended[k] = []byte(v[0])
}
if k == "Response-Content-Disposition" {
@ -415,6 +415,10 @@ func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool
}
}
if ce := r.Header.Get("Content-Encoding"); ce != "" {
metadata["Content-Encoding"] = []byte(ce)
}
//acp-owner
acpOwner := r.Header.Get(s3_constants.ExtAmzOwnerKey)
if len(acpOwner) > 0 {

2
weed/shell/command_collection_delete.go

@ -10,7 +10,7 @@ import (
)
func init() {
Commands = append(Commands, &commandCollectionDelete{})
//Commands = append(Commands, &commandCollectionDelete{})
}
type commandCollectionDelete struct {

2
weed/shell/command_fs_mv.go

@ -10,7 +10,7 @@ import (
)
func init() {
Commands = append(Commands, &commandFsMv{})
//Commands = append(Commands, &commandFsMv{})
}
type commandFsMv struct {

2
weed/shell/command_fs_rm.go

@ -11,7 +11,7 @@ import (
)
func init() {
Commands = append(Commands, &commandFsRm{})
//Commands = append(Commands, &commandFsRm{})
}
type commandFsRm struct {

2
weed/shell/command_s3_bucket_delete.go

@ -11,7 +11,7 @@ import (
)
func init() {
Commands = append(Commands, &commandS3BucketDelete{})
//Commands = append(Commands, &commandS3BucketDelete{})
}
type commandS3BucketDelete struct {

2
weed/shell/command_volume_balance_test.go

@ -252,7 +252,7 @@ func TestIsGoodMove(t *testing.T) {
func TestBalance(t *testing.T) {
topologyInfo := parseOutput(topoData)
volumeServers := collectVolumeServersByDc(topologyInfo, "")
volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo, nil)
diskTypes := collectVolumeDiskTypes(topologyInfo)
if err := balanceVolumeServers(nil, diskTypes, volumeReplicas, volumeServers, "ALL_COLLECTIONS", false); err != nil {

2
weed/shell/command_volume_delete.go

@ -9,7 +9,7 @@ import (
)
func init() {
Commands = append(Commands, &commandVolumeDelete{})
//Commands = append(Commands, &commandVolumeDelete{})
}
type commandVolumeDelete struct {

17
weed/util/http/http_global_client_util.go

@ -5,8 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/util/mem"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/mem"
"io"
"net/http"
"net/url"
@ -418,7 +418,11 @@ func CloseResponse(resp *http.Response) {
io.Copy(io.Discard, reader)
resp.Body.Close()
if reader.BytesRead > 0 {
glog.V(1).Infof("response leftover %d bytes", reader.BytesRead)
if resp.Request != nil && resp.Request.URL != nil {
glog.V(1).Infof("response leftover %d bytes, url: %s", reader.BytesRead, resp.Request.URL.RequestURI())
} else {
glog.V(1).Infof("response leftover %d bytes", reader.BytesRead)
}
}
}
@ -427,7 +431,12 @@ func CloseRequest(req *http.Request) {
io.Copy(io.Discard, reader)
req.Body.Close()
if reader.BytesRead > 0 {
glog.V(1).Infof("request leftover %d bytes", reader.BytesRead)
if req.URL != nil {
glog.V(1).Infof("request leftover %d bytes, url: %s", reader.BytesRead, req.URL.RequestURI())
} else {
glog.V(1).Infof("request leftover %d bytes", reader.BytesRead)
}
}
}
@ -477,4 +486,4 @@ func RetriedFetchChunkData(buffer []byte, urlStrings []string, cipherKey []byte,
return n, err
}
}
Loading…
Cancel
Save