diff --git a/Makefile b/Makefile index 315a62428..6ca6cae0d 100644 --- a/Makefile +++ b/Makefile @@ -10,5 +10,8 @@ install: full_install: cd weed; go install -tags "elastic gocdk sqlite ydb tikv" -tests: +server: install + weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json + +test: cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./... diff --git a/docker/compose/s3.json b/docker/compose/s3.json index 64dedb681..ce230863b 100644 --- a/docker/compose/s3.json +++ b/docker/compose/s3.json @@ -40,7 +40,10 @@ "List", "Tagging", "Write" - ] + ], + "account": { + "id": "testid" + } }, { "name": "s3_tests_alt", @@ -101,5 +104,12 @@ "Write" ] } - ] + ], + "accounts": [ + { + "id" : "testid", + "displayName": "M. Tester", + "emailAddress": "tester@ceph.com" + } + ] } \ No newline at end of file diff --git a/docker/compose/s3tests.conf b/docker/compose/s3tests.conf index 2bffe20d4..68d9ddeb7 100644 --- a/docker/compose/s3tests.conf +++ b/docker/compose/s3tests.conf @@ -18,10 +18,10 @@ bucket prefix = yournamehere-{random}- [s3 main] # main display_name set in vstart.sh -display_name = s3_tests +display_name = M. Tester # main user_idname set in vstart.sh -user_id = s3_tests +user_id = testid # main email set in vstart.sh email = tester@ceph.com diff --git a/weed/pb/iam.proto b/weed/pb/iam.proto index 1a6027292..99bb65ef2 100644 --- a/weed/pb/iam.proto +++ b/weed/pb/iam.proto @@ -16,13 +16,14 @@ service SeaweedIdentityAccessManagement { message S3ApiConfiguration { repeated Identity identities = 1; + repeated Account accounts = 2; } message Identity { string name = 1; repeated Credential credentials = 2; repeated string actions = 3; - string accountId = 4; + Account account = 4; } message Credential { @@ -32,6 +33,12 @@ message Credential { // bool is_disabled = 4; } +message Account { + string id = 1; + string display_name = 2; + string email_address = 3; +} + /* message Policy { repeated Statement statements = 1; diff --git a/weed/pb/iam_pb/iam.pb.go b/weed/pb/iam_pb/iam.pb.go index e94454e47..074e255e6 100644 --- a/weed/pb/iam_pb/iam.pb.go +++ b/weed/pb/iam_pb/iam.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 -// protoc v3.21.12 +// protoc-gen-go v1.30.0 +// protoc v4.23.2 // source: iam.proto package iam_pb @@ -26,6 +26,7 @@ type S3ApiConfiguration struct { unknownFields protoimpl.UnknownFields Identities []*Identity `protobuf:"bytes,1,rep,name=identities,proto3" json:"identities,omitempty"` + Accounts []*Account `protobuf:"bytes,2,rep,name=accounts,proto3" json:"accounts,omitempty"` } func (x *S3ApiConfiguration) Reset() { @@ -67,6 +68,13 @@ func (x *S3ApiConfiguration) GetIdentities() []*Identity { return nil } +func (x *S3ApiConfiguration) GetAccounts() []*Account { + if x != nil { + return x.Accounts + } + return nil +} + type Identity struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -75,7 +83,7 @@ type Identity struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Credentials []*Credential `protobuf:"bytes,2,rep,name=credentials,proto3" json:"credentials,omitempty"` Actions []string `protobuf:"bytes,3,rep,name=actions,proto3" json:"actions,omitempty"` - AccountId string `protobuf:"bytes,4,opt,name=accountId,proto3" json:"accountId,omitempty"` + Account *Account `protobuf:"bytes,4,opt,name=account,proto3" json:"account,omitempty"` } func (x *Identity) Reset() { @@ -131,11 +139,11 @@ func (x *Identity) GetActions() []string { return nil } -func (x *Identity) GetAccountId() string { +func (x *Identity) GetAccount() *Account { if x != nil { - return x.AccountId + return x.Account } - return "" + return nil } type Credential struct { @@ -193,36 +201,109 @@ func (x *Credential) GetSecretKey() string { return "" } +type Account struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + DisplayName string `protobuf:"bytes,2,opt,name=display_name,json=displayName,proto3" json:"display_name,omitempty"` + EmailAddress string `protobuf:"bytes,3,opt,name=email_address,json=emailAddress,proto3" json:"email_address,omitempty"` +} + +func (x *Account) Reset() { + *x = Account{} + if protoimpl.UnsafeEnabled { + mi := &file_iam_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Account) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Account) ProtoMessage() {} + +func (x *Account) ProtoReflect() protoreflect.Message { + mi := &file_iam_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Account.ProtoReflect.Descriptor instead. +func (*Account) Descriptor() ([]byte, []int) { + return file_iam_proto_rawDescGZIP(), []int{3} +} + +func (x *Account) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Account) GetDisplayName() string { + if x != nil { + return x.DisplayName + } + return "" +} + +func (x *Account) GetEmailAddress() string { + if x != nil { + return x.EmailAddress + } + return "" +} + var File_iam_proto protoreflect.FileDescriptor var file_iam_proto_rawDesc = []byte{ 0x0a, 0x09, 0x69, 0x61, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x69, 0x61, 0x6d, - 0x5f, 0x70, 0x62, 0x22, 0x46, 0x0a, 0x12, 0x53, 0x33, 0x41, 0x70, 0x69, 0x43, 0x6f, 0x6e, 0x66, + 0x5f, 0x70, 0x62, 0x22, 0x73, 0x0a, 0x12, 0x53, 0x33, 0x41, 0x70, 0x69, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, - 0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x8c, 0x01, 0x0a, 0x08, - 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b, - 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, - 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x4a, 0x0a, 0x0a, 0x43, 0x72, - 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, - 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, - 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65, - 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x63, - 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, - 0x64, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65, 0x61, - 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08, 0x49, - 0x61, 0x6d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, - 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, - 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x2b, 0x0a, 0x08, 0x61, + 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, + 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x08, + 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x22, 0x99, 0x01, 0x0a, 0x08, 0x49, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b, 0x63, 0x72, 0x65, + 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x12, + 0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x29, 0x0a, 0x07, 0x61, 0x63, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x69, 0x61, 0x6d, + 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x07, 0x61, 0x63, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x4a, 0x0a, 0x0a, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, + 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, + 0x22, 0x61, 0x0a, 0x07, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x64, + 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x64, 0x69, 0x73, 0x70, 0x6c, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, + 0x0a, 0x0d, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x41, 0x64, 0x64, 0x72, + 0x65, 0x73, 0x73, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x49, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, + 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08, 0x49, 0x61, 0x6d, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, + 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x69, 0x61, 0x6d, + 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -237,20 +318,23 @@ func file_iam_proto_rawDescGZIP() []byte { return file_iam_proto_rawDescData } -var file_iam_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_iam_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_iam_proto_goTypes = []interface{}{ (*S3ApiConfiguration)(nil), // 0: iam_pb.S3ApiConfiguration (*Identity)(nil), // 1: iam_pb.Identity (*Credential)(nil), // 2: iam_pb.Credential + (*Account)(nil), // 3: iam_pb.Account } var file_iam_proto_depIdxs = []int32{ 1, // 0: iam_pb.S3ApiConfiguration.identities:type_name -> iam_pb.Identity - 2, // 1: iam_pb.Identity.credentials:type_name -> iam_pb.Credential - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 3, // 1: iam_pb.S3ApiConfiguration.accounts:type_name -> iam_pb.Account + 2, // 2: iam_pb.Identity.credentials:type_name -> iam_pb.Credential + 3, // 3: iam_pb.Identity.account:type_name -> iam_pb.Account + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_iam_proto_init() } @@ -295,6 +379,18 @@ func file_iam_proto_init() { return nil } } + file_iam_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Account); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -302,7 +398,7 @@ func file_iam_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_iam_proto_rawDesc, NumEnums: 0, - NumMessages: 3, + NumMessages: 4, NumExtensions: 0, NumServices: 1, }, diff --git a/weed/pb/iam_pb/iam_grpc.pb.go b/weed/pb/iam_pb/iam_grpc.pb.go index ea4e1bb41..c4fe7becc 100644 --- a/weed/pb/iam_pb/iam_grpc.pb.go +++ b/weed/pb/iam_pb/iam_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.12 +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.23.2 // source: iam.proto package iam_pb @@ -15,6 +15,8 @@ import ( // Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const () + // SeaweedIdentityAccessManagementClient is the client API for SeaweedIdentityAccessManagement service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index f79448dcc..e7fff0dce 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -2,7 +2,6 @@ package s3api import ( "fmt" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "net/http" "os" "strings" @@ -17,11 +16,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" ) -var IdentityAnonymous = &Identity{ - Name: s3account.AccountAnonymous.Name, - AccountId: s3account.AccountAnonymous.Id, -} - type Action string type Iam interface { @@ -31,27 +25,64 @@ type Iam interface { type IdentityAccessManagement struct { m sync.RWMutex - identities []*Identity - isAuthEnabled bool - domain string + identities []*Identity + accessKeyIdent map[string]*Identity + accounts map[string]*Account + emailAccount map[string]*Account + hashes map[string]*sync.Pool + hashCounters map[string]*int32 + identityAnonymous *Identity + hashMu sync.RWMutex + domain string + isAuthEnabled bool } type Identity struct { Name string - AccountId string + Account *Account Credentials []*Credential Actions []Action } -func (i *Identity) isAnonymous() bool { - return i.Name == s3account.AccountAnonymous.Name +// Account represents a system user, a system user can +// configure multiple IAM-Users, IAM-Users can configure +// permissions respectively, and each IAM-User can +// configure multiple security credentials +type Account struct { + //Name is also used to display the "DisplayName" as the owner of the bucket or object + DisplayName string + EmailAddress string + + //Id is used to identify an Account when granting cross-account access(ACLs) to buckets and objects + Id string } +// Predefined Accounts +var ( + // AccountAdmin is used as the default account for IAM-Credentials access without Account configured + AccountAdmin = Account{ + DisplayName: "admin", + EmailAddress: "admin@example.com", + Id: s3_constants.AccountAdminId, + } + + // AccountAnonymous is used to represent the account for anonymous access + AccountAnonymous = Account{ + DisplayName: "anonymous", + EmailAddress: "anonymous@example.com", + Id: s3_constants.AccountAnonymousId, + } +) + type Credential struct { AccessKey string SecretKey string } +func (i *Identity) isAnonymous() bool { + return i.Account.Id == s3_constants.AccountAnonymousId +} + func (action Action) isAdmin() bool { return strings.HasPrefix(string(action), s3_constants.ACTION_ADMIN) } @@ -64,14 +95,19 @@ func (action Action) overBucket(bucket string) bool { return strings.HasSuffix(string(action), ":"+bucket) || strings.HasSuffix(string(action), ":*") } +// "Permission": "FULL_CONTROL"|"WRITE"|"WRITE_ACP"|"READ"|"READ_ACP" func (action Action) getPermission() Permission { switch act := strings.Split(string(action), ":")[0]; act { case s3_constants.ACTION_ADMIN: return Permission("FULL_CONTROL") case s3_constants.ACTION_WRITE: return Permission("WRITE") + case s3_constants.ACTION_WRITE_ACP: + return Permission("WRITE_ACP") case s3_constants.ACTION_READ: return Permission("READ") + case s3_constants.ACTION_READ_ACP: + return Permission("READ_ACP") default: return Permission("") } @@ -79,7 +115,9 @@ func (action Action) getPermission() Permission { func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement { iam := &IdentityAccessManagement{ - domain: option.DomainName, + domain: option.DomainName, + hashes: make(map[string]*sync.Pool), + hashCounters: make(map[string]*int32), } if option.Config != "" { if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil { @@ -133,26 +171,71 @@ func (iam *IdentityAccessManagement) LoadS3ApiConfigurationFromBytes(content []b func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3ApiConfiguration) error { var identities []*Identity + var identityAnonymous *Identity + accessKeyIdent := make(map[string]*Identity) + accounts := make(map[string]*Account) + emailAccount := make(map[string]*Account) + foundAccountAdmin := false + foundAccountAnonymous := false + + for _, account := range config.Accounts { + switch account.Id { + case AccountAdmin.Id: + AccountAdmin = Account{ + Id: account.Id, + DisplayName: account.DisplayName, + EmailAddress: account.EmailAddress, + } + accounts[account.Id] = &AccountAdmin + foundAccountAdmin = true + case AccountAnonymous.Id: + AccountAnonymous = Account{ + Id: account.Id, + DisplayName: account.DisplayName, + EmailAddress: account.EmailAddress, + } + accounts[account.Id] = &AccountAnonymous + foundAccountAnonymous = true + default: + t := Account{ + Id: account.Id, + DisplayName: account.DisplayName, + EmailAddress: account.EmailAddress, + } + accounts[account.Id] = &t + } + if account.EmailAddress != "" { + emailAccount[account.EmailAddress] = accounts[account.Id] + } + } + if !foundAccountAdmin { + accounts[AccountAdmin.Id] = &AccountAdmin + emailAccount[AccountAdmin.EmailAddress] = &AccountAdmin + } + if !foundAccountAnonymous { + accounts[AccountAnonymous.Id] = &AccountAnonymous + emailAccount[AccountAnonymous.EmailAddress] = &AccountAnonymous + } for _, ident := range config.Identities { t := &Identity{ Name: ident.Name, - AccountId: s3account.AccountAdmin.Id, Credentials: nil, Actions: nil, } - - if ident.Name == s3account.AccountAnonymous.Name { - if ident.AccountId != "" && ident.AccountId != s3account.AccountAnonymous.Id { - glog.Warningf("anonymous identity is associated with a non-anonymous account ID, the association is invalid") - } - t.AccountId = s3account.AccountAnonymous.Id - IdentityAnonymous = t - } else { - if len(ident.AccountId) > 0 { - t.AccountId = ident.AccountId + switch { + case ident.Name == AccountAnonymous.Id: + t.Account = &AccountAnonymous + identityAnonymous = t + case ident.Account == nil: + t.Account = &AccountAdmin + default: + if account, ok := accounts[ident.Account.Id]; ok { + t.Account = account + } else { + t.Account = &AccountAdmin + glog.Warningf("identity %s is associated with a non exist account ID, the association is invalid", ident.Name) } } - for _, action := range ident.Actions { t.Actions = append(t.Actions, Action(action)) } @@ -161,17 +244,22 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api AccessKey: cred.AccessKey, SecretKey: cred.SecretKey, }) + accessKeyIdent[cred.AccessKey] = t } identities = append(identities, t) } - iam.m.Lock() // atomically switch iam.identities = identities + iam.identityAnonymous = identityAnonymous + iam.accounts = accounts + iam.emailAccount = emailAccount + iam.accessKeyIdent = accessKeyIdent if !iam.isAuthEnabled { // one-directional, no toggling iam.isAuthEnabled = len(identities) > 0 } iam.m.Unlock() + return nil } @@ -180,14 +268,12 @@ func (iam *IdentityAccessManagement) isEnabled() bool { } func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) { - iam.m.RLock() defer iam.m.RUnlock() - for _, ident := range iam.identities { - for _, cred := range ident.Credentials { - // println("checking", ident.Name, cred.AccessKey) - if cred.AccessKey == accessKey { - return ident, cred, true + if ident, ok := iam.accessKeyIdent[accessKey]; ok { + for _, credential := range ident.Credentials { + if credential.AccessKey == accessKey { + return ident, credential, true } } } @@ -198,14 +284,30 @@ func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identi func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, found bool) { iam.m.RLock() defer iam.m.RUnlock() - for _, ident := range iam.identities { - if ident.isAnonymous() { - return ident, true - } + if iam.identityAnonymous != nil { + return iam.identityAnonymous, true } return nil, false } +func (iam *IdentityAccessManagement) GetAccountNameById(canonicalId string) string { + iam.m.RLock() + defer iam.m.RUnlock() + if account, ok := iam.accounts[canonicalId]; ok { + return account.DisplayName + } + return "" +} + +func (iam *IdentityAccessManagement) GetAccountIdByEmail(email string) string { + iam.m.RLock() + defer iam.m.RUnlock() + if account, ok := iam.emailAccount[email]; ok { + return account.Id + } + return "" +} + func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) http.HandlerFunc { return Auth(iam, nil, f, action, false) } @@ -243,6 +345,7 @@ func Auth(iam *IdentityAccessManagement, br *BucketRegistry, f http.HandlerFunc, } } +// check whether the request has valid access keys func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) { return authRequest(iam, nil, r, action, false) } @@ -286,8 +389,7 @@ func authRequest(iam *IdentityAccessManagement, br *BucketRegistry, r *http.Requ } } authType = "Anonymous" - identity = IdentityAnonymous - if len(identity.Actions) == 0 { + if identity, found = iam.lookupAnonymous(); !found || len(identity.Actions) == 0 { r.Header.Set(s3_constants.AmzAuthType, authType) return identity, s3err.ErrAccessDenied } @@ -302,7 +404,7 @@ func authRequest(iam *IdentityAccessManagement, br *BucketRegistry, r *http.Requ return identity, s3Err } - glog.V(3).Infof("user name: %v account id: %v actions: %v, action: %v", identity.Name, identity.AccountId, identity.Actions, action) + glog.V(3).Infof("user name: %v actions: %v, action: %v", identity.Name, identity.Actions, action) bucket, object := s3_constants.GetBucketAndObject(r) @@ -310,9 +412,8 @@ func authRequest(iam *IdentityAccessManagement, br *BucketRegistry, r *http.Requ return identity, s3err.ErrAccessDenied } - if !identity.isAnonymous() { - r.Header.Set(s3_constants.AmzAccountId, identity.AccountId) - } + r.Header.Set(s3_constants.AmzAccountId, identity.Account.Id) + return identity, s3err.ErrNone } diff --git a/weed/s3api/auth_credentials_test.go b/weed/s3api/auth_credentials_test.go index 1f0ffc1cc..5d1823537 100644 --- a/weed/s3api/auth_credentials_test.go +++ b/weed/s3api/auth_credentials_test.go @@ -2,14 +2,12 @@ package s3api import ( . "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/stretchr/testify/assert" "reflect" "testing" - jsonpb "google.golang.org/protobuf/encoding/protojson" - "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" + jsonpb "google.golang.org/protobuf/encoding/protojson" ) func TestIdentityListFileFormat(t *testing.T) { @@ -129,11 +127,22 @@ func TestCanDo(t *testing.T) { } type LoadS3ApiConfigurationTestCase struct { + pbAccount *iam_pb.Account pbIdent *iam_pb.Identity expectIdent *Identity } func TestLoadS3ApiConfiguration(t *testing.T) { + specifiedAccount := Account{ + Id: "specifiedAccountID", + DisplayName: "specifiedAccountName", + EmailAddress: "specifiedAccounEmail@example.com", + } + pbSpecifiedAccount := iam_pb.Account{ + Id: "specifiedAccountID", + DisplayName: "specifiedAccountName", + EmailAddress: "specifiedAccounEmail@example.com", + } testCases := map[string]*LoadS3ApiConfigurationTestCase{ "notSpecifyAccountId": { pbIdent: &iam_pb.Identity{ @@ -150,8 +159,8 @@ func TestLoadS3ApiConfiguration(t *testing.T) { }, }, expectIdent: &Identity{ - Name: "notSpecifyAccountId", - AccountId: s3account.AccountAdmin.Id, + Name: "notSpecifyAccountId", + Account: &AccountAdmin, Actions: []Action{ "Read", "Write", @@ -165,17 +174,18 @@ func TestLoadS3ApiConfiguration(t *testing.T) { }, }, "specifiedAccountID": { + pbAccount: &pbSpecifiedAccount, pbIdent: &iam_pb.Identity{ - Name: "specifiedAccountID", - AccountId: "specifiedAccountID", + Name: "specifiedAccountID", + Account: &pbSpecifiedAccount, Actions: []string{ "Read", "Write", }, }, expectIdent: &Identity{ - Name: "specifiedAccountID", - AccountId: "specifiedAccountID", + Name: "specifiedAccountID", + Account: &specifiedAccount, Actions: []Action{ "Read", "Write", @@ -191,8 +201,8 @@ func TestLoadS3ApiConfiguration(t *testing.T) { }, }, expectIdent: &Identity{ - Name: "anonymous", - AccountId: "anonymous", + Name: "anonymous", + Account: &AccountAnonymous, Actions: []Action{ "Read", "Write", @@ -206,6 +216,9 @@ func TestLoadS3ApiConfiguration(t *testing.T) { } for _, v := range testCases { config.Identities = append(config.Identities, v.pbIdent) + if v.pbAccount != nil { + config.Accounts = append(config.Accounts, v.pbAccount) + } } iam := IdentityAccessManagement{} @@ -217,7 +230,7 @@ func TestLoadS3ApiConfiguration(t *testing.T) { for _, ident := range iam.identities { tc := testCases[ident.Name] if !reflect.DeepEqual(ident, tc.expectIdent) { - t.Error("not expect") + t.Errorf("not expect for ident name %s", ident.Name) } } } diff --git a/weed/s3api/bucket_metadata.go b/weed/s3api/bucket_metadata.go index d3e47810d..fcf3d187d 100644 --- a/weed/s3api/bucket_metadata.go +++ b/weed/s3api/bucket_metadata.go @@ -6,7 +6,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/util" "math" @@ -19,7 +18,7 @@ var loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*B return nil, err } - return buildBucketMetadata(r.s3a.accountManager, entry), nil + return buildBucketMetadata(r.s3a.iam, entry), nil } type BucketMetaData struct { @@ -73,13 +72,13 @@ func (r *BucketRegistry) init() error { } func (r *BucketRegistry) LoadBucketMetadata(entry *filer_pb.Entry) { - bucketMetadata := buildBucketMetadata(r.s3a.accountManager, entry) + bucketMetadata := buildBucketMetadata(r.s3a.iam, entry) r.metadataCacheLock.Lock() defer r.metadataCacheLock.Unlock() r.metadataCache[entry.Name] = bucketMetadata } -func buildBucketMetadata(accountManager *s3account.AccountManager, entry *filer_pb.Entry) *BucketMetaData { +func buildBucketMetadata(accountManager AccountManager, entry *filer_pb.Entry) *BucketMetaData { entryJson, _ := json.Marshal(entry) glog.V(3).Infof("build bucket metadata,entry=%s", entryJson) bucketMetadata := &BucketMetaData{ @@ -90,8 +89,8 @@ func buildBucketMetadata(accountManager *s3account.AccountManager, entry *filer_ // Default owner: `AccountAdmin` Owner: &s3.Owner{ - ID: &s3account.AccountAdmin.Id, - DisplayName: &s3account.AccountAdmin.Name, + ID: &AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, }, } if entry.Extended != nil { @@ -111,8 +110,8 @@ func buildBucketMetadata(accountManager *s3account.AccountManager, entry *filer_ acpOwnerBytes, ok := entry.Extended[s3_constants.ExtAmzOwnerKey] if ok && len(acpOwnerBytes) > 0 { ownerAccountId := string(acpOwnerBytes) - ownerAccountName, exists := accountManager.IdNameMapping[ownerAccountId] - if !exists { + ownerAccountName := accountManager.GetAccountNameById(ownerAccountId) + if ownerAccountName == "" { glog.Warningf("owner[id=%s] is invalid, bucket: %s", ownerAccountId, bucketMetadata.Name) } else { bucketMetadata.Owner = &s3.Owner{ diff --git a/weed/s3api/bucket_metadata_test.go b/weed/s3api/bucket_metadata_test.go index 98c569dea..960f6d3ee 100644 --- a/weed/s3api/bucket_metadata_test.go +++ b/weed/s3api/bucket_metadata_test.go @@ -5,8 +5,8 @@ import ( "fmt" "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "reflect" "sync" @@ -31,7 +31,7 @@ var ( Name: "entryWithValidAcp", Extended: map[string][]byte{ s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced), - s3_constants.ExtAmzOwnerKey: []byte(s3account.AccountAdmin.Name), + s3_constants.ExtAmzOwnerKey: []byte(AccountAdmin.DisplayName), s3_constants.ExtAmzAclKey: goodEntryAcl, }, } @@ -88,8 +88,8 @@ var tcs = []*BucketMetadataTestCase{ Name: badEntry.Name, ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: nil, }, @@ -99,8 +99,8 @@ var tcs = []*BucketMetadataTestCase{ Name: goodEntry.Name, ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: s3_constants.PublicRead, }, @@ -110,8 +110,8 @@ var tcs = []*BucketMetadataTestCase{ Name: ownershipEmptyStr.Name, ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: []*s3.Grant{ { @@ -129,8 +129,8 @@ var tcs = []*BucketMetadataTestCase{ Name: ownershipValid.Name, ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: []*s3.Grant{ { @@ -148,8 +148,8 @@ var tcs = []*BucketMetadataTestCase{ Name: acpEmptyStr.Name, ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: []*s3.Grant{ { @@ -167,8 +167,8 @@ var tcs = []*BucketMetadataTestCase{ Name: acpEmptyObject.Name, ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: []*s3.Grant{ { @@ -186,8 +186,8 @@ var tcs = []*BucketMetadataTestCase{ Name: acpOwnerNil.Name, ObjectOwnership: s3_constants.DefaultObjectOwnership, Owner: &s3.Owner{ - DisplayName: &s3account.AccountAdmin.Name, - ID: &s3account.AccountAdmin.Id, + DisplayName: &AccountAdmin.DisplayName, + ID: &AccountAdmin.Id, }, Acl: make([]*s3.Grant, 0), }, @@ -195,14 +195,10 @@ var tcs = []*BucketMetadataTestCase{ } func TestBuildBucketMetadata(t *testing.T) { - accountManager := &s3account.AccountManager{ - IdNameMapping: map[string]string{ - s3account.AccountAdmin.Id: s3account.AccountAdmin.Name, - s3account.AccountAnonymous.Id: s3account.AccountAnonymous.Name, - }, - } + iam := &IdentityAccessManagement{} + _ = iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{}) for _, tc := range tcs { - resultBucketMetadata := buildBucketMetadata(accountManager, tc.filerEntry) + resultBucketMetadata := buildBucketMetadata(iam, tc.filerEntry) if !reflect.DeepEqual(resultBucketMetadata, tc.expectBucketMetadata) { t.Fatalf("result is unexpect: \nresult: %v, \nexpect: %v", resultBucketMetadata, tc.expectBucketMetadata) } diff --git a/weed/s3api/s3_constants/s3_acp.go b/weed/s3api/s3_constants/s3_acp.go new file mode 100644 index 000000000..d24e07e24 --- /dev/null +++ b/weed/s3api/s3_constants/s3_acp.go @@ -0,0 +1,6 @@ +package s3_constants + +const ( + AccountAnonymousId = "anonymous" + AccountAdminId = "admin" +) diff --git a/weed/s3api/s3account/s3_account.go b/weed/s3api/s3account/s3_account.go deleted file mode 100644 index 9b1b01123..000000000 --- a/weed/s3api/s3account/s3_account.go +++ /dev/null @@ -1,70 +0,0 @@ -package s3account - -import ( - "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" - "sync" -) - -//Predefined Accounts -var ( - // AccountAdmin is used as the default account for IAM-Credentials access without Account configured - AccountAdmin = Account{ - Name: "admin", - EmailAddress: "admin@example.com", - Id: "admin", - } - - // AccountAnonymous is used to represent the account for anonymous access - AccountAnonymous = Account{ - Name: "anonymous", - EmailAddress: "anonymous@example.com", - Id: "anonymous", - } -) - -//Account represents a system user, a system user can -//configure multiple IAM-Users, IAM-Users can configure -//permissions respectively, and each IAM-User can -//configure multiple security credentials -type Account struct { - //Name is also used to display the "DisplayName" as the owner of the bucket or object - Name string - EmailAddress string - - //Id is used to identify an Account when granting cross-account access(ACLs) to buckets and objects - Id string -} - -type AccountManager struct { - sync.Mutex - filerClient filer_pb.FilerClient - - IdNameMapping map[string]string - EmailIdMapping map[string]string -} - -func NewAccountManager(filerClient filer_pb.FilerClient) *AccountManager { - am := &AccountManager{ - filerClient: filerClient, - IdNameMapping: make(map[string]string), - EmailIdMapping: make(map[string]string), - } - am.initialize() - return am -} - -func (am *AccountManager) GetAccountNameById(canonicalId string) string { - return am.IdNameMapping[canonicalId] -} - -func (am *AccountManager) GetAccountIdByEmail(email string) string { - return am.EmailIdMapping[email] -} - -func (am *AccountManager) initialize() { - // load predefined Accounts - for _, account := range []Account{AccountAdmin, AccountAnonymous} { - am.IdNameMapping[account.Id] = account.Name - am.EmailIdMapping[account.EmailAddress] = account.Id - } -} diff --git a/weed/s3api/s3acl/acl_helper.go b/weed/s3api/s3api_acl_helper.go similarity index 95% rename from weed/s3api/s3acl/acl_helper.go rename to weed/s3api/s3api_acl_helper.go index 1e8ee9524..e68e49b78 100644 --- a/weed/s3api/s3acl/acl_helper.go +++ b/weed/s3api/s3api_acl_helper.go @@ -1,4 +1,4 @@ -package s3acl +package s3api import ( "encoding/json" @@ -10,7 +10,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/util" "net/http" @@ -19,11 +18,16 @@ import ( var customAclHeaders = []string{s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp} +type AccountManager interface { + GetAccountNameById(canonicalId string) string + GetAccountIdByEmail(email string) string +} + // GetAccountId get AccountId from request headers, AccountAnonymousId will be return if not presen func GetAccountId(r *http.Request) string { id := r.Header.Get(s3_constants.AmzAccountId) if len(id) == 0 { - return s3account.AccountAnonymous.Id + return s3_constants.AccountAnonymousId } else { return id } @@ -40,7 +44,7 @@ func ValidateAccount(requestAccountId string, allowedAccounts ...string) bool { } // ExtractBucketAcl extracts the acl from the request body, or from the header if request body is empty -func ExtractBucketAcl(r *http.Request, accountManager *s3account.AccountManager, objectOwnership, bucketOwnerId, requestAccountId string, createBucket bool) (grants []*s3.Grant, errCode s3err.ErrorCode) { +func ExtractBucketAcl(r *http.Request, accountManager AccountManager, objectOwnership, bucketOwnerId, requestAccountId string, createBucket bool) (grants []*s3.Grant, errCode s3err.ErrorCode) { cannedAclPresent := false if r.Header.Get(s3_constants.AmzCannedAcl) != "" { cannedAclPresent = true @@ -96,7 +100,7 @@ func ExtractBucketAcl(r *http.Request, accountManager *s3account.AccountManager, } // ExtractObjectAcl extracts the acl from the request body, or from the header if request body is empty -func ExtractObjectAcl(r *http.Request, accountManager *s3account.AccountManager, objectOwnership, bucketOwnerId, requestAccountId string, createObject bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) { +func ExtractObjectAcl(r *http.Request, accountManager AccountManager, objectOwnership, bucketOwnerId, requestAccountId string, createObject bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) { cannedAclPresent := false if r.Header.Get(s3_constants.AmzCannedAcl) != "" { cannedAclPresent = true @@ -342,8 +346,8 @@ func ExtractObjectCannedAcl(request *http.Request, objectOwnership, bucketOwnerI return } -// ValidateAndTransferGrants validate grant entity exists and transfer Email-Grant to Id-Grant -func ValidateAndTransferGrants(accountManager *s3account.AccountManager, grants []*s3.Grant) ([]*s3.Grant, s3err.ErrorCode) { +// ValidateAndTransferGrants validate grant & transfer Email-Grant to Id-Grant +func ValidateAndTransferGrants(accountManager AccountManager, grants []*s3.Grant) ([]*s3.Grant, s3err.ErrorCode) { var result []*s3.Grant for _, grant := range grants { grantee := grant.Grantee @@ -369,8 +373,8 @@ func ValidateAndTransferGrants(accountManager *s3account.AccountManager, grants glog.Warning("invalid canonical grantee! account id is nil") return nil, s3err.ErrInvalidRequest } - _, ok := accountManager.IdNameMapping[*grantee.ID] - if !ok { + name := accountManager.GetAccountNameById(*grantee.ID) + if len(name) == 0 { glog.Warningf("invalid canonical grantee! account id[%s] is not exists", *grantee.ID) return nil, s3err.ErrInvalidRequest } @@ -380,8 +384,8 @@ func ValidateAndTransferGrants(accountManager *s3account.AccountManager, grants glog.Warning("invalid email grantee! email address is nil") return nil, s3err.ErrInvalidRequest } - accountId, ok := accountManager.EmailIdMapping[*grantee.EmailAddress] - if !ok { + accountId := accountManager.GetAccountIdByEmail(*grantee.EmailAddress) + if len(accountId) == 0 { glog.Warningf("invalid email grantee! email address[%s] is not exists", *grantee.EmailAddress) return nil, s3err.ErrInvalidRequest } @@ -462,7 +466,7 @@ func DetermineRequiredGrants(accountId, permission string) (grants []*s3.Grant) }) // group grantee (AuthenticateUsers) - if accountId != s3account.AccountAnonymous.Id { + if accountId != s3_constants.AccountAnonymousId { grants = append(grants, &s3.Grant{ Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, diff --git a/weed/s3api/s3acl/acl_helper_test.go b/weed/s3api/s3api_acl_helper_test.go similarity index 80% rename from weed/s3api/s3acl/acl_helper_test.go rename to weed/s3api/s3api_acl_helper_test.go index ebf253851..58f5ac5d8 100644 --- a/weed/s3api/s3acl/acl_helper_test.go +++ b/weed/s3api/s3api_acl_helper_test.go @@ -1,34 +1,38 @@ -package s3acl +package s3api import ( "bytes" "encoding/json" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "io" "net/http" "testing" ) -var ( - accountManager = &s3account.AccountManager{ - IdNameMapping: map[string]string{ - s3account.AccountAdmin.Id: s3account.AccountAdmin.Name, - s3account.AccountAnonymous.Id: s3account.AccountAnonymous.Name, - "accountA": "accountA", - "accountB": "accountB", - }, - EmailIdMapping: map[string]string{ - s3account.AccountAdmin.EmailAddress: s3account.AccountAdmin.Id, - s3account.AccountAnonymous.EmailAddress: s3account.AccountAnonymous.Id, - "accountA@example.com": "accountA", - "accountBexample.com": "accountB", +var accountManager *IdentityAccessManagement + +func init() { + accountManager = &IdentityAccessManagement{} + _ = accountManager.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{ + Accounts: []*iam_pb.Account{ + { + Id: "accountA", + DisplayName: "accountAName", + EmailAddress: "accountA@example.com", + }, + { + Id: "accountB", + DisplayName: "accountBName", + EmailAddress: "accountB@example.com", + }, }, - } -) + }) +} func TestGetAccountId(t *testing.T) { req := &http.Request{ @@ -36,26 +40,345 @@ func TestGetAccountId(t *testing.T) { } //case1 //accountId: "admin" - req.Header.Set(s3_constants.AmzAccountId, s3account.AccountAdmin.Id) - if GetAccountId(req) != s3account.AccountAdmin.Id { + req.Header.Set(s3_constants.AmzAccountId, s3_constants.AccountAdminId) + if GetAccountId(req) != s3_constants.AccountAdminId { t.Fatal("expect accountId: admin") } //case2 //accountId: "anoymous" - req.Header.Set(s3_constants.AmzAccountId, s3account.AccountAnonymous.Id) - if GetAccountId(req) != s3account.AccountAnonymous.Id { + req.Header.Set(s3_constants.AmzAccountId, s3_constants.AccountAnonymousId) + if GetAccountId(req) != s3_constants.AccountAnonymousId { t.Fatal("expect accountId: anonymous") } //case3 //accountId is nil => "anonymous" req.Header.Del(s3_constants.AmzAccountId) - if GetAccountId(req) != s3account.AccountAnonymous.Id { + if GetAccountId(req) != s3_constants.AccountAnonymousId { t.Fatal("expect accountId: anonymous") } } +func TestExtractAcl(t *testing.T) { + type Case struct { + id int + resultErrCode, expectErrCode s3err.ErrorCode + resultGrants, expectGrants []*s3.Grant + } + testCases := make([]*Case, 0) + accountAdminId := "admin" + { + //case1 (good case) + //parse acp from request body + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + objectWriter := "accountA" + grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter) + testCases = append(testCases, &Case{ + 1, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &accountAdminId, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + + { + //case2 (good case) + //parse acp from header (cannedAcl) + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = nil + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate) + objectWriter := "accountA" + grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter) + testCases = append(testCases, &Case{ + 2, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &objectWriter, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + + { + //case3 (bad case) + //parse acp from request body (content is invalid) + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = io.NopCloser(bytes.NewReader([]byte("zdfsaf"))) + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate) + objectWriter := "accountA" + _, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter) + testCases = append(testCases, &Case{ + id: 3, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + //case4 (bad case) + //parse acp from header (cannedAcl is invalid) + req := &http.Request{ + Header: make(map[string][]string), + } + req.Body = nil + req.Header.Set(s3_constants.AmzCannedAcl, "dfaksjfk") + objectWriter := "accountA" + _, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, "", objectWriter) + testCases = append(testCases, &Case{ + id: 4, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + + { + //case5 (bad case) + //parse acp from request body: owner is inconsistent + req.Body = io.NopCloser(bytes.NewReader([]byte(` + + + admin + admin + + + + + admin + + FULL_CONTROL + + + + http://acs.amazonaws.com/groups/global/AllUsers + + FULL_CONTROL + + + + `))) + objectWriter = "accountA" + _, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, objectWriter, objectWriter) + testCases = append(testCases, &Case{ + id: 5, + resultErrCode: errCode, expectErrCode: s3err.ErrAccessDenied, + }) + } + + for _, tc := range testCases { + if tc.resultErrCode != tc.expectErrCode { + t.Fatalf("case[%d]: errorCode not expect", tc.id) + } + if !grantsEquals(tc.resultGrants, tc.expectGrants) { + t.Fatalf("case[%d]: grants not expect", tc.id) + } + } +} + +func TestParseAndValidateAclHeaders(t *testing.T) { + type Case struct { + id int + resultOwner, expectOwner string + resultErrCode, expectErrCode s3err.ErrorCode + resultGrants, expectGrants []*s3.Grant + } + testCases := make([]*Case, 0) + bucketOwner := "admin" + + { + //case1 (good case) + //parse custom acl + req := &http.Request{ + Header: make(map[string][]string), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example.com"`) + ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + 1, + ownerId, objectWriter, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeGroup, + URI: &s3_constants.GranteeGroupAllUsers, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: aws.String(s3_constants.AccountAnonymousId), + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: aws.String(s3_constants.AccountAdminId), + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //case2 (good case) + //parse canned acl (ownership=ObjectWriter) + req := &http.Request{ + Header: make(map[string][]string), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + 2, + ownerId, objectWriter, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &objectWriter, + }, + Permission: &s3_constants.PermissionFullControl, + }, + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwner, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //case3 (good case) + //parse canned acl (ownership=OwnershipBucketOwnerPreferred) + req := &http.Request{ + Header: make(map[string][]string), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl) + ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + 3, + ownerId, bucketOwner, + errCode, s3err.ErrNone, + grants, []*s3.Grant{ + { + Grantee: &s3.Grantee{ + Type: &s3_constants.GrantTypeCanonicalUser, + ID: &bucketOwner, + }, + Permission: &s3_constants.PermissionFullControl, + }, + }, + }) + } + { + //case4 (bad case) + //parse custom acl (grantee id not exists) + req := &http.Request{ + Header: make(map[string][]string), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="notExistsAccount", emailAddress="admin@example.com"`) + _, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + id: 4, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + { + //case5 (bad case) + //parse custom acl (invalid format) + req := &http.Request{ + Header: make(map[string][]string), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzAclFullControl, `uri="http:sfasf"`) + _, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + id: 5, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + { + //case6 (bad case) + //parse canned acl (invalid value) + req := &http.Request{ + Header: make(map[string][]string), + } + objectWriter := "accountA" + req.Header.Set(s3_constants.AmzCannedAcl, `uri="http:sfasf"`) + _, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false) + testCases = append(testCases, &Case{ + id: 5, + resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest, + }) + } + + for _, tc := range testCases { + if tc.expectErrCode != tc.resultErrCode { + t.Errorf("case[%d]: errCode unexpect", tc.id) + } + if tc.resultOwner != tc.expectOwner { + t.Errorf("case[%d]: ownerId unexpect", tc.id) + } + if !grantsEquals(tc.resultGrants, tc.expectGrants) { + t.Fatalf("case[%d]: grants not expect", tc.id) + } + } +} + func grantsEquals(a, b []*s3.Grant) bool { if len(a) != len(b) { return false @@ -71,7 +394,7 @@ func grantsEquals(a, b []*s3.Grant) bool { func TestDetermineReqGrants(t *testing.T) { { //case1: request account is anonymous - accountId := s3account.AccountAnonymous.Id + accountId := s3_constants.AccountAnonymousId reqPermission := s3_constants.PermissionRead resultGrants := DetermineRequiredGrants(accountId, reqPermission) @@ -176,7 +499,7 @@ func TestAssembleEntryWithAcp(t *testing.T) { Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), URI: &s3_constants.GranteeGroupAllUsers, }, }, @@ -249,13 +572,13 @@ func TestGrantEquals(t *testing.T) { GrantEquals(&s3.Grant{ Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ - ID: &s3account.AccountAdmin.Id, - EmailAddress: &s3account.AccountAdmin.EmailAddress, + ID: aws.String(s3_constants.AccountAdminId), + //EmailAddress: &s3account.AccountAdmin.EmailAddress, }, }, &s3.Grant{ Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), }, }): true, @@ -303,13 +626,13 @@ func TestGrantEquals(t *testing.T) { Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), }, }, &s3.Grant{ Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), }, }): true, @@ -317,14 +640,14 @@ func TestGrantEquals(t *testing.T) { Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), URI: &s3_constants.GranteeGroupAllUsers, }, }, &s3.Grant{ Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), }, }): false, @@ -332,7 +655,7 @@ func TestGrantEquals(t *testing.T) { Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), URI: &s3_constants.GranteeGroupAllUsers, }, }, &s3.Grant{ @@ -372,7 +695,7 @@ func TestSetAcpGrantsHeader(t *testing.T) { Permission: &s3_constants.PermissionRead, Grantee: &s3.Grantee{ Type: &s3_constants.GrantTypeGroup, - ID: &s3account.AccountAdmin.Id, + ID: aws.String(s3_constants.AccountAdminId), URI: &s3_constants.GranteeGroupAllUsers, }, }, diff --git a/weed/s3api/s3api_acp.go b/weed/s3api/s3api_acp.go index bed718055..6fe2b624a 100644 --- a/weed/s3api/s3api_acp.go +++ b/weed/s3api/s3api_acp.go @@ -5,8 +5,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" "github.com/seaweedfs/seaweedfs/weed/util" "net/http" @@ -16,7 +14,7 @@ import ( func getAccountId(r *http.Request) string { id := r.Header.Get(s3_constants.AmzAccountId) if len(id) == 0 { - return s3account.AccountAnonymous.Id + return AccountAnonymous.Id } else { return id } diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 1fd212d7f..fd049fd11 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -6,8 +6,6 @@ import ( "errors" "fmt" "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3acl" "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket" "github.com/seaweedfs/seaweedfs/weed/util" "math" diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 40783ea73..fb99afd30 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3account" "net" "net/http" "strings" @@ -41,7 +40,6 @@ type S3ApiServer struct { randomClientId int32 filerGuard *security.Guard client *http.Client - accountManager *s3account.AccountManager bucketRegistry *BucketRegistry } @@ -62,7 +60,6 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer filerGuard: security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec), cb: NewCircuitBreaker(option), } - s3ApiServer.accountManager = s3account.NewAccountManager(s3ApiServer) s3ApiServer.bucketRegistry = NewBucketRegistry(s3ApiServer) if option.LocalFilerSocket == "" { s3ApiServer.client = &http.Client{Transport: &http.Transport{