From f41925b60bd066048217a6de23185a6e6cfb75a7 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 14 Dec 2025 16:02:06 -0800 Subject: [PATCH] Embed IAM API into S3 server (#7740) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Embed IAM API into S3 server This change simplifies the S3 and IAM deployment by embedding the IAM API directly into the S3 server, following the patterns used by MinIO and Ceph RGW. Changes: - Add -iam flag to S3 server (enabled by default) - Create embedded IAM API handler in s3api package - Register IAM routes (POST to /) in S3 server when enabled - Deprecate standalone 'weed iam' command with warning Benefits: - Single binary, single port for both S3 and IAM APIs - Simpler deployment and configuration - Shared credential manager between S3 and IAM - Backward compatible: 'weed iam' still works with deprecation warning Usage: - weed s3 -port=8333 # S3 + IAM on same port (default) - weed s3 -iam=false # S3 only, disable embedded IAM - weed iam -port=8111 # Deprecated, shows warning * Fix nil pointer panic: add s3.iam flag to weed server command The enableIam field was not initialized when running S3 via 'weed server', causing a nil pointer dereference when checking *s3opt.enableIam. * Fix nil pointer panic: add s3.iam flag to weed filer command The enableIam field was not initialized when running S3 via 'weed filer -s3', causing a nil pointer dereference when checking *s3opt.enableIam. * Add integration tests for embedded IAM API Tests cover: - CreateUser, ListUsers, GetUser, UpdateUser, DeleteUser - CreateAccessKey, DeleteAccessKey, ListAccessKeys - CreatePolicy, PutUserPolicy, GetUserPolicy - Implicit username extraction from authorization header - Full user lifecycle workflow test These tests validate the embedded IAM API functionality that was added in the S3 server, ensuring IAM operations work correctly when served from the same port as S3. * Security: Use crypto/rand for IAM credential generation SECURITY FIX: Replace math/rand with crypto/rand for generating access keys and secret keys. Using math/rand is not cryptographically secure and can lead to predictable credentials. This change: 1. Replaces math/rand with crypto/rand in both: - weed/s3api/s3api_embedded_iam.go (embedded IAM) - weed/iamapi/iamapi_management_handlers.go (standalone IAM) 2. Removes the seededRand variable that was initialized with time-based seed (predictable) 3. Updates StringWithCharset/iamStringWithCharset to: - Use crypto/rand.Int() for secure random index generation - Return an error for proper error handling 4. Updates CreateAccessKey to handle the new error return 5. Updates DoActions handlers to propagate errors properly * Fix critical bug: DeleteUserPolicy was deleting entire user instead of policy BUG FIX: DeleteUserPolicy was incorrectly deleting the entire user identity from s3cfg.Identities instead of just clearing the user's inline policy (Actions). Before (wrong): s3cfg.Identities = append(s3cfg.Identities[:i], s3cfg.Identities[i+1:]...) After (correct): ident.Actions = nil Also: - Added proper iamDeleteUserPolicyResponse / DeleteUserPolicyResponse types - Fixed return type from iamPutUserPolicyResponse to iamDeleteUserPolicyResponse Affected files: - weed/s3api/s3api_embedded_iam.go (embedded IAM) - weed/iamapi/iamapi_management_handlers.go (standalone IAM) - weed/iamapi/iamapi_response.go (response types) * Add tests for DeleteUserPolicy to prevent regression Added two tests: 1. TestEmbeddedIamDeleteUserPolicy - Verifies that: - User is NOT deleted (identity still exists) - Credentials are NOT deleted - Only Actions (policy) are cleared to nil 2. TestEmbeddedIamDeleteUserPolicyUserNotFound - Verifies: - Returns 404 when user doesn't exist These tests ensure the bug fixed in the previous commit (deleting user instead of policy) doesn't regress. * Fix race condition: Add mutex lock to IAM DoActions The DoActions function performs a read-modify-write operation on the shared IAM configuration without any locking. This could lead to race conditions and data loss if multiple requests modify the IAM config concurrently. Added mutex lock at the start of DoActions in both: - weed/s3api/s3api_embedded_iam.go (embedded IAM) - weed/iamapi/iamapi_management_handlers.go (standalone IAM) The lock protects the entire read-modify-write cycle: 1. GetS3ApiConfiguration (read) 2. Modify s3cfg based on action 3. PutS3ApiConfiguration (write) * Fix action comparison and document CreatePolicy limitation 1. Replace reflect.DeepEqual with order-independent string slice comparison - Added iamStringSlicesEqual/stringSlicesEqual helper functions - Prevents duplicate policy statements when actions are in different order 2. Document CreatePolicy limitation in embedded IAM - Added TODO comment explaining that managed policies are not persisted - Users should use PutUserPolicy for inline policies 3. Fix deadlock in standalone IAM's CreatePolicy - Removed nested lock acquisition (DoActions already holds the lock) Files changed: - weed/s3api/s3api_embedded_iam.go - weed/iamapi/iamapi_management_handlers.go * Add rate limiting to embedded IAM endpoint Apply circuit breaker rate limiting to the IAM endpoint to prevent abuse. Also added request tracking for IAM operations. The IAM endpoint now follows the same pattern as other S3 endpoints: - track() for request metrics - s3a.iam.Auth() for authentication - s3a.cb.Limit() for rate limiting * Fix handleImplicitUsername to properly look up username from AccessKeyId According to AWS spec, when UserName is not specified in an IAM request, IAM should determine the username implicitly based on the AccessKeyId signing the request. Previously, the code incorrectly extracted s[2] (region field) from the SigV4 credential string as the username. This fix: 1. Extracts the AccessKeyId from s[0] of the credential string 2. Looks up the AccessKeyId in the credential store using LookupByAccessKey 3. Uses the identity's Name field as the username if found Also: - Added exported LookupByAccessKey wrapper method to IdentityAccessManagement - Updated tests to verify correct access key lookup behavior - Applied fix to both embedded IAM and standalone IAM implementations * Fix CreatePolicy to not trigger unnecessary save CreatePolicy validates the policy document and returns metadata but does not actually store the policy (SeaweedFS uses inline policies attached via PutUserPolicy). However, 'changed' was left as true, triggering an unnecessary save operation. Set changed = false after successful CreatePolicy validation in both embedded IAM and standalone IAM implementations. * Improve embedded IAM test quality - Remove unused mock types (mockCredentialManager, mockEmbeddedIamApi) - Use proto.Clone instead of proto.Merge for proper deep copy semantics - Replace brittle regex-based XML error extraction with proper XML unmarshalling - Remove unused regexp import - Add state and field assertions to tests: - CreateUser: verify username in response and user persisted in config - ListUsers: verify response contains expected users - GetUser: verify username in response - CreatePolicy: verify policy metadata in response - PutUserPolicy: verify actions were attached to user - CreateAccessKey: verify credentials in response and persisted in config * Remove shared test state and improve executeEmbeddedIamRequest - Remove package-level embeddedIamApi variable to avoid shared test state - Update executeEmbeddedIamRequest to accept API instance as parameter - Only call xml.Unmarshal when v != nil, making nil-v cases explicit - Return unmarshal error properly instead of always returning it - Update all tests to create their own EmbeddedIamApiForTest instance - Each test now has isolated state, preventing test interdependencies * Add comprehensive test coverage for embedded IAM Added tests for previously uncovered functions: - iamStringSlicesEqual: 0% → 100% - iamMapToStatementAction: 40% → 100% - iamMapToIdentitiesAction: 30% → 70% - iamHash: 100% - iamStringWithCharset: 85.7% - GetPolicyDocument: 75% → 100% - CreatePolicy: 91.7% → 100% - DeleteUser: 83.3% → 100% - GetUser: 83.3% → 100% - ListAccessKeys: 55.6% → 88.9% New test cases for helper functions, error handling, and edge cases. * Document IAM code duplication and reference GitHub issue #7747 Added comments to both IAM implementations noting the code duplication and referencing the tracking issue for future refactoring: - weed/s3api/s3api_embedded_iam.go (embedded IAM) - weed/iamapi/iamapi_management_handlers.go (standalone IAM) See: https://github.com/seaweedfs/seaweedfs/issues/7747 * Implement granular IAM authorization for self-service operations Previously, all IAM actions required ACTION_ADMIN permission, which was overly restrictive. This change implements AWS-like granular permissions: Self-service operations (allowed without admin for own resources): - CreateAccessKey (on own user) - DeleteAccessKey (on own user) - ListAccessKeys (on own user) - GetUser (on own user) - UpdateAccessKey (on own user) Admin-only operations: - CreateUser, DeleteUser, UpdateUser - PutUserPolicy, GetUserPolicy, DeleteUserPolicy - CreatePolicy - ListUsers - Operations on other users The new AuthIam middleware: 1. Authenticates the request (signature verification) 2. Parses the IAM Action and target UserName 3. For self-service actions, allows if user is operating on own resources 4. For all other actions or operations on other users, requires admin * Fix misleading comment in standalone IAM CreatePolicy The comment incorrectly stated that CreatePolicy only validates the policy document. In the standalone IAM server, CreatePolicy actually persists the policy via iama.s3ApiConfig.PutPolicies(). The changed flag is false because it doesn't modify s3cfg.Identities, not because nothing is stored. * Simplify IAM auth and add RequestId to responses - Remove redundant ACTION_ADMIN fallback in AuthIam: The action parameter in authRequest is for permission checking, not signature verification. If auth fails with ACTION_READ, it will fail with ACTION_ADMIN too. - Add SetRequestId() call before writing IAM responses for AWS compatibility. All IAM response structs embed iamCommonResponse which has SetRequestId(). * Address code review feedback for IAM implementation 1. auth_credentials.go: Add documentation warning that LookupByAccessKey returns internal pointers that should not be mutated. 2. iamapi_management_handlers.go & s3api_embedded_iam.go: Add input guards for StringWithCharset/iamStringWithCharset when length <= 0 or charset is empty to avoid runtime errors from rand.Int. 3. s3api_embedded_iam_test.go: Don't ignore xml.Marshal errors in test DoActions handler. Return proper error response if marshaling fails. 4. s3api_embedded_iam_test.go: Use obviously fake access key IDs (AKIATESTFAKEKEY*) to avoid CI secret scanner false positives. * Address code review feedback for IAM implementation (batch 2) 1. iamapi/iamapi_management_handlers.go: - Redact Authorization header log (security: avoid exposing signature) - Add nil-guard for iama.iam before LookupByAccessKey call 2. iamapi/iamapi_test.go: - Replace real-looking access keys with obviously fake ones (AKIATESTFAKEKEY*) to avoid CI secret scanner false positives 3. s3api/s3api_embedded_iam.go - CreateUser: - Validate UserName is not empty (return ErrCodeInvalidInputException) - Check for duplicate users (return ErrCodeEntityAlreadyExistsException) 4. s3api/s3api_embedded_iam.go - CreateAccessKey: - Return ErrCodeNoSuchEntityException if user doesn't exist - Removed implicit user creation behavior 5. s3api/s3api_embedded_iam.go - getActions: - Fix S3 ARN parsing for bucket/path patterns - Handle mybucket, mybucket/*, mybucket/path/* correctly - Return error if no valid actions found in policy 6. s3api/s3api_embedded_iam.go - handleImplicitUsername: - Redact Authorization header log - Add nil-guard for e.iam 7. s3api/s3api_embedded_iam.go - DoActions: - Reload in-memory IAM maps after credential mutations - Call LoadS3ApiConfigurationFromCredentialManager after save 8. s3api/auth_credentials.go - AuthSignatureOnly: - Add new signature-only authentication method - Bypasses S3 authorization checks for IAM operations - Used by AuthIam to properly separate signature verification from IAM-specific permission checks * Fix nil pointer dereference and error handling in IAM 1. AuthIam: Add nil check for identity after AuthSignatureOnly - AuthSignatureOnly can return nil identity with ErrNone for authTypePostPolicy or authTypeStreamingUnsigned - Now returns ErrAccessDenied if identity is nil 2. writeIamErrorResponse: Add missing error code cases - ErrCodeEntityAlreadyExistsException -> HTTP 409 Conflict - ErrCodeInvalidInputException -> HTTP 400 Bad Request 3. UpdateUser: Use consistent error handling - Changed from direct ErrInvalidRequest to writeIamErrorResponse - Now returns correct HTTP status codes based on error type * Add IAM config reload to standalone IAM server after mutations Match the behavior of embedded IAM (s3api_embedded_iam.go) by reloading the in-memory identity maps after persisting configuration changes. This ensures newly created access keys are visible to LookupByAccessKey immediately without requiring a server restart. * Minor improvements to test helpers and log masking 1. iamapi_test.go: Update mustMarshalJSON to use t.Helper() and t.Fatal() instead of panic() for better test diagnostics 2. s3api_embedded_iam.go: Mask access key in 'not found' log message to avoid exposing full access key IDs in logs * Mask access key in standalone IAM log message for consistency Match the embedded IAM version by masking the access key ID in the 'not found' log message (show only first 4 chars). --- .gitignore | 1 + weed/command/filer.go | 1 + weed/command/iam.go | 27 +- weed/command/s3.go | 3 + weed/command/server.go | 1 + weed/iamapi/iamapi_management_handlers.go | 157 +++- weed/iamapi/iamapi_response.go | 5 + weed/iamapi/iamapi_test.go | 52 +- weed/s3api/auth_credentials.go | 70 ++ weed/s3api/s3api_embedded_iam.go | 922 ++++++++++++++++++ weed/s3api/s3api_embedded_iam_test.go | 1028 +++++++++++++++++++++ weed/s3api/s3api_server.go | 18 + 12 files changed, 2241 insertions(+), 44 deletions(-) create mode 100644 weed/s3api/s3api_embedded_iam.go create mode 100644 weed/s3api/s3api_embedded_iam_test.go diff --git a/.gitignore b/.gitignore index cd240ab6d..2ddff99a8 100644 --- a/.gitignore +++ b/.gitignore @@ -124,3 +124,4 @@ ADVANCED_IAM_DEVELOPMENT_PLAN.md *.log weed-iam test/kafka/kafka-client-loadtest/weed-linux-arm64 +coverage.out diff --git a/weed/command/filer.go b/weed/command/filer.go index 3f616e624..f07c605d7 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -134,6 +134,7 @@ func init() { filerS3Options.idleTimeout = cmdFiler.Flag.Int("s3.idleTimeout", 120, "connection idle seconds") filerS3Options.concurrentUploadLimitMB = cmdFiler.Flag.Int("s3.concurrentUploadLimitMB", 0, "limit total concurrent upload size for S3, 0 means unlimited") filerS3Options.concurrentFileUploadLimit = cmdFiler.Flag.Int("s3.concurrentFileUploadLimit", 0, "limit number of concurrent file uploads for S3, 0 means unlimited") + filerS3Options.enableIam = cmdFiler.Flag.Bool("s3.iam", true, "enable embedded IAM API on the same S3 port") // start webdav on filer filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway") diff --git a/weed/command/iam.go b/weed/command/iam.go index 8fae7ec96..77f3a9014 100644 --- a/weed/command/iam.go +++ b/weed/command/iam.go @@ -44,13 +44,34 @@ func init() { var cmdIam = &Command{ UsageLine: "iam [-port=8111] [-filer=[,]...] [-master=,]", - Short: "start a iam API compatible server", - Long: `start a iam API compatible server. + Short: "[DEPRECATED] start a standalone iam API compatible server", + Long: `[DEPRECATED] start a standalone iam API compatible server. + + DEPRECATION NOTICE: + The standalone 'weed iam' command is deprecated and will be removed in a future release. + + The IAM API is now embedded in the S3 server by default. Simply use 'weed s3' instead, + which provides both S3 and IAM APIs on the same port (enabled by default with -iam=true). + + This simplifies deployment by running a single server instead of two separate servers, + following the pattern used by MinIO and Ceph RGW. + + To use the embedded IAM API: + weed s3 -port=8333 # IAM API is available on the same port + + To disable the embedded IAM API (if you prefer the old behavior): + weed s3 -iam=false # Run S3 without IAM + weed iam -port=8111 # Run IAM separately (deprecated) Multiple filer addresses can be specified for high availability, separated by commas.`, } func runIam(cmd *Command, args []string) bool { + glog.Warningf("================================================================================") + glog.Warningf("DEPRECATION WARNING: 'weed iam' is deprecated and will be removed in a future release.") + glog.Warningf("The IAM API is now embedded in 'weed s3' by default (use -iam=true, which is the default).") + glog.Warningf("Please migrate to using 'weed s3' which provides both S3 and IAM APIs on the same port.") + glog.Warningf("================================================================================") return iamStandaloneOptions.startIamServer() } @@ -89,7 +110,7 @@ func (iamopt *IamOptions) startIamServer() bool { if iamApiServer_err != nil { glog.Fatalf("IAM API Server startup error: %v", iamApiServer_err) } - + // Register shutdown handler to prevent goroutine leak grace.OnInterrupt(func() { iamApiServer.Shutdown() diff --git a/weed/command/s3.go b/weed/command/s3.go index 5f62e8e58..5691489f4 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -58,6 +58,7 @@ type S3Options struct { idleTimeout *int concurrentUploadLimitMB *int concurrentFileUploadLimit *int + enableIam *bool } func init() { @@ -86,6 +87,7 @@ func init() { s3StandaloneOptions.idleTimeout = cmdS3.Flag.Int("idleTimeout", 120, "connection idle seconds") s3StandaloneOptions.concurrentUploadLimitMB = cmdS3.Flag.Int("concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited") s3StandaloneOptions.concurrentFileUploadLimit = cmdS3.Flag.Int("concurrentFileUploadLimit", 0, "limit number of concurrent file uploads, 0 means unlimited") + s3StandaloneOptions.enableIam = cmdS3.Flag.Bool("iam", true, "enable embedded IAM API on the same port") } var cmdS3 = &Command{ @@ -279,6 +281,7 @@ func (s3opt *S3Options) startS3Server() bool { IamConfig: iamConfigPath, // Advanced IAM config (optional) ConcurrentUploadLimit: int64(*s3opt.concurrentUploadLimitMB) * 1024 * 1024, ConcurrentFileUploadLimit: int64(*s3opt.concurrentFileUploadLimit), + EnableIam: *s3opt.enableIam, // Embedded IAM API (enabled by default) }) if s3ApiServer_err != nil { glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err) diff --git a/weed/command/server.go b/weed/command/server.go index 49aa15c6e..954e3b93f 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -173,6 +173,7 @@ func init() { s3Options.idleTimeout = cmdServer.Flag.Int("s3.idleTimeout", 120, "connection idle seconds") s3Options.concurrentUploadLimitMB = cmdServer.Flag.Int("s3.concurrentUploadLimitMB", 0, "limit total concurrent upload size for S3, 0 means unlimited") s3Options.concurrentFileUploadLimit = cmdServer.Flag.Int("s3.concurrentFileUploadLimit", 0, "limit number of concurrent file uploads for S3, 0 means unlimited") + s3Options.enableIam = cmdServer.Flag.Bool("s3.iam", true, "enable embedded IAM API on the same S3 port") sftpOptions.port = cmdServer.Flag.Int("sftp.port", 2022, "SFTP server listen port") sftpOptions.sshPrivateKey = cmdServer.Flag.String("sftp.sshPrivateKey", "", "path to the SSH private key file for host authentication") diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index 1a8f852cd..1985b042f 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -1,17 +1,22 @@ package iamapi +// This file provides IAM API handlers for the standalone IAM server. +// NOTE: There is code duplication with weed/s3api/s3api_embedded_iam.go. +// See GitHub issue #7747 for the planned refactoring to extract common IAM logic +// into a shared package. + import ( + "crypto/rand" "crypto/sha1" "encoding/json" "errors" "fmt" - "math/rand" + "math/big" "net/http" "net/url" - "reflect" + "sort" "strings" "sync" - "time" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -38,8 +43,6 @@ const ( ) var ( - seededRand *rand.Rand = rand.New( - rand.NewSource(time.Now().UnixNano())) policyDocuments = map[string]*policy_engine.PolicyDocument{} policyLock = sync.RWMutex{} ) @@ -104,12 +107,45 @@ func Hash(s *string) string { return fmt.Sprintf("%x", h.Sum(nil)) } -func StringWithCharset(length int, charset string) string { +// StringWithCharset generates a cryptographically secure random string. +// Uses crypto/rand for security-sensitive credential generation. +func StringWithCharset(length int, charset string) (string, error) { + if length <= 0 { + return "", fmt.Errorf("length must be positive, got %d", length) + } + if charset == "" { + return "", fmt.Errorf("charset must not be empty") + } b := make([]byte, length) for i := range b { - b[i] = charset[seededRand.Intn(len(charset))] + n, err := rand.Int(rand.Reader, big.NewInt(int64(len(charset)))) + if err != nil { + return "", fmt.Errorf("failed to generate random index: %w", err) + } + b[i] = charset[n.Int64()] } - return string(b) + return string(b), nil +} + +// stringSlicesEqual compares two string slices for equality, ignoring order. +// This is used instead of reflect.DeepEqual to avoid order-dependent comparisons. +func stringSlicesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + // Make copies to avoid modifying the originals + aCopy := make([]string, len(a)) + bCopy := make([]string, len(b)) + copy(aCopy, a) + copy(bCopy, b) + sort.Strings(aCopy) + sort.Strings(bCopy) + for i := range aCopy { + if aCopy[i] != bCopy[i] { + return false + } + } + return true } func (iama *IamApiServer) ListUsers(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp ListUsersResponse) { @@ -199,8 +235,7 @@ func (iama *IamApiServer) CreatePolicy(s3cfg *iam_pb.S3ApiConfiguration, values resp.CreatePolicyResult.Policy.Arn = &arn resp.CreatePolicyResult.Policy.PolicyId = &policyId policies := Policies{} - policyLock.Lock() - defer policyLock.Unlock() + // Note: Lock is already held by DoActions, no need to acquire here if err = iama.s3ApiConfig.GetPolicies(&policies); err != nil { return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: err} } @@ -273,7 +308,8 @@ func (iama *IamApiServer) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values for resource, actions := range statements { isEqAction := false for i, statement := range policyDocument.Statement { - if reflect.DeepEqual(statement.Action.Strings(), actions) { + // Use order-independent comparison to avoid duplicates from different action orderings + if stringSlicesEqual(statement.Action.Strings(), actions) { policyDocument.Statement[i].Resource = policy_engine.NewStringOrStringSlice(append( policyDocument.Statement[i].Resource.Strings(), resource)...) isEqAction = true @@ -300,11 +336,12 @@ func (iama *IamApiServer) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values return resp, &IamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(USER_DOES_NOT_EXIST, userName)} } -func (iama *IamApiServer) DeleteUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp PutUserPolicyResponse, err *IamError) { +// DeleteUserPolicy removes the inline policy from a user (clears their actions). +func (iama *IamApiServer) DeleteUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp DeleteUserPolicyResponse, err *IamError) { userName := values.Get("UserName") - for i, ident := range s3cfg.Identities { + for _, ident := range s3cfg.Identities { if ident.Name == userName { - s3cfg.Identities = append(s3cfg.Identities[:i], s3cfg.Identities[i+1:]...) + ident.Actions = nil return resp, nil } } @@ -348,11 +385,19 @@ func GetActions(policy *policy_engine.PolicyDocument) ([]string, error) { return actions, nil } -func (iama *IamApiServer) CreateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp CreateAccessKeyResponse) { +func (iama *IamApiServer) CreateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp CreateAccessKeyResponse, iamErr *IamError) { userName := values.Get("UserName") status := iam.StatusTypeActive - accessKeyId := StringWithCharset(21, charsetUpper) - secretAccessKey := StringWithCharset(42, charset) + + accessKeyId, err := StringWithCharset(21, charsetUpper) + if err != nil { + return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: fmt.Errorf("failed to generate access key: %w", err)} + } + secretAccessKey, err := StringWithCharset(42, charset) + if err != nil { + return resp, &IamError{Code: iam.ErrCodeServiceFailureException, Error: fmt.Errorf("failed to generate secret key: %w", err)} + } + resp.CreateAccessKeyResult.AccessKey.AccessKeyId = &accessKeyId resp.CreateAccessKeyResult.AccessKey.SecretAccessKey = &secretAccessKey resp.CreateAccessKeyResult.AccessKey.UserName = &userName @@ -379,7 +424,7 @@ func (iama *IamApiServer) CreateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu }, ) } - return resp + return resp, nil } func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (resp DeleteAccessKeyResponse) { @@ -399,36 +444,60 @@ func (iama *IamApiServer) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, valu return resp } -// handleImplicitUsername adds username who signs the request to values if 'username' is not specified -// According to https://awscli.amazonaws.com/v2/documentation/api/latest/reference/iam/create-access-key.html/ -// "If you do not specify a user name, IAM determines the user name implicitly based on the Amazon Web -// Services access key ID signing the request." -func handleImplicitUsername(r *http.Request, values url.Values) { +// handleImplicitUsername adds username who signs the request to values if 'username' is not specified. +// According to AWS documentation: "If you do not specify a user name, IAM determines the user name +// implicitly based on the Amazon Web Services access key ID signing the request." +// This function extracts the AccessKeyId from the SigV4 credential and looks up the corresponding +// identity name in the credential store. +func (iama *IamApiServer) handleImplicitUsername(r *http.Request, values url.Values) { if len(r.Header["Authorization"]) == 0 || values.Get("UserName") != "" { return } - // get username who signs the request. For a typical Authorization: - // "AWS4-HMAC-SHA256 Credential=197FSAQ7HHTA48X64O3A/20220420/test1/iam/aws4_request, SignedHeaders=content-type; - // host;x-amz-date, Signature=6757dc6b3d7534d67e17842760310e99ee695408497f6edc4fdb84770c252dc8", - // the "test1" will be extracted as the username - glog.V(4).Infof("Authorization field: %v", r.Header["Authorization"][0]) + // Log presence of auth header without exposing sensitive signature material + glog.V(4).Infof("Authorization header present, extracting access key") + // Parse AWS SigV4 Authorization header format: + // "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/iam/aws4_request, ..." s := strings.Split(r.Header["Authorization"][0], "Credential=") if len(s) < 2 { return } s = strings.Split(s[1], ",") - if len(s) < 2 { + if len(s) < 1 { return } s = strings.Split(s[0], "/") - if len(s) < 5 { + if len(s) < 1 { + return + } + // s[0] is the AccessKeyId + accessKeyId := s[0] + if accessKeyId == "" { + return + } + // Nil-guard: ensure iam is initialized before lookup + if iama.iam == nil { + glog.V(4).Infof("IAM not initialized, cannot look up access key") + return + } + // Look up the identity by access key to get the username + identity, _, found := iama.iam.LookupByAccessKey(accessKeyId) + if !found { + // Mask access key in logs - show only first 4 chars + maskedKey := accessKeyId + if len(accessKeyId) > 4 { + maskedKey = accessKeyId[:4] + "***" + } + glog.V(4).Infof("Access key %s not found in credential store", maskedKey) return } - userName := s[2] - values.Set("UserName", userName) + values.Set("UserName", identity.Name) } func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { + // Lock to prevent concurrent read-modify-write race conditions + policyLock.Lock() + defer policyLock.Unlock() + if err := r.ParseForm(); err != nil { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) return @@ -449,7 +518,7 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { response = iama.ListUsers(s3cfg, values) changed = false case "ListAccessKeys": - handleImplicitUsername(r, values) + iama.handleImplicitUsername(r, values) response = iama.ListAccessKeys(s3cfg, values) changed = false case "CreateUser": @@ -477,10 +546,15 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { return } case "CreateAccessKey": - handleImplicitUsername(r, values) - response = iama.CreateAccessKey(s3cfg, values) + iama.handleImplicitUsername(r, values) + response, iamError = iama.CreateAccessKey(s3cfg, values) + if iamError != nil { + glog.Errorf("CreateAccessKey: %+v", iamError.Error) + writeIamErrorResponse(w, r, iamError) + return + } case "DeleteAccessKey": - handleImplicitUsername(r, values) + iama.handleImplicitUsername(r, values) response = iama.DeleteAccessKey(s3cfg, values) case "CreatePolicy": response, iamError = iama.CreatePolicy(s3cfg, values) @@ -489,6 +563,9 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) return } + // CreatePolicy persists the policy document via iama.s3ApiConfig.PutPolicies(). + // The `changed` flag is false because this does not modify the main s3cfg.Identities configuration. + changed = false case "PutUserPolicy": var iamError *IamError response, iamError = iama.PutUserPolicy(s3cfg, values) @@ -525,6 +602,14 @@ func (iama *IamApiServer) DoActions(w http.ResponseWriter, r *http.Request) { writeIamErrorResponse(w, r, &iamError) return } + // Reload in-memory identity maps so subsequent LookupByAccessKey calls + // can see newly created or deleted keys immediately + if iama.iam != nil { + if err := iama.iam.LoadS3ApiConfigurationFromCredentialManager(); err != nil { + glog.Warningf("Failed to reload IAM configuration after mutation: %v", err) + // Don't fail the request since the persistent save succeeded + } + } } s3err.WriteXMLResponse(w, r, http.StatusOK, response) } diff --git a/weed/iamapi/iamapi_response.go b/weed/iamapi/iamapi_response.go index df9443f0d..fc68ce5a5 100644 --- a/weed/iamapi/iamapi_response.go +++ b/weed/iamapi/iamapi_response.go @@ -84,6 +84,11 @@ type PutUserPolicyResponse struct { XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ PutUserPolicyResponse"` } +type DeleteUserPolicyResponse struct { + CommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteUserPolicyResponse"` +} + type GetUserPolicyResponse struct { CommonResponse XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetUserPolicyResponse"` diff --git a/weed/iamapi/iamapi_test.go b/weed/iamapi/iamapi_test.go index 94c48aa7f..fa04d1ce9 100644 --- a/weed/iamapi/iamapi_test.go +++ b/weed/iamapi/iamapi_test.go @@ -1,6 +1,7 @@ package iamapi import ( + "encoding/json" "encoding/xml" "net/http" "net/http/httptest" @@ -14,6 +15,7 @@ import ( "github.com/gorilla/mux" "github.com/jinzhu/copier" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api" "github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine" "github.com/stretchr/testify/assert" ) @@ -244,22 +246,62 @@ func executeRequest(req *http.Request, v interface{}) (*httptest.ResponseRecorde } func TestHandleImplicitUsername(t *testing.T) { + // Create a mock IamApiServer with credential store + // The handleImplicitUsername function now looks up the username from the + // credential store based on AccessKeyId, not from the region field in the auth header. + // Note: Using obviously fake access keys to avoid CI secret scanner false positives + + // Create IAM directly as struct literal (same pattern as other tests) + iam := &s3api.IdentityAccessManagement{} + + // Load test credentials - map access key to identity name + testConfig := &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + { + Name: "testuser1", + Credentials: []*iam_pb.Credential{ + {AccessKey: "AKIATESTFAKEKEY000001", SecretKey: "testsecretfake"}, + }, + }, + }, + } + err := iam.LoadS3ApiConfigurationFromBytes(mustMarshalJSON(t, testConfig)) + if err != nil { + t.Fatalf("Failed to load test config: %v", err) + } + + iama := &IamApiServer{ + iam: iam, + } + var tests = []struct { r *http.Request values url.Values userName string }{ + // No authorization header - should not set username {&http.Request{}, url.Values{}, ""}, - {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=197FSAQ7HHTA48X64O3A/20220420/test1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=6757dc6b3d7534d67e17842760310e99ee695408497f6edc4fdb84770c252dc8"}}}, url.Values{}, "test1"}, - {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 =197FSAQ7HHTA48X64O3A/20220420/test1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=6757dc6b3d7534d67e17842760310e99ee695408497f6edc4fdb84770c252dc8"}}}, url.Values{}, ""}, - {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=197FSAQ7HHTA48X64O3A/20220420/test1/iam/aws4_request SignedHeaders=content-type;host;x-amz-date Signature=6757dc6b3d7534d67e17842760310e99ee695408497f6edc4fdb84770c252dc8"}}}, url.Values{}, ""}, - {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=197FSAQ7HHTA48X64O3A/20220420/test1/iam, SignedHeaders=content-type;host;x-amz-date, Signature=6757dc6b3d7534d67e17842760310e99ee695408497f6edc4fdb84770c252dc8"}}}, url.Values{}, ""}, + // Valid auth header with known access key - should look up and find "testuser1" + {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=AKIATESTFAKEKEY000001/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, "testuser1"}, + // Malformed auth header (no Credential=) - should not set username + {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 =AKIATESTFAKEKEY000001/20220420/test1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""}, + // Unknown access key - should not set username + {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=AKIATESTUNKNOWN000000/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""}, } for i, test := range tests { - handleImplicitUsername(test.r, test.values) + iama.handleImplicitUsername(test.r, test.values) if un := test.values.Get("UserName"); un != test.userName { t.Errorf("No.%d: Got: %v, Expected: %v", i, un, test.userName) } } } + +func mustMarshalJSON(t *testing.T, v interface{}) []byte { + t.Helper() + data, err := json.Marshal(v) + if err != nil { + t.Fatalf("failed to marshal JSON: %v", err) + } + return data +} diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 7b5d9a262..c81fb3a88 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -427,6 +427,16 @@ func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identi return nil, nil, false } +// LookupByAccessKey is an exported wrapper for lookupByAccessKey. +// It returns the identity and credential associated with the given access key. +// +// WARNING: The returned pointers reference internal data structures. +// Callers MUST NOT modify the returned Identity or Credential objects. +// If mutation is needed, make a copy first. +func (iam *IdentityAccessManagement) LookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) { + return iam.lookupByAccessKey(accessKey) +} + func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, found bool) { iam.m.RLock() defer iam.m.RUnlock() @@ -633,6 +643,66 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) } +// AuthSignatureOnly performs only signature verification without any authorization checks. +// This is used for IAM API operations where authorization is handled separately based on +// the specific IAM action (e.g., self-service vs admin operations). +// Returns the authenticated identity and any signature verification error. +func (iam *IdentityAccessManagement) AuthSignatureOnly(r *http.Request) (*Identity, s3err.ErrorCode) { + var identity *Identity + var s3Err s3err.ErrorCode + var authType string + switch getRequestAuthType(r) { + case authTypeUnknown: + glog.V(3).Infof("unknown auth type") + r.Header.Set(s3_constants.AmzAuthType, "Unknown") + return identity, s3err.ErrAccessDenied + case authTypePresignedV2, authTypeSignedV2: + glog.V(3).Infof("v2 auth type") + identity, s3Err = iam.isReqAuthenticatedV2(r) + authType = "SigV2" + case authTypeStreamingSigned, authTypeSigned, authTypePresigned: + glog.V(3).Infof("v4 auth type") + identity, s3Err = iam.reqSignatureV4Verify(r) + authType = "SigV4" + case authTypePostPolicy: + glog.V(3).Infof("post policy auth type") + r.Header.Set(s3_constants.AmzAuthType, "PostPolicy") + return identity, s3err.ErrNone + case authTypeStreamingUnsigned: + glog.V(3).Infof("unsigned streaming upload") + return identity, s3err.ErrNone + case authTypeJWT: + glog.V(3).Infof("jwt auth type detected, iamIntegration != nil? %t", iam.iamIntegration != nil) + r.Header.Set(s3_constants.AmzAuthType, "Jwt") + if iam.iamIntegration != nil { + identity, s3Err = iam.authenticateJWTWithIAM(r) + authType = "Jwt" + } else { + glog.V(2).Infof("IAM integration is nil, returning ErrNotImplemented") + return identity, s3err.ErrNotImplemented + } + case authTypeAnonymous: + // Anonymous users cannot use IAM API + return identity, s3err.ErrAccessDenied + default: + return identity, s3err.ErrNotImplemented + } + + if len(authType) > 0 { + r.Header.Set(s3_constants.AmzAuthType, authType) + } + if s3Err != s3err.ErrNone { + return identity, s3Err + } + + // Set account ID header for downstream handlers + if identity != nil && identity.Account != nil { + r.Header.Set(s3_constants.AmzAccountId, identity.Account.Id) + } + + return identity, s3err.ErrNone +} + func (identity *Identity) canDo(action Action, bucket string, objectKey string) bool { if identity.isAdmin() { return true diff --git a/weed/s3api/s3api_embedded_iam.go b/weed/s3api/s3api_embedded_iam.go new file mode 100644 index 000000000..e7a30b4c1 --- /dev/null +++ b/weed/s3api/s3api_embedded_iam.go @@ -0,0 +1,922 @@ +package s3api + +// This file provides IAM API functionality embedded in the S3 server. +// NOTE: There is code duplication with weed/iamapi/iamapi_management_handlers.go. +// See GitHub issue #7747 for the planned refactoring to extract common IAM logic +// into a shared package. + +import ( + "context" + "crypto/rand" + "crypto/sha1" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "math/big" + "net/http" + "net/url" + "sort" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go/service/iam" + "github.com/seaweedfs/seaweedfs/weed/credential" + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine" + . "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "google.golang.org/protobuf/proto" +) + +// EmbeddedIamApi provides IAM API functionality embedded in the S3 server. +// This allows running a single server that handles both S3 and IAM requests. +type EmbeddedIamApi struct { + credentialManager *credential.CredentialManager + iam *IdentityAccessManagement + policyLock sync.RWMutex +} + +// NewEmbeddedIamApi creates a new embedded IAM API handler. +func NewEmbeddedIamApi(credentialManager *credential.CredentialManager, iam *IdentityAccessManagement) *EmbeddedIamApi { + return &EmbeddedIamApi{ + credentialManager: credentialManager, + iam: iam, + } +} + +// IAM response types +type iamCommonResponse struct { + ResponseMetadata struct { + RequestId string `xml:"RequestId"` + } `xml:"ResponseMetadata"` +} + +func (r *iamCommonResponse) SetRequestId() { + r.ResponseMetadata.RequestId = fmt.Sprintf("%d", time.Now().UnixNano()) +} + +type iamListUsersResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListUsersResponse"` + ListUsersResult struct { + Users []*iam.User `xml:"Users>member"` + IsTruncated bool `xml:"IsTruncated"` + } `xml:"ListUsersResult"` +} + +type iamListAccessKeysResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ListAccessKeysResponse"` + ListAccessKeysResult struct { + AccessKeyMetadata []*iam.AccessKeyMetadata `xml:"AccessKeyMetadata>member"` + IsTruncated bool `xml:"IsTruncated"` + } `xml:"ListAccessKeysResult"` +} + +type iamDeleteAccessKeyResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteAccessKeyResponse"` +} + +type iamCreatePolicyResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreatePolicyResponse"` + CreatePolicyResult struct { + Policy iam.Policy `xml:"Policy"` + } `xml:"CreatePolicyResult"` +} + +type iamCreateUserResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateUserResponse"` + CreateUserResult struct { + User iam.User `xml:"User"` + } `xml:"CreateUserResult"` +} + +type iamDeleteUserResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteUserResponse"` +} + +type iamGetUserResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetUserResponse"` + GetUserResult struct { + User iam.User `xml:"User"` + } `xml:"GetUserResult"` +} + +type iamUpdateUserResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ UpdateUserResponse"` +} + +type iamCreateAccessKeyResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ CreateAccessKeyResponse"` + CreateAccessKeyResult struct { + AccessKey iam.AccessKey `xml:"AccessKey"` + } `xml:"CreateAccessKeyResult"` +} + +type iamPutUserPolicyResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ PutUserPolicyResponse"` +} + +type iamDeleteUserPolicyResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ DeleteUserPolicyResponse"` +} + +type iamGetUserPolicyResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ GetUserPolicyResponse"` + GetUserPolicyResult struct { + UserName string `xml:"UserName"` + PolicyName string `xml:"PolicyName"` + PolicyDocument string `xml:"PolicyDocument"` + } `xml:"GetUserPolicyResult"` +} + +type iamErrorResponse struct { + iamCommonResponse + XMLName xml.Name `xml:"https://iam.amazonaws.com/doc/2010-05-08/ ErrorResponse"` + Error struct { + iam.ErrorDetails + Type string `xml:"Type"` + } `xml:"Error"` +} + +type iamError struct { + Code string + Error error +} + +// Policies stores IAM policies +type iamPolicies struct { + Policies map[string]policy_engine.PolicyDocument `json:"policies"` +} + +const ( + iamCharsetUpper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + iamCharset = iamCharsetUpper + "abcdefghijklmnopqrstuvwxyz/" + iamPolicyDocumentVersion = "2012-10-17" + iamUserDoesNotExist = "the user with name %s cannot be found." +) + +// Statement action constants +const ( + iamStatementActionAdmin = "*" + iamStatementActionWrite = "Put*" + iamStatementActionWriteAcp = "PutBucketAcl" + iamStatementActionRead = "Get*" + iamStatementActionReadAcp = "GetBucketAcl" + iamStatementActionList = "List*" + iamStatementActionTagging = "Tagging*" + iamStatementActionDelete = "DeleteBucket*" +) + +func iamHash(s *string) string { + h := sha1.New() + h.Write([]byte(*s)) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +// iamStringWithCharset generates a cryptographically secure random string. +// Uses crypto/rand for security-sensitive credential generation. +func iamStringWithCharset(length int, charset string) (string, error) { + if length <= 0 { + return "", fmt.Errorf("length must be positive, got %d", length) + } + if charset == "" { + return "", fmt.Errorf("charset must not be empty") + } + b := make([]byte, length) + for i := range b { + n, err := rand.Int(rand.Reader, big.NewInt(int64(len(charset)))) + if err != nil { + return "", fmt.Errorf("failed to generate random index: %w", err) + } + b[i] = charset[n.Int64()] + } + return string(b), nil +} + +// iamStringSlicesEqual compares two string slices for equality, ignoring order. +// This is used instead of reflect.DeepEqual to avoid order-dependent comparisons. +func iamStringSlicesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + // Make copies to avoid modifying the originals + aCopy := make([]string, len(a)) + bCopy := make([]string, len(b)) + copy(aCopy, a) + copy(bCopy, b) + sort.Strings(aCopy) + sort.Strings(bCopy) + for i := range aCopy { + if aCopy[i] != bCopy[i] { + return false + } + } + return true +} + +func iamMapToStatementAction(action string) string { + switch action { + case iamStatementActionAdmin: + return ACTION_ADMIN + case iamStatementActionWrite: + return ACTION_WRITE + case iamStatementActionWriteAcp: + return ACTION_WRITE_ACP + case iamStatementActionRead: + return ACTION_READ + case iamStatementActionReadAcp: + return ACTION_READ_ACP + case iamStatementActionList: + return ACTION_LIST + case iamStatementActionTagging: + return ACTION_TAGGING + case iamStatementActionDelete: + return ACTION_DELETE_BUCKET + default: + return "" + } +} + +func iamMapToIdentitiesAction(action string) string { + switch action { + case ACTION_ADMIN: + return iamStatementActionAdmin + case ACTION_WRITE: + return iamStatementActionWrite + case ACTION_WRITE_ACP: + return iamStatementActionWriteAcp + case ACTION_READ: + return iamStatementActionRead + case ACTION_READ_ACP: + return iamStatementActionReadAcp + case ACTION_LIST: + return iamStatementActionList + case ACTION_TAGGING: + return iamStatementActionTagging + case ACTION_DELETE_BUCKET: + return iamStatementActionDelete + default: + return "" + } +} + +func newIamErrorResponse(errCode string, errMsg string) iamErrorResponse { + errorResp := iamErrorResponse{} + errorResp.Error.Type = "Sender" + errorResp.Error.Code = &errCode + errorResp.Error.Message = &errMsg + return errorResp +} + +func (e *EmbeddedIamApi) writeIamErrorResponse(w http.ResponseWriter, r *http.Request, iamErr *iamError) { + if iamErr == nil { + glog.Errorf("No error found") + return + } + + errCode := iamErr.Code + errMsg := iamErr.Error.Error() + glog.Errorf("IAM Response %+v", errMsg) + + errorResp := newIamErrorResponse(errCode, errMsg) + internalErrorResponse := newIamErrorResponse(iam.ErrCodeServiceFailureException, "Internal server error") + + switch errCode { + case iam.ErrCodeNoSuchEntityException: + s3err.WriteXMLResponse(w, r, http.StatusNotFound, errorResp) + case iam.ErrCodeEntityAlreadyExistsException: + s3err.WriteXMLResponse(w, r, http.StatusConflict, errorResp) + case iam.ErrCodeMalformedPolicyDocumentException, iam.ErrCodeInvalidInputException: + s3err.WriteXMLResponse(w, r, http.StatusBadRequest, errorResp) + case iam.ErrCodeServiceFailureException: + s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, internalErrorResponse) + default: + s3err.WriteXMLResponse(w, r, http.StatusInternalServerError, internalErrorResponse) + } +} + +// GetS3ApiConfiguration loads the S3 API configuration from the credential manager. +func (e *EmbeddedIamApi) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) error { + config, err := e.credentialManager.LoadConfiguration(context.Background()) + if err != nil { + return fmt.Errorf("failed to load configuration: %w", err) + } + proto.Merge(s3cfg, config) + return nil +} + +// PutS3ApiConfiguration saves the S3 API configuration to the credential manager. +func (e *EmbeddedIamApi) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) error { + return e.credentialManager.SaveConfiguration(context.Background(), s3cfg) +} + +// ListUsers lists all IAM users. +func (e *EmbeddedIamApi) ListUsers(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) iamListUsersResponse { + var resp iamListUsersResponse + for _, ident := range s3cfg.Identities { + resp.ListUsersResult.Users = append(resp.ListUsersResult.Users, &iam.User{UserName: &ident.Name}) + } + return resp +} + +// ListAccessKeys lists access keys for a user. +func (e *EmbeddedIamApi) ListAccessKeys(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) iamListAccessKeysResponse { + var resp iamListAccessKeysResponse + status := iam.StatusTypeActive + userName := values.Get("UserName") + for _, ident := range s3cfg.Identities { + if userName != "" && userName != ident.Name { + continue + } + for _, cred := range ident.Credentials { + resp.ListAccessKeysResult.AccessKeyMetadata = append(resp.ListAccessKeysResult.AccessKeyMetadata, + &iam.AccessKeyMetadata{UserName: &ident.Name, AccessKeyId: &cred.AccessKey, Status: &status}, + ) + } + } + return resp +} + +// CreateUser creates a new IAM user. +func (e *EmbeddedIamApi) CreateUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (iamCreateUserResponse, *iamError) { + var resp iamCreateUserResponse + userName := values.Get("UserName") + + // Validate UserName is not empty + if userName == "" { + return resp, &iamError{Code: iam.ErrCodeInvalidInputException, Error: fmt.Errorf("UserName is required")} + } + + // Check for duplicate user + for _, ident := range s3cfg.Identities { + if ident.Name == userName { + return resp, &iamError{Code: iam.ErrCodeEntityAlreadyExistsException, Error: fmt.Errorf("user %s already exists", userName)} + } + } + + resp.CreateUserResult.User.UserName = &userName + s3cfg.Identities = append(s3cfg.Identities, &iam_pb.Identity{Name: userName}) + return resp, nil +} + +// DeleteUser deletes an IAM user. +func (e *EmbeddedIamApi) DeleteUser(s3cfg *iam_pb.S3ApiConfiguration, userName string) (iamDeleteUserResponse, *iamError) { + var resp iamDeleteUserResponse + for i, ident := range s3cfg.Identities { + if userName == ident.Name { + s3cfg.Identities = append(s3cfg.Identities[:i], s3cfg.Identities[i+1:]...) + return resp, nil + } + } + return resp, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(iamUserDoesNotExist, userName)} +} + +// GetUser gets an IAM user. +func (e *EmbeddedIamApi) GetUser(s3cfg *iam_pb.S3ApiConfiguration, userName string) (iamGetUserResponse, *iamError) { + var resp iamGetUserResponse + for _, ident := range s3cfg.Identities { + if userName == ident.Name { + resp.GetUserResult.User = iam.User{UserName: &ident.Name} + return resp, nil + } + } + return resp, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(iamUserDoesNotExist, userName)} +} + +// UpdateUser updates an IAM user. +func (e *EmbeddedIamApi) UpdateUser(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (iamUpdateUserResponse, *iamError) { + var resp iamUpdateUserResponse + userName := values.Get("UserName") + newUserName := values.Get("NewUserName") + if newUserName != "" { + for _, ident := range s3cfg.Identities { + if userName == ident.Name { + ident.Name = newUserName + return resp, nil + } + } + } else { + return resp, nil + } + return resp, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(iamUserDoesNotExist, userName)} +} + +// CreateAccessKey creates an access key for a user. +func (e *EmbeddedIamApi) CreateAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (iamCreateAccessKeyResponse, *iamError) { + var resp iamCreateAccessKeyResponse + userName := values.Get("UserName") + status := iam.StatusTypeActive + + accessKeyId, err := iamStringWithCharset(21, iamCharsetUpper) + if err != nil { + return resp, &iamError{Code: iam.ErrCodeServiceFailureException, Error: fmt.Errorf("failed to generate access key: %w", err)} + } + secretAccessKey, err := iamStringWithCharset(42, iamCharset) + if err != nil { + return resp, &iamError{Code: iam.ErrCodeServiceFailureException, Error: fmt.Errorf("failed to generate secret key: %w", err)} + } + + resp.CreateAccessKeyResult.AccessKey.AccessKeyId = &accessKeyId + resp.CreateAccessKeyResult.AccessKey.SecretAccessKey = &secretAccessKey + resp.CreateAccessKeyResult.AccessKey.UserName = &userName + resp.CreateAccessKeyResult.AccessKey.Status = &status + + for _, ident := range s3cfg.Identities { + if userName == ident.Name { + ident.Credentials = append(ident.Credentials, + &iam_pb.Credential{AccessKey: accessKeyId, SecretKey: secretAccessKey}) + return resp, nil + } + } + // User not found - return error instead of implicitly creating the user + return resp, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(iamUserDoesNotExist, userName)} +} + +// DeleteAccessKey deletes an access key for a user. +func (e *EmbeddedIamApi) DeleteAccessKey(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) iamDeleteAccessKeyResponse { + var resp iamDeleteAccessKeyResponse + userName := values.Get("UserName") + accessKeyId := values.Get("AccessKeyId") + for _, ident := range s3cfg.Identities { + if userName == ident.Name { + for i, cred := range ident.Credentials { + if cred.AccessKey == accessKeyId { + ident.Credentials = append(ident.Credentials[:i], ident.Credentials[i+1:]...) + break + } + } + break + } + } + return resp +} + +// GetPolicyDocument parses a policy document string. +func (e *EmbeddedIamApi) GetPolicyDocument(policy *string) (policy_engine.PolicyDocument, error) { + var policyDocument policy_engine.PolicyDocument + if err := json.Unmarshal([]byte(*policy), &policyDocument); err != nil { + return policy_engine.PolicyDocument{}, err + } + return policyDocument, nil +} + +// CreatePolicy validates and creates a new IAM managed policy. +// NOTE: Currently this only validates the policy document and returns policy metadata. +// The policy is not persisted to a managed policy store. To apply permissions to a user, +// use PutUserPolicy which stores the policy inline on the user's identity. +// TODO: Implement managed policy storage for full AWS IAM compatibility (ListPolicies, GetPolicy, AttachUserPolicy). +func (e *EmbeddedIamApi) CreatePolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (iamCreatePolicyResponse, *iamError) { + var resp iamCreatePolicyResponse + policyName := values.Get("PolicyName") + policyDocumentString := values.Get("PolicyDocument") + _, err := e.GetPolicyDocument(&policyDocumentString) + if err != nil { + return resp, &iamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err} + } + policyId := iamHash(&policyDocumentString) + arn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName) + resp.CreatePolicyResult.Policy.PolicyName = &policyName + resp.CreatePolicyResult.Policy.Arn = &arn + resp.CreatePolicyResult.Policy.PolicyId = &policyId + return resp, nil +} + +// getActions extracts actions from a policy document. +// S3 ARN format: arn:aws:s3:::bucket or arn:aws:s3:::bucket/path/* +// res[5] contains the bucket and optional path after ::: +func (e *EmbeddedIamApi) getActions(policy *policy_engine.PolicyDocument) ([]string, error) { + var actions []string + + for _, statement := range policy.Statement { + if statement.Effect != policy_engine.PolicyEffectAllow { + return nil, fmt.Errorf("not a valid effect: '%s'. Only 'Allow' is possible", statement.Effect) + } + for _, resource := range statement.Resource.Strings() { + res := strings.Split(resource, ":") + if len(res) != 6 || res[0] != "arn" || res[1] != "aws" || res[2] != "s3" { + continue + } + for _, action := range statement.Action.Strings() { + act := strings.Split(action, ":") + if len(act) != 2 || act[0] != "s3" { + continue + } + statementAction := iamMapToStatementAction(act[1]) + if statementAction == "" { + return nil, fmt.Errorf("not a valid action: '%s'", act[1]) + } + + resourcePath := res[5] + if resourcePath == "*" { + // Wildcard - applies to all buckets + actions = append(actions, statementAction) + continue + } + + // Parse bucket and optional object path + // Examples: "mybucket", "mybucket/*", "mybucket/prefix/*" + bucket, objectPath, hasSep := strings.Cut(resourcePath, "/") + if bucket == "" { + continue // Invalid: empty bucket name + } + + if !hasSep || objectPath == "" || objectPath == "*" { + // Bucket-level or bucket/* - use just bucket name + actions = append(actions, fmt.Sprintf("%s:%s", statementAction, bucket)) + } else { + // Path-specific: bucket/path/* -> Action:bucket/path + // Remove trailing /* if present for cleaner action format + objectPath = strings.TrimSuffix(objectPath, "/*") + objectPath = strings.TrimSuffix(objectPath, "*") + if objectPath == "" { + actions = append(actions, fmt.Sprintf("%s:%s", statementAction, bucket)) + } else { + actions = append(actions, fmt.Sprintf("%s:%s/%s", statementAction, bucket, objectPath)) + } + } + } + } + } + + if len(actions) == 0 { + return nil, fmt.Errorf("no valid actions found in policy document") + } + return actions, nil +} + +// PutUserPolicy attaches a policy to a user. +func (e *EmbeddedIamApi) PutUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (iamPutUserPolicyResponse, *iamError) { + var resp iamPutUserPolicyResponse + userName := values.Get("UserName") + policyDocumentString := values.Get("PolicyDocument") + policyDocument, err := e.GetPolicyDocument(&policyDocumentString) + if err != nil { + return resp, &iamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err} + } + actions, err := e.getActions(&policyDocument) + if err != nil { + return resp, &iamError{Code: iam.ErrCodeMalformedPolicyDocumentException, Error: err} + } + glog.V(3).Infof("PutUserPolicy: actions=%v", actions) + for _, ident := range s3cfg.Identities { + if userName != ident.Name { + continue + } + ident.Actions = actions + return resp, nil + } + return resp, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("the user with name %s cannot be found", userName)} +} + +// GetUserPolicy gets the policy attached to a user. +func (e *EmbeddedIamApi) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (iamGetUserPolicyResponse, *iamError) { + var resp iamGetUserPolicyResponse + userName := values.Get("UserName") + policyName := values.Get("PolicyName") + for _, ident := range s3cfg.Identities { + if userName != ident.Name { + continue + } + + resp.GetUserPolicyResult.UserName = userName + resp.GetUserPolicyResult.PolicyName = policyName + if len(ident.Actions) == 0 { + return resp, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: errors.New("no actions found")} + } + + policyDocument := policy_engine.PolicyDocument{Version: iamPolicyDocumentVersion} + statements := make(map[string][]string) + for _, action := range ident.Actions { + // Action format: "ActionType" (global) or "ActionType:bucket" or "ActionType:bucket/path" + actionType, bucketPath, hasPath := strings.Cut(action, ":") + var resource string + if !hasPath { + // Global action (no bucket specified) + resource = "*" + } else if strings.Contains(bucketPath, "/") { + // Path-specific: bucket/path -> arn:aws:s3:::bucket/path/* + resource = fmt.Sprintf("arn:aws:s3:::%s/*", bucketPath) + } else { + // Bucket-level: bucket -> arn:aws:s3:::bucket/* + resource = fmt.Sprintf("arn:aws:s3:::%s/*", bucketPath) + } + statements[resource] = append(statements[resource], + fmt.Sprintf("s3:%s", iamMapToIdentitiesAction(actionType)), + ) + } + for resource, actions := range statements { + isEqAction := false + for i, statement := range policyDocument.Statement { + // Use order-independent comparison to avoid duplicates from different action orderings + if iamStringSlicesEqual(statement.Action.Strings(), actions) { + policyDocument.Statement[i].Resource = policy_engine.NewStringOrStringSlice(append( + policyDocument.Statement[i].Resource.Strings(), resource)...) + isEqAction = true + break + } + } + if isEqAction { + continue + } + policyDocumentStatement := policy_engine.PolicyStatement{ + Effect: policy_engine.PolicyEffectAllow, + Action: policy_engine.NewStringOrStringSlice(actions...), + Resource: policy_engine.NewStringOrStringSlice(resource), + } + policyDocument.Statement = append(policyDocument.Statement, policyDocumentStatement) + } + policyDocumentJSON, err := json.Marshal(policyDocument) + if err != nil { + return resp, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err} + } + resp.GetUserPolicyResult.PolicyDocument = string(policyDocumentJSON) + return resp, nil + } + return resp, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(iamUserDoesNotExist, userName)} +} + +// DeleteUserPolicy removes the inline policy from a user (clears their actions). +func (e *EmbeddedIamApi) DeleteUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values url.Values) (iamDeleteUserPolicyResponse, *iamError) { + var resp iamDeleteUserPolicyResponse + userName := values.Get("UserName") + for _, ident := range s3cfg.Identities { + if ident.Name == userName { + ident.Actions = nil + return resp, nil + } + } + return resp, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf(iamUserDoesNotExist, userName)} +} + +// handleImplicitUsername adds username who signs the request to values if 'username' is not specified. +// According to AWS documentation: "If you do not specify a user name, IAM determines the user name +// implicitly based on the Amazon Web Services access key ID signing the request." +// This function extracts the AccessKeyId from the SigV4 credential and looks up the corresponding +// identity name in the credential store. +func (e *EmbeddedIamApi) handleImplicitUsername(r *http.Request, values url.Values) { + if len(r.Header["Authorization"]) == 0 || values.Get("UserName") != "" { + return + } + // Log presence of auth header without exposing sensitive signature material + glog.V(4).Infof("Authorization header present, extracting access key") + // Parse AWS SigV4 Authorization header format: + // "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/iam/aws4_request, ..." + s := strings.Split(r.Header["Authorization"][0], "Credential=") + if len(s) < 2 { + return + } + s = strings.Split(s[1], ",") + if len(s) < 1 { + return + } + s = strings.Split(s[0], "/") + if len(s) < 1 { + return + } + // s[0] is the AccessKeyId + accessKeyId := s[0] + if accessKeyId == "" { + return + } + // Nil-guard: ensure iam is initialized before lookup + if e.iam == nil { + glog.V(4).Infof("IAM not initialized, cannot look up access key") + return + } + // Look up the identity by access key to get the username + identity, _, found := e.iam.LookupByAccessKey(accessKeyId) + if !found { + // Mask access key in logs - show only first 4 chars + maskedKey := accessKeyId + if len(accessKeyId) > 4 { + maskedKey = accessKeyId[:4] + "***" + } + glog.V(4).Infof("Access key %s not found in credential store", maskedKey) + return + } + values.Set("UserName", identity.Name) +} + +// iamSelfServiceActions are actions that users can perform on their own resources without admin rights. +// According to AWS IAM, users can manage their own access keys without requiring full admin permissions. +var iamSelfServiceActions = map[string]bool{ + "CreateAccessKey": true, + "DeleteAccessKey": true, + "ListAccessKeys": true, + "GetUser": true, + "UpdateAccessKey": true, +} + +// iamRequiresAdminForOthers returns true if the action requires admin rights when operating on other users. +func iamRequiresAdminForOthers(action string) bool { + return iamSelfServiceActions[action] +} + +// AuthIam provides IAM-specific authentication that allows self-service operations. +// Users can manage their own access keys without admin rights, but need admin for operations on other users. +// The action parameter is accepted for interface compatibility with cb.Limit but is not used +// since IAM permission checking is done based on the IAM Action parameter in the request. +func (e *EmbeddedIamApi) AuthIam(f http.HandlerFunc, _ Action) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // If auth is not enabled, allow all + if !e.iam.isEnabled() { + f(w, r) + return + } + + // Parse form to get Action and UserName + if err := r.ParseForm(); err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) + return + } + + action := r.Form.Get("Action") + targetUserName := r.PostForm.Get("UserName") + + // Authenticate the request using signature-only verification. + // This bypasses S3 authorization checks (identity.canDo) since IAM operations + // have their own permission model based on self-service vs admin operations. + identity, errCode := e.iam.AuthSignatureOnly(r) + if errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, errCode) + return + } + + // IAM API requests must be authenticated - reject nil identity + // (can happen for authTypePostPolicy or authTypeStreamingUnsigned) + if identity == nil { + s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied) + return + } + + // Store identity in context + if identity != nil && identity.Name != "" { + ctx := SetIdentityNameInContext(r.Context(), identity.Name) + ctx = SetIdentityInContext(ctx, identity) + r = r.WithContext(ctx) + } + + // Check permissions based on action type + if iamRequiresAdminForOthers(action) { + // Self-service action: allow if operating on own resources or no target specified + if targetUserName == "" || targetUserName == identity.Name { + // Self-service: allowed + f(w, r) + return + } + // Operating on another user: require admin + if !identity.isAdmin() { + s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied) + return + } + } else { + // All other IAM actions require admin (CreateUser, DeleteUser, PutUserPolicy, etc.) + if !identity.isAdmin() { + s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied) + return + } + } + + f(w, r) + } +} + +// DoActions handles IAM API actions. +func (e *EmbeddedIamApi) DoActions(w http.ResponseWriter, r *http.Request) { + // Lock to prevent concurrent read-modify-write race conditions + e.policyLock.Lock() + defer e.policyLock.Unlock() + + if err := r.ParseForm(); err != nil { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) + return + } + values := r.PostForm + s3cfg := &iam_pb.S3ApiConfiguration{} + if err := e.GetS3ApiConfiguration(s3cfg); err != nil && !errors.Is(err, filer_pb.ErrNotFound) { + s3err.WriteErrorResponse(w, r, s3err.ErrInternalError) + return + } + + glog.V(4).Infof("IAM DoActions: %+v", values) + var response interface{} + var iamErr *iamError + changed := true + switch r.Form.Get("Action") { + case "ListUsers": + response = e.ListUsers(s3cfg, values) + changed = false + case "ListAccessKeys": + e.handleImplicitUsername(r, values) + response = e.ListAccessKeys(s3cfg, values) + changed = false + case "CreateUser": + response, iamErr = e.CreateUser(s3cfg, values) + if iamErr != nil { + e.writeIamErrorResponse(w, r, iamErr) + return + } + case "GetUser": + userName := values.Get("UserName") + response, iamErr = e.GetUser(s3cfg, userName) + if iamErr != nil { + e.writeIamErrorResponse(w, r, iamErr) + return + } + changed = false + case "UpdateUser": + response, iamErr = e.UpdateUser(s3cfg, values) + if iamErr != nil { + e.writeIamErrorResponse(w, r, iamErr) + return + } + case "DeleteUser": + userName := values.Get("UserName") + response, iamErr = e.DeleteUser(s3cfg, userName) + if iamErr != nil { + e.writeIamErrorResponse(w, r, iamErr) + return + } + case "CreateAccessKey": + e.handleImplicitUsername(r, values) + response, iamErr = e.CreateAccessKey(s3cfg, values) + if iamErr != nil { + glog.Errorf("CreateAccessKey: %+v", iamErr.Error) + e.writeIamErrorResponse(w, r, iamErr) + return + } + case "DeleteAccessKey": + e.handleImplicitUsername(r, values) + response = e.DeleteAccessKey(s3cfg, values) + case "CreatePolicy": + response, iamErr = e.CreatePolicy(s3cfg, values) + if iamErr != nil { + glog.Errorf("CreatePolicy: %+v", iamErr.Error) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest) + return + } + // CreatePolicy only validates the policy document and returns metadata. + // Policies are not stored separately; they are attached inline via PutUserPolicy. + changed = false + case "PutUserPolicy": + response, iamErr = e.PutUserPolicy(s3cfg, values) + if iamErr != nil { + glog.Errorf("PutUserPolicy: %+v", iamErr.Error) + e.writeIamErrorResponse(w, r, iamErr) + return + } + case "GetUserPolicy": + response, iamErr = e.GetUserPolicy(s3cfg, values) + if iamErr != nil { + e.writeIamErrorResponse(w, r, iamErr) + return + } + changed = false + case "DeleteUserPolicy": + response, iamErr = e.DeleteUserPolicy(s3cfg, values) + if iamErr != nil { + e.writeIamErrorResponse(w, r, iamErr) + return + } + default: + errNotImplemented := s3err.GetAPIError(s3err.ErrNotImplemented) + errorResponse := iamErrorResponse{} + errorResponse.Error.Code = &errNotImplemented.Code + errorResponse.Error.Message = &errNotImplemented.Description + s3err.WriteXMLResponse(w, r, errNotImplemented.HTTPStatusCode, errorResponse) + return + } + if changed { + if err := e.PutS3ApiConfiguration(s3cfg); err != nil { + iamErr = &iamError{Code: iam.ErrCodeServiceFailureException, Error: err} + e.writeIamErrorResponse(w, r, iamErr) + return + } + // Reload in-memory identity maps so subsequent LookupByAccessKey calls + // can see newly created or deleted keys immediately + if err := e.iam.LoadS3ApiConfigurationFromCredentialManager(); err != nil { + glog.Warningf("Failed to reload IAM configuration after mutation: %v", err) + // Don't fail the request since the persistent save succeeded + } + } + // Set RequestId for AWS compatibility + if r, ok := response.(interface{ SetRequestId() }); ok { + r.SetRequestId() + } + s3err.WriteXMLResponse(w, r, http.StatusOK, response) +} diff --git a/weed/s3api/s3api_embedded_iam_test.go b/weed/s3api/s3api_embedded_iam_test.go new file mode 100644 index 000000000..81839084b --- /dev/null +++ b/weed/s3api/s3api_embedded_iam_test.go @@ -0,0 +1,1028 @@ +package s3api + +import ( + "encoding/json" + "encoding/xml" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/proto" +) + +// EmbeddedIamApiForTest is a testable version of EmbeddedIamApi +type EmbeddedIamApiForTest struct { + *EmbeddedIamApi + mockConfig *iam_pb.S3ApiConfiguration +} + +func NewEmbeddedIamApiForTest() *EmbeddedIamApiForTest { + e := &EmbeddedIamApiForTest{ + EmbeddedIamApi: &EmbeddedIamApi{ + iam: &IdentityAccessManagement{}, + }, + mockConfig: &iam_pb.S3ApiConfiguration{}, + } + return e +} + +// Override GetS3ApiConfiguration for testing +func (e *EmbeddedIamApiForTest) GetS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) error { + // Use proto.Clone for proper deep copy semantics + if e.mockConfig != nil { + cloned := proto.Clone(e.mockConfig).(*iam_pb.S3ApiConfiguration) + proto.Merge(s3cfg, cloned) + } + return nil +} + +// Override PutS3ApiConfiguration for testing +func (e *EmbeddedIamApiForTest) PutS3ApiConfiguration(s3cfg *iam_pb.S3ApiConfiguration) error { + // Use proto.Clone for proper deep copy semantics + e.mockConfig = proto.Clone(s3cfg).(*iam_pb.S3ApiConfiguration) + return nil +} + +// DoActions handles IAM API actions for testing +func (e *EmbeddedIamApiForTest) DoActions(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + http.Error(w, "Invalid request", http.StatusBadRequest) + return + } + values := r.PostForm + s3cfg := &iam_pb.S3ApiConfiguration{} + if err := e.GetS3ApiConfiguration(s3cfg); err != nil { + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + + var response interface{} + var iamErr *iamError + changed := true + + switch r.Form.Get("Action") { + case "ListUsers": + response = e.ListUsers(s3cfg, values) + changed = false + case "ListAccessKeys": + e.handleImplicitUsername(r, values) + response = e.ListAccessKeys(s3cfg, values) + changed = false + case "CreateUser": + response, iamErr = e.CreateUser(s3cfg, values) + if iamErr != nil { + e.writeIamErrorResponse(w, r, iamErr) + return + } + case "GetUser": + userName := values.Get("UserName") + response, iamErr = e.GetUser(s3cfg, userName) + if iamErr != nil { + e.writeIamErrorResponse(w, r, iamErr) + return + } + changed = false + case "UpdateUser": + response, iamErr = e.UpdateUser(s3cfg, values) + if iamErr != nil { + http.Error(w, "Invalid request", http.StatusBadRequest) + return + } + case "DeleteUser": + userName := values.Get("UserName") + response, iamErr = e.DeleteUser(s3cfg, userName) + if iamErr != nil { + e.writeIamErrorResponse(w, r, iamErr) + return + } + case "CreateAccessKey": + e.handleImplicitUsername(r, values) + response, iamErr = e.CreateAccessKey(s3cfg, values) + if iamErr != nil { + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + case "DeleteAccessKey": + e.handleImplicitUsername(r, values) + response = e.DeleteAccessKey(s3cfg, values) + case "CreatePolicy": + response, iamErr = e.CreatePolicy(s3cfg, values) + if iamErr != nil { + http.Error(w, "Invalid request", http.StatusBadRequest) + return + } + case "PutUserPolicy": + response, iamErr = e.PutUserPolicy(s3cfg, values) + if iamErr != nil { + e.writeIamErrorResponse(w, r, iamErr) + return + } + case "GetUserPolicy": + response, iamErr = e.GetUserPolicy(s3cfg, values) + if iamErr != nil { + e.writeIamErrorResponse(w, r, iamErr) + return + } + changed = false + case "DeleteUserPolicy": + response, iamErr = e.DeleteUserPolicy(s3cfg, values) + if iamErr != nil { + e.writeIamErrorResponse(w, r, iamErr) + return + } + default: + http.Error(w, "Not implemented", http.StatusNotImplemented) + return + } + + if changed { + if err := e.PutS3ApiConfiguration(s3cfg); err != nil { + http.Error(w, "Internal error", http.StatusInternalServerError) + return + } + } + + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + xmlBytes, err := xml.Marshal(response) + if err != nil { + // This should not happen in tests, but log it for debugging + http.Error(w, "Internal error: failed to marshal response", http.StatusInternalServerError) + return + } + _, _ = w.Write(xmlBytes) +} + +// executeEmbeddedIamRequest executes an IAM request against the given API instance. +// If v is non-nil, the response body is unmarshalled into it. +func executeEmbeddedIamRequest(api *EmbeddedIamApiForTest, req *http.Request, v interface{}) (*httptest.ResponseRecorder, error) { + rr := httptest.NewRecorder() + apiRouter := mux.NewRouter().SkipClean(true) + apiRouter.Path("/").Methods(http.MethodPost).HandlerFunc(api.DoActions) + apiRouter.ServeHTTP(rr, req) + if v != nil { + if err := xml.Unmarshal(rr.Body.Bytes(), v); err != nil { + return rr, err + } + } + return rr, nil +} + +// embeddedIamErrorResponseForTest is used for parsing IAM error responses in tests +type embeddedIamErrorResponseForTest struct { + Error struct { + Code string `xml:"Code"` + Message string `xml:"Message"` + } `xml:"Error"` +} + +func extractEmbeddedIamErrorCodeAndMessage(response *httptest.ResponseRecorder) (string, string) { + var er embeddedIamErrorResponseForTest + if err := xml.Unmarshal(response.Body.Bytes(), &er); err != nil { + return "", "" + } + return er.Error.Code, er.Error.Message +} + +// TestEmbeddedIamCreateUser tests creating a user via the embedded IAM API +func TestEmbeddedIamCreateUser(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{} + + userName := aws.String("TestUser") + params := &iam.CreateUserInput{UserName: userName} + req, _ := iam.New(session.New()).CreateUserRequest(params) + _ = req.Build() + out := iamCreateUserResponse{} + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, &out) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) + + // Verify response contains correct username + assert.NotNil(t, out.CreateUserResult.User.UserName) + assert.Equal(t, "TestUser", *out.CreateUserResult.User.UserName) + + // Verify user was persisted in config + assert.Len(t, api.mockConfig.Identities, 1) + assert.Equal(t, "TestUser", api.mockConfig.Identities[0].Name) +} + +// TestEmbeddedIamListUsers tests listing users via the embedded IAM API +func TestEmbeddedIamListUsers(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + {Name: "User1"}, + {Name: "User2"}, + }, + } + + params := &iam.ListUsersInput{} + req, _ := iam.New(session.New()).ListUsersRequest(params) + _ = req.Build() + out := iamListUsersResponse{} + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, &out) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) + + // Verify response contains the users + assert.Len(t, out.ListUsersResult.Users, 2) +} + +// TestEmbeddedIamListAccessKeys tests listing access keys via the embedded IAM API +func TestEmbeddedIamListAccessKeys(t *testing.T) { + api := NewEmbeddedIamApiForTest() + svc := iam.New(session.New()) + params := &iam.ListAccessKeysInput{} + req, _ := svc.ListAccessKeysRequest(params) + _ = req.Build() + out := iamListAccessKeysResponse{} + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, &out) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) +} + +// TestEmbeddedIamGetUser tests getting a user via the embedded IAM API +func TestEmbeddedIamGetUser(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + {Name: "TestUser"}, + }, + } + + userName := aws.String("TestUser") + params := &iam.GetUserInput{UserName: userName} + req, _ := iam.New(session.New()).GetUserRequest(params) + _ = req.Build() + out := iamGetUserResponse{} + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, &out) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) + + // Verify response contains correct username + assert.NotNil(t, out.GetUserResult.User.UserName) + assert.Equal(t, "TestUser", *out.GetUserResult.User.UserName) +} + +// TestEmbeddedIamCreatePolicy tests creating a policy via the embedded IAM API +func TestEmbeddedIamCreatePolicy(t *testing.T) { + api := NewEmbeddedIamApiForTest() + params := &iam.CreatePolicyInput{ + PolicyName: aws.String("S3-read-only-example-bucket"), + PolicyDocument: aws.String(` + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:Get*", + "s3:List*" + ], + "Resource": [ + "arn:aws:s3:::EXAMPLE-BUCKET", + "arn:aws:s3:::EXAMPLE-BUCKET/*" + ] + } + ] + }`), + } + req, _ := iam.New(session.New()).CreatePolicyRequest(params) + _ = req.Build() + out := iamCreatePolicyResponse{} + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, &out) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) + + // Verify response contains policy metadata + assert.NotNil(t, out.CreatePolicyResult.Policy.PolicyName) + assert.Equal(t, "S3-read-only-example-bucket", *out.CreatePolicyResult.Policy.PolicyName) + assert.NotNil(t, out.CreatePolicyResult.Policy.Arn) + assert.NotNil(t, out.CreatePolicyResult.Policy.PolicyId) +} + +// TestEmbeddedIamPutUserPolicy tests attaching a policy to a user +func TestEmbeddedIamPutUserPolicy(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + {Name: "TestUser"}, + }, + } + + userName := aws.String("TestUser") + params := &iam.PutUserPolicyInput{ + UserName: userName, + PolicyName: aws.String("S3-read-only-example-bucket"), + PolicyDocument: aws.String( + `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:Get*", + "s3:List*" + ], + "Resource": [ + "arn:aws:s3:::EXAMPLE-BUCKET", + "arn:aws:s3:::EXAMPLE-BUCKET/*" + ] + } + ] + }`), + } + req, _ := iam.New(session.New()).PutUserPolicyRequest(params) + _ = req.Build() + out := iamPutUserPolicyResponse{} + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, &out) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) + + // Verify policy was attached to the user (actions should be set) + assert.Len(t, api.mockConfig.Identities, 1) + assert.NotEmpty(t, api.mockConfig.Identities[0].Actions) +} + +// TestEmbeddedIamPutUserPolicyError tests error handling when user doesn't exist +func TestEmbeddedIamPutUserPolicyError(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{} + + userName := aws.String("InvalidUser") + params := &iam.PutUserPolicyInput{ + UserName: userName, + PolicyName: aws.String("S3-read-only-example-bucket"), + PolicyDocument: aws.String( + `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:Get*", + "s3:List*" + ], + "Resource": [ + "arn:aws:s3:::EXAMPLE-BUCKET", + "arn:aws:s3:::EXAMPLE-BUCKET/*" + ] + } + ] + }`), + } + req, _ := iam.New(session.New()).PutUserPolicyRequest(params) + _ = req.Build() + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusNotFound, response.Code) + + expectedCode := "NoSuchEntity" + code, _ := extractEmbeddedIamErrorCodeAndMessage(response) + assert.Equal(t, expectedCode, code) +} + +// TestEmbeddedIamGetUserPolicy tests getting a user's policy +func TestEmbeddedIamGetUserPolicy(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + { + Name: "TestUser", + Actions: []string{"Read", "List"}, + }, + }, + } + + userName := aws.String("TestUser") + params := &iam.GetUserPolicyInput{ + UserName: userName, + PolicyName: aws.String("S3-read-only-example-bucket"), + } + req, _ := iam.New(session.New()).GetUserPolicyRequest(params) + _ = req.Build() + out := iamGetUserPolicyResponse{} + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, &out) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) +} + +// TestEmbeddedIamDeleteUserPolicy tests deleting a user's policy (clears actions) +func TestEmbeddedIamDeleteUserPolicy(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + { + Name: "TestUser", + Actions: []string{"Read", "Write", "List"}, + Credentials: []*iam_pb.Credential{ + {AccessKey: "AKIATEST12345", SecretKey: "secret"}, + }, + }, + }, + } + + // Use direct form post for DeleteUserPolicy + form := url.Values{} + form.Set("Action", "DeleteUserPolicy") + form.Set("UserName", "TestUser") + form.Set("PolicyName", "TestPolicy") + + req, _ := http.NewRequest("POST", "/", nil) + req.PostForm = form + req.Form = form + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + rr := httptest.NewRecorder() + apiRouter := mux.NewRouter().SkipClean(true) + apiRouter.Path("/").Methods(http.MethodPost).HandlerFunc(api.DoActions) + apiRouter.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code) + + // CRITICAL: Verify user still exists (was NOT deleted) + assert.Len(t, api.mockConfig.Identities, 1, "User should NOT be deleted") + assert.Equal(t, "TestUser", api.mockConfig.Identities[0].Name) + + // Verify credentials are still intact + assert.Len(t, api.mockConfig.Identities[0].Credentials, 1, "Credentials should NOT be deleted") + assert.Equal(t, "AKIATEST12345", api.mockConfig.Identities[0].Credentials[0].AccessKey) + + // Verify actions/policy was cleared + assert.Nil(t, api.mockConfig.Identities[0].Actions, "Actions should be cleared") +} + +// TestEmbeddedIamDeleteUserPolicyUserNotFound tests error when user doesn't exist +func TestEmbeddedIamDeleteUserPolicyUserNotFound(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{} + + form := url.Values{} + form.Set("Action", "DeleteUserPolicy") + form.Set("UserName", "NonExistentUser") + form.Set("PolicyName", "TestPolicy") + + req, _ := http.NewRequest("POST", "/", nil) + req.PostForm = form + req.Form = form + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + rr := httptest.NewRecorder() + apiRouter := mux.NewRouter().SkipClean(true) + apiRouter.Path("/").Methods(http.MethodPost).HandlerFunc(api.DoActions) + apiRouter.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusNotFound, rr.Code) +} + +// TestEmbeddedIamUpdateUser tests updating a user +func TestEmbeddedIamUpdateUser(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + {Name: "TestUser"}, + }, + } + + userName := aws.String("TestUser") + newUserName := aws.String("TestUser-New") + params := &iam.UpdateUserInput{NewUserName: newUserName, UserName: userName} + req, _ := iam.New(session.New()).UpdateUserRequest(params) + _ = req.Build() + out := iamUpdateUserResponse{} + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, &out) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) +} + +// TestEmbeddedIamDeleteUser tests deleting a user +func TestEmbeddedIamDeleteUser(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + {Name: "TestUser-New"}, + }, + } + + userName := aws.String("TestUser-New") + params := &iam.DeleteUserInput{UserName: userName} + req, _ := iam.New(session.New()).DeleteUserRequest(params) + _ = req.Build() + out := iamDeleteUserResponse{} + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, &out) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) +} + +// TestEmbeddedIamCreateAccessKey tests creating an access key +func TestEmbeddedIamCreateAccessKey(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + {Name: "TestUser"}, + }, + } + + userName := aws.String("TestUser") + params := &iam.CreateAccessKeyInput{UserName: userName} + req, _ := iam.New(session.New()).CreateAccessKeyRequest(params) + _ = req.Build() + out := iamCreateAccessKeyResponse{} + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, &out) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) + + // Verify response contains access key credentials + assert.NotNil(t, out.CreateAccessKeyResult.AccessKey.AccessKeyId) + assert.NotEmpty(t, *out.CreateAccessKeyResult.AccessKey.AccessKeyId) + assert.NotNil(t, out.CreateAccessKeyResult.AccessKey.SecretAccessKey) + assert.NotEmpty(t, *out.CreateAccessKeyResult.AccessKey.SecretAccessKey) + assert.NotNil(t, out.CreateAccessKeyResult.AccessKey.UserName) + assert.Equal(t, "TestUser", *out.CreateAccessKeyResult.AccessKey.UserName) + + // Verify credentials were persisted + assert.Len(t, api.mockConfig.Identities[0].Credentials, 1) +} + +// TestEmbeddedIamDeleteAccessKey tests deleting an access key via direct form post +func TestEmbeddedIamDeleteAccessKey(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + { + Name: "TestUser", + Credentials: []*iam_pb.Credential{ + {AccessKey: "AKIATEST12345", SecretKey: "secret"}, + }, + }, + }, + } + + // Use direct form post since AWS SDK may format differently + form := url.Values{} + form.Set("Action", "DeleteAccessKey") + form.Set("UserName", "TestUser") + form.Set("AccessKeyId", "AKIATEST12345") + + req, _ := http.NewRequest("POST", "/", nil) + req.PostForm = form + req.Form = form + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + rr := httptest.NewRecorder() + apiRouter := mux.NewRouter().SkipClean(true) + apiRouter.Path("/").Methods(http.MethodPost).HandlerFunc(api.DoActions) + apiRouter.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code) + + // Verify the access key was deleted + assert.Len(t, api.mockConfig.Identities[0].Credentials, 0) +} + +// TestEmbeddedIamHandleImplicitUsername tests implicit username extraction from authorization header +func TestEmbeddedIamHandleImplicitUsername(t *testing.T) { + // Create IAM with test credentials - the handleImplicitUsername function now looks + // up the username from the credential store based on AccessKeyId + // Note: Using obviously fake access keys to avoid secret scanner false positives + iam := &IdentityAccessManagement{} + testConfig := &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + { + Name: "testuser1", + Credentials: []*iam_pb.Credential{ + {AccessKey: "AKIATESTFAKEKEY000001", SecretKey: "testsecretfake"}, + }, + }, + }, + } + err := iam.LoadS3ApiConfigurationFromBytes(mustMarshalJSON(testConfig)) + if err != nil { + t.Fatalf("Failed to load test config: %v", err) + } + + embeddedApi := &EmbeddedIamApi{ + iam: iam, + } + + var tests = []struct { + r *http.Request + values url.Values + userName string + }{ + // No authorization header - should not set username + {&http.Request{}, url.Values{}, ""}, + // Valid auth header with known access key - should look up and find "testuser1" + {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=AKIATESTFAKEKEY000001/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, "testuser1"}, + // Malformed auth header (no Credential=) - should not set username + {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 =AKIATESTFAKEKEY000001/20220420/test1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""}, + // Unknown access key - should not set username + {&http.Request{Header: http.Header{"Authorization": []string{"AWS4-HMAC-SHA256 Credential=AKIATESTUNKNOWN000000/20220420/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=fakesignature0123456789abcdef"}}}, url.Values{}, ""}, + } + + for i, test := range tests { + embeddedApi.handleImplicitUsername(test.r, test.values) + if un := test.values.Get("UserName"); un != test.userName { + t.Errorf("No.%d: Got: %v, Expected: %v", i, un, test.userName) + } + } +} + +func mustMarshalJSON(v interface{}) []byte { + data, err := json.Marshal(v) + if err != nil { + panic(err) + } + return data +} + +// TestEmbeddedIamFullWorkflow tests a complete user lifecycle +func TestEmbeddedIamFullWorkflow(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{} + + // 1. Create user + t.Run("CreateUser", func(t *testing.T) { + userName := aws.String("WorkflowUser") + params := &iam.CreateUserInput{UserName: userName} + req, _ := iam.New(session.New()).CreateUserRequest(params) + _ = req.Build() + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) + }) + + // 2. Create access key for user + t.Run("CreateAccessKey", func(t *testing.T) { + userName := aws.String("WorkflowUser") + params := &iam.CreateAccessKeyInput{UserName: userName} + req, _ := iam.New(session.New()).CreateAccessKeyRequest(params) + _ = req.Build() + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) + }) + + // 3. Attach policy to user + t.Run("PutUserPolicy", func(t *testing.T) { + params := &iam.PutUserPolicyInput{ + UserName: aws.String("WorkflowUser"), + PolicyName: aws.String("ReadWritePolicy"), + PolicyDocument: aws.String(`{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:Get*", "s3:Put*"], + "Resource": ["arn:aws:s3:::*"] + }] + }`), + } + req, _ := iam.New(session.New()).PutUserPolicyRequest(params) + _ = req.Build() + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) + }) + + // 4. List users to verify + t.Run("ListUsers", func(t *testing.T) { + params := &iam.ListUsersInput{} + req, _ := iam.New(session.New()).ListUsersRequest(params) + _ = req.Build() + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) + }) + + // 5. Delete user + t.Run("DeleteUser", func(t *testing.T) { + params := &iam.DeleteUserInput{UserName: aws.String("WorkflowUser")} + req, _ := iam.New(session.New()).DeleteUserRequest(params) + _ = req.Build() + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, nil) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) + }) +} + +// TestIamStringSlicesEqual tests the iamStringSlicesEqual helper function +func TestIamStringSlicesEqual(t *testing.T) { + tests := []struct { + name string + a []string + b []string + expected bool + }{ + {"both empty", []string{}, []string{}, true}, + {"both nil", nil, nil, true}, + {"same elements same order", []string{"a", "b", "c"}, []string{"a", "b", "c"}, true}, + {"same elements different order", []string{"c", "a", "b"}, []string{"a", "b", "c"}, true}, + {"different lengths", []string{"a", "b"}, []string{"a", "b", "c"}, false}, + {"different elements", []string{"a", "b", "c"}, []string{"a", "b", "d"}, false}, + {"one empty one not", []string{}, []string{"a"}, false}, + {"duplicates same", []string{"a", "a", "b"}, []string{"a", "b", "a"}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := iamStringSlicesEqual(tt.a, tt.b) + assert.Equal(t, tt.expected, result) + }) + } +} + +// TestIamHash tests the iamHash function +func TestIamHash(t *testing.T) { + input := "test-policy-document" + hash := iamHash(&input) + + // Hash should be non-empty + assert.NotEmpty(t, hash) + + // Same input should produce same hash + hash2 := iamHash(&input) + assert.Equal(t, hash, hash2) + + // Different input should produce different hash + input2 := "different-policy" + hash3 := iamHash(&input2) + assert.NotEqual(t, hash, hash3) +} + +// TestIamStringWithCharset tests the cryptographically secure random string generator +func TestIamStringWithCharset(t *testing.T) { + charset := "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + length := 20 + + str, err := iamStringWithCharset(length, charset) + assert.NoError(t, err) + assert.Len(t, str, length) + + // All characters should be from the charset + for _, c := range str { + assert.Contains(t, charset, string(c)) + } + + // Two calls should produce different strings (with very high probability) + str2, err := iamStringWithCharset(length, charset) + assert.NoError(t, err) + assert.NotEqual(t, str, str2) +} + +// TestIamMapToStatementAction tests action mapping +func TestIamMapToStatementAction(t *testing.T) { + // iamMapToStatementAction maps IAM statement action patterns to internal action names + tests := []struct { + input string + expected string + }{ + {"*", "Admin"}, + {"Get*", "Read"}, + {"Put*", "Write"}, + {"List*", "List"}, + {"Tagging*", "Tagging"}, + {"DeleteBucket*", "DeleteBucket"}, + {"PutBucketAcl", "WriteAcp"}, + {"GetBucketAcl", "ReadAcp"}, + {"InvalidAction", ""}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := iamMapToStatementAction(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +// TestIamMapToIdentitiesAction tests reverse action mapping +func TestIamMapToIdentitiesAction(t *testing.T) { + // iamMapToIdentitiesAction maps internal action names to IAM statement action patterns + tests := []struct { + input string + expected string + }{ + {"Admin", "*"}, + {"Read", "Get*"}, + {"Write", "Put*"}, + {"List", "List*"}, + {"Tagging", "Tagging*"}, + {"Unknown", ""}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := iamMapToIdentitiesAction(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +// TestEmbeddedIamGetUserNotFound tests GetUser with non-existent user +func TestEmbeddedIamGetUserNotFound(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + {Name: "ExistingUser"}, + }, + } + + userName := aws.String("NonExistentUser") + params := &iam.GetUserInput{UserName: userName} + req, _ := iam.New(session.New()).GetUserRequest(params) + _ = req.Build() + response, _ := executeEmbeddedIamRequest(api, req.HTTPRequest, nil) + assert.Equal(t, http.StatusNotFound, response.Code) +} + +// TestEmbeddedIamDeleteUserNotFound tests DeleteUser with non-existent user +func TestEmbeddedIamDeleteUserNotFound(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{} + + userName := aws.String("NonExistentUser") + params := &iam.DeleteUserInput{UserName: userName} + req, _ := iam.New(session.New()).DeleteUserRequest(params) + _ = req.Build() + response, _ := executeEmbeddedIamRequest(api, req.HTTPRequest, nil) + assert.Equal(t, http.StatusNotFound, response.Code) +} + +// TestEmbeddedIamUpdateUserNotFound tests UpdateUser with non-existent user +func TestEmbeddedIamUpdateUserNotFound(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{} + + params := &iam.UpdateUserInput{ + UserName: aws.String("NonExistentUser"), + NewUserName: aws.String("NewName"), + } + req, _ := iam.New(session.New()).UpdateUserRequest(params) + _ = req.Build() + response, _ := executeEmbeddedIamRequest(api, req.HTTPRequest, nil) + assert.Equal(t, http.StatusBadRequest, response.Code) +} + +// TestEmbeddedIamCreateAccessKeyForExistingUser tests CreateAccessKey creates credentials for existing user +func TestEmbeddedIamCreateAccessKeyForExistingUser(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + {Name: "ExistingUser"}, + }, + } + + // Use direct form post + form := url.Values{} + form.Set("Action", "CreateAccessKey") + form.Set("UserName", "ExistingUser") + + req, _ := http.NewRequest("POST", "/", nil) + req.PostForm = form + req.Form = form + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + rr := httptest.NewRecorder() + apiRouter := mux.NewRouter().SkipClean(true) + apiRouter.Path("/").Methods(http.MethodPost).HandlerFunc(api.DoActions) + apiRouter.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code) + // Verify credentials were created + assert.Len(t, api.mockConfig.Identities[0].Credentials, 1) +} + +// TestEmbeddedIamGetUserPolicyUserNotFound tests GetUserPolicy with non-existent user +func TestEmbeddedIamGetUserPolicyUserNotFound(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{} + + params := &iam.GetUserPolicyInput{ + UserName: aws.String("NonExistentUser"), + PolicyName: aws.String("TestPolicy"), + } + req, _ := iam.New(session.New()).GetUserPolicyRequest(params) + _ = req.Build() + response, _ := executeEmbeddedIamRequest(api, req.HTTPRequest, nil) + assert.Equal(t, http.StatusNotFound, response.Code) +} + +// TestEmbeddedIamCreatePolicyMalformed tests CreatePolicy with invalid policy document +func TestEmbeddedIamCreatePolicyMalformed(t *testing.T) { + api := NewEmbeddedIamApiForTest() + + params := &iam.CreatePolicyInput{ + PolicyName: aws.String("TestPolicy"), + PolicyDocument: aws.String("invalid json"), + } + req, _ := iam.New(session.New()).CreatePolicyRequest(params) + _ = req.Build() + response, _ := executeEmbeddedIamRequest(api, req.HTTPRequest, nil) + assert.Equal(t, http.StatusBadRequest, response.Code) +} + +// TestEmbeddedIamListAccessKeysForUser tests listing access keys for a specific user +func TestEmbeddedIamListAccessKeysForUser(t *testing.T) { + api := NewEmbeddedIamApiForTest() + api.mockConfig = &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{ + { + Name: "TestUser", + Credentials: []*iam_pb.Credential{ + {AccessKey: "AKIATEST1", SecretKey: "secret1"}, + {AccessKey: "AKIATEST2", SecretKey: "secret2"}, + }, + }, + }, + } + + params := &iam.ListAccessKeysInput{UserName: aws.String("TestUser")} + req, _ := iam.New(session.New()).ListAccessKeysRequest(params) + _ = req.Build() + out := iamListAccessKeysResponse{} + response, err := executeEmbeddedIamRequest(api, req.HTTPRequest, &out) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, response.Code) + + // Verify both access keys are listed + assert.Len(t, out.ListAccessKeysResult.AccessKeyMetadata, 2) +} + +// TestEmbeddedIamNotImplementedAction tests handling of unimplemented actions +func TestEmbeddedIamNotImplementedAction(t *testing.T) { + api := NewEmbeddedIamApiForTest() + + form := url.Values{} + form.Set("Action", "SomeUnknownAction") + + req, _ := http.NewRequest("POST", "/", nil) + req.PostForm = form + req.Form = form + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + rr := httptest.NewRecorder() + apiRouter := mux.NewRouter().SkipClean(true) + apiRouter.Path("/").Methods(http.MethodPost).HandlerFunc(api.DoActions) + apiRouter.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusNotImplemented, rr.Code) +} + +// TestGetPolicyDocument tests parsing of policy documents +func TestGetPolicyDocument(t *testing.T) { + api := NewEmbeddedIamApiForTest() + + validPolicy := `{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": ["arn:aws:s3:::bucket/*"] + }] + }` + + doc, err := api.GetPolicyDocument(&validPolicy) + assert.NoError(t, err) + assert.Equal(t, "2012-10-17", doc.Version) + assert.Len(t, doc.Statement, 1) + + // Test invalid JSON + invalidPolicy := "not valid json" + _, err = api.GetPolicyDocument(&invalidPolicy) + assert.Error(t, err) +} + +// TestEmbeddedIamGetActionsFromPolicy tests action extraction from policy documents +func TestEmbeddedIamGetActionsFromPolicy(t *testing.T) { + api := NewEmbeddedIamApiForTest() + + // Actions must use wildcards (Get*, Put*, List*, etc.) as expected by the mapper + policyDoc := `{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:Get*", "s3:Put*"], + "Resource": ["arn:aws:s3:::mybucket/*"] + }] + }` + + policy, err := api.GetPolicyDocument(&policyDoc) + assert.NoError(t, err) + + actions, err := api.getActions(&policy) + assert.NoError(t, err) + assert.NotEmpty(t, actions) + // Should have Read and Write actions for the bucket + // arn:aws:s3:::mybucket/* means all objects in mybucket, represented as "Action:mybucket" + assert.Contains(t, actions, "Read:mybucket") + assert.Contains(t, actions, "Write:mybucket") +} + diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 4a8368409..bf1a44e54 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -50,6 +50,7 @@ type S3ApiServerOption struct { IamConfig string // Advanced IAM configuration file path ConcurrentUploadLimit int64 ConcurrentFileUploadLimit int64 + EnableIam bool // Enable embedded IAM API on the same port } type S3ApiServer struct { @@ -69,6 +70,7 @@ type S3ApiServer struct { inFlightDataSize int64 inFlightUploads int64 inFlightDataLimitCond *sync.Cond + embeddedIam *EmbeddedIamApi // Embedded IAM API server (when enabled) } func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) { @@ -186,6 +188,12 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl } } + // Initialize embedded IAM API if enabled + if option.EnableIam { + s3ApiServer.embeddedIam = NewEmbeddedIamApi(s3ApiServer.credentialManager, iam) + glog.V(0).Infof("Embedded IAM API initialized (use -iam=false to disable)") + } + if option.Config != "" { grace.OnReload(func() { if err := s3ApiServer.iam.loadS3ApiConfigurationFromFile(option.Config); err != nil { @@ -594,6 +602,16 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { } }) + // Embedded IAM API (POST to "/" with Action parameter) + // This must be before ListBuckets since IAM uses POST and ListBuckets uses GET + // Uses AuthIam for granular permission checking: + // - Self-service operations (own access keys) don't require admin + // - Operations on other users require admin privileges + if s3a.embeddedIam != nil { + apiRouter.Methods(http.MethodPost).Path("/").HandlerFunc(track(s3a.embeddedIam.AuthIam(s3a.cb.Limit(s3a.embeddedIam.DoActions, ACTION_WRITE)), "IAM")) + glog.V(0).Infof("Embedded IAM API enabled on S3 port") + } + // ListBuckets apiRouter.Methods(http.MethodGet).Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_LIST), "LIST"))