add debugging for InvalidAccessKeyId

weed/credential/filer_etc/filer_etc_identity.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/credential"
 	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
 )
@@ -14,24 +15,63 @@ import (
 func (store *FilerEtcStore) LoadConfiguration(ctx context.Context) (*iam_pb.S3ApiConfiguration, error) {
 	s3cfg := &iam_pb.S3ApiConfiguration{}
 
+	glog.V(1).Infof("Loading IAM configuration from %s/%s (filer: %s)", 
+		filer.IamConfigDirectory, filer.IamIdentityFile, store.filerGrpcAddress)
+
 	err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 		// Use ReadInsideFiler instead of ReadEntry since identity.json is small
 		// and stored inline. ReadEntry requires a master client for chunked files,
 		// but ReadInsideFiler only reads inline content.
 		content, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory, filer.IamIdentityFile)
 		if err != nil {
-			if err != filer_pb.ErrNotFound {
-				return err
+			if err == filer_pb.ErrNotFound {
+				glog.V(1).Infof("IAM identity file not found at %s/%s, no credentials loaded", 
+					filer.IamConfigDirectory, filer.IamIdentityFile)
+				return nil
 			}
+			glog.Errorf("Failed to read IAM identity file from %s/%s: %v", 
+				filer.IamConfigDirectory, filer.IamIdentityFile, err)
+			return err
+		}
+		
+		if len(content) == 0 {
+			glog.V(1).Infof("IAM identity file at %s/%s is empty", 
+				filer.IamConfigDirectory, filer.IamIdentityFile)
 			return nil
 		}
-		if len(content) > 0 {
-			return filer.ParseS3ConfigurationFromBytes(content, s3cfg)
+		
+		glog.V(2).Infof("Read %d bytes from %s/%s", 
+			len(content), filer.IamConfigDirectory, filer.IamIdentityFile)
+		
+		if err := filer.ParseS3ConfigurationFromBytes(content, s3cfg); err != nil {
+			glog.Errorf("Failed to parse IAM configuration from %s/%s: %v", 
+				filer.IamConfigDirectory, filer.IamIdentityFile, err)
+			return err
 		}
+		
+		glog.V(1).Infof("Successfully parsed IAM configuration with %d identities and %d accounts", 
+			len(s3cfg.Identities), len(s3cfg.Accounts))
 		return nil
 	})
 
-	return s3cfg, err
+	if err != nil {
+		return s3cfg, err
+	}
+
+	// Log loaded identities for debugging
+	if glog.V(2) {
+		for _, identity := range s3cfg.Identities {
+			credCount := len(identity.Credentials)
+			actionCount := len(identity.Actions)
+			glog.V(2).Infof("  Identity: %s (credentials: %d, actions: %d)", 
+				identity.Name, credCount, actionCount)
+			for _, cred := range identity.Credentials {
+				glog.V(3).Infof("    Access Key: %s", cred.AccessKey)
+			}
+		}
+	}
+
+	return s3cfg, nil
 }
 
 func (store *FilerEtcStore) SaveConfiguration(ctx context.Context, config *iam_pb.S3ApiConfiguration) error {

weed/credential/filer_etc/filer_etc_policy.go

@@ -27,6 +27,9 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_
 		return policiesCollection.Policies, nil
 	}
 
+	glog.V(2).Infof("Loading IAM policies from %s/%s (filer: %s)", 
+		filer.IamConfigDirectory, filer.IamPoliciesFile, store.filerGrpcAddress)
+
 	err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 		// Use ReadInsideFiler instead of ReadEntry since policies.json is small
 		// and stored inline. ReadEntry requires a master client for chunked files,
@@ -34,16 +37,32 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_
 		content, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory, filer.IamPoliciesFile)
 		if err != nil {
 			if err == filer_pb.ErrNotFound {
-				glog.V(1).Infof("Policies file not found at %s/%s, returning empty policies", filer.IamConfigDirectory, filer.IamPoliciesFile)
+				glog.V(1).Infof("Policies file not found at %s/%s, returning empty policies", 
+					filer.IamConfigDirectory, filer.IamPoliciesFile)
 				// If file doesn't exist, return empty collection
 				return nil
 			}
+			glog.Errorf("Failed to read IAM policies file from %s/%s: %v", 
+				filer.IamConfigDirectory, filer.IamPoliciesFile, err)
 			return err
 		}
 
-		if len(content) > 0 {
-			return json.Unmarshal(content, policiesCollection)
+		if len(content) == 0 {
+			glog.V(2).Infof("IAM policies file at %s/%s is empty", 
+				filer.IamConfigDirectory, filer.IamPoliciesFile)
+			return nil
+		}
+
+		glog.V(2).Infof("Read %d bytes from %s/%s", 
+			len(content), filer.IamConfigDirectory, filer.IamPoliciesFile)
+
+		if err := json.Unmarshal(content, policiesCollection); err != nil {
+			glog.Errorf("Failed to parse IAM policies from %s/%s: %v", 
+				filer.IamConfigDirectory, filer.IamPoliciesFile, err)
+			return err
 		}
+
+		glog.V(1).Infof("Successfully loaded %d IAM policies", len(policiesCollection.Policies))
 		return nil
 	})
 
@@ -51,6 +70,13 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_
 		return nil, err
 	}
 
+	// Log policy names for debugging
+	if glog.V(2) && len(policiesCollection.Policies) > 0 {
+		for policyName := range policiesCollection.Policies {
+			glog.V(2).Infof("  Policy: %s", policyName)
+		}
+	}
+
 	return policiesCollection.Policies, nil
 }
 

weed/s3api/auth_credentials.go

@@ -349,6 +349,17 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
 	}
 	iam.m.Unlock()
 
+	// Log configuration summary
+	glog.V(1).Infof("Loaded %d identities, %d accounts, %d access keys. Auth enabled: %v",
+		len(identities), len(accounts), len(accessKeyIdent), iam.isAuthEnabled)
+	
+	if glog.V(2) {
+		glog.V(2).Infof("Access key to identity mapping:")
+		for accessKey, identity := range accessKeyIdent {
+			glog.V(2).Infof("  %s -> %s (actions: %d)", accessKey, identity.Name, len(identity.Actions))
+		}
+	}
+
 	return nil
 }
 
@@ -359,14 +370,29 @@ func (iam *IdentityAccessManagement) isEnabled() bool {
 func (iam *IdentityAccessManagement) lookupByAccessKey(accessKey string) (identity *Identity, cred *Credential, found bool) {
 	iam.m.RLock()
 	defer iam.m.RUnlock()
+	
+	glog.V(3).Infof("Looking up access key: %s (total keys registered: %d)", accessKey, len(iam.accessKeyIdent))
+	
 	if ident, ok := iam.accessKeyIdent[accessKey]; ok {
 		for _, credential := range ident.Credentials {
 			if credential.AccessKey == accessKey {
+				glog.V(2).Infof("Found access key %s for identity %s", accessKey, ident.Name)
 				return ident, credential, true
 			}
 		}
 	}
-	glog.V(1).Infof("could not find accessKey %s", accessKey)
+	
+	glog.V(1).Infof("Could not find access key %s. Available keys: %d, Auth enabled: %v", 
+		accessKey, len(iam.accessKeyIdent), iam.isAuthEnabled)
+	
+	// Log all registered access keys at higher verbosity for debugging
+	if glog.V(3) {
+		glog.V(3).Infof("Registered access keys:")
+		for key := range iam.accessKeyIdent {
+			glog.V(3).Infof("  - %s", key)
+		}
+	}
+	
 	return nil, nil, false
 }
 
@@ -649,12 +675,24 @@ func (iam *IdentityAccessManagement) GetCredentialManager() *credential.Credenti
 
 // LoadS3ApiConfigurationFromCredentialManager loads configuration using the credential manager
 func (iam *IdentityAccessManagement) LoadS3ApiConfigurationFromCredentialManager() error {
+	glog.V(1).Infof("Loading S3 API configuration from credential manager")
+	
 	s3ApiConfiguration, err := iam.credentialManager.LoadConfiguration(context.Background())
 	if err != nil {
+		glog.Errorf("Failed to load configuration from credential manager: %v", err)
 		return fmt.Errorf("failed to load configuration from credential manager: %w", err)
 	}
 
-	return iam.loadS3ApiConfiguration(s3ApiConfiguration)
+	glog.V(2).Infof("Credential manager returned %d identities and %d accounts", 
+		len(s3ApiConfiguration.Identities), len(s3ApiConfiguration.Accounts))
+
+	if err := iam.loadS3ApiConfiguration(s3ApiConfiguration); err != nil {
+		glog.Errorf("Failed to load S3 API configuration: %v", err)
+		return err
+	}
+
+	glog.V(1).Infof("Successfully loaded S3 API configuration from credential manager")
+	return nil
 }
 
 // initializeKMSFromConfig loads KMS configuration from TOML format

weed/s3api/auth_signature_v2.go

@@ -28,6 +28,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
 )
@@ -76,6 +77,14 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(formValues http.
 
 	identity, cred, found := iam.lookupByAccessKey(accessKey)
 	if !found {
+		// Log detailed error information for InvalidAccessKeyId (V2 POST)
+		iam.m.RLock()
+		availableKeyCount := len(iam.accessKeyIdent)
+		iam.m.RUnlock()
+		
+		glog.Warningf("InvalidAccessKeyId (V2 POST): attempted key '%s' not found. Available keys: %d, Auth enabled: %v",
+			accessKey, availableKeyCount, iam.isAuthEnabled)
+		
 		return s3err.ErrInvalidAccessKeyID
 	}
 
@@ -113,6 +122,14 @@ func (iam *IdentityAccessManagement) doesSignV2Match(r *http.Request) (*Identity
 
 	identity, cred, found := iam.lookupByAccessKey(accessKey)
 	if !found {
+		// Log detailed error information for InvalidAccessKeyId (V2 signed)
+		iam.m.RLock()
+		availableKeyCount := len(iam.accessKeyIdent)
+		iam.m.RUnlock()
+		
+		glog.Warningf("InvalidAccessKeyId (V2 signed): attempted key '%s' not found. Available keys: %d, Auth enabled: %v",
+			accessKey, availableKeyCount, iam.isAuthEnabled)
+		
 		return nil, s3err.ErrInvalidAccessKeyID
 	}
 
@@ -145,6 +162,7 @@ func (iam *IdentityAccessManagement) doesPresignV2SignatureMatch(r *http.Request
 
 	accessKey := query.Get("AWSAccessKeyId")
 	if accessKey == "" {
+		glog.Warningf("InvalidAccessKeyId (V2 presigned): empty access key in request")
 		return nil, s3err.ErrInvalidAccessKeyID
 	}
 
@@ -155,6 +173,14 @@ func (iam *IdentityAccessManagement) doesPresignV2SignatureMatch(r *http.Request
 
 	identity, cred, found := iam.lookupByAccessKey(accessKey)
 	if !found {
+		// Log detailed error information for InvalidAccessKeyId (V2 presigned)
+		iam.m.RLock()
+		availableKeyCount := len(iam.accessKeyIdent)
+		iam.m.RUnlock()
+		
+		glog.Warningf("InvalidAccessKeyId (V2 presigned): attempted key '%s' not found. Available keys: %d, Auth enabled: %v",
+			accessKey, availableKeyCount, iam.isAuthEnabled)
+		
 		return nil, s3err.ErrInvalidAccessKeyID
 	}
 

weed/s3api/auth_signature_v4.go

@@ -207,6 +207,21 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh
 	// 2. Lookup user and credentials
 	identity, cred, found := iam.lookupByAccessKey(authInfo.AccessKey)
 	if !found {
+		// Log detailed error information for InvalidAccessKeyId
+		iam.m.RLock()
+		availableKeys := make([]string, 0, len(iam.accessKeyIdent))
+		for key := range iam.accessKeyIdent {
+			availableKeys = append(availableKeys, key)
+		}
+		iam.m.RUnlock()
+		
+		glog.Warningf("InvalidAccessKeyId: attempted key '%s' not found. Available keys: %d, Auth enabled: %v",
+			authInfo.AccessKey, len(availableKeys), iam.isAuthEnabled)
+		
+		if glog.V(2) && len(availableKeys) > 0 {
+			glog.V(2).Infof("Available access keys: %v", availableKeys)
+		}
+		
 		return nil, nil, "", nil, s3err.ErrInvalidAccessKeyID
 	}
 
@@ -543,6 +558,14 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(formValues http.
 
 	identity, cred, found := iam.lookupByAccessKey(credHeader.accessKey)
 	if !found {
+		// Log detailed error information for InvalidAccessKeyId (POST policy)
+		iam.m.RLock()
+		availableKeyCount := len(iam.accessKeyIdent)
+		iam.m.RUnlock()
+		
+		glog.Warningf("InvalidAccessKeyId (POST policy): attempted key '%s' not found. Available keys: %d, Auth enabled: %v",
+			credHeader.accessKey, availableKeyCount, iam.isAuthEnabled)
+		
 		return s3err.ErrInvalidAccessKeyID
 	}