From e3d3c495abe9062755a7ee29b78a4f0e8dbde855 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 Jul 2025 02:05:26 -0700 Subject: [PATCH] S3 API: simpler way to start s3 with credentials (#7030) * simpler way to start s3 with credentials * AWS_ACCESS_KEY_ID=access_key AWS_SECRET_ACCESS_KEY=secret_key weed s3 * last adding credentials from env variables * Update weed/s3api/auth_credentials.go Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * simplify * adjust doc --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- weed/command/s3.go | 8 +++ weed/s3api/auth_credentials.go | 72 +++++++++++++++------- weed/s3api/auth_credentials_test.go | 93 +++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 20 deletions(-) diff --git a/weed/command/s3.go b/weed/command/s3.go index 25b75e9da..8dcb681b9 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -160,6 +160,14 @@ var cmdS3 = &Command{ ] } + Alternatively, you can use environment variables to supplement admin credentials: + + AWS_ACCESS_KEY_ID=your_access_key AWS_SECRET_ACCESS_KEY=your_secret_key weed s3 + + This will add admin credentials from environment variables to any existing + configuration. Environment variables are added after loading file/filer + configurations and are skipped if the same access key already exists. + `, } diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 742f3eede..4f00639ff 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -85,22 +85,6 @@ type Credential struct { SecretKey string } -func (i *Identity) isAnonymous() bool { - return i.Account.Id == s3_constants.AccountAnonymousId -} - -func (action Action) isAdmin() bool { - return strings.HasPrefix(string(action), s3_constants.ACTION_ADMIN) -} - -func (action Action) isOwner(bucket string) bool { - return string(action) == s3_constants.ACTION_ADMIN+":"+bucket -} - -func (action Action) overBucket(bucket string) bool { - return strings.HasSuffix(string(action), ":"+bucket) || strings.HasSuffix(string(action), ":*") -} - // "Permission": "FULL_CONTROL"|"WRITE"|"WRITE_ACP"|"READ"|"READ_ACP" func (action Action) getPermission() Permission { switch act := strings.Split(string(action), ":")[0]; act { @@ -147,6 +131,7 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitSto iam.credentialManager = credentialManager + // First, load configurations from file or filer if option.Config != "" { glog.V(3).Infof("loading static config file %s", option.Config) if err := iam.loadS3ApiConfigurationFromFile(option.Config); err != nil { @@ -158,6 +143,57 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitSto glog.Warningf("fail to load config: %v", err) } } + + // Then, add admin credentials from environment variables if available +// This supplements the configuration by adding admin credentials from environment variables if they don't already exist. + accessKeyId := os.Getenv("AWS_ACCESS_KEY_ID") + secretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY") + + if accessKeyId != "" && secretAccessKey != "" { + glog.V(0).Infof("Adding S3 admin credentials from AWS environment variables") + + // Check if an identity with this access key already exists + iam.m.RLock() + _, accessKeyExists := iam.accessKeyIdent[accessKeyId] + iam.m.RUnlock() + + if !accessKeyExists { + // Create environment variable identity name + identityNameSuffix := accessKeyId + if len(accessKeyId) > 8 { + identityNameSuffix = accessKeyId[:8] + } + + // Create admin identity with environment variable credentials + envIdentity := &Identity{ + Name: "admin-" + identityNameSuffix, + Account: &AccountAdmin, + Credentials: []*Credential{ + { + AccessKey: accessKeyId, + SecretKey: secretAccessKey, + }, + }, + Actions: []Action{ + s3_constants.ACTION_ADMIN, + }, + } + + // Add to existing configuration + iam.m.Lock() + iam.identities = append(iam.identities, envIdentity) + iam.accessKeyIdent[accessKeyId] = envIdentity + if !iam.isAuthEnabled { + iam.isAuthEnabled = true + } + iam.m.Unlock() + + glog.V(0).Infof("Added admin identity from AWS environment variables: %s", envIdentity.Name) + } else { + glog.V(0).Infof("Access key %s already exists, skipping environment variable credentials", accessKeyId) + } + } + return iam } @@ -538,9 +574,5 @@ func (iam *IdentityAccessManagement) LoadS3ApiConfigurationFromCredentialManager return fmt.Errorf("failed to load configuration from credential manager: %w", err) } - if len(s3ApiConfiguration.Identities) == 0 { - return fmt.Errorf("no identities found") - } - return iam.loadS3ApiConfiguration(s3ApiConfiguration) } diff --git a/weed/s3api/auth_credentials_test.go b/weed/s3api/auth_credentials_test.go index dbc431332..0ed6e65db 100644 --- a/weed/s3api/auth_credentials_test.go +++ b/weed/s3api/auth_credentials_test.go @@ -1,9 +1,11 @@ package s3api import ( + "os" "reflect" "testing" + "github.com/seaweedfs/seaweedfs/weed/credential" . "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/stretchr/testify/assert" @@ -264,3 +266,94 @@ func TestLoadS3ApiConfiguration(t *testing.T) { } } } + +func TestNewIdentityAccessManagementWithStoreEnvVars(t *testing.T) { + // Save original environment + originalAccessKeyId := os.Getenv("AWS_ACCESS_KEY_ID") + originalSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY") + + // Clean up after test + defer func() { + if originalAccessKeyId != "" { + os.Setenv("AWS_ACCESS_KEY_ID", originalAccessKeyId) + } else { + os.Unsetenv("AWS_ACCESS_KEY_ID") + } + if originalSecretAccessKey != "" { + os.Setenv("AWS_SECRET_ACCESS_KEY", originalSecretAccessKey) + } else { + os.Unsetenv("AWS_SECRET_ACCESS_KEY") + } + }() + + tests := []struct { + name string + accessKeyId string + secretAccessKey string + expectEnvIdentity bool + expectedName string + }{ + { + name: "Both env vars set", + accessKeyId: "AKIA1234567890ABCDEF", + secretAccessKey: "secret123456789012345678901234567890abcdef12", + expectEnvIdentity: true, + expectedName: "admin-AKIA1234", + }, + { + name: "Short access key", + accessKeyId: "SHORT", + secretAccessKey: "secret123456789012345678901234567890abcdef12", + expectEnvIdentity: true, + expectedName: "admin-SHORT", + }, + { + name: "No env vars set", + accessKeyId: "", + secretAccessKey: "", + expectEnvIdentity: false, + expectedName: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set up environment variables + if tt.accessKeyId != "" { + os.Setenv("AWS_ACCESS_KEY_ID", tt.accessKeyId) + } else { + os.Unsetenv("AWS_ACCESS_KEY_ID") + } + if tt.secretAccessKey != "" { + os.Setenv("AWS_SECRET_ACCESS_KEY", tt.secretAccessKey) + } else { + os.Unsetenv("AWS_SECRET_ACCESS_KEY") + } + + // Create IAM instance with memory store for testing + option := &S3ApiServerOption{ + Config: "", // No config file, should use environment variables + } + iam := NewIdentityAccessManagementWithStore(option, string(credential.StoreTypeMemory)) + + if tt.expectEnvIdentity { + // Check that environment variable identity was created + found := false + for _, identity := range iam.identities { + if identity.Name == tt.expectedName { + found = true + assert.Len(t, identity.Credentials, 1, "Should have one credential") + assert.Equal(t, tt.accessKeyId, identity.Credentials[0].AccessKey, "Access key should match environment variable") + assert.Equal(t, tt.secretAccessKey, identity.Credentials[0].SecretKey, "Secret key should match environment variable") + assert.Contains(t, identity.Actions, Action(ACTION_ADMIN), "Should have admin action") + break + } + } + assert.True(t, found, "Should find identity created from environment variables") + } else { + // When no env vars, should have no identities (since no config file) + assert.Len(t, iam.identities, 0, "Should have no identities when no env vars and no config file") + } + }) + } +}