Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

test: add Polaris S3 tables integration tests (#8489) 

* test: add polaris integration test harness

* test: add polaris integration coverage

* ci: run polaris s3 tables tests

* test: harden polaris harness

* test: DRY polaris integration tests

* ci: pre-pull Polaris image

* test: extend Polaris pull timeout

* test: refine polaris credentials selection

* test: keep Polaris tables inside allowed location

* test: use fresh context for polaris cleanup

* test: prefer specific Polaris storage credential

* test: tolerate Polaris credential variants

* test: request Polaris vended credentials

* test: load Polaris table credentials

* test: allow polaris vended access via bucket policy

* test: align Polaris object keys with table location

* test: rename Polaris vended role references

* test: simplify Polaris vended credential extraction

* test: marshal Polaris bucket policy
master
Chris Lu 15 hours ago
committed by GitHub
parent
479da50433
commit
fb944f0071
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
2 changed files with 41 additions and 82 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      test/s3tables/polaris/polaris_env_test.go
  2. 119
      test/s3tables/polaris/polaris_test.go

4
test/s3tables/polaris/polaris_env_test.go
View File

@ -28,7 +28,7 @@ const (
polarisRootClientID = "root"
polarisRootClientSecret = "s3cr3t"
polarisRegion = "us-east-1"
polarisRoleArn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
polarisRoleArn = "arn:aws:iam::000000000000:role/PolarisVendedRole"
polarisSigningKey = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz" // gitleaks:allow - test signing key
)
@ -140,7 +140,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
},
"roles": [
{
"roleName": "LakekeeperVendedRole",
"roleName": "PolarisVendedRole",
"roleArn": "%s",
"trustPolicy": {
"Version": "2012-10-17",

119
test/s3tables/polaris/polaris_test.go
View File

@ -321,10 +321,28 @@ func newPolarisSession(t *testing.T, ctx context.Context, env *TestEnvironment)
}); err != nil {
t.Fatalf("CreateBucket failed: %v", err)
}
policy := fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Sid":"AllowPolarisVendedAccess","Effect":"Allow","Principal":"*","Action":"s3:*","Resource":["arn:aws:s3:::%s","arn:aws:s3:::%s/polaris/*"]}]}`, bucketName, bucketName)
policyDoc := map[string]interface{}{
"Version": "2012-10-17",
"Statement": []map[string]interface{}{
{
"Sid": "AllowPolarisVendedAccess",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:*",
"Resource": []string{
fmt.Sprintf("arn:aws:s3:::%s", bucketName),
fmt.Sprintf("arn:aws:s3:::%s/polaris/*", bucketName),
},
},
},
}
policyBytes, err := json.Marshal(policyDoc)
if err != nil {
t.Fatalf("Failed to marshal bucket policy: %v", err)
}
if _, err := adminS3.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
Bucket: aws.String(bucketName),
Policy: aws.String(policy),
Policy: aws.String(string(policyBytes)),
}); err != nil {
t.Fatalf("PutBucketPolicy failed: %v", err)
}
@ -623,36 +641,31 @@ func (c *polarisCatalogClient) LoadCredentials(ctx context.Context, namespace, t
}
func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoint, fallbackRegion string) (aws.Credentials, string, string, bool, error) {
configMap, err := selectStorageConfig(load, targetPrefix)
credentialConfig, err := selectStorageConfig(load, targetPrefix)
if err != nil {
return aws.Credentials{}, "", "", false, err
}
lookup := func(keys ...string) string {
for _, key := range keys {
if val, ok := configMap[key]; ok && val != "" {
return val
lookupConfig := func(key string) string {
if load != nil && load.Config != nil {
if val, ok := load.Config[key]; ok && strings.TrimSpace(val) != "" {
return strings.TrimSpace(val)
}
}
if val, ok := credentialConfig[key]; ok && strings.TrimSpace(val) != "" {
return strings.TrimSpace(val)
}
return ""
}
accessKey := lookup("s3.access-key-id", "aws.access-key-id", "s3.accessKeyId", "aws.accessKeyId", "accessKeyId", "access_key_id")
secretKey := lookup("s3.secret-access-key", "aws.secret-access-key", "s3.secretAccessKey", "aws.secretAccessKey", "secretAccessKey", "secret_access_key")
sessionToken := lookup("s3.session-token", "aws.session-token", "s3.sessionToken", "aws.sessionToken", "sessionToken", "session_token")
accessKey := strings.TrimSpace(credentialConfig["s3.access-key-id"])
secretKey := strings.TrimSpace(credentialConfig["s3.secret-access-key"])
sessionToken := strings.TrimSpace(credentialConfig["s3.session-token"])
if accessKey == "" || secretKey == "" {
if altConfig := findConfigWithKeys(load, targetPrefix); altConfig != nil {
configMap = altConfig
accessKey = lookup("s3.access-key-id", "aws.access-key-id", "s3.accessKeyId", "aws.accessKeyId", "accessKeyId", "access_key_id")
secretKey = lookup("s3.secret-access-key", "aws.secret-access-key", "s3.secretAccessKey", "aws.secretAccessKey", "secretAccessKey", "secret_access_key")
sessionToken = lookup("s3.session-token", "aws.session-token", "s3.sessionToken", "aws.sessionToken", "sessionToken", "session_token")
}
}
if accessKey == "" || secretKey == "" {
return aws.Credentials{}, "", "", false, fmt.Errorf("missing access key or secret in storage credentials")
return aws.Credentials{}, "", "", false, fmt.Errorf("missing s3.access-key-id or s3.secret-access-key in selected storage credential")
}
endpoint := lookup("s3.endpoint", "s3.endpoint-url", "aws.endpoint")
endpoint := lookupConfig("s3.endpoint")
if endpoint == "" {
endpoint = fallbackEndpoint
}
@ -660,13 +673,13 @@ func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoin
endpoint = "http://" + endpoint
}
region := lookup("s3.region", "aws.region")
region := lookupConfig("client.region")
if region == "" {
region = fallbackRegion
}
pathStyle := true
if value := lookup("s3.path-style-access", "s3.pathStyleAccess", "path-style-access"); value != "" {
if value := lookupConfig("s3.path-style-access"); value != "" {
pathStyle = strings.EqualFold(value, "true")
}
@ -679,6 +692,10 @@ func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoin
}
func selectStorageConfig(load *loadTableResponse, targetPrefix string) (map[string]string, error) {
if load == nil {
return nil, fmt.Errorf("load table response is nil")
}
switch len(load.StorageCredentials) {
case 0:
if load.Config == nil {
@ -733,64 +750,6 @@ func normalizePrefix(prefix string) string {
return p
}
func findConfigWithKeys(load *loadTableResponse, targetPrefix string) map[string]string {
normalizedTarget := normalizePrefix(targetPrefix)
if normalizedTarget == "" {
return nil
}
hasKeys := func(config map[string]string) bool {
if config == nil {
return false
}
accessKey := lookupValue(config, "s3.access-key-id", "aws.access-key-id", "s3.accessKeyId", "aws.accessKeyId", "accessKeyId", "access_key_id")
secretKey := lookupValue(config, "s3.secret-access-key", "aws.secret-access-key", "s3.secretAccessKey", "aws.secretAccessKey", "secretAccessKey", "secret_access_key")
return accessKey != "" && secretKey != ""
}
bestConfig := map[string]string(nil)
bestLen := -1
for _, cred := range load.StorageCredentials {
if !hasKeys(cred.Config) {
continue
}
prefix := normalizePrefix(cred.Prefix)
if prefix == "" {
if bestLen < 0 {
bestLen = 0
bestConfig = cred.Config
}
continue
}
if normalizedTarget == prefix || strings.HasPrefix(normalizedTarget, prefix+"/") {
if len(prefix) > bestLen {
bestLen = len(prefix)
bestConfig = cred.Config
}
}
}
if bestConfig != nil {
return bestConfig
}
if len(load.StorageCredentials) == 0 && hasKeys(load.Config) {
return load.Config
}
if hasKeys(load.Config) {
return load.Config
}
return nil
}
func lookupValue(config map[string]string, keys ...string) string {
for _, key := range keys {
if val, ok := config[key]; ok && val != "" {
return val
}
}
return ""
}
func s3URIToKeyPrefix(uri, bucket string) (string, error) {
prefix := "s3://" + bucket + "/"
if !strings.HasPrefix(uri, prefix) {

Write Preview
Loading…
Cancel
Save