From fb944f00718d174e869821c9f65e242221c735ee Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 2 Mar 2026 12:10:03 -0800 Subject: [PATCH] test: add Polaris S3 tables integration tests (#8489) * test: add polaris integration test harness * test: add polaris integration coverage * ci: run polaris s3 tables tests * test: harden polaris harness * test: DRY polaris integration tests * ci: pre-pull Polaris image * test: extend Polaris pull timeout * test: refine polaris credentials selection * test: keep Polaris tables inside allowed location * test: use fresh context for polaris cleanup * test: prefer specific Polaris storage credential * test: tolerate Polaris credential variants * test: request Polaris vended credentials * test: load Polaris table credentials * test: allow polaris vended access via bucket policy * test: align Polaris object keys with table location * test: rename Polaris vended role references * test: simplify Polaris vended credential extraction * test: marshal Polaris bucket policy --- test/s3tables/polaris/polaris_env_test.go | 4 +- test/s3tables/polaris/polaris_test.go | 119 +++++++--------------- 2 files changed, 41 insertions(+), 82 deletions(-) diff --git a/test/s3tables/polaris/polaris_env_test.go b/test/s3tables/polaris/polaris_env_test.go index 2a3496e2f..414a4781d 100644 --- a/test/s3tables/polaris/polaris_env_test.go +++ b/test/s3tables/polaris/polaris_env_test.go @@ -28,7 +28,7 @@ const ( polarisRootClientID = "root" polarisRootClientSecret = "s3cr3t" polarisRegion = "us-east-1" - polarisRoleArn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole" + polarisRoleArn = "arn:aws:iam::000000000000:role/PolarisVendedRole" polarisSigningKey = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz" // gitleaks:allow - test signing key ) @@ -140,7 +140,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { }, "roles": [ { - "roleName": "LakekeeperVendedRole", + "roleName": "PolarisVendedRole", "roleArn": "%s", "trustPolicy": { "Version": "2012-10-17", diff --git a/test/s3tables/polaris/polaris_test.go b/test/s3tables/polaris/polaris_test.go index db1fead63..61f34d8c6 100644 --- a/test/s3tables/polaris/polaris_test.go +++ b/test/s3tables/polaris/polaris_test.go @@ -321,10 +321,28 @@ func newPolarisSession(t *testing.T, ctx context.Context, env *TestEnvironment) }); err != nil { t.Fatalf("CreateBucket failed: %v", err) } - policy := fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Sid":"AllowPolarisVendedAccess","Effect":"Allow","Principal":"*","Action":"s3:*","Resource":["arn:aws:s3:::%s","arn:aws:s3:::%s/polaris/*"]}]}`, bucketName, bucketName) + policyDoc := map[string]interface{}{ + "Version": "2012-10-17", + "Statement": []map[string]interface{}{ + { + "Sid": "AllowPolarisVendedAccess", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:*", + "Resource": []string{ + fmt.Sprintf("arn:aws:s3:::%s", bucketName), + fmt.Sprintf("arn:aws:s3:::%s/polaris/*", bucketName), + }, + }, + }, + } + policyBytes, err := json.Marshal(policyDoc) + if err != nil { + t.Fatalf("Failed to marshal bucket policy: %v", err) + } if _, err := adminS3.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{ Bucket: aws.String(bucketName), - Policy: aws.String(policy), + Policy: aws.String(string(policyBytes)), }); err != nil { t.Fatalf("PutBucketPolicy failed: %v", err) } @@ -623,36 +641,31 @@ func (c *polarisCatalogClient) LoadCredentials(ctx context.Context, namespace, t } func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoint, fallbackRegion string) (aws.Credentials, string, string, bool, error) { - configMap, err := selectStorageConfig(load, targetPrefix) + credentialConfig, err := selectStorageConfig(load, targetPrefix) if err != nil { return aws.Credentials{}, "", "", false, err } - lookup := func(keys ...string) string { - for _, key := range keys { - if val, ok := configMap[key]; ok && val != "" { - return val + lookupConfig := func(key string) string { + if load != nil && load.Config != nil { + if val, ok := load.Config[key]; ok && strings.TrimSpace(val) != "" { + return strings.TrimSpace(val) } } + if val, ok := credentialConfig[key]; ok && strings.TrimSpace(val) != "" { + return strings.TrimSpace(val) + } return "" } - accessKey := lookup("s3.access-key-id", "aws.access-key-id", "s3.accessKeyId", "aws.accessKeyId", "accessKeyId", "access_key_id") - secretKey := lookup("s3.secret-access-key", "aws.secret-access-key", "s3.secretAccessKey", "aws.secretAccessKey", "secretAccessKey", "secret_access_key") - sessionToken := lookup("s3.session-token", "aws.session-token", "s3.sessionToken", "aws.sessionToken", "sessionToken", "session_token") + accessKey := strings.TrimSpace(credentialConfig["s3.access-key-id"]) + secretKey := strings.TrimSpace(credentialConfig["s3.secret-access-key"]) + sessionToken := strings.TrimSpace(credentialConfig["s3.session-token"]) if accessKey == "" || secretKey == "" { - if altConfig := findConfigWithKeys(load, targetPrefix); altConfig != nil { - configMap = altConfig - accessKey = lookup("s3.access-key-id", "aws.access-key-id", "s3.accessKeyId", "aws.accessKeyId", "accessKeyId", "access_key_id") - secretKey = lookup("s3.secret-access-key", "aws.secret-access-key", "s3.secretAccessKey", "aws.secretAccessKey", "secretAccessKey", "secret_access_key") - sessionToken = lookup("s3.session-token", "aws.session-token", "s3.sessionToken", "aws.sessionToken", "sessionToken", "session_token") - } - } - if accessKey == "" || secretKey == "" { - return aws.Credentials{}, "", "", false, fmt.Errorf("missing access key or secret in storage credentials") + return aws.Credentials{}, "", "", false, fmt.Errorf("missing s3.access-key-id or s3.secret-access-key in selected storage credential") } - endpoint := lookup("s3.endpoint", "s3.endpoint-url", "aws.endpoint") + endpoint := lookupConfig("s3.endpoint") if endpoint == "" { endpoint = fallbackEndpoint } @@ -660,13 +673,13 @@ func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoin endpoint = "http://" + endpoint } - region := lookup("s3.region", "aws.region") + region := lookupConfig("client.region") if region == "" { region = fallbackRegion } pathStyle := true - if value := lookup("s3.path-style-access", "s3.pathStyleAccess", "path-style-access"); value != "" { + if value := lookupConfig("s3.path-style-access"); value != "" { pathStyle = strings.EqualFold(value, "true") } @@ -679,6 +692,10 @@ func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoin } func selectStorageConfig(load *loadTableResponse, targetPrefix string) (map[string]string, error) { + if load == nil { + return nil, fmt.Errorf("load table response is nil") + } + switch len(load.StorageCredentials) { case 0: if load.Config == nil { @@ -733,64 +750,6 @@ func normalizePrefix(prefix string) string { return p } -func findConfigWithKeys(load *loadTableResponse, targetPrefix string) map[string]string { - normalizedTarget := normalizePrefix(targetPrefix) - if normalizedTarget == "" { - return nil - } - - hasKeys := func(config map[string]string) bool { - if config == nil { - return false - } - accessKey := lookupValue(config, "s3.access-key-id", "aws.access-key-id", "s3.accessKeyId", "aws.accessKeyId", "accessKeyId", "access_key_id") - secretKey := lookupValue(config, "s3.secret-access-key", "aws.secret-access-key", "s3.secretAccessKey", "aws.secretAccessKey", "secretAccessKey", "secret_access_key") - return accessKey != "" && secretKey != "" - } - - bestConfig := map[string]string(nil) - bestLen := -1 - for _, cred := range load.StorageCredentials { - if !hasKeys(cred.Config) { - continue - } - prefix := normalizePrefix(cred.Prefix) - if prefix == "" { - if bestLen < 0 { - bestLen = 0 - bestConfig = cred.Config - } - continue - } - if normalizedTarget == prefix || strings.HasPrefix(normalizedTarget, prefix+"/") { - if len(prefix) > bestLen { - bestLen = len(prefix) - bestConfig = cred.Config - } - } - } - if bestConfig != nil { - return bestConfig - } - - if len(load.StorageCredentials) == 0 && hasKeys(load.Config) { - return load.Config - } - if hasKeys(load.Config) { - return load.Config - } - return nil -} - -func lookupValue(config map[string]string, keys ...string) string { - for _, key := range keys { - if val, ok := config[key]; ok && val != "" { - return val - } - } - return "" -} - func s3URIToKeyPrefix(uri, bucket string) (string, error) { prefix := "s3://" + bucket + "/" if !strings.HasPrefix(uri, prefix) {