diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index 385bbf59f..3ccf43575 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -334,6 +334,12 @@ jobs: pip install tox pip install -e . + - name: Fix S3 tests bucket creation conflicts + run: | + python3 test/s3/fix_s3_tests_bucket_conflicts.py + env: + S3_TESTS_PATH: s3-tests + - name: Run S3 Object Lock, Retention, and Versioning tests timeout-minutes: 15 shell: bash diff --git a/test/s3/fix_s3_tests_bucket_conflicts.py b/test/s3/fix_s3_tests_bucket_conflicts.py new file mode 100755 index 000000000..c64ea3d70 --- /dev/null +++ b/test/s3/fix_s3_tests_bucket_conflicts.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +""" +Fix S3 tests to handle bucket creation conflicts properly. + +This script patches the s3-tests repository to handle BucketAlreadyExists errors +by retrying with new bucket names instead of failing. + +This is needed because SeaweedFS now properly returns BucketAlreadyExists errors +per S3 specification, but the original s3-tests code doesn't handle this case. +""" + +import os +import sys +import re + +def patch_s3_tests_init_file(file_path): + """Patch the s3tests_boto3/functional/__init__.py file to handle bucket conflicts.""" + + if not os.path.exists(file_path): + print(f"Error: File {file_path} not found") + return False + + print(f"Patching {file_path}...") + + with open(file_path, 'r') as f: + content = f.read() + + # Check if already patched + if 'max_retries = 10' in content: + print("File already patched, skipping...") + return True + + # Patch get_new_bucket_resource function + old_resource_func = '''def get_new_bucket_resource(name=None): + """ + Get a bucket that exists and is empty. + + Always recreates a bucket from scratch. This is useful to also + reset ACLs and such. + """ + s3 = boto3.resource('s3', + aws_access_key_id=config.main_access_key, + aws_secret_access_key=config.main_secret_key, + endpoint_url=config.default_endpoint, + use_ssl=config.default_is_secure, + verify=config.default_ssl_verify) + if name is None: + name = get_new_bucket_name() + bucket = s3.Bucket(name) + bucket_location = bucket.create() + return bucket''' + + new_resource_func = '''def get_new_bucket_resource(name=None): + """ + Get a bucket that exists and is empty. + + Always recreates a bucket from scratch. This is useful to also + reset ACLs and such. + """ + s3 = boto3.resource('s3', + aws_access_key_id=config.main_access_key, + aws_secret_access_key=config.main_secret_key, + endpoint_url=config.default_endpoint, + use_ssl=config.default_is_secure, + verify=config.default_ssl_verify) + + # Retry bucket creation if name conflicts occur + max_retries = 10 + for attempt in range(max_retries): + if name is None or attempt > 0: + name = get_new_bucket_name() + + bucket = s3.Bucket(name) + try: + bucket_location = bucket.create() + return bucket + except ClientError as e: + error_code = e.response['Error']['Code'] + if error_code in ['BucketAlreadyExists', 'BucketAlreadyOwnedByYou']: + # Bucket name conflict, try again with a new name + if attempt == max_retries - 1: + raise Exception(f"Failed to create unique bucket after {max_retries} attempts") + continue + else: + # Other error, re-raise + raise''' + + # Patch get_new_bucket function + old_client_func = '''def get_new_bucket(client=None, name=None): + """ + Get a bucket that exists and is empty. + + Always recreates a bucket from scratch. This is useful to also + reset ACLs and such. + """ + if client is None: + client = get_client() + if name is None: + name = get_new_bucket_name() + + client.create_bucket(Bucket=name) + return name''' + + new_client_func = '''def get_new_bucket(client=None, name=None): + """ + Get a bucket that exists and is empty. + + Always recreates a bucket from scratch. This is useful to also + reset ACLs and such. + """ + if client is None: + client = get_client() + + # Retry bucket creation if name conflicts occur + max_retries = 10 + for attempt in range(max_retries): + if name is None or attempt > 0: + name = get_new_bucket_name() + + try: + client.create_bucket(Bucket=name) + return name + except ClientError as e: + error_code = e.response['Error']['Code'] + if error_code in ['BucketAlreadyExists', 'BucketAlreadyOwnedByYou']: + # Bucket name conflict, try again with a new name + if attempt == max_retries - 1: + raise Exception(f"Failed to create unique bucket after {max_retries} attempts") + continue + else: + # Other error, re-raise + raise''' + + # Apply patches + content = content.replace(old_resource_func, new_resource_func) + content = content.replace(old_client_func, new_client_func) + + # Write back the patched content + with open(file_path, 'w') as f: + f.write(content) + + print("Successfully patched s3-tests to handle bucket creation conflicts!") + return True + +def main(): + """Main function to apply the patch.""" + + # Default path for s3-tests in GitHub Actions + s3_tests_path = os.environ.get('S3_TESTS_PATH', 's3-tests') + init_file_path = os.path.join(s3_tests_path, 's3tests_boto3', 'functional', '__init__.py') + + print("🔧 Fixing S3 tests bucket creation conflicts...") + print(f"Looking for s3-tests at: {s3_tests_path}") + + if not os.path.exists(s3_tests_path): + print(f"Error: s3-tests directory not found at {s3_tests_path}") + print("Make sure to check out the s3-tests repository first") + return 1 + + if patch_s3_tests_init_file(init_file_path): + print("✅ S3 tests successfully patched!") + print("The tests will now handle BucketAlreadyExists errors properly") + return 0 + else: + print("❌ Failed to patch s3-tests") + return 1 + +if __name__ == '__main__': + sys.exit(main())