From 33f3f81759a0e86017e8915cfec4d84e3614eb85 Mon Sep 17 00:00:00 2001 From: Drew Short Date: Sat, 25 Dec 2021 14:08:02 -0600 Subject: [PATCH] Cleanup and bugfix --- acm.py | 78 ++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/acm.py b/acm.py index 048eb2b..d25a849 100755 --- a/acm.py +++ b/acm.py @@ -18,7 +18,7 @@ from minio.error import NoSuchKey # Size of the buffer to read files with BUF_SIZE = 4096 -#Application Version +# Application Version VERSION = "1.4.0" @@ -80,7 +80,8 @@ def run_asyncio_commands(tasks, max_concurrent_tasks=0): num_chunks = len(chunks) else: chunks = make_chunks(tasks=tasks, chunk_size=max_concurrent_tasks) - num_chunks = len(list(make_chunks(tasks=tasks, chunk_size=max_concurrent_tasks))) + num_chunks = len( + list(make_chunks(tasks=tasks, chunk_size=max_concurrent_tasks))) if asyncio.get_event_loop().is_closed(): asyncio.set_event_loop(asyncio.new_event_loop()) @@ -196,7 +197,7 @@ def add_nested_key(config: Dict[str, any], path: List[str], value: str) -> bool: else: if target not in config: config[target] = {} - add_nested_key(config[target], path[1:],value) + add_nested_key(config[target], path[1:], value) return False @@ -204,7 +205,8 @@ def read_env_config(prefix, separator='__') -> any: prefix = prefix+separator env_config = {} - environment_variables = [env for env in os.environ.keys() if env.startswith(prefix)] + environment_variables = [ + env for env in os.environ.keys() if env.startswith(prefix)] for env in environment_variables: path = env[len(prefix):].split('__') @@ -212,16 +214,16 @@ def read_env_config(prefix, separator='__') -> any: return env_config + def load_config(path: str) -> any: combined_config = {} with open( - os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'acm-config-default.json'), - 'r') as combined_config_file: + os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'acm-config-default.json'), + 'r') as combined_config_file: combined_config = json.load(combined_config_file) - config = {} with open(path, 'r') as config_file: config = json.load(config_file) @@ -236,10 +238,12 @@ def load_config(path: str) -> any: update(combined_config, read_env_config('ACM')) # Calculate profiles hash - profile_hashes={} - profile_hashes['all'] = get_string_sha256sum(json.dumps(combined_config['profiles'])) + profile_hashes = {} + profile_hashes['all'] = get_string_sha256sum( + json.dumps(combined_config['profiles'])) for profile in combined_config['profiles'].keys(): - profile_hashes[profile] = get_string_sha256sum(json.dumps(combined_config['profiles'][profile])) + profile_hashes[profile] = get_string_sha256sum( + json.dumps(combined_config['profiles'][profile])) combined_config['profileHashes'] = profile_hashes @@ -356,7 +360,8 @@ def check_matched_files_hashes(ctx, context, print_identity, profile, files): try: file_object = s3.get_object(s3_bucket, file_identity) stored_data = json.load(file_object) - stored_profile_hash, stored_file_hash, calculated_file_hash = get_file_sha256sum(stored_data, profile, file) + stored_profile_hash, stored_file_hash, calculated_file_hash = get_file_sha256sum( + stored_data, profile, file) if calculated_file_hash == stored_file_hash \ and ctx.obj['CONFIG']['profileHashes'][profile] == stored_profile_hash: if print_identity: @@ -392,7 +397,8 @@ def check_changed_files_hashes(ctx, context, profile, files): try: file_object = s3.get_object(s3_bucket, file_identity) stored_data = json.load(file_object) - stored_profile_hash, stored_file_hash, calculated_file_hash = get_file_sha256sum(stored_data, profile, file) + stored_profile_hash, stored_file_hash, calculated_file_hash = get_file_sha256sum( + stored_data, profile, file) if calculated_file_hash != stored_file_hash \ or ctx.obj['CONFIG']['profileHashes'][profile] != stored_profile_hash: changed_files.append(file) @@ -485,7 +491,8 @@ def store_files(ctx, context, files): content_type="application/octet-stream" ) if 'ADD_PREFIX' in ctx.obj and ctx.obj['ADD_PREFIX'] is not None: - stored_files.append(os.path.join(ctx.obj['ADD_PREFIX'], file_identity)) + stored_files.append(os.path.join( + ctx.obj['ADD_PREFIX'], file_identity)) else: stored_files.append(file) except ResponseError as e: @@ -523,7 +530,8 @@ def retrieve_files(ctx, context, destination, files): ) retrieved_files.append(file_destination) except NoSuchKey as e: - print(f'ERROR: {file_identity} {file_destination} {e}', file=sys.stderr) + print( + f'ERROR: {file_identity} {file_destination} {e}', file=sys.stderr) except ResponseError as e: print(f'ERROR: {file_destination} {e}', file=sys.stderr) @@ -587,14 +595,15 @@ def clean_files(ctx, context, context_data, dry_run, files): found_data_objects: List[str] = [] for obj in s3.list_objects_v2(s3_data_bucket, recursive=False): if obj.is_dir: - found_data_objects.extend(list_s3_dir(s3, s3_data_bucket, obj.object_name)) + found_data_objects.extend(list_s3_dir( + s3, s3_data_bucket, obj.object_name)) else: found_data_objects.append(obj.object_name) # print(os.linesep.join(found_data_objects)) for file_identity in found_objects: - if not file_identity in found_files: + if file_identity not in found_files: if dry_run: removed_files.append(f'{s3_bucket}:{file_identity}') else: @@ -602,18 +611,21 @@ def clean_files(ctx, context, context_data, dry_run, files): s3.remove_object(s3_bucket, file_identity) removed_files.append(f'{s3_bucket}:{file_identity}') except ResponseError as e: - print(f'ERROR: {s3_bucket}:{file_identity} {e}', file=sys.stderr) + print( + f'ERROR: {s3_bucket}:{file_identity} {e}', file=sys.stderr) for file_data_identity in found_data_objects: - if not file_data_identity in found_data_files: + if file_data_identity not in found_data_files: if dry_run: removed_files.append(f'{s3_data_bucket}:{file_data_identity}') else: try: s3.remove_object(s3_data_bucket, file_data_identity) - removed_files.append(f'{s3_data_bucket}:{file_data_identity}') + removed_files.append( + f'{s3_data_bucket}:{file_data_identity}') except ResponseError as e: - print(f'ERROR: {s3_data_bucket}:{file_data_identity} {e}', file=sys.stderr) + print( + f'ERROR: {s3_data_bucket}:{file_data_identity} {e}', file=sys.stderr) print(os.linesep.join(removed_files)) @@ -691,7 +703,8 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity """ if keep_smaller_input: command = f"cp {input_file} {output_file}" - return lambda: + + def task(): input_size = os.path.getsize(input_file) output_size = os.path.getsize(output_file) if output_size > input_size: @@ -706,9 +719,9 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity )] ) ) + return task return None - for input_file in files: for content_configuration in content_configurations: if any([input_file.endswith(extension) for extension in content_configuration['extensions']]): @@ -721,26 +734,31 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity and content_configuration['preserveInputExtension']: output_file = os.path.join(destination, file) else: - output_file_without_ext = os.path.splitext(os.path.join(destination, file))[0] + output_file_without_ext = os.path.splitext( + os.path.join(destination, file))[0] output_file = f'{output_file_without_ext}.{content_configuration["outputExtension"]}' - output_file_identity = get_file_identity({'REMOVE_PREFIX': destination}, output_file) + output_file_identity = get_file_identity( + {'REMOVE_PREFIX': destination}, output_file) output_file_dir = os.path.dirname(output_file) os.makedirs(output_file_dir, exist_ok=True) if 'preserveSmallerInput' in content_configuration: - preserve_smaller_input = bool(content_configuration['preserveSmallerInput']) + preserve_smaller_input = bool( + content_configuration['preserveSmallerInput']) else: preserve_smaller_input = True if 'forcePreserveSmallerInput' in content_configuration: - force_preserve_smaller_input = bool(content_configuration['forcePreserveSmallerInput']) + force_preserve_smaller_input = bool( + content_configuration['forcePreserveSmallerInput']) else: force_preserve_smaller_input = False # Only preserve the input if requested AND the extensions of the input and the output match - preserve_smaller_input = preserve_smaller_input and (force_preserve_smaller_input or file_extension == content_configuration["outputExtension"]) + preserve_smaller_input = preserve_smaller_input and ( + force_preserve_smaller_input or file_extension == content_configuration["outputExtension"]) command: str = content_configuration['command'] \ .replace('{input_file}', f'\'{input_file}\'') \ @@ -754,7 +772,7 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity on_success=[store_filename( task_output, f'{input_file}\t{output_file_identity}' if print_input_and_identity else output_file - ),queue_follow_up_task_if_keep_smaller_input( + ), queue_follow_up_task_if_keep_smaller_input( follow_up_tasks, input_file, output_file,