warricksothr
/
Asset-Compression-Manager
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Cleanup and bugfix

master 1.4.1
Drew Short 2 years ago
parent
ed7554f972
commit
33f3f81759
1 changed files with 48 additions and 30 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 78
      acm.py

78
acm.py
View File

@ -18,7 +18,7 @@ from minio.error import NoSuchKey
# Size of the buffer to read files with
BUF_SIZE = 4096
#Application Version
# Application Version
VERSION = "1.4.0"
@ -80,7 +80,8 @@ def run_asyncio_commands(tasks, max_concurrent_tasks=0):
num_chunks = len(chunks)
else:
chunks = make_chunks(tasks=tasks, chunk_size=max_concurrent_tasks)
num_chunks = len(list(make_chunks(tasks=tasks, chunk_size=max_concurrent_tasks)))
num_chunks = len(
list(make_chunks(tasks=tasks, chunk_size=max_concurrent_tasks)))
if asyncio.get_event_loop().is_closed():
asyncio.set_event_loop(asyncio.new_event_loop())
@ -196,7 +197,7 @@ def add_nested_key(config: Dict[str, any], path: List[str], value: str) -> bool:
else:
if target not in config:
config[target] = {}
add_nested_key(config[target], path[1:],value)
add_nested_key(config[target], path[1:], value)
return False
@ -204,7 +205,8 @@ def read_env_config(prefix, separator='__') -> any:
prefix = prefix+separator
env_config = {}
environment_variables = [env for env in os.environ.keys() if env.startswith(prefix)]
environment_variables = [
env for env in os.environ.keys() if env.startswith(prefix)]
for env in environment_variables:
path = env[len(prefix):].split('__')
@ -212,16 +214,16 @@ def read_env_config(prefix, separator='__') -> any:
return env_config
def load_config(path: str) -> any:
combined_config = {}
with open(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'acm-config-default.json'),
'r') as combined_config_file:
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'acm-config-default.json'),
'r') as combined_config_file:
combined_config = json.load(combined_config_file)
config = {}
with open(path, 'r') as config_file:
config = json.load(config_file)
@ -236,10 +238,12 @@ def load_config(path: str) -> any:
update(combined_config, read_env_config('ACM'))
# Calculate profiles hash
profile_hashes={}
profile_hashes['all'] = get_string_sha256sum(json.dumps(combined_config['profiles']))
profile_hashes = {}
profile_hashes['all'] = get_string_sha256sum(
json.dumps(combined_config['profiles']))
for profile in combined_config['profiles'].keys():
profile_hashes[profile] = get_string_sha256sum(json.dumps(combined_config['profiles'][profile]))
profile_hashes[profile] = get_string_sha256sum(
json.dumps(combined_config['profiles'][profile]))
combined_config['profileHashes'] = profile_hashes
@ -356,7 +360,8 @@ def check_matched_files_hashes(ctx, context, print_identity, profile, files):
try:
file_object = s3.get_object(s3_bucket, file_identity)
stored_data = json.load(file_object)
stored_profile_hash, stored_file_hash, calculated_file_hash = get_file_sha256sum(stored_data, profile, file)
stored_profile_hash, stored_file_hash, calculated_file_hash = get_file_sha256sum(
stored_data, profile, file)
if calculated_file_hash == stored_file_hash \
and ctx.obj['CONFIG']['profileHashes'][profile] == stored_profile_hash:
if print_identity:
@ -392,7 +397,8 @@ def check_changed_files_hashes(ctx, context, profile, files):
try:
file_object = s3.get_object(s3_bucket, file_identity)
stored_data = json.load(file_object)
stored_profile_hash, stored_file_hash, calculated_file_hash = get_file_sha256sum(stored_data, profile, file)
stored_profile_hash, stored_file_hash, calculated_file_hash = get_file_sha256sum(
stored_data, profile, file)
if calculated_file_hash != stored_file_hash \
or ctx.obj['CONFIG']['profileHashes'][profile] != stored_profile_hash:
changed_files.append(file)
@ -485,7 +491,8 @@ def store_files(ctx, context, files):
content_type="application/octet-stream"
)
if 'ADD_PREFIX' in ctx.obj and ctx.obj['ADD_PREFIX'] is not None:
stored_files.append(os.path.join(ctx.obj['ADD_PREFIX'], file_identity))
stored_files.append(os.path.join(
ctx.obj['ADD_PREFIX'], file_identity))
else:
stored_files.append(file)
except ResponseError as e:
@ -523,7 +530,8 @@ def retrieve_files(ctx, context, destination, files):
)
retrieved_files.append(file_destination)
except NoSuchKey as e:
print(f'ERROR: {file_identity} {file_destination} {e}', file=sys.stderr)
print(
f'ERROR: {file_identity} {file_destination} {e}', file=sys.stderr)
except ResponseError as e:
print(f'ERROR: {file_destination} {e}', file=sys.stderr)
@ -587,14 +595,15 @@ def clean_files(ctx, context, context_data, dry_run, files):
found_data_objects: List[str] = []
for obj in s3.list_objects_v2(s3_data_bucket, recursive=False):
if obj.is_dir:
found_data_objects.extend(list_s3_dir(s3, s3_data_bucket, obj.object_name))
found_data_objects.extend(list_s3_dir(
s3, s3_data_bucket, obj.object_name))
else:
found_data_objects.append(obj.object_name)
# print(os.linesep.join(found_data_objects))
for file_identity in found_objects:
if not file_identity in found_files:
if file_identity not in found_files:
if dry_run:
removed_files.append(f'{s3_bucket}:{file_identity}')
else:
@ -602,18 +611,21 @@ def clean_files(ctx, context, context_data, dry_run, files):
s3.remove_object(s3_bucket, file_identity)
removed_files.append(f'{s3_bucket}:{file_identity}')
except ResponseError as e:
print(f'ERROR: {s3_bucket}:{file_identity} {e}', file=sys.stderr)
print(
f'ERROR: {s3_bucket}:{file_identity} {e}', file=sys.stderr)
for file_data_identity in found_data_objects:
if not file_data_identity in found_data_files:
if file_data_identity not in found_data_files:
if dry_run:
removed_files.append(f'{s3_data_bucket}:{file_data_identity}')
else:
try:
s3.remove_object(s3_data_bucket, file_data_identity)
removed_files.append(f'{s3_data_bucket}:{file_data_identity}')
removed_files.append(
f'{s3_data_bucket}:{file_data_identity}')
except ResponseError as e:
print(f'ERROR: {s3_data_bucket}:{file_data_identity} {e}', file=sys.stderr)
print(
f'ERROR: {s3_data_bucket}:{file_data_identity} {e}', file=sys.stderr)
print(os.linesep.join(removed_files))
@ -691,7 +703,8 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
"""
if keep_smaller_input:
command = f"cp {input_file} {output_file}"
return lambda:
def task():
input_size = os.path.getsize(input_file)
output_size = os.path.getsize(output_file)
if output_size > input_size:
@ -706,9 +719,9 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
)]
)
)
return task
return None
for input_file in files:
for content_configuration in content_configurations:
if any([input_file.endswith(extension) for extension in content_configuration['extensions']]):
@ -721,26 +734,31 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
and content_configuration['preserveInputExtension']:
output_file = os.path.join(destination, file)
else:
output_file_without_ext = os.path.splitext(os.path.join(destination, file))[0]
output_file_without_ext = os.path.splitext(
os.path.join(destination, file))[0]
output_file = f'{output_file_without_ext}.{content_configuration["outputExtension"]}'
output_file_identity = get_file_identity({'REMOVE_PREFIX': destination}, output_file)
output_file_identity = get_file_identity(
{'REMOVE_PREFIX': destination}, output_file)
output_file_dir = os.path.dirname(output_file)
os.makedirs(output_file_dir, exist_ok=True)
if 'preserveSmallerInput' in content_configuration:
preserve_smaller_input = bool(content_configuration['preserveSmallerInput'])
preserve_smaller_input = bool(
content_configuration['preserveSmallerInput'])
else:
preserve_smaller_input = True
if 'forcePreserveSmallerInput' in content_configuration:
force_preserve_smaller_input = bool(content_configuration['forcePreserveSmallerInput'])
force_preserve_smaller_input = bool(
content_configuration['forcePreserveSmallerInput'])
else:
force_preserve_smaller_input = False
# Only preserve the input if requested AND the extensions of the input and the output match
preserve_smaller_input = preserve_smaller_input and (force_preserve_smaller_input or file_extension == content_configuration["outputExtension"])
preserve_smaller_input = preserve_smaller_input and (
force_preserve_smaller_input or file_extension == content_configuration["outputExtension"])
command: str = content_configuration['command'] \
.replace('{input_file}', f'\'{input_file}\'') \
@ -754,7 +772,7 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
on_success=[store_filename(
task_output,
f'{input_file}\t{output_file_identity}' if print_input_and_identity else output_file
),queue_follow_up_task_if_keep_smaller_input(
), queue_follow_up_task_if_keep_smaller_input(
follow_up_tasks,
input_file,
output_file,

Write Preview
Loading…
Cancel
Save