From 567df9bef9abfd25b1c23fb742ef48aab88bd6db Mon Sep 17 00:00:00 2001 From: Drew Short Date: Mon, 27 Dec 2021 10:27:05 -0600 Subject: [PATCH] Fix minio breaking changes and optimize container image layers --- Dockerfile | 11 +++++---- acm.py | 69 +++++++++++++++++++++++++++++++----------------------- 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/Dockerfile b/Dockerfile index efff4a0..93673bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,10 +45,8 @@ RUN poetry config virtualenvs.create false WORKDIR /app -# Copy application -COPY . /app/ - # Install application requirements +COPY pyproject.toml poetry.lock /app/ RUN poetry install # Cleanup image programs and cache @@ -56,7 +54,11 @@ RUN apt-get remove -y \ build-essential \ curl \ python3-pip \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /tmp/* + +# Copy application +COPY . /app/ WORKDIR /bin @@ -70,6 +72,7 @@ VOLUME ["/input", "/output"] WORKDIR /app RUN mv docker/* . \ + && chmod +x entrypoint.sh \ && rm -rf docker \ && mv acm-config.json.example acm-config.json diff --git a/acm.py b/acm.py index 5bf4fb2..a199cb7 100755 --- a/acm.py +++ b/acm.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import asyncio import collections.abc import hashlib @@ -13,14 +13,14 @@ import tempfile from typing import List, Dict, Callable import click -from minio import Minio, ResponseError -from minio.error import NoSuchKey +from minio import Minio, InvalidResponseError +from minio.error import S3Error # Size of the buffer to read files with BUF_SIZE = 4096 # Application Version -VERSION = "1.4.4" +VERSION = "1.5.0" LOG = logging.getLogger("acm") LOG.setLevel(logging.ERROR) @@ -381,10 +381,13 @@ def check_matched_files_hashes(ctx, context, print_identity, profile, files): matching_files.append(stored_data['storedAssetIdentity']) else: matching_files.append(file) - except NoSuchKey as e: - continue - except ValueError or ResponseError as e: - print(f'ERROR: {file} {e}') + except S3Error as e: + if e.code == "NoSuchKey": + continue + else: + LOG.error(e) + except ValueError or InvalidResponseError as e: + LOG.error(f'ERROR: {file} {e}') print(os.linesep.join(matching_files)) @@ -415,10 +418,13 @@ def check_changed_files_hashes(ctx, context, profile, files): if calculated_file_hash != stored_file_hash \ or ctx.obj['CONFIG']['profileHashes'][profile] != stored_profile_hash: changed_files.append(file) - except NoSuchKey as e: - changed_files.append(file) - except ValueError or ResponseError as e: - print(f'ERROR: {file} {e}') + except S3Error as e: + if e.code == "NoSuchKey": + changed_files.append(file) + else: + LOG.error(e) + except ValueError or InvalidResponseError as e: + LOG.error(f'ERROR: {file} {e}') print(os.linesep.join(changed_files)) @@ -473,8 +479,8 @@ def update_changed_files_hashes(ctx, context, input_and_identity, profile, files metadata={} ) updated_files.append(file) - except ValueError or ResponseError as e: - print(f'ERROR: {file} {e}') + except ValueError or InvalidResponseError as e: + LOG.error(f'ERROR: {file} {e}') print(os.linesep.join(updated_files)) @@ -508,8 +514,8 @@ def store_files(ctx, context, files): ctx.obj['ADD_PREFIX'], file_identity)) else: stored_files.append(file) - except ResponseError as e: - print(f'ERROR: {file} {e}', file=sys.stderr) + except InvalidResponseError as e: + LOG.error(f'ERROR: {file} {e}', file=sys.stderr) print(os.linesep.join(stored_files)) @@ -542,11 +548,13 @@ def retrieve_files(ctx, context, destination, files): file_destination ) retrieved_files.append(file_destination) - except NoSuchKey as e: - print( - f'ERROR: {file_identity} {file_destination} {e}', file=sys.stderr) - except ResponseError as e: - print(f'ERROR: {file_destination} {e}', file=sys.stderr) + except S3Error as e: + if e.code == "NoSuchKey": + LOG.error(f'ERROR: {file_identity} {file_destination} {e}', file=sys.stderr) + else: + LOG.error(e) + except InvalidResponseError as e: + LOG.error(f'ERROR: {file_destination} {e}', file=sys.stderr) print(os.linesep.join(retrieved_files)) @@ -584,10 +592,13 @@ def clean_files(ctx, context, context_data, dry_run, files): else: file_object = s3.get_object(s3_bucket, file_identity) found_files.append(file_identity) - except ResponseError as e: - print(f'ERROR: ResponseError {file_identity} {e}', file=sys.stderr) - except NoSuchKey as e: - print(f'ERROR: NoSuchKey {file_identity}', file=sys.stderr) + except InvalidResponseError as e: + LOG.error(f'ERROR: InvalidResponseError {file_identity} {e}', file=sys.stderr) + except S3Error as e: + if e.code == "NoSuchKey": + LOG.error(f'ERROR: NoSuchKey {file_identity}', file=sys.stderr) + else: + LOG.error(e) # print(os.linesep.join(found_objects)) # print(os.linesep.join(found_objects)) @@ -623,8 +634,8 @@ def clean_files(ctx, context, context_data, dry_run, files): try: s3.remove_object(s3_bucket, file_identity) removed_files.append(f'{s3_bucket}:{file_identity}') - except ResponseError as e: - print( + except InvalidResponseError as e: + LOG.error( f'ERROR: {s3_bucket}:{file_identity} {e}', file=sys.stderr) for file_data_identity in found_data_objects: @@ -636,8 +647,8 @@ def clean_files(ctx, context, context_data, dry_run, files): s3.remove_object(s3_data_bucket, file_data_identity) removed_files.append( f'{s3_data_bucket}:{file_data_identity}') - except ResponseError as e: - print( + except InvalidResponseError as e: + LOG.error( f'ERROR: {s3_data_bucket}:{file_data_identity} {e}', file=sys.stderr) print(os.linesep.join(removed_files))