Browse Source

Fix minio breaking changes and optimize container image layers

acm-debugging-and-enhancements
Drew Short 9 months ago
parent
commit
567df9bef9
  1. 11
      Dockerfile
  2. 69
      acm.py

11
Dockerfile

@ -45,10 +45,8 @@ RUN poetry config virtualenvs.create false
WORKDIR /app
# Copy application
COPY . /app/
# Install application requirements
COPY pyproject.toml poetry.lock /app/
RUN poetry install
# Cleanup image programs and cache
@ -56,7 +54,11 @@ RUN apt-get remove -y \
build-essential \
curl \
python3-pip \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /tmp/*
# Copy application
COPY . /app/
WORKDIR /bin
@ -70,6 +72,7 @@ VOLUME ["/input", "/output"]
WORKDIR /app
RUN mv docker/* . \
&& chmod +x entrypoint.sh \
&& rm -rf docker \
&& mv acm-config.json.example acm-config.json

69
acm.py

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import asyncio
import collections.abc
import hashlib
@ -13,14 +13,14 @@ import tempfile
from typing import List, Dict, Callable
import click
from minio import Minio, ResponseError
from minio.error import NoSuchKey
from minio import Minio, InvalidResponseError
from minio.error import S3Error
# Size of the buffer to read files with
BUF_SIZE = 4096
# Application Version
VERSION = "1.4.4"
VERSION = "1.5.0"
LOG = logging.getLogger("acm")
LOG.setLevel(logging.ERROR)
@ -381,10 +381,13 @@ def check_matched_files_hashes(ctx, context, print_identity, profile, files):
matching_files.append(stored_data['storedAssetIdentity'])
else:
matching_files.append(file)
except NoSuchKey as e:
continue
except ValueError or ResponseError as e:
print(f'ERROR: {file} {e}')
except S3Error as e:
if e.code == "NoSuchKey":
continue
else:
LOG.error(e)
except ValueError or InvalidResponseError as e:
LOG.error(f'ERROR: {file} {e}')
print(os.linesep.join(matching_files))
@ -415,10 +418,13 @@ def check_changed_files_hashes(ctx, context, profile, files):
if calculated_file_hash != stored_file_hash \
or ctx.obj['CONFIG']['profileHashes'][profile] != stored_profile_hash:
changed_files.append(file)
except NoSuchKey as e:
changed_files.append(file)
except ValueError or ResponseError as e:
print(f'ERROR: {file} {e}')
except S3Error as e:
if e.code == "NoSuchKey":
changed_files.append(file)
else:
LOG.error(e)
except ValueError or InvalidResponseError as e:
LOG.error(f'ERROR: {file} {e}')
print(os.linesep.join(changed_files))
@ -473,8 +479,8 @@ def update_changed_files_hashes(ctx, context, input_and_identity, profile, files
metadata={}
)
updated_files.append(file)
except ValueError or ResponseError as e:
print(f'ERROR: {file} {e}')
except ValueError or InvalidResponseError as e:
LOG.error(f'ERROR: {file} {e}')
print(os.linesep.join(updated_files))
@ -508,8 +514,8 @@ def store_files(ctx, context, files):
ctx.obj['ADD_PREFIX'], file_identity))
else:
stored_files.append(file)
except ResponseError as e:
print(f'ERROR: {file} {e}', file=sys.stderr)
except InvalidResponseError as e:
LOG.error(f'ERROR: {file} {e}', file=sys.stderr)
print(os.linesep.join(stored_files))
@ -542,11 +548,13 @@ def retrieve_files(ctx, context, destination, files):
file_destination
)
retrieved_files.append(file_destination)
except NoSuchKey as e:
print(
f'ERROR: {file_identity} {file_destination} {e}', file=sys.stderr)
except ResponseError as e:
print(f'ERROR: {file_destination} {e}', file=sys.stderr)
except S3Error as e:
if e.code == "NoSuchKey":
LOG.error(f'ERROR: {file_identity} {file_destination} {e}', file=sys.stderr)
else:
LOG.error(e)
except InvalidResponseError as e:
LOG.error(f'ERROR: {file_destination} {e}', file=sys.stderr)
print(os.linesep.join(retrieved_files))
@ -584,10 +592,13 @@ def clean_files(ctx, context, context_data, dry_run, files):
else:
file_object = s3.get_object(s3_bucket, file_identity)
found_files.append(file_identity)
except ResponseError as e:
print(f'ERROR: ResponseError {file_identity} {e}', file=sys.stderr)
except NoSuchKey as e:
print(f'ERROR: NoSuchKey {file_identity}', file=sys.stderr)
except InvalidResponseError as e:
LOG.error(f'ERROR: InvalidResponseError {file_identity} {e}', file=sys.stderr)
except S3Error as e:
if e.code == "NoSuchKey":
LOG.error(f'ERROR: NoSuchKey {file_identity}', file=sys.stderr)
else:
LOG.error(e)
# print(os.linesep.join(found_objects))
# print(os.linesep.join(found_objects))
@ -623,8 +634,8 @@ def clean_files(ctx, context, context_data, dry_run, files):
try:
s3.remove_object(s3_bucket, file_identity)
removed_files.append(f'{s3_bucket}:{file_identity}')
except ResponseError as e:
print(
except InvalidResponseError as e:
LOG.error(
f'ERROR: {s3_bucket}:{file_identity} {e}', file=sys.stderr)
for file_data_identity in found_data_objects:
@ -636,8 +647,8 @@ def clean_files(ctx, context, context_data, dry_run, files):
s3.remove_object(s3_data_bucket, file_data_identity)
removed_files.append(
f'{s3_data_bucket}:{file_data_identity}')
except ResponseError as e:
print(
except InvalidResponseError as e:
LOG.error(
f'ERROR: {s3_data_bucket}:{file_data_identity} {e}', file=sys.stderr)
print(os.linesep.join(removed_files))

Loading…
Cancel
Save