warricksothr
/
Asset-Compression-Manager
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Fix minio breaking changes and optimize container image layers

acm-debugging-and-enhancements
Drew Short 2 years ago
parent
ef3b2dac22
commit
567df9bef9
2 changed files with 47 additions and 33 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 11
      Dockerfile
  2. 69
      acm.py

11
Dockerfile
View File

@ -45,10 +45,8 @@ RUN poetry config virtualenvs.create false
WORKDIR /app
# Copy application
COPY . /app/
# Install application requirements
COPY pyproject.toml poetry.lock /app/
RUN poetry install
# Cleanup image programs and cache
@ -56,7 +54,11 @@ RUN apt-get remove -y \
build-essential \
curl \
python3-pip \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* \
&& rm -rf /tmp/*
# Copy application
COPY . /app/
WORKDIR /bin
@ -70,6 +72,7 @@ VOLUME ["/input", "/output"]
WORKDIR /app
RUN mv docker/* . \
&& chmod +x entrypoint.sh \
&& rm -rf docker \
&& mv acm-config.json.example acm-config.json

69
acm.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import asyncio
import collections.abc
import hashlib
@ -13,14 +13,14 @@ import tempfile
from typing import List, Dict, Callable
import click
from minio import Minio, ResponseError
from minio.error import NoSuchKey
from minio import Minio, InvalidResponseError
from minio.error import S3Error
# Size of the buffer to read files with
BUF_SIZE = 4096
# Application Version
VERSION = "1.4.4"
VERSION = "1.5.0"
LOG = logging.getLogger("acm")
LOG.setLevel(logging.ERROR)
@ -381,10 +381,13 @@ def check_matched_files_hashes(ctx, context, print_identity, profile, files):
matching_files.append(stored_data['storedAssetIdentity'])
else:
matching_files.append(file)
except NoSuchKey as e:
continue
except ValueError or ResponseError as e:
print(f'ERROR: {file} {e}')
except S3Error as e:
if e.code == "NoSuchKey":
continue
else:
LOG.error(e)
except ValueError or InvalidResponseError as e:
LOG.error(f'ERROR: {file} {e}')
print(os.linesep.join(matching_files))
@ -415,10 +418,13 @@ def check_changed_files_hashes(ctx, context, profile, files):
if calculated_file_hash != stored_file_hash \
or ctx.obj['CONFIG']['profileHashes'][profile] != stored_profile_hash:
changed_files.append(file)
except NoSuchKey as e:
changed_files.append(file)
except ValueError or ResponseError as e:
print(f'ERROR: {file} {e}')
except S3Error as e:
if e.code == "NoSuchKey":
changed_files.append(file)
else:
LOG.error(e)
except ValueError or InvalidResponseError as e:
LOG.error(f'ERROR: {file} {e}')
print(os.linesep.join(changed_files))
@ -473,8 +479,8 @@ def update_changed_files_hashes(ctx, context, input_and_identity, profile, files
metadata={}
)
updated_files.append(file)
except ValueError or ResponseError as e:
print(f'ERROR: {file} {e}')
except ValueError or InvalidResponseError as e:
LOG.error(f'ERROR: {file} {e}')
print(os.linesep.join(updated_files))
@ -508,8 +514,8 @@ def store_files(ctx, context, files):
ctx.obj['ADD_PREFIX'], file_identity))
else:
stored_files.append(file)
except ResponseError as e:
print(f'ERROR: {file} {e}', file=sys.stderr)
except InvalidResponseError as e:
LOG.error(f'ERROR: {file} {e}', file=sys.stderr)
print(os.linesep.join(stored_files))
@ -542,11 +548,13 @@ def retrieve_files(ctx, context, destination, files):
file_destination
)
retrieved_files.append(file_destination)
except NoSuchKey as e:
print(
f'ERROR: {file_identity} {file_destination} {e}', file=sys.stderr)
except ResponseError as e:
print(f'ERROR: {file_destination} {e}', file=sys.stderr)
except S3Error as e:
if e.code == "NoSuchKey":
LOG.error(f'ERROR: {file_identity} {file_destination} {e}', file=sys.stderr)
else:
LOG.error(e)
except InvalidResponseError as e:
LOG.error(f'ERROR: {file_destination} {e}', file=sys.stderr)
print(os.linesep.join(retrieved_files))
@ -584,10 +592,13 @@ def clean_files(ctx, context, context_data, dry_run, files):
else:
file_object = s3.get_object(s3_bucket, file_identity)
found_files.append(file_identity)
except ResponseError as e:
print(f'ERROR: ResponseError {file_identity} {e}', file=sys.stderr)
except NoSuchKey as e:
print(f'ERROR: NoSuchKey {file_identity}', file=sys.stderr)
except InvalidResponseError as e:
LOG.error(f'ERROR: InvalidResponseError {file_identity} {e}', file=sys.stderr)
except S3Error as e:
if e.code == "NoSuchKey":
LOG.error(f'ERROR: NoSuchKey {file_identity}', file=sys.stderr)
else:
LOG.error(e)
# print(os.linesep.join(found_objects))
# print(os.linesep.join(found_objects))
@ -623,8 +634,8 @@ def clean_files(ctx, context, context_data, dry_run, files):
try:
s3.remove_object(s3_bucket, file_identity)
removed_files.append(f'{s3_bucket}:{file_identity}')
except ResponseError as e:
print(
except InvalidResponseError as e:
LOG.error(
f'ERROR: {s3_bucket}:{file_identity} {e}', file=sys.stderr)
for file_data_identity in found_data_objects:
@ -636,8 +647,8 @@ def clean_files(ctx, context, context_data, dry_run, files):
s3.remove_object(s3_data_bucket, file_data_identity)
removed_files.append(
f'{s3_data_bucket}:{file_data_identity}')
except ResponseError as e:
print(
except InvalidResponseError as e:
LOG.error(
f'ERROR: {s3_data_bucket}:{file_data_identity} {e}', file=sys.stderr)
print(os.linesep.join(removed_files))

Write Preview
Loading…
Cancel
Save