diff --git a/.dockerignore b/.dockerignore index 1b42b4b..b357cff 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,7 @@ .git/ .idea/ - scripts/ +venv/ .gitignore Dockerfile diff --git a/.gitignore b/.gitignore index 1157c04..a2d7998 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ .idea/ - venv/ l_venv/ diff --git a/Dockerfile b/Dockerfile index 93673bf..89ad54f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:20.04 LABEL maintainer="Drew Short " \ name="acm" \ - version="1.5.0" \ + version="2.0.0" \ description="Prepackaged ACM with defaults and tooling" ENV LC_ALL=C.UTF-8 diff --git a/acm-config-default.json b/acm-config-default.json deleted file mode 100644 index f4f6043..0000000 --- a/acm-config-default.json +++ /dev/null @@ -1,242 +0,0 @@ -{ - "concurrency": 0, - "profiles": { - "default": { - "jpeg": { - "version": "1.5.0", - "processors": [ - "cjpeg" - ], - "extensions": [ - "jpg", - "jpeg" - ], - "outputExtension": "jpg", - "forcePreserveSmallerInput": true, - "command": "cjpeg -optimize -quality 90 -progressive -outfile {output_file} {input_file}" - }, - "png": { - "version": "1.5.0", - "processors": [ - "optipng" - ], - "extensions": [ - "png" - ], - "outputExtension": "png", - "forcePreserveSmallerInput": true, - "command": "optipng -o2 -strip all -out {output_file} {input_file}" - }, - "video": { - "version": "1.5.0", - "processors": [ - "ffmpeg" - ], - "extensions": [ - "mp4", - "webm" - ], - "outputExtension": "webm", - "command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -c:v libvpx-vp9 -b:v 0 -crf 29 -c:a libopus {output_file}" - }, - "audio": { - "version": "1.5.0", - "processors": [ - "ffmpeg", - "opusenc" - ], - "extensions": [ - "wav", - "mp3" - ], - "outputExtension": "ogg", - "command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}" - } - }, - "placebo": { - "jpeg": { - "version": "1.5.0", - "processors": [ - "cp" - ], - "extensions": [ - "jpg", - "jpeg" - ], - "outputExtension": "jpg", - "preserveInputExtension": true, - "preserveSmallerInput": false, - "command": "cp {input_file} {output_file}" - }, - "png": { - "version": "1.5.0", - "processors": [ - "cp" - ], - "extensions": [ - "png" - ], - "outputExtension": "png", - "preserveInputExtension": true, - "preserveSmallerInput": false, - "command": "cp {input_file} {output_file}" - }, - "video": { - "version": "1.5.0", - "processors": [ - "cp" - ], - "extensions": [ - "mp4", - "webm" - ], - "outputExtension": "mp4", - "preserveInputExtension": true, - "preserveSmallerInput": false, - "command": "cp {input_file} {output_file}" - }, - "audio": { - "version": "1.5.0", - "processors": [ - "cp" - ], - "extensions": [ - "wav", - "mp3" - ], - "outputExtension": "ogg", - "preserveInputExtension": true, - "preserveSmallerInput": false, - "command": "cp {input_file} {output_file}" - } - }, - "webp": { - "jpeg": { - "version": "1.5.0", - "processors": [ - "cwebp" - ], - "extensions": [ - "jpg", - "jpeg" - ], - "outputExtension": "webp", - "command": "cwebp -jpeg_like -q 90 -o {output_file} {input_file}" - }, - "png": { - "version": "1.5.0", - "processors": [ - "cwebp" - ], - "extensions": [ - "png" - ], - "outputExtension": "webp", - "command": "cwebp -lossless -o {output_file} {input_file}" - } - }, - "aggressive": { - "jpeg": { - "version": "1.5.0", - "processors": [ - "ffmpeg", - "cjpeg" - ], - "extensions": [ - "jpg", - "jpeg" - ], - "outputExtension": "jpg", - "forcePreserveSmallerInput": true, - "command": "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cjpeg -optimize -quality 75 -progressive -outfile {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}" - }, - "png": { - "version": "1.5.0", - "processors": [ - "optipng" - ], - "extensions": [ - "png" - ], - "outputExtension": "png", - "forcePreserveSmallerInput": true, - "command": "optipng -o2 -strip all -out {output_file} {input_file}" - }, - "video": { - "version": "1.5.0", - "processors": [ - "ffmpeg" - ], - "extensions": [ - "mp4", - "webm" - ], - "outputExtension": "webm", - "command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}" - }, - "audio": { - "version": "1.5.0", - "processors": [ - "ffmpeg", - "opusenc" - ], - "extensions": [ - "wav", - "mp3" - ], - "outputExtension": "ogg", - "command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}" - } - }, - "aggressive-webp": { - "jpeg": { - "version": "1.5.0", - "processors": [ - "cwebp" - ], - "extensions": [ - "jpg", - "jpeg" - ], - "outputExtension": "webp", - "command": "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cwebp -jpeg_like -q 75 -o {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}" - }, - "png": { - "version": "1.5.0", - "processors": [ - "cwebp" - ], - "extensions": [ - "png" - ], - "outputExtension": "webp", - "command": "cwebp -o {output_file} ${input_file}" - }, - "video": { - "version": "1.5.0", - "processors": [ - "ffmpeg" - ], - "extensions": [ - "mp4", - "webm" - ], - "outputExtension": "webm", - "command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}" - }, - "audio": { - "version": "1.5.0", - "processors": [ - "ffmpeg", - "opusenc" - ], - "extensions": [ - "wav", - "mp3" - ], - "outputExtension": "ogg", - "command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}" - } - } - } -} \ No newline at end of file diff --git a/acm.py b/acm.py index 3ddead7..0305b56 100755 --- a/acm.py +++ b/acm.py @@ -16,14 +16,13 @@ import click from minio import Minio, InvalidResponseError from minio.error import S3Error +from acm.config import VERSION, default_config from acm.logging import setup_basic_logging, update_logging_level +from acm.utility import get_string_sha256sum # Size of the buffer to read files with BUF_SIZE = 4096 -# Application Version -VERSION = "2.0.0" - LOG = setup_basic_logging("acm") ########### @@ -189,14 +188,6 @@ def get_file_sha256sum(stored_data, profile, file): return stored_profile_hash, stored_file_hash, calculated_file_hash -def get_string_sha256sum(string: str, encoding='utf-8') -> str: - sha256sum = hashlib.sha256() - with io.BytesIO(json.dumps(string).encode(encoding)) as c: - for byte_block in iter(lambda: c.read(BUF_SIZE), b''): - sha256sum.update(byte_block) - return sha256sum.hexdigest() - - def add_nested_key(config: Dict[str, any], path: List[str], value: str) -> bool: target = path[0].lower() if len(path) == 1: @@ -270,7 +261,7 @@ def cli(ctx, debug, config, stdin, remove_prefix, add_prefix): # Propagate the global configs ctx.obj['DEBUG'] = debug - ctx.obj['CONFIG'] = load_config(config) + # ctx.obj['CONFIG'] = load_config(config) ctx.obj['READ_STDIN'] = stdin ctx.obj['REMOVE_PREFIX'] = remove_prefix ctx.obj['ADD_PREFIX'] = add_prefix @@ -296,6 +287,18 @@ def print_config(ctx): print(json.dumps(ctx.obj['CONFIG'], indent=2, sort_keys=True)) +@cli.command(name="default-config") +@click.argument('profile', default="all") +@click.pass_context +def print_default_config(ctx, profile): + """ + Print the configuration + """ + if profile == "all": + print(default_config().json(exclude_none=True, indent=2, sort_keys=True)) + else: + config = default_config() + ############################### # S3 Storage Focused Commands # ############################### diff --git a/acm/config.py b/acm/config.py index e255ff8..37df568 100644 --- a/acm/config.py +++ b/acm/config.py @@ -1,44 +1,298 @@ +import json import logging import typing -from pydantic import BaseModel, validator +from pydantic import BaseModel, BaseSettings, validator + +from acm.utility import get_string_sha256sum, get_string_xor LOG = logging.getLogger("acm.config") +# Application Version +VERSION = "2.0.0" + + +class ACMProfileProcessorOptions(BaseModel): + force_preserve_smaller_input: bool = False -class ACMProfileTarget(BaseModel): + +class ACMProfileProcessor(BaseModel): name: str - version: str + version: typing.Optional[str] processors: typing.List[str] extensions: typing.List[str] output_extension: str - force_preserve_smaller_input: bool + options: ACMProfileProcessorOptions command: str signature: typing.Optional[str] + @validator('version', always=True) + def version_validator(cls, v, values) -> str: + # TODO Set the version to the app version if not provided + if v is None: + return VERSION + @validator('signature', always=True) def signature_validator(cls, v, values) -> str: - # TODO calculate the hash for the profile target - return "" + signature_keys = ["name", "version", "processors", "extensions", "output_extension", "command"] + signature_values = [value for key, value in values.items() if key in signature_keys] + return get_string_sha256sum(json.dumps(signature_values)) class ACMProfile(BaseModel): name: str - processors: typing.List[ACMProfileTarget] + version: typing.Optional[str] + processors: typing.List[ACMProfileProcessor] signature: typing.Optional[str] + @validator('version', always=True) + def version_validator(cls, v, values) -> str: + if v is None: + return VERSION + + # @validator('processors', always=True) + # def processors_validator(cls, v, values) -> str: + # # Collapse the same named processors into a single processor at the correct index + @validator('signature', always=True) def hash_signature_validator(cls, v, values) -> str: - # TODO calculate the hash for the profile - return "" + signature_keys = ["name", "version"] + signature_values = [value for key, value in values.items() if key in signature_keys] + signature = get_string_sha256sum(json.dumps(signature_values)) + + processor_signatures = [processor.signature for processor in values["processors"]] + if len(processor_signatures) > 1: + combined_processor_signature = get_string_xor(*processor_signatures) + else: + combined_processor_signature = processor_signatures[0] + + return get_string_sha256sum(signature + combined_processor_signature) + +class ACMS3(BaseModel): + secure: bool = False, + host: str = "127.0.0.1:9000" + access_key: typing.Optional[str] + secret_key: typing.Optional[str] -class ACMConfig(BaseModel): + +class ACMConfig(BaseSettings): concurrency: int = 0 + debug: bool = False + s3: typing.Optional[ACMS3] + version: typing.Optional[str] profiles: typing.List[ACMProfile] signature: typing.Optional[str] + @validator('version', always=True) + def version_validator(cls, v, values) -> str: + if v is None: + return VERSION + @validator('signature', always=True) def signature_validator(cls, v, values) -> str: - # TODO calculate the hash for the config - return "" + signature_keys = ["version"] + signature_values = [value for key, value in values.items() if key in signature_keys] + signature = get_string_sha256sum(json.dumps(signature_values)) + + profiles_signatures = [profiles.signature for profiles in values["profiles"]] + if len(profiles_signatures) > 1: + combined_profiles_signature = get_string_xor(*profiles_signatures) + else: + combined_profiles_signature = profiles_signatures[0] + + return get_string_sha256sum(signature + combined_profiles_signature) + + class Config: + env_prefix = 'ACM_' + env_nested_delimiter = '__' + + +def default_config(): + """ + Returns the default ACM config + """ + acm_profiles = [] + + # default # + acm_default_processors = [] + acm_default_processors.append(ACMProfileProcessor( + name = "jpeg", + processors = ["cjpeg"], + extensions = ["jpg", "jpeg"], + output_extension = "jpg", + options = ACMProfileProcessorOptions(force_preserve_smaller_input=True), + command = "cjpeg -optimize -quality 90 -progressive -outfile {output_file} {input_file}" + )) + acm_default_processors.append(ACMProfileProcessor( + name = "png", + processors = ["optipng"], + extensions = ["png"], + output_extension = "png", + options = ACMProfileProcessorOptions(force_preserve_smaller_input=True), + command = "optipng -o2 -strip all -out {output_file} {input_file}" + )) + acm_default_processors.append(ACMProfileProcessor( + name = "video", + processors = ["ffmpeg"], + extensions = ["mp4","webm"], + output_extension = "webm", + options = ACMProfileProcessorOptions(), + command = "optipng -o2 -strip all -out {output_file} {input_file}" + )) + acm_default_processors.append(ACMProfileProcessor( + name = "audio", + processors = ["ffmpeg","opusenc"], + extensions = ["wav","mp3"], + output_extension = "ogg", + options = ACMProfileProcessorOptions(), + command = "optipng -o2 -strip all -out {output_file} {input_file}" + )) + acm_profiles.append(ACMProfile( + name = "default", + processors = acm_default_processors + )) + + # placebo # + acm_placebo_processors = [] + acm_placebo_processors.append(ACMProfileProcessor( + name = "jpeg", + processors = ["cjpeg"], + extensions = ["jpg", "jpeg"], + output_extension = "jpg", + options = ACMProfileProcessorOptions(), + command = "cp {input_file} {output_file}" + )) + acm_placebo_processors.append(ACMProfileProcessor( + name = "png", + processors = ["optipng"], + extensions = ["png"], + output_extension = "png", + options = ACMProfileProcessorOptions(), + command = "cp {input_file} {output_file}" + )) + acm_placebo_processors.append(ACMProfileProcessor( + name = "video", + processors = ["ffmpeg"], + extensions = ["mp4","webm"], + output_extension = "webm", + options = ACMProfileProcessorOptions(), + command = "cp {input_file} {output_file}" + )) + acm_placebo_processors.append(ACMProfileProcessor( + name = "audio", + processors = ["ffmpeg","opusenc"], + extensions = ["wav","mp3"], + output_extension = "ogg", + options = ACMProfileProcessorOptions(), + command = "cp {input_file} {output_file}" + )) + acm_profiles.append(ACMProfile( + name = "placebo", + processors = acm_placebo_processors + )) + + # webp # + acm_webp_processors = [] + acm_webp_processors.append(ACMProfileProcessor( + name = "jpeg", + processors = ["cwebp"], + extensions = ["jpg", "jpeg"], + output_extension = "jpg", + options = ACMProfileProcessorOptions(), + command = "cwebp -jpeg_like -q 90 -o {output_file} {input_file}" + )) + acm_webp_processors.append(ACMProfileProcessor( + name = "png", + processors = ["cwebp"], + extensions = ["png"], + output_extension = "png", + options = ACMProfileProcessorOptions(), + command = "cwebp -lossless -o {output_file} {input_file}" + )) + acm_profiles.append(ACMProfile( + name = "webp", + processors = acm_webp_processors + )) + + # aggressive # + acm_aggressive_processors = [] + acm_aggressive_processors.append(ACMProfileProcessor( + name = "jpeg", + processors = ["cjpeg"], + extensions = ["jpg", "jpeg"], + output_extension = "jpg", + options = ACMProfileProcessorOptions(force_preserve_smaller_input=True), + command = "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cjpeg -optimize -quality 75 -progressive -outfile {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}" + )) + acm_aggressive_processors.append(ACMProfileProcessor( + name = "png", + processors = ["optipng"], + extensions = ["png"], + output_extension = "png", + options = ACMProfileProcessorOptions(force_preserve_smaller_input=True), + command = "optipng -o2 -strip all -out {output_file} {input_file}" + )) + acm_aggressive_processors.append(ACMProfileProcessor( + name = "video", + processors = ["ffmpeg"], + extensions = ["mp4","webm"], + output_extension = "webm", + options = ACMProfileProcessorOptions(), + command = "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}" + )) + acm_aggressive_processors.append(ACMProfileProcessor( + name = "audio", + processors = ["ffmpeg","opusenc"], + extensions = ["wav","mp3"], + output_extension = "ogg", + options = ACMProfileProcessorOptions(), + command = "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}" + )) + acm_profiles.append(ACMProfile( + name = "aggressive", + processors = acm_aggressive_processors + )) + + # aggressive-webp # + acm_aggressive_webp_processors = [] + acm_aggressive_webp_processors.append(ACMProfileProcessor( + name = "jpeg", + processors = ["cwebp"], + extensions = ["jpg", "jpeg"], + output_extension = "jpg", + options = ACMProfileProcessorOptions(), + command = "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cwebp -jpeg_like -q 75 -o {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}" + )) + acm_aggressive_webp_processors.append(ACMProfileProcessor( + name = "png", + processors = ["optipng"], + extensions = ["png"], + output_extension = "png", + options = ACMProfileProcessorOptions(), + command = "cwebp -o {output_file} ${input_file}" + )) + acm_aggressive_webp_processors.append(ACMProfileProcessor( + name = "video", + processors = ["ffmpeg"], + extensions = ["mp4","webm"], + output_extension = "webm", + options = ACMProfileProcessorOptions(), + command = "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}" + )) + acm_aggressive_webp_processors.append(ACMProfileProcessor( + name = "audio", + processors = ["ffmpeg","opusenc"], + extensions = ["wav","mp3"], + output_extension = "ogg", + options = ACMProfileProcessorOptions(), + command = "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}" + )) + acm_profiles.append(ACMProfile( + name = "aggressive-webp", + processors = acm_aggressive_webp_processors + )) + + return ACMConfig( + profiles=acm_profiles + ) \ No newline at end of file diff --git a/acm/utility.py b/acm/utility.py new file mode 100644 index 0000000..5fe2d24 --- /dev/null +++ b/acm/utility.py @@ -0,0 +1,31 @@ +import hashlib +import io +import logging + +# Size of the buffer to read files with +BUF_SIZE = 4096 + +LOG = logging.getLogger("acm.utility") + + +def get_string_sha256sum(content: str, encoding='utf-8') -> str: + sha256sum = hashlib.sha256() + with io.BytesIO(content.encode(encoding)) as c: + for byte_block in iter(lambda: c.read(BUF_SIZE), b''): + sha256sum.update(byte_block) + return sha256sum.hexdigest() + + +def get_string_hex(content: str) -> hex: + return hex(int(content, base=16)) + + +def get_hex_xor(first: hex, second: hex) -> hex: + return hex(int(first, base=16) ^ int(second, base=16)) + + +def get_string_xor(first: str, second: str, *extra: str) -> str: + result = get_hex_xor(get_string_hex(first), get_string_hex(second)) + for next_hex in extra: + result = get_hex_xor(result, get_string_hex(next_hex)) + return str(result) \ No newline at end of file