Browse Source

Migrate to structured config

acm-debugging-and-enhancements
Drew Short 2 years ago
parent
commit
898aefd1fe
  1. 2
      .dockerignore
  2. 1
      .gitignore
  3. 2
      Dockerfile
  4. 242
      acm-config-default.json
  5. 27
      acm.py
  6. 278
      acm/config.py
  7. 31
      acm/utility.py

2
.dockerignore

@ -1,7 +1,7 @@
.git/
.idea/
scripts/
venv/
.gitignore
Dockerfile

1
.gitignore

@ -1,5 +1,4 @@
.idea/
venv/
l_venv/

2
Dockerfile

@ -2,7 +2,7 @@ FROM ubuntu:20.04
LABEL maintainer="Drew Short <warrick@sothr.com>" \
name="acm" \
version="1.5.0" \
version="2.0.0" \
description="Prepackaged ACM with defaults and tooling"
ENV LC_ALL=C.UTF-8

242
acm-config-default.json

@ -1,242 +0,0 @@
{
"concurrency": 0,
"profiles": {
"default": {
"jpeg": {
"version": "1.5.0",
"processors": [
"cjpeg"
],
"extensions": [
"jpg",
"jpeg"
],
"outputExtension": "jpg",
"forcePreserveSmallerInput": true,
"command": "cjpeg -optimize -quality 90 -progressive -outfile {output_file} {input_file}"
},
"png": {
"version": "1.5.0",
"processors": [
"optipng"
],
"extensions": [
"png"
],
"outputExtension": "png",
"forcePreserveSmallerInput": true,
"command": "optipng -o2 -strip all -out {output_file} {input_file}"
},
"video": {
"version": "1.5.0",
"processors": [
"ffmpeg"
],
"extensions": [
"mp4",
"webm"
],
"outputExtension": "webm",
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -c:v libvpx-vp9 -b:v 0 -crf 29 -c:a libopus {output_file}"
},
"audio": {
"version": "1.5.0",
"processors": [
"ffmpeg",
"opusenc"
],
"extensions": [
"wav",
"mp3"
],
"outputExtension": "ogg",
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}"
}
},
"placebo": {
"jpeg": {
"version": "1.5.0",
"processors": [
"cp"
],
"extensions": [
"jpg",
"jpeg"
],
"outputExtension": "jpg",
"preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}"
},
"png": {
"version": "1.5.0",
"processors": [
"cp"
],
"extensions": [
"png"
],
"outputExtension": "png",
"preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}"
},
"video": {
"version": "1.5.0",
"processors": [
"cp"
],
"extensions": [
"mp4",
"webm"
],
"outputExtension": "mp4",
"preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}"
},
"audio": {
"version": "1.5.0",
"processors": [
"cp"
],
"extensions": [
"wav",
"mp3"
],
"outputExtension": "ogg",
"preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}"
}
},
"webp": {
"jpeg": {
"version": "1.5.0",
"processors": [
"cwebp"
],
"extensions": [
"jpg",
"jpeg"
],
"outputExtension": "webp",
"command": "cwebp -jpeg_like -q 90 -o {output_file} {input_file}"
},
"png": {
"version": "1.5.0",
"processors": [
"cwebp"
],
"extensions": [
"png"
],
"outputExtension": "webp",
"command": "cwebp -lossless -o {output_file} {input_file}"
}
},
"aggressive": {
"jpeg": {
"version": "1.5.0",
"processors": [
"ffmpeg",
"cjpeg"
],
"extensions": [
"jpg",
"jpeg"
],
"outputExtension": "jpg",
"forcePreserveSmallerInput": true,
"command": "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cjpeg -optimize -quality 75 -progressive -outfile {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}"
},
"png": {
"version": "1.5.0",
"processors": [
"optipng"
],
"extensions": [
"png"
],
"outputExtension": "png",
"forcePreserveSmallerInput": true,
"command": "optipng -o2 -strip all -out {output_file} {input_file}"
},
"video": {
"version": "1.5.0",
"processors": [
"ffmpeg"
],
"extensions": [
"mp4",
"webm"
],
"outputExtension": "webm",
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}"
},
"audio": {
"version": "1.5.0",
"processors": [
"ffmpeg",
"opusenc"
],
"extensions": [
"wav",
"mp3"
],
"outputExtension": "ogg",
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}"
}
},
"aggressive-webp": {
"jpeg": {
"version": "1.5.0",
"processors": [
"cwebp"
],
"extensions": [
"jpg",
"jpeg"
],
"outputExtension": "webp",
"command": "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cwebp -jpeg_like -q 75 -o {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}"
},
"png": {
"version": "1.5.0",
"processors": [
"cwebp"
],
"extensions": [
"png"
],
"outputExtension": "webp",
"command": "cwebp -o {output_file} ${input_file}"
},
"video": {
"version": "1.5.0",
"processors": [
"ffmpeg"
],
"extensions": [
"mp4",
"webm"
],
"outputExtension": "webm",
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}"
},
"audio": {
"version": "1.5.0",
"processors": [
"ffmpeg",
"opusenc"
],
"extensions": [
"wav",
"mp3"
],
"outputExtension": "ogg",
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}"
}
}
}
}

27
acm.py

@ -16,14 +16,13 @@ import click
from minio import Minio, InvalidResponseError
from minio.error import S3Error
from acm.config import VERSION, default_config
from acm.logging import setup_basic_logging, update_logging_level
from acm.utility import get_string_sha256sum
# Size of the buffer to read files with
BUF_SIZE = 4096
# Application Version
VERSION = "2.0.0"
LOG = setup_basic_logging("acm")
###########
@ -189,14 +188,6 @@ def get_file_sha256sum(stored_data, profile, file):
return stored_profile_hash, stored_file_hash, calculated_file_hash
def get_string_sha256sum(string: str, encoding='utf-8') -> str:
sha256sum = hashlib.sha256()
with io.BytesIO(json.dumps(string).encode(encoding)) as c:
for byte_block in iter(lambda: c.read(BUF_SIZE), b''):
sha256sum.update(byte_block)
return sha256sum.hexdigest()
def add_nested_key(config: Dict[str, any], path: List[str], value: str) -> bool:
target = path[0].lower()
if len(path) == 1:
@ -270,7 +261,7 @@ def cli(ctx, debug, config, stdin, remove_prefix, add_prefix):
# Propagate the global configs
ctx.obj['DEBUG'] = debug
ctx.obj['CONFIG'] = load_config(config)
# ctx.obj['CONFIG'] = load_config(config)
ctx.obj['READ_STDIN'] = stdin
ctx.obj['REMOVE_PREFIX'] = remove_prefix
ctx.obj['ADD_PREFIX'] = add_prefix
@ -296,6 +287,18 @@ def print_config(ctx):
print(json.dumps(ctx.obj['CONFIG'], indent=2, sort_keys=True))
@cli.command(name="default-config")
@click.argument('profile', default="all")
@click.pass_context
def print_default_config(ctx, profile):
"""
Print the configuration
"""
if profile == "all":
print(default_config().json(exclude_none=True, indent=2, sort_keys=True))
else:
config = default_config()
###############################
# S3 Storage Focused Commands #
###############################

278
acm/config.py

@ -1,44 +1,298 @@
import json
import logging
import typing
from pydantic import BaseModel, validator
from pydantic import BaseModel, BaseSettings, validator
from acm.utility import get_string_sha256sum, get_string_xor
LOG = logging.getLogger("acm.config")
# Application Version
VERSION = "2.0.0"
class ACMProfileProcessorOptions(BaseModel):
force_preserve_smaller_input: bool = False
class ACMProfileTarget(BaseModel):
class ACMProfileProcessor(BaseModel):
name: str
version: str
version: typing.Optional[str]
processors: typing.List[str]
extensions: typing.List[str]
output_extension: str
force_preserve_smaller_input: bool
options: ACMProfileProcessorOptions
command: str
signature: typing.Optional[str]
@validator('version', always=True)
def version_validator(cls, v, values) -> str:
# TODO Set the version to the app version if not provided
if v is None:
return VERSION
@validator('signature', always=True)
def signature_validator(cls, v, values) -> str:
# TODO calculate the hash for the profile target
return ""
signature_keys = ["name", "version", "processors", "extensions", "output_extension", "command"]
signature_values = [value for key, value in values.items() if key in signature_keys]
return get_string_sha256sum(json.dumps(signature_values))
class ACMProfile(BaseModel):
name: str
processors: typing.List[ACMProfileTarget]
version: typing.Optional[str]
processors: typing.List[ACMProfileProcessor]
signature: typing.Optional[str]
@validator('version', always=True)
def version_validator(cls, v, values) -> str:
if v is None:
return VERSION
# @validator('processors', always=True)
# def processors_validator(cls, v, values) -> str:
# # Collapse the same named processors into a single processor at the correct index
@validator('signature', always=True)
def hash_signature_validator(cls, v, values) -> str:
# TODO calculate the hash for the profile
return ""
signature_keys = ["name", "version"]
signature_values = [value for key, value in values.items() if key in signature_keys]
signature = get_string_sha256sum(json.dumps(signature_values))
processor_signatures = [processor.signature for processor in values["processors"]]
if len(processor_signatures) > 1:
combined_processor_signature = get_string_xor(*processor_signatures)
else:
combined_processor_signature = processor_signatures[0]
return get_string_sha256sum(signature + combined_processor_signature)
class ACMConfig(BaseModel):
class ACMS3(BaseModel):
secure: bool = False,
host: str = "127.0.0.1:9000"
access_key: typing.Optional[str]
secret_key: typing.Optional[str]
class ACMConfig(BaseSettings):
concurrency: int = 0
debug: bool = False
s3: typing.Optional[ACMS3]
version: typing.Optional[str]
profiles: typing.List[ACMProfile]
signature: typing.Optional[str]
@validator('version', always=True)
def version_validator(cls, v, values) -> str:
if v is None:
return VERSION
@validator('signature', always=True)
def signature_validator(cls, v, values) -> str:
# TODO calculate the hash for the config
return ""
signature_keys = ["version"]
signature_values = [value for key, value in values.items() if key in signature_keys]
signature = get_string_sha256sum(json.dumps(signature_values))
profiles_signatures = [profiles.signature for profiles in values["profiles"]]
if len(profiles_signatures) > 1:
combined_profiles_signature = get_string_xor(*profiles_signatures)
else:
combined_profiles_signature = profiles_signatures[0]
return get_string_sha256sum(signature + combined_profiles_signature)
class Config:
env_prefix = 'ACM_'
env_nested_delimiter = '__'
def default_config():
"""
Returns the default ACM config
"""
acm_profiles = []
# default #
acm_default_processors = []
acm_default_processors.append(ACMProfileProcessor(
name = "jpeg",
processors = ["cjpeg"],
extensions = ["jpg", "jpeg"],
output_extension = "jpg",
options = ACMProfileProcessorOptions(force_preserve_smaller_input=True),
command = "cjpeg -optimize -quality 90 -progressive -outfile {output_file} {input_file}"
))
acm_default_processors.append(ACMProfileProcessor(
name = "png",
processors = ["optipng"],
extensions = ["png"],
output_extension = "png",
options = ACMProfileProcessorOptions(force_preserve_smaller_input=True),
command = "optipng -o2 -strip all -out {output_file} {input_file}"
))
acm_default_processors.append(ACMProfileProcessor(
name = "video",
processors = ["ffmpeg"],
extensions = ["mp4","webm"],
output_extension = "webm",
options = ACMProfileProcessorOptions(),
command = "optipng -o2 -strip all -out {output_file} {input_file}"
))
acm_default_processors.append(ACMProfileProcessor(
name = "audio",
processors = ["ffmpeg","opusenc"],
extensions = ["wav","mp3"],
output_extension = "ogg",
options = ACMProfileProcessorOptions(),
command = "optipng -o2 -strip all -out {output_file} {input_file}"
))
acm_profiles.append(ACMProfile(
name = "default",
processors = acm_default_processors
))
# placebo #
acm_placebo_processors = []
acm_placebo_processors.append(ACMProfileProcessor(
name = "jpeg",
processors = ["cjpeg"],
extensions = ["jpg", "jpeg"],
output_extension = "jpg",
options = ACMProfileProcessorOptions(),
command = "cp {input_file} {output_file}"
))
acm_placebo_processors.append(ACMProfileProcessor(
name = "png",
processors = ["optipng"],
extensions = ["png"],
output_extension = "png",
options = ACMProfileProcessorOptions(),
command = "cp {input_file} {output_file}"
))
acm_placebo_processors.append(ACMProfileProcessor(
name = "video",
processors = ["ffmpeg"],
extensions = ["mp4","webm"],
output_extension = "webm",
options = ACMProfileProcessorOptions(),
command = "cp {input_file} {output_file}"
))
acm_placebo_processors.append(ACMProfileProcessor(
name = "audio",
processors = ["ffmpeg","opusenc"],
extensions = ["wav","mp3"],
output_extension = "ogg",
options = ACMProfileProcessorOptions(),
command = "cp {input_file} {output_file}"
))
acm_profiles.append(ACMProfile(
name = "placebo",
processors = acm_placebo_processors
))
# webp #
acm_webp_processors = []
acm_webp_processors.append(ACMProfileProcessor(
name = "jpeg",
processors = ["cwebp"],
extensions = ["jpg", "jpeg"],
output_extension = "jpg",
options = ACMProfileProcessorOptions(),
command = "cwebp -jpeg_like -q 90 -o {output_file} {input_file}"
))
acm_webp_processors.append(ACMProfileProcessor(
name = "png",
processors = ["cwebp"],
extensions = ["png"],
output_extension = "png",
options = ACMProfileProcessorOptions(),
command = "cwebp -lossless -o {output_file} {input_file}"
))
acm_profiles.append(ACMProfile(
name = "webp",
processors = acm_webp_processors
))
# aggressive #
acm_aggressive_processors = []
acm_aggressive_processors.append(ACMProfileProcessor(
name = "jpeg",
processors = ["cjpeg"],
extensions = ["jpg", "jpeg"],
output_extension = "jpg",
options = ACMProfileProcessorOptions(force_preserve_smaller_input=True),
command = "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cjpeg -optimize -quality 75 -progressive -outfile {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}"
))
acm_aggressive_processors.append(ACMProfileProcessor(
name = "png",
processors = ["optipng"],
extensions = ["png"],
output_extension = "png",
options = ACMProfileProcessorOptions(force_preserve_smaller_input=True),
command = "optipng -o2 -strip all -out {output_file} {input_file}"
))
acm_aggressive_processors.append(ACMProfileProcessor(
name = "video",
processors = ["ffmpeg"],
extensions = ["mp4","webm"],
output_extension = "webm",
options = ACMProfileProcessorOptions(),
command = "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}"
))
acm_aggressive_processors.append(ACMProfileProcessor(
name = "audio",
processors = ["ffmpeg","opusenc"],
extensions = ["wav","mp3"],
output_extension = "ogg",
options = ACMProfileProcessorOptions(),
command = "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}"
))
acm_profiles.append(ACMProfile(
name = "aggressive",
processors = acm_aggressive_processors
))
# aggressive-webp #
acm_aggressive_webp_processors = []
acm_aggressive_webp_processors.append(ACMProfileProcessor(
name = "jpeg",
processors = ["cwebp"],
extensions = ["jpg", "jpeg"],
output_extension = "jpg",
options = ACMProfileProcessorOptions(),
command = "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cwebp -jpeg_like -q 75 -o {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}"
))
acm_aggressive_webp_processors.append(ACMProfileProcessor(
name = "png",
processors = ["optipng"],
extensions = ["png"],
output_extension = "png",
options = ACMProfileProcessorOptions(),
command = "cwebp -o {output_file} ${input_file}"
))
acm_aggressive_webp_processors.append(ACMProfileProcessor(
name = "video",
processors = ["ffmpeg"],
extensions = ["mp4","webm"],
output_extension = "webm",
options = ACMProfileProcessorOptions(),
command = "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}"
))
acm_aggressive_webp_processors.append(ACMProfileProcessor(
name = "audio",
processors = ["ffmpeg","opusenc"],
extensions = ["wav","mp3"],
output_extension = "ogg",
options = ACMProfileProcessorOptions(),
command = "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}"
))
acm_profiles.append(ACMProfile(
name = "aggressive-webp",
processors = acm_aggressive_webp_processors
))
return ACMConfig(
profiles=acm_profiles
)

31
acm/utility.py

@ -0,0 +1,31 @@
import hashlib
import io
import logging
# Size of the buffer to read files with
BUF_SIZE = 4096
LOG = logging.getLogger("acm.utility")
def get_string_sha256sum(content: str, encoding='utf-8') -> str:
sha256sum = hashlib.sha256()
with io.BytesIO(content.encode(encoding)) as c:
for byte_block in iter(lambda: c.read(BUF_SIZE), b''):
sha256sum.update(byte_block)
return sha256sum.hexdigest()
def get_string_hex(content: str) -> hex:
return hex(int(content, base=16))
def get_hex_xor(first: hex, second: hex) -> hex:
return hex(int(first, base=16) ^ int(second, base=16))
def get_string_xor(first: str, second: str, *extra: str) -> str:
result = get_hex_xor(get_string_hex(first), get_string_hex(second))
for next_hex in extra:
result = get_hex_xor(result, get_string_hex(next_hex))
return str(result)
Loading…
Cancel
Save