warricksothr
/
Asset-Compression-Manager
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Migrate to structured config

acm-debugging-and-enhancements
Drew Short 3 years ago
parent
def0761d96
commit
898aefd1fe
7 changed files with 314 additions and 269 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      .dockerignore
  2. 1
      .gitignore
  3. 2
      Dockerfile
  4. 242
      acm-config-default.json
  5. 27
      acm.py
  6. 278
      acm/config.py
  7. 31
      acm/utility.py

2
.dockerignore
View File

@ -1,7 +1,7 @@
.git/ .git/
.idea/ .idea/
scripts/ scripts/
venv/
.gitignore .gitignore
Dockerfile Dockerfile

1
.gitignore
View File

@ -1,5 +1,4 @@
.idea/ .idea/
venv/ venv/
l_venv/ l_venv/

2
Dockerfile
View File

@ -2,7 +2,7 @@ FROM ubuntu:20.04
LABEL maintainer="Drew Short <warrick@sothr.com>" \ LABEL maintainer="Drew Short <warrick@sothr.com>" \
name="acm" \ name="acm" \
version="1.5.0" \
version="2.0.0" \
description="Prepackaged ACM with defaults and tooling" description="Prepackaged ACM with defaults and tooling"
ENV LC_ALL=C.UTF-8 ENV LC_ALL=C.UTF-8

242
acm-config-default.json
View File

@ -1,242 +0,0 @@
{
"concurrency": 0,
"profiles": {
"default": {
"jpeg": {
"version": "1.5.0",
"processors": [
"cjpeg"
],
"extensions": [
"jpg",
"jpeg"
],
"outputExtension": "jpg",
"forcePreserveSmallerInput": true,
"command": "cjpeg -optimize -quality 90 -progressive -outfile {output_file} {input_file}"
},
"png": {
"version": "1.5.0",
"processors": [
"optipng"
],
"extensions": [
"png"
],
"outputExtension": "png",
"forcePreserveSmallerInput": true,
"command": "optipng -o2 -strip all -out {output_file} {input_file}"
},
"video": {
"version": "1.5.0",
"processors": [
"ffmpeg"
],
"extensions": [
"mp4",
"webm"
],
"outputExtension": "webm",
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -c:v libvpx-vp9 -b:v 0 -crf 29 -c:a libopus {output_file}"
},
"audio": {
"version": "1.5.0",
"processors": [
"ffmpeg",
"opusenc"
],
"extensions": [
"wav",
"mp3"
],
"outputExtension": "ogg",
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}"
}
},
"placebo": {
"jpeg": {
"version": "1.5.0",
"processors": [
"cp"
],
"extensions": [
"jpg",
"jpeg"
],
"outputExtension": "jpg",
"preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}"
},
"png": {
"version": "1.5.0",
"processors": [
"cp"
],
"extensions": [
"png"
],
"outputExtension": "png",
"preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}"
},
"video": {
"version": "1.5.0",
"processors": [
"cp"
],
"extensions": [
"mp4",
"webm"
],
"outputExtension": "mp4",
"preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}"
},
"audio": {
"version": "1.5.0",
"processors": [
"cp"
],
"extensions": [
"wav",
"mp3"
],
"outputExtension": "ogg",
"preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}"
}
},
"webp": {
"jpeg": {
"version": "1.5.0",
"processors": [
"cwebp"
],
"extensions": [
"jpg",
"jpeg"
],
"outputExtension": "webp",
"command": "cwebp -jpeg_like -q 90 -o {output_file} {input_file}"
},
"png": {
"version": "1.5.0",
"processors": [
"cwebp"
],
"extensions": [
"png"
],
"outputExtension": "webp",
"command": "cwebp -lossless -o {output_file} {input_file}"
}
},
"aggressive": {
"jpeg": {
"version": "1.5.0",
"processors": [
"ffmpeg",
"cjpeg"
],
"extensions": [
"jpg",
"jpeg"
],
"outputExtension": "jpg",
"forcePreserveSmallerInput": true,
"command": "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cjpeg -optimize -quality 75 -progressive -outfile {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}"
},
"png": {
"version": "1.5.0",
"processors": [
"optipng"
],
"extensions": [
"png"
],
"outputExtension": "png",
"forcePreserveSmallerInput": true,
"command": "optipng -o2 -strip all -out {output_file} {input_file}"
},
"video": {
"version": "1.5.0",
"processors": [
"ffmpeg"
],
"extensions": [
"mp4",
"webm"
],
"outputExtension": "webm",
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}"
},
"audio": {
"version": "1.5.0",
"processors": [
"ffmpeg",
"opusenc"
],
"extensions": [
"wav",
"mp3"
],
"outputExtension": "ogg",
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}"
}
},
"aggressive-webp": {
"jpeg": {
"version": "1.5.0",
"processors": [
"cwebp"
],
"extensions": [
"jpg",
"jpeg"
],
"outputExtension": "webp",
"command": "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cwebp -jpeg_like -q 75 -o {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}"
},
"png": {
"version": "1.5.0",
"processors": [
"cwebp"
],
"extensions": [
"png"
],
"outputExtension": "webp",
"command": "cwebp -o {output_file} ${input_file}"
},
"video": {
"version": "1.5.0",
"processors": [
"ffmpeg"
],
"extensions": [
"mp4",
"webm"
],
"outputExtension": "webm",
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}"
},
"audio": {
"version": "1.5.0",
"processors": [
"ffmpeg",
"opusenc"
],
"extensions": [
"wav",
"mp3"
],
"outputExtension": "ogg",
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}"
}
}
}
}

27
acm.py
View File

@ -16,14 +16,13 @@ import click
from minio import Minio, InvalidResponseError from minio import Minio, InvalidResponseError
from minio.error import S3Error from minio.error import S3Error
from acm.config import VERSION, default_config
from acm.logging import setup_basic_logging, update_logging_level from acm.logging import setup_basic_logging, update_logging_level
from acm.utility import get_string_sha256sum
# Size of the buffer to read files with # Size of the buffer to read files with
BUF_SIZE = 4096 BUF_SIZE = 4096
# Application Version
VERSION = "2.0.0"
LOG = setup_basic_logging("acm") LOG = setup_basic_logging("acm")
########### ###########
@ -189,14 +188,6 @@ def get_file_sha256sum(stored_data, profile, file):
return stored_profile_hash, stored_file_hash, calculated_file_hash return stored_profile_hash, stored_file_hash, calculated_file_hash
def get_string_sha256sum(string: str, encoding='utf-8') -> str:
sha256sum = hashlib.sha256()
with io.BytesIO(json.dumps(string).encode(encoding)) as c:
for byte_block in iter(lambda: c.read(BUF_SIZE), b''):
sha256sum.update(byte_block)
return sha256sum.hexdigest()
def add_nested_key(config: Dict[str, any], path: List[str], value: str) -> bool: def add_nested_key(config: Dict[str, any], path: List[str], value: str) -> bool:
target = path[0].lower() target = path[0].lower()
if len(path) == 1: if len(path) == 1:
@ -270,7 +261,7 @@ def cli(ctx, debug, config, stdin, remove_prefix, add_prefix):
# Propagate the global configs # Propagate the global configs
ctx.obj['DEBUG'] = debug ctx.obj['DEBUG'] = debug
ctx.obj['CONFIG'] = load_config(config)
# ctx.obj['CONFIG'] = load_config(config)
ctx.obj['READ_STDIN'] = stdin ctx.obj['READ_STDIN'] = stdin
ctx.obj['REMOVE_PREFIX'] = remove_prefix ctx.obj['REMOVE_PREFIX'] = remove_prefix
ctx.obj['ADD_PREFIX'] = add_prefix ctx.obj['ADD_PREFIX'] = add_prefix
@ -296,6 +287,18 @@ def print_config(ctx):
print(json.dumps(ctx.obj['CONFIG'], indent=2, sort_keys=True)) print(json.dumps(ctx.obj['CONFIG'], indent=2, sort_keys=True))
@cli.command(name="default-config")
@click.argument('profile', default="all")
@click.pass_context
def print_default_config(ctx, profile):
"""
Print the configuration
"""
if profile == "all":
print(default_config().json(exclude_none=True, indent=2, sort_keys=True))
else:
config = default_config()
############################### ###############################
# S3 Storage Focused Commands # # S3 Storage Focused Commands #
############################### ###############################

278
acm/config.py
View File

@ -1,44 +1,298 @@
import json
import logging import logging
import typing import typing
from pydantic import BaseModel, validator
from pydantic import BaseModel, BaseSettings, validator
from acm.utility import get_string_sha256sum, get_string_xor
LOG = logging.getLogger("acm.config") LOG = logging.getLogger("acm.config")
# Application Version
VERSION = "2.0.0"
class ACMProfileProcessorOptions(BaseModel):
force_preserve_smaller_input: bool = False
class ACMProfileTarget(BaseModel):
class ACMProfileProcessor(BaseModel):
name: str name: str
version: str
version: typing.Optional[str]
processors: typing.List[str] processors: typing.List[str]
extensions: typing.List[str] extensions: typing.List[str]
output_extension: str output_extension: str
force_preserve_smaller_input: bool
options: ACMProfileProcessorOptions
command: str command: str
signature: typing.Optional[str] signature: typing.Optional[str]
@validator('version', always=True)
def version_validator(cls, v, values) -> str:
# TODO Set the version to the app version if not provided
if v is None:
return VERSION
@validator('signature', always=True) @validator('signature', always=True)
def signature_validator(cls, v, values) -> str: def signature_validator(cls, v, values) -> str:
# TODO calculate the hash for the profile target
return ""
signature_keys = ["name", "version", "processors", "extensions", "output_extension", "command"]
signature_values = [value for key, value in values.items() if key in signature_keys]
return get_string_sha256sum(json.dumps(signature_values))
class ACMProfile(BaseModel): class ACMProfile(BaseModel):
name: str name: str
processors: typing.List[ACMProfileTarget]
version: typing.Optional[str]
processors: typing.List[ACMProfileProcessor]
signature: typing.Optional[str] signature: typing.Optional[str]
@validator('version', always=True)
def version_validator(cls, v, values) -> str:
if v is None:
return VERSION
# @validator('processors', always=True)
# def processors_validator(cls, v, values) -> str:
# # Collapse the same named processors into a single processor at the correct index
@validator('signature', always=True) @validator('signature', always=True)
def hash_signature_validator(cls, v, values) -> str: def hash_signature_validator(cls, v, values) -> str:
# TODO calculate the hash for the profile
return ""
signature_keys = ["name", "version"]
signature_values = [value for key, value in values.items() if key in signature_keys]
signature = get_string_sha256sum(json.dumps(signature_values))
processor_signatures = [processor.signature for processor in values["processors"]]
if len(processor_signatures) > 1:
combined_processor_signature = get_string_xor(*processor_signatures)
else:
combined_processor_signature = processor_signatures[0]
return get_string_sha256sum(signature + combined_processor_signature)
class ACMConfig(BaseModel):
class ACMS3(BaseModel):
secure: bool = False,
host: str = "127.0.0.1:9000"
access_key: typing.Optional[str]
secret_key: typing.Optional[str]
class ACMConfig(BaseSettings):
concurrency: int = 0 concurrency: int = 0
debug: bool = False
s3: typing.Optional[ACMS3]
version: typing.Optional[str]
profiles: typing.List[ACMProfile] profiles: typing.List[ACMProfile]
signature: typing.Optional[str] signature: typing.Optional[str]
@validator('version', always=True)
def version_validator(cls, v, values) -> str:
if v is None:
return VERSION
@validator('signature', always=True) @validator('signature', always=True)
def signature_validator(cls, v, values) -> str: def signature_validator(cls, v, values) -> str:
# TODO calculate the hash for the config
return ""
signature_keys = ["version"]
signature_values = [value for key, value in values.items() if key in signature_keys]
signature = get_string_sha256sum(json.dumps(signature_values))
profiles_signatures = [profiles.signature for profiles in values["profiles"]]
if len(profiles_signatures) > 1:
combined_profiles_signature = get_string_xor(*profiles_signatures)
else:
combined_profiles_signature = profiles_signatures[0]
return get_string_sha256sum(signature + combined_profiles_signature)
class Config:
env_prefix = 'ACM_'
env_nested_delimiter = '__'
def default_config():
"""
Returns the default ACM config
"""
acm_profiles = []
# default #
acm_default_processors = []
acm_default_processors.append(ACMProfileProcessor(
name = "jpeg",
processors = ["cjpeg"],
extensions = ["jpg", "jpeg"],
output_extension = "jpg",
options = ACMProfileProcessorOptions(force_preserve_smaller_input=True),
command = "cjpeg -optimize -quality 90 -progressive -outfile {output_file} {input_file}"
))
acm_default_processors.append(ACMProfileProcessor(
name = "png",
processors = ["optipng"],
extensions = ["png"],
output_extension = "png",
options = ACMProfileProcessorOptions(force_preserve_smaller_input=True),
command = "optipng -o2 -strip all -out {output_file} {input_file}"
))
acm_default_processors.append(ACMProfileProcessor(
name = "video",
processors = ["ffmpeg"],
extensions = ["mp4","webm"],
output_extension = "webm",
options = ACMProfileProcessorOptions(),
command = "optipng -o2 -strip all -out {output_file} {input_file}"
))
acm_default_processors.append(ACMProfileProcessor(
name = "audio",
processors = ["ffmpeg","opusenc"],
extensions = ["wav","mp3"],
output_extension = "ogg",
options = ACMProfileProcessorOptions(),
command = "optipng -o2 -strip all -out {output_file} {input_file}"
))
acm_profiles.append(ACMProfile(
name = "default",
processors = acm_default_processors
))
# placebo #
acm_placebo_processors = []
acm_placebo_processors.append(ACMProfileProcessor(
name = "jpeg",
processors = ["cjpeg"],
extensions = ["jpg", "jpeg"],
output_extension = "jpg",
options = ACMProfileProcessorOptions(),
command = "cp {input_file} {output_file}"
))
acm_placebo_processors.append(ACMProfileProcessor(
name = "png",
processors = ["optipng"],
extensions = ["png"],
output_extension = "png",
options = ACMProfileProcessorOptions(),
command = "cp {input_file} {output_file}"
))
acm_placebo_processors.append(ACMProfileProcessor(
name = "video",
processors = ["ffmpeg"],
extensions = ["mp4","webm"],
output_extension = "webm",
options = ACMProfileProcessorOptions(),
command = "cp {input_file} {output_file}"
))
acm_placebo_processors.append(ACMProfileProcessor(
name = "audio",
processors = ["ffmpeg","opusenc"],
extensions = ["wav","mp3"],
output_extension = "ogg",
options = ACMProfileProcessorOptions(),
command = "cp {input_file} {output_file}"
))
acm_profiles.append(ACMProfile(
name = "placebo",
processors = acm_placebo_processors
))
# webp #
acm_webp_processors = []
acm_webp_processors.append(ACMProfileProcessor(
name = "jpeg",
processors = ["cwebp"],
extensions = ["jpg", "jpeg"],
output_extension = "jpg",
options = ACMProfileProcessorOptions(),
command = "cwebp -jpeg_like -q 90 -o {output_file} {input_file}"
))
acm_webp_processors.append(ACMProfileProcessor(
name = "png",
processors = ["cwebp"],
extensions = ["png"],
output_extension = "png",
options = ACMProfileProcessorOptions(),
command = "cwebp -lossless -o {output_file} {input_file}"
))
acm_profiles.append(ACMProfile(
name = "webp",
processors = acm_webp_processors
))
# aggressive #
acm_aggressive_processors = []
acm_aggressive_processors.append(ACMProfileProcessor(
name = "jpeg",
processors = ["cjpeg"],
extensions = ["jpg", "jpeg"],
output_extension = "jpg",
options = ACMProfileProcessorOptions(force_preserve_smaller_input=True),
command = "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cjpeg -optimize -quality 75 -progressive -outfile {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}"
))
acm_aggressive_processors.append(ACMProfileProcessor(
name = "png",
processors = ["optipng"],
extensions = ["png"],
output_extension = "png",
options = ACMProfileProcessorOptions(force_preserve_smaller_input=True),
command = "optipng -o2 -strip all -out {output_file} {input_file}"
))
acm_aggressive_processors.append(ACMProfileProcessor(
name = "video",
processors = ["ffmpeg"],
extensions = ["mp4","webm"],
output_extension = "webm",
options = ACMProfileProcessorOptions(),
command = "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}"
))
acm_aggressive_processors.append(ACMProfileProcessor(
name = "audio",
processors = ["ffmpeg","opusenc"],
extensions = ["wav","mp3"],
output_extension = "ogg",
options = ACMProfileProcessorOptions(),
command = "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}"
))
acm_profiles.append(ACMProfile(
name = "aggressive",
processors = acm_aggressive_processors
))
# aggressive-webp #
acm_aggressive_webp_processors = []
acm_aggressive_webp_processors.append(ACMProfileProcessor(
name = "jpeg",
processors = ["cwebp"],
extensions = ["jpg", "jpeg"],
output_extension = "jpg",
options = ACMProfileProcessorOptions(),
command = "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cwebp -jpeg_like -q 75 -o {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}"
))
acm_aggressive_webp_processors.append(ACMProfileProcessor(
name = "png",
processors = ["optipng"],
extensions = ["png"],
output_extension = "png",
options = ACMProfileProcessorOptions(),
command = "cwebp -o {output_file} ${input_file}"
))
acm_aggressive_webp_processors.append(ACMProfileProcessor(
name = "video",
processors = ["ffmpeg"],
extensions = ["mp4","webm"],
output_extension = "webm",
options = ACMProfileProcessorOptions(),
command = "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}"
))
acm_aggressive_webp_processors.append(ACMProfileProcessor(
name = "audio",
processors = ["ffmpeg","opusenc"],
extensions = ["wav","mp3"],
output_extension = "ogg",
options = ACMProfileProcessorOptions(),
command = "ffmpeg -hide_banner -loglevel panic -i {input_file} -f wav -| opusenc --bitrate 64 --vbr --downmix-stereo --discard-comments --discard-pictures - {output_file}"
))
acm_profiles.append(ACMProfile(
name = "aggressive-webp",
processors = acm_aggressive_webp_processors
))
return ACMConfig(
profiles=acm_profiles
)

31
acm/utility.py
View File

@ -0,0 +1,31 @@
import hashlib
import io
import logging
# Size of the buffer to read files with
BUF_SIZE = 4096
LOG = logging.getLogger("acm.utility")
def get_string_sha256sum(content: str, encoding='utf-8') -> str:
sha256sum = hashlib.sha256()
with io.BytesIO(content.encode(encoding)) as c:
for byte_block in iter(lambda: c.read(BUF_SIZE), b''):
sha256sum.update(byte_block)
return sha256sum.hexdigest()
def get_string_hex(content: str) -> hex:
return hex(int(content, base=16))
def get_hex_xor(first: hex, second: hex) -> hex:
return hex(int(first, base=16) ^ int(second, base=16))
def get_string_xor(first: str, second: str, *extra: str) -> str:
result = get_hex_xor(get_string_hex(first), get_string_hex(second))
for next_hex in extra:
result = get_hex_xor(result, get_string_hex(next_hex))
return str(result)
Write Preview
Loading…
Cancel
Save