Browse Source

Added functionlaity to preserve smaller images

add-file-preservation 1.4.0
Drew Short 2 years ago
parent
commit
ed7554f972
  1. 2
      Dockerfile
  2. 46
      acm-config-default.json
  3. 66
      acm.py
  4. 2
      setup.py

2
Dockerfile

@ -2,7 +2,7 @@ FROM ubuntu:20.04
LABEL maintainer="Drew Short <warrick@sothr.com>" \
name="acm" \
version="1.2.0" \
version="1.4.0" \
description="Prepackaged ACM with defaults and tooling"
ENV LC_ALL=C.UTF-8

46
acm-config-default.json

@ -3,26 +3,28 @@
"profiles": {
"default": {
"jpeg": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cjpeg"],
"extensions": [
"jpg",
"jpeg"
],
"outputExtension": "jpg",
"forcePreserveSmallerInput": true,
"command": "cjpeg -optimize -quality 90 -progressive -outfile {output_file} {input_file}"
},
"png": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["optipng"],
"extensions": [
"png"
],
"outputExtension": "png",
"forcePreserveSmallerInput": true,
"command": "optipng -o2 -strip all -out {output_file} {input_file}"
},
"video": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg"],
"extensions": [
"mp4",
@ -32,7 +34,7 @@
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -c:v libvpx-vp9 -b:v 0 -crf 29 -c:a libopus {output_file}"
},
"audio": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg", "opusenc"],
"extensions": [
"wav",
@ -44,7 +46,7 @@
},
"placebo": {
"jpeg": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cp"],
"extensions": [
"jpg",
@ -52,20 +54,22 @@
],
"outputExtension": "jpg",
"preserveInputExtension": true,
"command": "cp {input_file} {output_file}"
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}",
},
"png": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cp"],
"extensions": [
"png"
],
"outputExtension": "png",
"preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}"
},
"video": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cp"],
"extensions": [
"mp4",
@ -73,10 +77,11 @@
],
"outputExtension": "mp4",
"preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}"
},
"audio": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cp"],
"extensions": [
"wav",
@ -84,12 +89,13 @@
],
"outputExtension": "ogg",
"preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}"
}
},
"webp": {
"jpeg": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cwebp"],
"extensions": [
"jpg",
@ -99,7 +105,7 @@
"command": "cwebp -jpeg_like -q 90 -o {output_file} {input_file}"
},
"png": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cwebp"],
"extensions": [
"png"
@ -110,26 +116,28 @@
},
"aggressive": {
"jpeg": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg", "cjpeg"],
"extensions": [
"jpg",
"jpeg"
],
"outputExtension": "jpg",
"forcePreserveSmallerInput": true,
"command": "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cjpeg -optimize -quality 75 -progressive -outfile {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}"
},
"png": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["optipng"],
"extensions": [
"png"
],
"outputExtension": "png",
"forcePreserveSmallerInput": true,
"command": "optipng -o2 -strip all -out {output_file} {input_file}"
},
"video": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg"],
"extensions": [
"mp4",
@ -139,7 +147,7 @@
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}"
},
"audio": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg", "opusenc"],
"extensions": [
"wav",
@ -151,7 +159,7 @@
},
"aggressive-webp": {
"jpeg": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cwebp"],
"extensions": [
"jpg",
@ -161,7 +169,7 @@
"command": "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cwebp -jpeg_like -q 75 -o {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}"
},
"png": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cwebp"],
"extensions": [
"png"
@ -170,7 +178,7 @@
"command": "cwebp -o {output_file} ${input_file}"
},
"video": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg"],
"extensions": [
"mp4",
@ -180,7 +188,7 @@
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}"
},
"audio": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg", "opusenc"],
"extensions": [
"wav",

66
acm.py

@ -5,6 +5,7 @@ import hashlib
import io
import json
import os
import pathlib
import platform
import sys
import tempfile
@ -18,7 +19,7 @@ from minio.error import NoSuchKey
BUF_SIZE = 4096
#Application Version
VERSION = "1.3.1"
VERSION = "1.4.0"
###########
@ -27,7 +28,7 @@ VERSION = "1.3.1"
async def run_command_shell(
command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, on_success: Callable = ()):
command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, on_success: List[Callable] = [()]):
"""Run command in subprocess (shell).
Note:
@ -41,7 +42,8 @@ async def run_command_shell(
process_stdout, process_stderr = await process.communicate()
if process.returncode == 0:
on_success()
for callable in on_success:
callable()
if stdout != asyncio.subprocess.DEVNULL:
result = process_stdout.decode().strip()
@ -669,8 +671,9 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
if destination is None:
destination = tempfile.mkdtemp()
compressed_files = []
task_output = []
tasks = []
follow_up_tasks = []
def store_filename(storage_list: List[str], filename: str):
"""
@ -682,10 +685,35 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
"""
return lambda: storage_list.append(filename)
def queue_follow_up_task_if_keep_smaller_input(follow_up_tasks, input_file: str, output_file: str, keep_smaller_input: bool = True):
"""
A lambda wrapper that handles keeping the smallest of the two files.
"""
if keep_smaller_input:
command = f"cp {input_file} {output_file}"
return lambda:
input_size = os.path.getsize(input_file)
output_size = os.path.getsize(output_file)
if output_size > input_size:
follow_up_tasks.append(
run_command_shell(
command,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
on_success=[store_filename(
task_output,
f'Preserved smaller "{input_file}" {output_size} > {input_size}'
)]
)
)
return None
for input_file in files:
for content_configuration in content_configurations:
if any([input_file.endswith(extension) for extension in content_configuration['extensions']]):
file = input_file
file_extension = pathlib.Path(input_file).suffix
if 'REMOVE_PREFIX' in ctx.obj and ctx.obj['REMOVE_PREFIX'] is not None:
file = strip_prefix(ctx.obj['REMOVE_PREFIX'], input_file)
@ -701,6 +729,19 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
output_file_dir = os.path.dirname(output_file)
os.makedirs(output_file_dir, exist_ok=True)
if 'preserveSmallerInput' in content_configuration:
preserve_smaller_input = bool(content_configuration['preserveSmallerInput'])
else:
preserve_smaller_input = True
if 'forcePreserveSmallerInput' in content_configuration:
force_preserve_smaller_input = bool(content_configuration['forcePreserveSmallerInput'])
else:
force_preserve_smaller_input = False
# Only preserve the input if requested AND the extensions of the input and the output match
preserve_smaller_input = preserve_smaller_input and (force_preserve_smaller_input or file_extension == content_configuration["outputExtension"])
command: str = content_configuration['command'] \
.replace('{input_file}', f'\'{input_file}\'') \
.replace('{output_file}', f'\'{output_file}\'')
@ -710,10 +751,15 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
command,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
on_success=store_filename(
compressed_files,
on_success=[store_filename(
task_output,
f'{input_file}\t{output_file_identity}' if print_input_and_identity else output_file
)
),queue_follow_up_task_if_keep_smaller_input(
follow_up_tasks,
input_file,
output_file,
preserve_smaller_input
)]
)
)
@ -721,7 +767,11 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
tasks, max_concurrent_tasks=ctx.obj['CONFIG']['concurrency']
)
print(os.linesep.join(compressed_files))
follow_up_results = run_asyncio_commands(
follow_up_tasks, max_concurrent_tasks=ctx.obj['CONFIG']['concurrency']
)
print(os.linesep.join(task_output))
if __name__ == '__main__':

2
setup.py

@ -4,7 +4,7 @@ from distutils.core import setup
setup(
name='Asset-Compression-Manager',
version='1.3.1',
version='1.4.0',
description='Helper Utility For Managing Compressed Assets',
author='Drew Short',
author_email='warrick@sothr.com'
Loading…
Cancel
Save