warricksothr
/
Asset-Compression-Manager
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Added functionlaity to preserve smaller images

add-file-preservation 1.4.0
Drew Short 3 years ago
parent
4889048b17
commit
ed7554f972
4 changed files with 87 additions and 29 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      Dockerfile
  2. 46
      acm-config-default.json
  3. 66
      acm.py
  4. 2
      setup.py

2
Dockerfile
View File

@ -2,7 +2,7 @@ FROM ubuntu:20.04
LABEL maintainer="Drew Short <warrick@sothr.com>" \ LABEL maintainer="Drew Short <warrick@sothr.com>" \
name="acm" \ name="acm" \
version="1.2.0" \
version="1.4.0" \
description="Prepackaged ACM with defaults and tooling" description="Prepackaged ACM with defaults and tooling"
ENV LC_ALL=C.UTF-8 ENV LC_ALL=C.UTF-8

46
acm-config-default.json
View File

@ -3,26 +3,28 @@
"profiles": { "profiles": {
"default": { "default": {
"jpeg": { "jpeg": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cjpeg"], "processors": ["cjpeg"],
"extensions": [ "extensions": [
"jpg", "jpg",
"jpeg" "jpeg"
], ],
"outputExtension": "jpg", "outputExtension": "jpg",
"forcePreserveSmallerInput": true,
"command": "cjpeg -optimize -quality 90 -progressive -outfile {output_file} {input_file}" "command": "cjpeg -optimize -quality 90 -progressive -outfile {output_file} {input_file}"
}, },
"png": { "png": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["optipng"], "processors": ["optipng"],
"extensions": [ "extensions": [
"png" "png"
], ],
"outputExtension": "png", "outputExtension": "png",
"forcePreserveSmallerInput": true,
"command": "optipng -o2 -strip all -out {output_file} {input_file}" "command": "optipng -o2 -strip all -out {output_file} {input_file}"
}, },
"video": { "video": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg"], "processors": ["ffmpeg"],
"extensions": [ "extensions": [
"mp4", "mp4",
@ -32,7 +34,7 @@
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -c:v libvpx-vp9 -b:v 0 -crf 29 -c:a libopus {output_file}" "command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -c:v libvpx-vp9 -b:v 0 -crf 29 -c:a libopus {output_file}"
}, },
"audio": { "audio": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg", "opusenc"], "processors": ["ffmpeg", "opusenc"],
"extensions": [ "extensions": [
"wav", "wav",
@ -44,7 +46,7 @@
}, },
"placebo": { "placebo": {
"jpeg": { "jpeg": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cp"], "processors": ["cp"],
"extensions": [ "extensions": [
"jpg", "jpg",
@ -52,20 +54,22 @@
], ],
"outputExtension": "jpg", "outputExtension": "jpg",
"preserveInputExtension": true, "preserveInputExtension": true,
"command": "cp {input_file} {output_file}"
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}",
}, },
"png": { "png": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cp"], "processors": ["cp"],
"extensions": [ "extensions": [
"png" "png"
], ],
"outputExtension": "png", "outputExtension": "png",
"preserveInputExtension": true, "preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}" "command": "cp {input_file} {output_file}"
}, },
"video": { "video": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cp"], "processors": ["cp"],
"extensions": [ "extensions": [
"mp4", "mp4",
@ -73,10 +77,11 @@
], ],
"outputExtension": "mp4", "outputExtension": "mp4",
"preserveInputExtension": true, "preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}" "command": "cp {input_file} {output_file}"
}, },
"audio": { "audio": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cp"], "processors": ["cp"],
"extensions": [ "extensions": [
"wav", "wav",
@ -84,12 +89,13 @@
], ],
"outputExtension": "ogg", "outputExtension": "ogg",
"preserveInputExtension": true, "preserveInputExtension": true,
"preserveSmallerInput": false,
"command": "cp {input_file} {output_file}" "command": "cp {input_file} {output_file}"
} }
}, },
"webp": { "webp": {
"jpeg": { "jpeg": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cwebp"], "processors": ["cwebp"],
"extensions": [ "extensions": [
"jpg", "jpg",
@ -99,7 +105,7 @@
"command": "cwebp -jpeg_like -q 90 -o {output_file} {input_file}" "command": "cwebp -jpeg_like -q 90 -o {output_file} {input_file}"
}, },
"png": { "png": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cwebp"], "processors": ["cwebp"],
"extensions": [ "extensions": [
"png" "png"
@ -110,26 +116,28 @@
}, },
"aggressive": { "aggressive": {
"jpeg": { "jpeg": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg", "cjpeg"], "processors": ["ffmpeg", "cjpeg"],
"extensions": [ "extensions": [
"jpg", "jpg",
"jpeg" "jpeg"
], ],
"outputExtension": "jpg", "outputExtension": "jpg",
"forcePreserveSmallerInput": true,
"command": "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cjpeg -optimize -quality 75 -progressive -outfile {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}" "command": "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cjpeg -optimize -quality 75 -progressive -outfile {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}"
}, },
"png": { "png": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["optipng"], "processors": ["optipng"],
"extensions": [ "extensions": [
"png" "png"
], ],
"outputExtension": "png", "outputExtension": "png",
"forcePreserveSmallerInput": true,
"command": "optipng -o2 -strip all -out {output_file} {input_file}" "command": "optipng -o2 -strip all -out {output_file} {input_file}"
}, },
"video": { "video": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg"], "processors": ["ffmpeg"],
"extensions": [ "extensions": [
"mp4", "mp4",
@ -139,7 +147,7 @@
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}" "command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}"
}, },
"audio": { "audio": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg", "opusenc"], "processors": ["ffmpeg", "opusenc"],
"extensions": [ "extensions": [
"wav", "wav",
@ -151,7 +159,7 @@
}, },
"aggressive-webp": { "aggressive-webp": {
"jpeg": { "jpeg": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cwebp"], "processors": ["cwebp"],
"extensions": [ "extensions": [
"jpg", "jpg",
@ -161,7 +169,7 @@
"command": "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cwebp -jpeg_like -q 75 -o {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}" "command": "export FILE={output_file} && export TEMP_FILE=${FILE}_tmp.jpg && ffmpeg -i {input_file} -vf scale=-1:720 ${TEMP_FILE} && cwebp -jpeg_like -q 75 -o {output_file} ${TEMP_FILE} && rm ${TEMP_FILE}"
}, },
"png": { "png": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["cwebp"], "processors": ["cwebp"],
"extensions": [ "extensions": [
"png" "png"
@ -170,7 +178,7 @@
"command": "cwebp -o {output_file} ${input_file}" "command": "cwebp -o {output_file} ${input_file}"
}, },
"video": { "video": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg"], "processors": ["ffmpeg"],
"extensions": [ "extensions": [
"mp4", "mp4",
@ -180,7 +188,7 @@
"command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}" "command": "ffmpeg -hide_banner -loglevel panic -i {input_file} -vf scale=-1:720 -c:v libvpx-vp9 -b:v 0 -crf 38 -c:a libopus {output_file}"
}, },
"audio": { "audio": {
"version": "1.3.1",
"version": "1.4.0",
"processors": ["ffmpeg", "opusenc"], "processors": ["ffmpeg", "opusenc"],
"extensions": [ "extensions": [
"wav", "wav",

66
acm.py
View File

@ -5,6 +5,7 @@ import hashlib
import io import io
import json import json
import os import os
import pathlib
import platform import platform
import sys import sys
import tempfile import tempfile
@ -18,7 +19,7 @@ from minio.error import NoSuchKey
BUF_SIZE = 4096 BUF_SIZE = 4096
#Application Version #Application Version
VERSION = "1.3.1"
VERSION = "1.4.0"
########### ###########
@ -27,7 +28,7 @@ VERSION = "1.3.1"
async def run_command_shell( async def run_command_shell(
command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, on_success: Callable = ()):
command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, on_success: List[Callable] = [()]):
"""Run command in subprocess (shell). """Run command in subprocess (shell).
Note: Note:
@ -41,7 +42,8 @@ async def run_command_shell(
process_stdout, process_stderr = await process.communicate() process_stdout, process_stderr = await process.communicate()
if process.returncode == 0: if process.returncode == 0:
on_success()
for callable in on_success:
callable()
if stdout != asyncio.subprocess.DEVNULL: if stdout != asyncio.subprocess.DEVNULL:
result = process_stdout.decode().strip() result = process_stdout.decode().strip()
@ -669,8 +671,9 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
if destination is None: if destination is None:
destination = tempfile.mkdtemp() destination = tempfile.mkdtemp()
compressed_files = []
task_output = []
tasks = [] tasks = []
follow_up_tasks = []
def store_filename(storage_list: List[str], filename: str): def store_filename(storage_list: List[str], filename: str):
""" """
@ -682,10 +685,35 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
""" """
return lambda: storage_list.append(filename) return lambda: storage_list.append(filename)
def queue_follow_up_task_if_keep_smaller_input(follow_up_tasks, input_file: str, output_file: str, keep_smaller_input: bool = True):
"""
A lambda wrapper that handles keeping the smallest of the two files.
"""
if keep_smaller_input:
command = f"cp {input_file} {output_file}"
return lambda:
input_size = os.path.getsize(input_file)
output_size = os.path.getsize(output_file)
if output_size > input_size:
follow_up_tasks.append(
run_command_shell(
command,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
on_success=[store_filename(
task_output,
f'Preserved smaller "{input_file}" {output_size} > {input_size}'
)]
)
)
return None
for input_file in files: for input_file in files:
for content_configuration in content_configurations: for content_configuration in content_configurations:
if any([input_file.endswith(extension) for extension in content_configuration['extensions']]): if any([input_file.endswith(extension) for extension in content_configuration['extensions']]):
file = input_file file = input_file
file_extension = pathlib.Path(input_file).suffix
if 'REMOVE_PREFIX' in ctx.obj and ctx.obj['REMOVE_PREFIX'] is not None: if 'REMOVE_PREFIX' in ctx.obj and ctx.obj['REMOVE_PREFIX'] is not None:
file = strip_prefix(ctx.obj['REMOVE_PREFIX'], input_file) file = strip_prefix(ctx.obj['REMOVE_PREFIX'], input_file)
@ -701,6 +729,19 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
output_file_dir = os.path.dirname(output_file) output_file_dir = os.path.dirname(output_file)
os.makedirs(output_file_dir, exist_ok=True) os.makedirs(output_file_dir, exist_ok=True)
if 'preserveSmallerInput' in content_configuration:
preserve_smaller_input = bool(content_configuration['preserveSmallerInput'])
else:
preserve_smaller_input = True
if 'forcePreserveSmallerInput' in content_configuration:
force_preserve_smaller_input = bool(content_configuration['forcePreserveSmallerInput'])
else:
force_preserve_smaller_input = False
# Only preserve the input if requested AND the extensions of the input and the output match
preserve_smaller_input = preserve_smaller_input and (force_preserve_smaller_input or file_extension == content_configuration["outputExtension"])
command: str = content_configuration['command'] \ command: str = content_configuration['command'] \
.replace('{input_file}', f'\'{input_file}\'') \ .replace('{input_file}', f'\'{input_file}\'') \
.replace('{output_file}', f'\'{output_file}\'') .replace('{output_file}', f'\'{output_file}\'')
@ -710,10 +751,15 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
command, command,
stdout=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
on_success=store_filename(
compressed_files,
on_success=[store_filename(
task_output,
f'{input_file}\t{output_file_identity}' if print_input_and_identity else output_file f'{input_file}\t{output_file_identity}' if print_input_and_identity else output_file
)
),queue_follow_up_task_if_keep_smaller_input(
follow_up_tasks,
input_file,
output_file,
preserve_smaller_input
)]
) )
) )
@ -721,7 +767,11 @@ def compress_assets(ctx, profile, content, destination, print_input_and_identity
tasks, max_concurrent_tasks=ctx.obj['CONFIG']['concurrency'] tasks, max_concurrent_tasks=ctx.obj['CONFIG']['concurrency']
) )
print(os.linesep.join(compressed_files))
follow_up_results = run_asyncio_commands(
follow_up_tasks, max_concurrent_tasks=ctx.obj['CONFIG']['concurrency']
)
print(os.linesep.join(task_output))
if __name__ == '__main__': if __name__ == '__main__':

2
setup.py
View File

@ -4,7 +4,7 @@ from distutils.core import setup
setup( setup(
name='Asset-Compression-Manager', name='Asset-Compression-Manager',
version='1.3.1',
version='1.4.0',
description='Helper Utility For Managing Compressed Assets', description='Helper Utility For Managing Compressed Assets',
author='Drew Short', author='Drew Short',
author_email='warrick@sothr.com' author_email='warrick@sothr.com'
Write Preview
Loading…
Cancel
Save