From 6a216aba5234ad4d7113cfa76806a877bddae9bc Mon Sep 17 00:00:00 2001
From: Deimos <deimos@tildes.net>
Date: Mon, 5 Oct 2020 18:20:41 -0600
Subject: [PATCH] Add invoke task to update pip package versions

First invoke task: uses pip-compile to update the versions of all the
pip packages in requirements.txt and requirements-dev.txt. It also
post-processes the output file and removes any comments that have a "-r"
reference in them, since those currently cause Salt to break (and are
kind of redundant anyway).

Unfortunately, as part of writing this I discovered that invoke can't
handle type annotations in the definitions of its task functions, so I
had to exclude tasks.py from being checked by mypy. That makes me a
little nervous about whether invoke is still being maintained. Relevant
issue (over 4 years old): https://github.com/pyinvoke/invoke/issues/357
---
 tildes/mypy.ini |  4 ++++
 tildes/tasks.py | 37 +++++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 tildes/tasks.py

diff --git a/tildes/mypy.ini b/tildes/mypy.ini
index 3d54ee7..55b6322 100644
--- a/tildes/mypy.ini
+++ b/tildes/mypy.ini
@@ -9,5 +9,9 @@ show_error_context = true
 warn_redundant_casts = true
 warn_unused_ignores = true
 
+# invoke crashes if task functions use type annotations, so we can't use them there
+[mypy-tasks]
+disallow_untyped_defs = false
+
 [mypy-tests.*]
 disallow_untyped_defs = false
diff --git a/tildes/tasks.py b/tildes/tasks.py
new file mode 100644
index 0000000..314858d
--- /dev/null
+++ b/tildes/tasks.py
@@ -0,0 +1,37 @@
+# Copyright (c) 2020 Tildes contributors <code@tildes.net>
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+"""Contains tasks that can be run through the invoke tool."""
+
+import re
+from pathlib import Path
+
+from invoke import task
+
+
+@task
+def update_pip_requirements(context):
+    """Use pip-tools to update package versions in the requirements files."""
+
+    def build_and_clean(context, name):
+        """Update a pip requirements file and clean up the result."""
+        in_filename = Path(name).with_suffix(".in")
+        out_filename = Path(name).with_suffix(".txt")
+
+        print(f"Updating package versions from {in_filename}")
+        context.run(f"pip-compile --no-header --quiet --upgrade {in_filename}")
+
+        # Salt's pip module is currently broken if any comments in the requirements
+        # file have "-r" in them, so we need to remove all of those comments
+        with open(out_filename, "r") as req_file:
+            req_lines = req_file.readlines()
+
+        # remove any comments that include an -r reference
+        # (meaning it's a package that was specifically installed, not a dependency)
+        cleaned_lines = [re.sub(r"\s+# via.*-r.*", "", line) for line in req_lines]
+
+        with open(out_filename, "w") as req_file:
+            req_file.writelines(cleaned_lines)
+
+    build_and_clean(context, "requirements")
+    build_and_clean(context, "requirements-dev")