Start using pip-tools and split dev dependencies

I'm going to start using pip-tools to manage dependencies:
https://github.com/jazzband/pip-tools

This makes updating the dependencies and virtualenv easier in a few
ways, and makes it simple to keep dev dependencies split out (so I can
stop installing them in production).

Now, to do a check and update all packages to their newest versions, the
main command is:

pip-compile --no-header --upgrade requirements.in

and again with requirements-dev.in to update that one as well. This will
update all the package versions in requirements.txt and
requirements-dev.txt. The virtualenv can then be updated to match those
versions by running:

pip-sync requirements.txt

(or requirements-dev.txt for dev environment). This currently needs to
be run with sudo, but I'm going to try to fix that shortly.

salt/salt/python.sls

@@ -32,7 +32,11 @@ psycopg2-deps:
 
 pip-installs:
   pip.installed:
+    {% if grains['id'] == 'dev' %}
+    - requirements: {{ app_dir }}/requirements-dev.txt
+    {% else %}
     - requirements: {{ app_dir }}/requirements.txt
+    {% endif %}
     - bin_env: {{ venv_dir }}
   require:
     - cmd: venv-setup

tildes/requirements-dev.in

@@ -0,0 +1,13 @@
+-r requirements.in
+black
+freezegun
+ipython
+mypy
+mypy-extensions
+prospector
+pyramid-debugtoolbar
+pyramid-ipython
+pytest
+pytest-mock
+testing.redis
+webtest

tildes/requirements-dev.txt

@@ -0,0 +1,122 @@
+ago==0.0.93
+alembic==1.2.1
+amqpy==0.13.1
+appdirs==1.4.3            # via black
+argon2-cffi==19.1.0
+astroid==2.2.5            # via prospector, pylint, pylint-celery, pylint-flask, requirements-detector
+atomicwrites==1.3.0       # via pytest
+attrs==19.3.0             # via black, pytest
+backcall==0.1.0           # via ipython
+beautifulsoup4==4.8.1
+black==19.3b0
+bleach==3.1.0
+certifi==2019.9.11        # via requests, sentry-sdk
+cffi==1.13.1              # via argon2-cffi, pygit2
+chardet==3.0.4            # via requests
+click==7.0
+cornice==3.6.0
+decorator==4.4.0          # via ipython, traitlets
+dodgy==0.1.9              # via prospector
+freezegun==0.3.12
+gunicorn==19.9.0
+html5lib==1.0.1
+hupper==1.9               # via pyramid
+idna==2.8                 # via requests
+importlib-metadata==0.23  # via pluggy, pytest
+ipython-genutils==0.2.0   # via traitlets
+ipython==7.8.0
+isort==4.3.21             # via pylint
+jedi==0.15.1              # via ipython
+jinja2==2.10.3            # via pyramid-jinja2
+lazy-object-proxy==1.4.2  # via astroid
+mako==1.1.0               # via alembic, pyramid-mako
+markupsafe==1.1.1         # via jinja2, mako, pyramid-jinja2
+marshmallow==2.20.5
+mccabe==0.6.1             # via prospector, pylint
+more-itertools==7.2.0     # via pytest, zipp
+mypy-extensions==0.4.3
+mypy==0.740
+packaging==19.2           # via pytest
+parso==0.5.1              # via jedi
+pastedeploy==2.0.1        # via plaster-pastedeploy
+pep8-naming==0.4.1        # via prospector
+pexpect==4.7.0            # via ipython
+pickleshare==0.7.5        # via ipython
+pillow==6.2.1
+pip-tools==4.2.0
+plaster-pastedeploy==0.7  # via pyramid
+plaster==1.0              # via plaster-pastedeploy, pyramid
+pluggy==0.13.0            # via pytest
+prometheus-client==0.7.1
+prompt-toolkit==2.0.10    # via ipython
+prospector==1.1.7
+psycopg2==2.8.4
+ptyprocess==0.6.0         # via pexpect
+publicsuffix2==2.20160818
+py==1.8.0                 # via pytest
+pycodestyle==2.4.0        # via prospector
+pycparser==2.19           # via cffi
+pydocstyle==4.0.1         # via prospector
+pyflakes==1.6.0           # via prospector
+pygit2==0.28.2
+pygments==2.4.2
+pylint-celery==0.3        # via prospector
+pylint-django==2.0.10     # via prospector
+pylint-flask==0.6         # via prospector
+pylint-plugin-utils==0.6  # via prospector, pylint-celery, pylint-django, pylint-flask
+pylint==2.3.1             # via prospector, pylint-celery, pylint-django, pylint-flask, pylint-plugin-utils
+pyotp==2.3.0
+pyparsing==2.4.2          # via packaging
+pyramid-debugtoolbar==4.5
+pyramid-ipython==0.2
+pyramid-jinja2==2.8
+pyramid-mako==1.1.0       # via pyramid-debugtoolbar
+pyramid-session-redis==1.5.0
+pyramid-tm==2.3
+pyramid-webassets==0.10
+pyramid==1.10.4
+pytest-mock==1.11.2
+pytest==5.2.1
+python-dateutil==2.8.0
+python-editor==1.0.4      # via alembic
+pyyaml==5.1.2
+qrcode==6.1
+redis==3.3.11
+repoze.lru==0.7           # via pyramid-debugtoolbar
+requests==2.22.0
+requirements-detector==0.6  # via prospector
+sentry-sdk==0.13.0
+setoptconf==0.2.0         # via prospector
+simplejson==3.16.0        # via cornice
+six==1.12.0               # via amqpy, argon2-cffi, astroid, bleach, cornice, freezegun, html5lib, packaging, pip-tools, prompt-toolkit, pygit2, pyramid-session-redis, pyramid-webassets, python-dateutil, qrcode, sqlalchemy-utils, traitlets, webtest
+snowballstemmer==2.0.0    # via pydocstyle
+soupsieve==1.9.4          # via beautifulsoup4
+sqlalchemy-utils==0.34.2
+sqlalchemy==1.3.10
+stripe==2.37.2
+testing.common.database==2.0.3  # via testing.redis
+testing.redis==1.1.1
+titlecase==0.12.0
+toml==0.10.0              # via black
+traitlets==4.3.3          # via ipython
+transaction==2.4.0        # via pyramid-tm, zope.sqlalchemy
+translationstring==1.3    # via pyramid
+typed-ast==1.4.0          # via astroid, mypy
+typing-extensions==3.7.4  # via mypy
+urllib3==1.25.6           # via requests, sentry-sdk
+venusian==1.2.0           # via cornice, pyramid
+waitress==1.3.1           # via webtest
+wcwidth==0.1.7            # via prompt-toolkit, pytest
+webargs==4.4.1
+webassets==0.12.1         # via pyramid-webassets
+webencodings==0.5.1       # via bleach, html5lib
+webob==1.8.5              # via pyramid, webtest
+webtest==2.0.33
+wrapt==1.11.2
+zipp==0.6.0               # via importlib-metadata
+zope.deprecation==4.4.0   # via pyramid, pyramid-jinja2
+zope.interface==4.6.0     # via pyramid, pyramid-webassets, transaction, zope.sqlalchemy
+zope.sqlalchemy==1.2
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools==41.4.0        # via ipython, plaster, pyramid, zope.deprecation, zope.interface, zope.sqlalchemy

tildes/requirements-to-freeze.txt → tildes/requirements.in

@@ -2,45 +2,37 @@ ago
 alembic
 amqpy
 argon2_cffi
-black
+beautifulsoup4
 bleach
 click
 cornice
-freezegun
 gunicorn
 html5lib
-ipython
 marshmallow<3.0  # 3.0+ requires significant updates
-mypy
-mypy-extensions
 Pillow
+pip-tools
 prometheus-client
-prospector
 psycopg2
 publicsuffix2==2.20160818
 pygit2
 Pygments
 pyotp
 pyramid
-pyramid-debugtoolbar
-pyramid-ipython
 pyramid-jinja2
 pyramid-session-redis==1.5.0  # 1.5.1 has a change that will invalidate current sessions
 pyramid-tm
 pyramid-webassets
-pytest
-pytest-mock
 python-dateutil
 PyYAML  # needs to be installed separately for webassets
 qrcode
+pip-tools
+redis
 requests
 sentry-sdk
 SQLAlchemy
 SQLAlchemy-Utils
 stripe
-testing.redis
 titlecase
 webargs<5.0  # 5.0.0 breaks many views, will require significant updates
-webtest
 wrapt
 zope.sqlalchemy

tildes/requirements.txt

@@ -1,118 +1,65 @@
 ago==0.0.93
 alembic==1.2.1
 amqpy==0.13.1
-appdirs==1.4.3
 argon2-cffi==19.1.0
-astroid==2.2.5
-atomicwrites==1.3.0
-attrs==19.3.0
-backcall==0.1.0
 beautifulsoup4==4.8.1
-black==19.3b0
 bleach==3.1.0
-certifi==2019.9.11
-cffi==1.13.0
-chardet==3.0.4
-Click==7.0
+certifi==2019.9.11        # via requests, sentry-sdk
+cffi==1.13.1              # via argon2-cffi, pygit2
+chardet==3.0.4            # via requests
+click==7.0
 cornice==3.6.0
-decorator==4.4.0
-dodgy==0.1.9
-freezegun==0.3.12
 gunicorn==19.9.0
 html5lib==1.0.1
-hupper==1.9
-idna==2.8
-importlib-metadata==0.23
-ipython==7.8.0
-ipython-genutils==0.2.0
-isort==4.3.21
-jedi==0.15.1
-Jinja2==2.10.3
-lazy-object-proxy==1.4.2
-Mako==1.1.0
-MarkupSafe==1.1.1
+hupper==1.9               # via pyramid
+idna==2.8                 # via requests
+jinja2==2.10.3            # via pyramid-jinja2
+mako==1.1.0               # via alembic
+markupsafe==1.1.1         # via jinja2, mako, pyramid-jinja2
 marshmallow==2.20.5
-mccabe==0.6.1
-more-itertools==7.2.0
-mypy==0.740
-mypy-extensions==0.4.2
-packaging==19.2
-parso==0.5.1
-PasteDeploy==2.0.1
-pep8-naming==0.4.1
-pexpect==4.7.0
-pickleshare==0.7.5
-Pillow==6.2.0
-plaster==1.0
-plaster-pastedeploy==0.7
-pluggy==0.13.0
+pastedeploy==2.0.1        # via plaster-pastedeploy
+pillow==6.2.1
+pip-tools==4.2.0
+plaster-pastedeploy==0.7  # via pyramid
+plaster==1.0              # via plaster-pastedeploy, pyramid
 prometheus-client==0.7.1
-prompt-toolkit==2.0.10
-prospector==1.1.7
-psycopg2==2.8.3
-ptyprocess==0.6.0
+psycopg2==2.8.4
 publicsuffix2==2.20160818
-py==1.8.0
-pycodestyle==2.4.0
-pycparser==2.19
-pydocstyle==4.0.1
-pyflakes==1.6.0
+pycparser==2.19           # via cffi
 pygit2==0.28.2
-Pygments==2.4.2
-pylint==2.3.1
-pylint-celery==0.3
-pylint-django==2.0.10
-pylint-flask==0.6
-pylint-plugin-utils==0.6
+pygments==2.4.2
 pyotp==2.3.0
-pyparsing==2.4.2
-pyramid==1.10.4
-pyramid-debugtoolbar==4.5
-pyramid-ipython==0.2
 pyramid-jinja2==2.8
-pyramid-mako==1.1.0
 pyramid-session-redis==1.5.0
 pyramid-tm==2.3
 pyramid-webassets==0.10
-pytest==5.2.1
-pytest-mock==1.11.1
+pyramid==1.10.4
 python-dateutil==2.8.0
-python-editor==1.0.4
-PyYAML==5.1.2
+python-editor==1.0.4      # via alembic
+pyyaml==5.1.2
 qrcode==6.1
 redis==3.3.11
-repoze.lru==0.7
 requests==2.22.0
-requirements-detector==0.6
-sentry-sdk==0.12.3
-setoptconf==0.2.0
-simplejson==3.16.0
-six==1.12.0
-snowballstemmer==2.0.0
-soupsieve==1.9.4
-SQLAlchemy==1.3.10
-SQLAlchemy-Utils==0.34.2
+sentry-sdk==0.13.0
+simplejson==3.16.0        # via cornice
+six==1.12.0               # via amqpy, argon2-cffi, bleach, cornice, html5lib, pip-tools, pygit2, pyramid-session-redis, pyramid-webassets, python-dateutil, qrcode, sqlalchemy-utils
+soupsieve==1.9.4          # via beautifulsoup4
+sqlalchemy-utils==0.34.2
+sqlalchemy==1.3.10
 stripe==2.37.2
-testing.common.database==2.0.3
-testing.redis==1.1.1
 titlecase==0.12.0
-toml==0.10.0
-traitlets==4.3.3
-transaction==2.4.0
-translationstring==1.3
-typed-ast==1.4.0
-typing-extensions==3.7.4
-urllib3==1.25.6
-venusian==1.2.0
-waitress==1.3.1
-wcwidth==0.1.7
+transaction==2.4.0        # via pyramid-tm, zope.sqlalchemy
+translationstring==1.3    # via pyramid
+urllib3==1.25.6           # via requests, sentry-sdk
+venusian==1.2.0           # via cornice, pyramid
 webargs==4.4.1
-webassets==0.12.1
-webencodings==0.5.1
-WebOb==1.8.5
-WebTest==2.0.33
+webassets==0.12.1         # via pyramid-webassets
+webencodings==0.5.1       # via bleach, html5lib
+webob==1.8.5              # via pyramid
 wrapt==1.11.2
-zipp==0.6.0
-zope.deprecation==4.4.0
-zope.interface==4.6.0
-zope.sqlalchemy==1.1
+zope.deprecation==4.4.0   # via pyramid, pyramid-jinja2
+zope.interface==4.6.0     # via pyramid, pyramid-webassets, transaction, zope.sqlalchemy
+zope.sqlalchemy==1.2
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools==41.4.0        # via plaster, pyramid, zope.deprecation, zope.interface, zope.sqlalchemy