From 31de48e447bb48b3933d407cdcb5c1a0a2e58645 Mon Sep 17 00:00:00 2001 From: Deimos Date: Thu, 24 Oct 2019 15:15:02 -0600 Subject: [PATCH] Start using pip-tools and split dev dependencies I'm going to start using pip-tools to manage dependencies: https://github.com/jazzband/pip-tools This makes updating the dependencies and virtualenv easier in a few ways, and makes it simple to keep dev dependencies split out (so I can stop installing them in production). Now, to do a check and update all packages to their newest versions, the main command is: pip-compile --no-header --upgrade requirements.in and again with requirements-dev.in to update that one as well. This will update all the package versions in requirements.txt and requirements-dev.txt. The virtualenv can then be updated to match those versions by running: pip-sync requirements.txt (or requirements-dev.txt for dev environment). This currently needs to be run with sudo, but I'm going to try to fix that shortly. --- salt/salt/python.sls | 4 + tildes/requirements-dev.in | 13 ++ tildes/requirements-dev.txt | 122 ++++++++++++++++ ...irements-to-freeze.txt => requirements.in} | 16 +-- tildes/requirements.txt | 131 ++++++------------ 5 files changed, 182 insertions(+), 104 deletions(-) create mode 100644 tildes/requirements-dev.in create mode 100644 tildes/requirements-dev.txt rename tildes/{requirements-to-freeze.txt => requirements.in} (81%) diff --git a/salt/salt/python.sls b/salt/salt/python.sls index f156993..f94b32e 100644 --- a/salt/salt/python.sls +++ b/salt/salt/python.sls @@ -32,7 +32,11 @@ psycopg2-deps: pip-installs: pip.installed: + {% if grains['id'] == 'dev' %} + - requirements: {{ app_dir }}/requirements-dev.txt + {% else %} - requirements: {{ app_dir }}/requirements.txt + {% endif %} - bin_env: {{ venv_dir }} require: - cmd: venv-setup diff --git a/tildes/requirements-dev.in b/tildes/requirements-dev.in new file mode 100644 index 0000000..d46fedd --- /dev/null +++ b/tildes/requirements-dev.in @@ -0,0 +1,13 @@ +-r requirements.in +black +freezegun +ipython +mypy +mypy-extensions +prospector +pyramid-debugtoolbar +pyramid-ipython +pytest +pytest-mock +testing.redis +webtest diff --git a/tildes/requirements-dev.txt b/tildes/requirements-dev.txt new file mode 100644 index 0000000..d0bbb57 --- /dev/null +++ b/tildes/requirements-dev.txt @@ -0,0 +1,122 @@ +ago==0.0.93 +alembic==1.2.1 +amqpy==0.13.1 +appdirs==1.4.3 # via black +argon2-cffi==19.1.0 +astroid==2.2.5 # via prospector, pylint, pylint-celery, pylint-flask, requirements-detector +atomicwrites==1.3.0 # via pytest +attrs==19.3.0 # via black, pytest +backcall==0.1.0 # via ipython +beautifulsoup4==4.8.1 +black==19.3b0 +bleach==3.1.0 +certifi==2019.9.11 # via requests, sentry-sdk +cffi==1.13.1 # via argon2-cffi, pygit2 +chardet==3.0.4 # via requests +click==7.0 +cornice==3.6.0 +decorator==4.4.0 # via ipython, traitlets +dodgy==0.1.9 # via prospector +freezegun==0.3.12 +gunicorn==19.9.0 +html5lib==1.0.1 +hupper==1.9 # via pyramid +idna==2.8 # via requests +importlib-metadata==0.23 # via pluggy, pytest +ipython-genutils==0.2.0 # via traitlets +ipython==7.8.0 +isort==4.3.21 # via pylint +jedi==0.15.1 # via ipython +jinja2==2.10.3 # via pyramid-jinja2 +lazy-object-proxy==1.4.2 # via astroid +mako==1.1.0 # via alembic, pyramid-mako +markupsafe==1.1.1 # via jinja2, mako, pyramid-jinja2 +marshmallow==2.20.5 +mccabe==0.6.1 # via prospector, pylint +more-itertools==7.2.0 # via pytest, zipp +mypy-extensions==0.4.3 +mypy==0.740 +packaging==19.2 # via pytest +parso==0.5.1 # via jedi +pastedeploy==2.0.1 # via plaster-pastedeploy +pep8-naming==0.4.1 # via prospector +pexpect==4.7.0 # via ipython +pickleshare==0.7.5 # via ipython +pillow==6.2.1 +pip-tools==4.2.0 +plaster-pastedeploy==0.7 # via pyramid +plaster==1.0 # via plaster-pastedeploy, pyramid +pluggy==0.13.0 # via pytest +prometheus-client==0.7.1 +prompt-toolkit==2.0.10 # via ipython +prospector==1.1.7 +psycopg2==2.8.4 +ptyprocess==0.6.0 # via pexpect +publicsuffix2==2.20160818 +py==1.8.0 # via pytest +pycodestyle==2.4.0 # via prospector +pycparser==2.19 # via cffi +pydocstyle==4.0.1 # via prospector +pyflakes==1.6.0 # via prospector +pygit2==0.28.2 +pygments==2.4.2 +pylint-celery==0.3 # via prospector +pylint-django==2.0.10 # via prospector +pylint-flask==0.6 # via prospector +pylint-plugin-utils==0.6 # via prospector, pylint-celery, pylint-django, pylint-flask +pylint==2.3.1 # via prospector, pylint-celery, pylint-django, pylint-flask, pylint-plugin-utils +pyotp==2.3.0 +pyparsing==2.4.2 # via packaging +pyramid-debugtoolbar==4.5 +pyramid-ipython==0.2 +pyramid-jinja2==2.8 +pyramid-mako==1.1.0 # via pyramid-debugtoolbar +pyramid-session-redis==1.5.0 +pyramid-tm==2.3 +pyramid-webassets==0.10 +pyramid==1.10.4 +pytest-mock==1.11.2 +pytest==5.2.1 +python-dateutil==2.8.0 +python-editor==1.0.4 # via alembic +pyyaml==5.1.2 +qrcode==6.1 +redis==3.3.11 +repoze.lru==0.7 # via pyramid-debugtoolbar +requests==2.22.0 +requirements-detector==0.6 # via prospector +sentry-sdk==0.13.0 +setoptconf==0.2.0 # via prospector +simplejson==3.16.0 # via cornice +six==1.12.0 # via amqpy, argon2-cffi, astroid, bleach, cornice, freezegun, html5lib, packaging, pip-tools, prompt-toolkit, pygit2, pyramid-session-redis, pyramid-webassets, python-dateutil, qrcode, sqlalchemy-utils, traitlets, webtest +snowballstemmer==2.0.0 # via pydocstyle +soupsieve==1.9.4 # via beautifulsoup4 +sqlalchemy-utils==0.34.2 +sqlalchemy==1.3.10 +stripe==2.37.2 +testing.common.database==2.0.3 # via testing.redis +testing.redis==1.1.1 +titlecase==0.12.0 +toml==0.10.0 # via black +traitlets==4.3.3 # via ipython +transaction==2.4.0 # via pyramid-tm, zope.sqlalchemy +translationstring==1.3 # via pyramid +typed-ast==1.4.0 # via astroid, mypy +typing-extensions==3.7.4 # via mypy +urllib3==1.25.6 # via requests, sentry-sdk +venusian==1.2.0 # via cornice, pyramid +waitress==1.3.1 # via webtest +wcwidth==0.1.7 # via prompt-toolkit, pytest +webargs==4.4.1 +webassets==0.12.1 # via pyramid-webassets +webencodings==0.5.1 # via bleach, html5lib +webob==1.8.5 # via pyramid, webtest +webtest==2.0.33 +wrapt==1.11.2 +zipp==0.6.0 # via importlib-metadata +zope.deprecation==4.4.0 # via pyramid, pyramid-jinja2 +zope.interface==4.6.0 # via pyramid, pyramid-webassets, transaction, zope.sqlalchemy +zope.sqlalchemy==1.2 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.4.0 # via ipython, plaster, pyramid, zope.deprecation, zope.interface, zope.sqlalchemy diff --git a/tildes/requirements-to-freeze.txt b/tildes/requirements.in similarity index 81% rename from tildes/requirements-to-freeze.txt rename to tildes/requirements.in index e811ed6..cd3b9b6 100644 --- a/tildes/requirements-to-freeze.txt +++ b/tildes/requirements.in @@ -2,45 +2,37 @@ ago alembic amqpy argon2_cffi -black +beautifulsoup4 bleach click cornice -freezegun gunicorn html5lib -ipython marshmallow<3.0 # 3.0+ requires significant updates -mypy -mypy-extensions Pillow +pip-tools prometheus-client -prospector psycopg2 publicsuffix2==2.20160818 pygit2 Pygments pyotp pyramid -pyramid-debugtoolbar -pyramid-ipython pyramid-jinja2 pyramid-session-redis==1.5.0 # 1.5.1 has a change that will invalidate current sessions pyramid-tm pyramid-webassets -pytest -pytest-mock python-dateutil PyYAML # needs to be installed separately for webassets qrcode +pip-tools +redis requests sentry-sdk SQLAlchemy SQLAlchemy-Utils stripe -testing.redis titlecase webargs<5.0 # 5.0.0 breaks many views, will require significant updates -webtest wrapt zope.sqlalchemy diff --git a/tildes/requirements.txt b/tildes/requirements.txt index a451995..2d74d3a 100644 --- a/tildes/requirements.txt +++ b/tildes/requirements.txt @@ -1,118 +1,65 @@ ago==0.0.93 alembic==1.2.1 amqpy==0.13.1 -appdirs==1.4.3 argon2-cffi==19.1.0 -astroid==2.2.5 -atomicwrites==1.3.0 -attrs==19.3.0 -backcall==0.1.0 beautifulsoup4==4.8.1 -black==19.3b0 bleach==3.1.0 -certifi==2019.9.11 -cffi==1.13.0 -chardet==3.0.4 -Click==7.0 +certifi==2019.9.11 # via requests, sentry-sdk +cffi==1.13.1 # via argon2-cffi, pygit2 +chardet==3.0.4 # via requests +click==7.0 cornice==3.6.0 -decorator==4.4.0 -dodgy==0.1.9 -freezegun==0.3.12 gunicorn==19.9.0 html5lib==1.0.1 -hupper==1.9 -idna==2.8 -importlib-metadata==0.23 -ipython==7.8.0 -ipython-genutils==0.2.0 -isort==4.3.21 -jedi==0.15.1 -Jinja2==2.10.3 -lazy-object-proxy==1.4.2 -Mako==1.1.0 -MarkupSafe==1.1.1 +hupper==1.9 # via pyramid +idna==2.8 # via requests +jinja2==2.10.3 # via pyramid-jinja2 +mako==1.1.0 # via alembic +markupsafe==1.1.1 # via jinja2, mako, pyramid-jinja2 marshmallow==2.20.5 -mccabe==0.6.1 -more-itertools==7.2.0 -mypy==0.740 -mypy-extensions==0.4.2 -packaging==19.2 -parso==0.5.1 -PasteDeploy==2.0.1 -pep8-naming==0.4.1 -pexpect==4.7.0 -pickleshare==0.7.5 -Pillow==6.2.0 -plaster==1.0 -plaster-pastedeploy==0.7 -pluggy==0.13.0 +pastedeploy==2.0.1 # via plaster-pastedeploy +pillow==6.2.1 +pip-tools==4.2.0 +plaster-pastedeploy==0.7 # via pyramid +plaster==1.0 # via plaster-pastedeploy, pyramid prometheus-client==0.7.1 -prompt-toolkit==2.0.10 -prospector==1.1.7 -psycopg2==2.8.3 -ptyprocess==0.6.0 +psycopg2==2.8.4 publicsuffix2==2.20160818 -py==1.8.0 -pycodestyle==2.4.0 -pycparser==2.19 -pydocstyle==4.0.1 -pyflakes==1.6.0 +pycparser==2.19 # via cffi pygit2==0.28.2 -Pygments==2.4.2 -pylint==2.3.1 -pylint-celery==0.3 -pylint-django==2.0.10 -pylint-flask==0.6 -pylint-plugin-utils==0.6 +pygments==2.4.2 pyotp==2.3.0 -pyparsing==2.4.2 -pyramid==1.10.4 -pyramid-debugtoolbar==4.5 -pyramid-ipython==0.2 pyramid-jinja2==2.8 -pyramid-mako==1.1.0 pyramid-session-redis==1.5.0 pyramid-tm==2.3 pyramid-webassets==0.10 -pytest==5.2.1 -pytest-mock==1.11.1 +pyramid==1.10.4 python-dateutil==2.8.0 -python-editor==1.0.4 -PyYAML==5.1.2 +python-editor==1.0.4 # via alembic +pyyaml==5.1.2 qrcode==6.1 redis==3.3.11 -repoze.lru==0.7 requests==2.22.0 -requirements-detector==0.6 -sentry-sdk==0.12.3 -setoptconf==0.2.0 -simplejson==3.16.0 -six==1.12.0 -snowballstemmer==2.0.0 -soupsieve==1.9.4 -SQLAlchemy==1.3.10 -SQLAlchemy-Utils==0.34.2 +sentry-sdk==0.13.0 +simplejson==3.16.0 # via cornice +six==1.12.0 # via amqpy, argon2-cffi, bleach, cornice, html5lib, pip-tools, pygit2, pyramid-session-redis, pyramid-webassets, python-dateutil, qrcode, sqlalchemy-utils +soupsieve==1.9.4 # via beautifulsoup4 +sqlalchemy-utils==0.34.2 +sqlalchemy==1.3.10 stripe==2.37.2 -testing.common.database==2.0.3 -testing.redis==1.1.1 titlecase==0.12.0 -toml==0.10.0 -traitlets==4.3.3 -transaction==2.4.0 -translationstring==1.3 -typed-ast==1.4.0 -typing-extensions==3.7.4 -urllib3==1.25.6 -venusian==1.2.0 -waitress==1.3.1 -wcwidth==0.1.7 +transaction==2.4.0 # via pyramid-tm, zope.sqlalchemy +translationstring==1.3 # via pyramid +urllib3==1.25.6 # via requests, sentry-sdk +venusian==1.2.0 # via cornice, pyramid webargs==4.4.1 -webassets==0.12.1 -webencodings==0.5.1 -WebOb==1.8.5 -WebTest==2.0.33 +webassets==0.12.1 # via pyramid-webassets +webencodings==0.5.1 # via bleach, html5lib +webob==1.8.5 # via pyramid wrapt==1.11.2 -zipp==0.6.0 -zope.deprecation==4.4.0 -zope.interface==4.6.0 -zope.sqlalchemy==1.1 +zope.deprecation==4.4.0 # via pyramid, pyramid-jinja2 +zope.interface==4.6.0 # via pyramid, pyramid-webassets, transaction, zope.sqlalchemy +zope.sqlalchemy==1.2 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools==41.4.0 # via plaster, pyramid, zope.deprecation, zope.interface, zope.sqlalchemy