From 036d46d5894a25fb9ca883695cc57dbcd0eb7d42 Mon Sep 17 00:00:00 2001 From: Deimos Date: Mon, 3 Aug 2020 14:37:08 -0600 Subject: [PATCH] Add marks to slower tests and don't run by default This uses pytest's "markers" system to add markers to two special types of tests: * webtest - ones that use the WebTest library and are testing the actual HTTP app, instead of executing code/functions directly * html_validation - ones that are generating HTML output (via webtest) and running it through the Nu HTML Checker to validate it. The "webtest" marker is added automatically by checking whether a test uses either of the webtest fixtures, and the html_validation one is currently added manually to the only module that has those tests. In the future, we could probably put HTML validation tests in their own folder and mark them automatically based on the module's path or something similar. This also changes the default arguments for pytest to exclude these two marked types of tests, and updates the git hooks so that webtests are run pre-commit (but not HTML validation), and all tests are run pre-push. Similar to the way we use prospector, this makes it so that the very slow tests are only run before pushing. --- git_hooks/pre-commit | 2 +- git_hooks/pre-push | 2 +- tildes/pytest.ini | 5 ++++- tildes/tests/conftest.py | 11 ++++++++++- tildes/tests/webtests/test_w3_validator.py | 6 ++++++ 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit index c240ebb..d3fa312 100755 --- a/git_hooks/pre-commit +++ b/git_hooks/pre-commit @@ -5,6 +5,6 @@ vagrant ssh -c ". activate \ && echo 'Checking mypy type annotations...' && mypy --no-error-summary . \ && echo 'Checking if Black would reformat any code...' && black --check . \ - && echo -n 'Running tests: ' && pytest -q \ + && echo -n 'Running tests: ' && pytest -q -m 'not html_validation' \ && echo 'Checking SCSS style...' && npm run --silent lint:scss \ && echo 'Checking JS style...' && npm run --silent lint:js" diff --git a/git_hooks/pre-push b/git_hooks/pre-push index 03f7a8c..e0b2908 100755 --- a/git_hooks/pre-push +++ b/git_hooks/pre-push @@ -5,7 +5,7 @@ vagrant ssh -c ". activate \ && echo 'Checking mypy type annotations...' && mypy --no-error-summary . \ && echo 'Checking if Black would reformat any code...' && black --check . \ - && echo -n 'Running tests: ' && pytest -q \ + && echo -n 'Running tests: ' && pytest -q -m '' \ && echo 'Checking SCSS style...' && npm run --silent lint:scss \ && echo 'Checking JS style...' && npm run --silent lint:js \ && echo 'Checking Python style fully (takes a while)...' && prospector -M" diff --git a/tildes/pytest.ini b/tildes/pytest.ini index 6162a49..4e2516a 100644 --- a/tildes/pytest.ini +++ b/tildes/pytest.ini @@ -1,7 +1,10 @@ [pytest] testpaths = tests -addopts = -p no:cacheprovider +addopts = -p no:cacheprovider --strict-markers -m "not (html_validation or webtest)" filterwarnings = ignore::DeprecationWarning ignore::PendingDeprecationWarning ignore::yaml.YAMLLoadWarning +markers = + html_validation: mark a test as one that validates HTML using the Nu HTML Checker (very slow) + webtest: mark a test as one that uses the WebTest library, which goes through the actual WSGI app and involves using HTTP/HTML (more of a "functional test" than "unit test") diff --git a/tildes/tests/conftest.py b/tildes/tests/conftest.py index fbda3de..fe87bda 100644 --- a/tildes/tests/conftest.py +++ b/tildes/tests/conftest.py @@ -5,7 +5,7 @@ import os from pyramid import testing from pyramid.paster import get_app, get_appsettings -from pytest import fixture +from pytest import fixture, mark from redis import Redis from sqlalchemy import create_engine from sqlalchemy.engine.url import make_url @@ -224,3 +224,12 @@ def webtest(base_app): def webtest_loggedout(base_app): """Create a logged-out webtest TestApp (function scope, so no state is retained).""" yield TestApp(base_app, extra_environ=WEBTEST_EXTRA_ENVIRON) + + +def pytest_collection_modifyitems(items): + """Add "webtest" marker to any tests that use either of the WebTest fixtures.""" + webtest_fixture_names = ("webtest", "webtest_loggedout") + + for item in items: + if any([fixture in item.fixturenames for fixture in webtest_fixture_names]): + item.add_marker(mark.webtest) diff --git a/tildes/tests/webtests/test_w3_validator.py b/tildes/tests/webtests/test_w3_validator.py index 35ef752..35f662e 100644 --- a/tildes/tests/webtests/test_w3_validator.py +++ b/tildes/tests/webtests/test_w3_validator.py @@ -3,6 +3,12 @@ import subprocess +from pytest import mark + + +# marks all tests in this module with "html_validation" marker +pytestmark = mark.html_validation + def test_homepage_html_loggedout(webtest_loggedout): """Validate HTML5 on the Tildes homepage, logged out."""