Add marks to slower tests and don't run by default

This uses pytest's "markers" system to add markers to two special types
of tests:

* webtest - ones that use the WebTest library and are testing the actual
  HTTP app, instead of executing code/functions directly
* html_validation - ones that are generating HTML output (via webtest)
  and running it through the Nu HTML Checker to validate it.

The "webtest" marker is added automatically by checking whether a test
uses either of the webtest fixtures, and the html_validation one is
currently added manually to the only module that has those tests. In the
future, we could probably put HTML validation tests in their own folder
and mark them automatically based on the module's path or something
similar.

This also changes the default arguments for pytest to exclude these two
marked types of tests, and updates the git hooks so that webtests are
run pre-commit (but not HTML validation), and all tests are run
pre-push. Similar to the way we use prospector, this makes it so that
the very slow tests are only run before pushing.

git_hooks/pre-commit

@@ -5,6 +5,6 @@
 vagrant ssh -c ". activate \
     && echo 'Checking mypy type annotations...' && mypy --no-error-summary . \
     && echo 'Checking if Black would reformat any code...' && black --check . \
-    && echo -n 'Running tests: ' && pytest -q \
+    && echo -n 'Running tests: ' && pytest -q -m 'not html_validation' \
     && echo 'Checking SCSS style...' && npm run --silent lint:scss \
     && echo 'Checking JS style...' && npm run --silent lint:js"

git_hooks/pre-push

@@ -5,7 +5,7 @@
 vagrant ssh -c ". activate \
     && echo 'Checking mypy type annotations...' && mypy --no-error-summary . \
     && echo 'Checking if Black would reformat any code...' && black --check . \
-    && echo -n 'Running tests: ' && pytest -q \
+    && echo -n 'Running tests: ' && pytest -q -m '' \
     && echo 'Checking SCSS style...' && npm run --silent lint:scss \
     && echo 'Checking JS style...' && npm run --silent lint:js \
     && echo 'Checking Python style fully (takes a while)...' && prospector -M"

tildes/pytest.ini

@@ -1,7 +1,10 @@
 [pytest]
 testpaths = tests
-addopts = -p no:cacheprovider
+addopts = -p no:cacheprovider --strict-markers -m "not (html_validation or webtest)"
 filterwarnings =
     ignore::DeprecationWarning
     ignore::PendingDeprecationWarning
     ignore::yaml.YAMLLoadWarning
+markers =
+    html_validation: mark a test as one that validates HTML using the Nu HTML Checker (very slow)
+    webtest: mark a test as one that uses the WebTest library, which goes through the actual WSGI app and involves using HTTP/HTML (more of a "functional test" than "unit test")

tildes/tests/conftest.py

@@ -5,7 +5,7 @@ import os
 
 from pyramid import testing
 from pyramid.paster import get_app, get_appsettings
-from pytest import fixture
+from pytest import fixture, mark
 from redis import Redis
 from sqlalchemy import create_engine
 from sqlalchemy.engine.url import make_url
@@ -224,3 +224,12 @@ def webtest(base_app):
 def webtest_loggedout(base_app):
     """Create a logged-out webtest TestApp (function scope, so no state is retained)."""
     yield TestApp(base_app, extra_environ=WEBTEST_EXTRA_ENVIRON)
+
+
+def pytest_collection_modifyitems(items):
+    """Add "webtest" marker to any tests that use either of the WebTest fixtures."""
+    webtest_fixture_names = ("webtest", "webtest_loggedout")
+
+    for item in items:
+        if any([fixture in item.fixturenames for fixture in webtest_fixture_names]):
+            item.add_marker(mark.webtest)

tildes/tests/webtests/test_w3_validator.py

@@ -3,6 +3,12 @@
 
 import subprocess
 
+from pytest import mark
+
+
+# marks all tests in this module with "html_validation" marker
+pytestmark = mark.html_validation
+
 
 def test_homepage_html_loggedout(webtest_loggedout):
     """Validate HTML5 on the Tildes homepage, logged out."""