Further "unknown marks warning" improvements (#5178)

Further "unknown marks warning" improvements

Author: nicoddemus

changelog/5023.feature.rst

@@ -0,0 +1,5 @@
+New flag ``--strict-markers`` that triggers an error when unknown markers (e.g. those not registered using the `markers option`_ in the configuration file) are used in the test suite.
+
+The existing ``--strict`` option has the same behavior currently, but can be augmented in the future for additional checks.
+
+.. _`markers option`: https://docs.pytest.org/en/latest/reference.html#confval-markers

doc/en/example/markers.rst

@@ -259,7 +259,7 @@ For an example on how to add and work with markers from a plugin, see
     * Asking for existing markers via ``pytest --markers`` gives good output
 
     * Typos in function markers are treated as an error if you use
-      the ``--strict`` option.
+      the ``--strict-markers`` option.
 
 .. _`scoped-marking`:
 

doc/en/mark.rst

@@ -41,15 +41,15 @@ marks by registering them in ``pytest.ini`` like this:
         slow
         serial
 
-When the ``--strict`` command-line flag is passed, any unknown marks applied
+When the ``--strict-markers`` command-line flag is passed, any unknown marks applied
 with the ``@pytest.mark.name_of_the_mark`` decorator will trigger an error.
 Marks added by pytest or by a plugin instead of the decorator will not trigger
-this error.  To enforce validation of markers, add ``--strict`` to ``addopts``:
+this error.  To enforce validation of markers, add ``--strict-markers`` to ``addopts``:
 
 .. code-block:: ini
 
     [pytest]
-    addopts = --strict
+    addopts = --strict-markers
     markers =
         slow
         serial

doc/en/reference.rst

@@ -1269,15 +1269,17 @@ passed multiple times. The expected format is ``name=value``. For example::
 
 .. confval:: markers
 
-    When the ``--strict`` command-line argument is used, only known markers -
-    defined in code by core pytest or some plugin - are allowed.
-    You can list additional markers in this setting to add them to the whitelist.
+    When the ``--strict-markers`` or ``--strict`` command-line arguments are used,
+    only known markers - defined in code by core pytest or some plugin - are allowed.
 
-    You can list one marker name per line, indented from the option name.
+    You can list additional markers in this setting to add them to the whitelist,
+    in which case you probably want to add ``--strict-markers`` to ``addopts``
+    to avoid future regressions:
 
     .. code-block:: ini
 
         [pytest]
+        addopts = --strict-markers
         markers =
             slow
             serial

src/_pytest/main.py

@@ -47,11 +47,6 @@ def pytest_addoption(parser):
         type="args",
         default=[],
     )
-    # parser.addini("dirpatterns",
-    #    "patterns specifying possible locations of test files",
-    #    type="linelist", default=["**/test_*.txt",
-    #            "**/test_*.py", "**/*_test.py"]
-    # )
     group = parser.getgroup("general", "running and selection options")
     group._addoption(
         "-x",
@@ -71,9 +66,10 @@ def pytest_addoption(parser):
         help="exit after first num failures or errors.",
     )
     group._addoption(
+        "--strict-markers",
         "--strict",
         action="store_true",
-        help="marks not registered in configuration file raise errors.",
+        help="markers not registered in the `markers` section of the configuration file raise errors.",
     )
     group._addoption(
         "-c",

src/_pytest/mark/structures.py

@@ -311,8 +311,11 @@ def __getattr__(self, name):
             # If the name is not in the set of known marks after updating,
             # then it really is time to issue a warning or an error.
             if name not in self._markers:
-                if self._config.option.strict:
-                    fail("{!r} is not a registered marker".format(name), pytrace=False)
+                if self._config.option.strict_markers:
+                    fail(
+                        "{!r} not found in `markers` configuration option".format(name),
+                        pytrace=False,
+                    )
                 else:
                     warnings.warn(
                         "Unknown pytest.mark.%s - is this a typo?  You can register "

testing/code/test_excinfo.py

@@ -11,7 +11,6 @@
 import py
 import six
 from six.moves import queue
-from test_source import astonly
 
 import _pytest
 import pytest
@@ -147,7 +146,6 @@ def test_traceback_entry_getsource(self):
         assert s.startswith("def f():")
         assert s.endswith("raise ValueError")
 
-    @astonly
     @failsonjython
     def test_traceback_entry_getsource_in_construct(self):
         source = _pytest._code.Source(

testing/code/test_source.py

@@ -16,7 +16,6 @@
 import pytest
 from _pytest._code import Source
 
-astonly = pytest.mark.nothing
 failsonjython = pytest.mark.xfail("sys.platform.startswith('java')")
 
 
@@ -227,7 +226,6 @@ def test_getstatementrange_triple_quoted(self):
         s = source.getstatement(1)
         assert s == str(source)
 
-    @astonly
     def test_getstatementrange_within_constructs(self):
         source = Source(
             """\
@@ -630,7 +628,6 @@ def test_multiline():
 
 
 class TestTry(object):
-    pytestmark = astonly
     source = """\
 try:
     raise ValueError
@@ -675,7 +672,6 @@ def test_finally(self):
 
 
 class TestIf(object):
-    pytestmark = astonly
     source = """\
 if 1:
     y = 3

testing/python/fixtures.py

@@ -1925,10 +1925,10 @@ def test_hello(arg1):
         reprec = testdir.inline_run()
         reprec.assertoutcome(passed=1)
 
-    @pytest.mark.issue(226)
     @pytest.mark.parametrize("param1", ["", "params=[1]"], ids=["p00", "p01"])
     @pytest.mark.parametrize("param2", ["", "params=[1]"], ids=["p10", "p11"])
     def test_ordering_dependencies_torndown_first(self, testdir, param1, param2):
+        """#226"""
         testdir.makepyfile(
             """
             import pytest
@@ -2707,9 +2707,9 @@ def test_3():
         reprec = testdir.inline_run("-v")
         reprec.assertoutcome(passed=5)
 
-    @pytest.mark.issue(246)
     @pytest.mark.parametrize("scope", ["session", "function", "module"])
     def test_finalizer_order_on_parametrization(self, scope, testdir):
+        """#246"""
         testdir.makepyfile(
             """
             import pytest
@@ -2744,8 +2744,8 @@ def test_other():
         reprec = testdir.inline_run("-lvs")
         reprec.assertoutcome(passed=3)
 
-    @pytest.mark.issue(396)
     def test_class_scope_parametrization_ordering(self, testdir):
+        """#396"""
         testdir.makepyfile(
             """
             import pytest
@@ -2865,8 +2865,8 @@ def test_foo(fix):
         res = testdir.runpytest("-v")
         res.stdout.fnmatch_lines(["*test_foo*alpha*", "*test_foo*beta*"])
 
-    @pytest.mark.issue(920)
     def test_deterministic_fixture_collection(self, testdir, monkeypatch):
+        """#920"""
         testdir.makepyfile(
             """
             import pytest
@@ -3649,7 +3649,6 @@ class TestScopeOrdering(object):
     """Class of tests that ensure fixtures are ordered based on their scopes (#2405)"""
 
     @pytest.mark.parametrize("variant", ["mark", "autouse"])
-    @pytest.mark.issue(github="#2405")
     def test_func_closure_module_auto(self, testdir, variant, monkeypatch):
         """Semantically identical to the example posted in #2405 when ``use_mark=True``"""
         monkeypatch.setenv("FIXTURE_ACTIVATION_VARIANT", variant)

testing/python/integration.py

@@ -393,8 +393,9 @@ def test_blah(self):
         assert not call.items
 
 
-@pytest.mark.issue(351)
 class TestParameterize(object):
+    """#351"""
+
     def test_idfn_marker(self, testdir):
         testdir.makepyfile(
             """