Ensure state tests only run against the intended fork

eth/tools/fixtures/loading.py

@@ -77,7 +77,7 @@ def load_fixture(fixture_path: str,
 def filter_fixtures(all_fixtures: Iterable[Any],
                     fixtures_base_dir: str,
                     mark_fn: Callable[[str, str], bool]=None,
-                    ignore_fn: Callable[[str, str], bool]=None) -> Any:
+                    ignore_fn: Callable[..., bool]=None) -> Any:
     """
     Helper function for filtering test fixtures.
 

tests/conftest.py

@@ -186,3 +186,7 @@ def chain_without_block_validation(
     }
     chain = klass.from_genesis(base_db, genesis_params, genesis_state)
     return chain
+
+
+def pytest_addoption(parser):
+    parser.addoption("--fork", type=str, required=False)

tests/json-fixtures/test_state.py

@@ -162,12 +162,33 @@ def mark_statetest_fixtures(fixture_path, fixture_key, fixture_fork, fixture_ind
         return pytest.mark.xfail(reason="Listed in INCORRECT_UPSTREAM_TESTS.")
 
 
+def generate_ignore_fn_for_fork(metafunc):
+    """
+    The statetest fixtures have different definitions for each fork and we must ensure to only run
+    test against against the intended fork (e.g run Constantinople state tests against
+    Constantinople VM).
+    We can not rely on `pytest -k` matching for that as that matches against an identification
+    string that includes the path and name of the test which in some cases also contains fork
+    fork names. A test file may be named "ConstantinopleSomething.json" but still contains
+    individual definitions per fork.
+    """
+    passed_fork = metafunc.config.getoption('fork')
+    if passed_fork:
+        passed_fork = passed_fork.lower()
+
+        def ignore_fn(fixture_path, fixture_key, fixture_fork, post_state_index):
+            return fixture_fork.lower() != passed_fork
+
+        return ignore_fn
+
+
 def pytest_generate_tests(metafunc):
     generate_fixture_tests(
         metafunc=metafunc,
         base_fixture_path=BASE_FIXTURE_PATH,
         preprocess_fn=expand_fixtures_forks,
         filter_fn=filter_fixtures(
+            ignore_fn=generate_ignore_fn_for_fork(metafunc),
             fixtures_base_dir=BASE_FIXTURE_PATH,
             mark_fn=mark_statetest_fixtures,
         ),

tox.ini

@@ -36,13 +36,13 @@ commands=
     transactions: pytest {posargs:tests/json-fixtures/test_transactions.py}
     vm: pytest {posargs:tests/json-fixtures/test_virtual_machine.py}
     native-blockchain: pytest {posargs:tests/json-fixtures/test_blockchain.py}
-    native-state-frontier: pytest {posargs:tests/json-fixtures/test_state.py -k Frontier}
-    native-state-homestead: pytest {posargs:tests/json-fixtures/test_state.py -k Homestead}
-    native-state-eip150: pytest {posargs:tests/json-fixtures/test_state.py -k EIP150}
-    native-state-eip158: pytest {posargs:tests/json-fixtures/test_state.py -k EIP158}
-    native-state-byzantium: pytest {posargs:tests/json-fixtures/test_state.py -k Byzantium}
-    native-state-constantinople: pytest {posargs:tests/json-fixtures/test_state.py -k Constantinople}
-    native-state-metropolis: pytest {posargs:tests/json-fixtures/test_state.py -k Metropolis}
+    native-state-frontier: pytest {posargs:tests/json-fixtures/test_state.py --fork Frontier}
+    native-state-homestead: pytest {posargs:tests/json-fixtures/test_state.py --fork Homestead}
+    native-state-eip150: pytest {posargs:tests/json-fixtures/test_state.py --fork EIP150}
+    native-state-eip158: pytest {posargs:tests/json-fixtures/test_state.py --fork EIP158}
+    native-state-byzantium: pytest {posargs:tests/json-fixtures/test_state.py --fork Byzantium}
+    native-state-constantinople: pytest {posargs:tests/json-fixtures/test_state.py --fork Constantinople}
+    native-state-metropolis: pytest {posargs:tests/json-fixtures/test_state.py --fork Metropolis}
     lightchain_integration: pytest --integration {posargs:tests/trinity/integration/test_lightchain_integration.py}
 
 deps = .[p2p,trinity,eth-extra,test]