From ba309fdad304a558804faadd656331759dbd98df Mon Sep 17 00:00:00 2001 From: Christoph Burgdorf Date: Thu, 15 Nov 2018 13:19:16 +0100 Subject: [PATCH] Ensure state test test against intended fork only The statetest fixtures have different definitions for each fork and we must ensure to only run test against against the intended fork (e.g run Constantinople state tests against Constantinople VM). We can not rely on `pytest -k` matching for that as that matches against an identification string that includes the path and name of the test file which in some cases also contains fork names. A test file may be named "ConstantinopleSomething.json" but still contains individual definitions per fork. --- eth/tools/fixtures/loading.py | 2 +- tests/conftest.py | 4 ++++ tests/json-fixtures/test_state.py | 21 +++++++++++++++++++++ tox.ini | 14 +++++++------- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/eth/tools/fixtures/loading.py b/eth/tools/fixtures/loading.py index 02cd96419d..0c2317951d 100644 --- a/eth/tools/fixtures/loading.py +++ b/eth/tools/fixtures/loading.py @@ -77,7 +77,7 @@ def load_fixture(fixture_path: str, def filter_fixtures(all_fixtures: Iterable[Any], fixtures_base_dir: str, mark_fn: Callable[[str, str], bool]=None, - ignore_fn: Callable[[str, str], bool]=None) -> Any: + ignore_fn: Callable[..., bool]=None) -> Any: """ Helper function for filtering test fixtures. diff --git a/tests/conftest.py b/tests/conftest.py index 5600587e4f..e298dc4d39 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -186,3 +186,7 @@ def chain_without_block_validation( } chain = klass.from_genesis(base_db, genesis_params, genesis_state) return chain + + +def pytest_addoption(parser): + parser.addoption("--fork", type=str, required=False) diff --git a/tests/json-fixtures/test_state.py b/tests/json-fixtures/test_state.py index b3855e41ee..bac660cfca 100644 --- a/tests/json-fixtures/test_state.py +++ b/tests/json-fixtures/test_state.py @@ -162,12 +162,33 @@ def mark_statetest_fixtures(fixture_path, fixture_key, fixture_fork, fixture_ind return pytest.mark.xfail(reason="Listed in INCORRECT_UPSTREAM_TESTS.") +def generate_ignore_fn_for_fork(metafunc): + """ + The statetest fixtures have different definitions for each fork and we must ensure to only run + test against against the intended fork (e.g run Constantinople state tests against + Constantinople VM). + We can not rely on `pytest -k` matching for that as that matches against an identification + string that includes the path and name of the test which in some cases also contains fork + fork names. A test file may be named "ConstantinopleSomething.json" but still contains + individual definitions per fork. + """ + passed_fork = metafunc.config.getoption('fork') + if passed_fork: + passed_fork = passed_fork.lower() + + def ignore_fn(fixture_path, fixture_key, fixture_fork, post_state_index): + return fixture_fork.lower() != passed_fork + + return ignore_fn + + def pytest_generate_tests(metafunc): generate_fixture_tests( metafunc=metafunc, base_fixture_path=BASE_FIXTURE_PATH, preprocess_fn=expand_fixtures_forks, filter_fn=filter_fixtures( + ignore_fn=generate_ignore_fn_for_fork(metafunc), fixtures_base_dir=BASE_FIXTURE_PATH, mark_fn=mark_statetest_fixtures, ), diff --git a/tox.ini b/tox.ini index 0b1f63d9ed..86e48878c8 100644 --- a/tox.ini +++ b/tox.ini @@ -36,13 +36,13 @@ commands= transactions: pytest {posargs:tests/json-fixtures/test_transactions.py} vm: pytest {posargs:tests/json-fixtures/test_virtual_machine.py} native-blockchain: pytest {posargs:tests/json-fixtures/test_blockchain.py} - native-state-frontier: pytest {posargs:tests/json-fixtures/test_state.py -k Frontier} - native-state-homestead: pytest {posargs:tests/json-fixtures/test_state.py -k Homestead} - native-state-eip150: pytest {posargs:tests/json-fixtures/test_state.py -k EIP150} - native-state-eip158: pytest {posargs:tests/json-fixtures/test_state.py -k EIP158} - native-state-byzantium: pytest {posargs:tests/json-fixtures/test_state.py -k Byzantium} - native-state-constantinople: pytest {posargs:tests/json-fixtures/test_state.py -k Constantinople} - native-state-metropolis: pytest {posargs:tests/json-fixtures/test_state.py -k Metropolis} + native-state-frontier: pytest {posargs:tests/json-fixtures/test_state.py --fork Frontier} + native-state-homestead: pytest {posargs:tests/json-fixtures/test_state.py --fork Homestead} + native-state-eip150: pytest {posargs:tests/json-fixtures/test_state.py --fork EIP150} + native-state-eip158: pytest {posargs:tests/json-fixtures/test_state.py --fork EIP158} + native-state-byzantium: pytest {posargs:tests/json-fixtures/test_state.py --fork Byzantium} + native-state-constantinople: pytest {posargs:tests/json-fixtures/test_state.py --fork Constantinople} + native-state-metropolis: pytest {posargs:tests/json-fixtures/test_state.py --fork Metropolis} lightchain_integration: pytest --integration {posargs:tests/trinity/integration/test_lightchain_integration.py} deps = .[p2p,trinity,eth-extra,test]