From 473af4dbad89043e0dc73a41cb6505ed12816a85 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 2 Sep 2025 16:02:50 +0200 Subject: [PATCH 1/4] Ensure `tox_extend_envs` list can be read twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously (PR #3591) a new hook point was added. The tests checked that it works well with the `tox config` and `tox list` commands. However, `tox run` turned out to have a problem — it would complain that there's no such env, when invoked: ```console ROOT: 170 E HandledError| provided environments not found in configuration file: pip-compile-tox-env-lock [tox/run.py:23] ``` Turned out, this was because the config object is being interated twice in some subcommands. This in turn iterates over the discovered additional ephemeral environments list object. The implementation passes an iterator into it and so when it's first accessed, it's exhausted and the second attempt does not give the same envs, causing inconsistency. The patch solves this by using `itertools.tee()`, making sure that the underlying iterable is always cached and it's possible to repeat iteration as many times as possible without loosing the data in the process. --- src/tox/config/main.py | 4 ++-- src/tox/session/state.py | 4 ++-- tests/plugin/test_inline.py | 8 ++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/tox/config/main.py b/src/tox/config/main.py index ff264ea21..f699ede4b 100644 --- a/src/tox/config/main.py +++ b/src/tox/config/main.py @@ -2,7 +2,7 @@ import os from collections import OrderedDict, defaultdict -from itertools import chain +from itertools import chain, tee from pathlib import Path from typing import TYPE_CHECKING, Any, Iterable, Iterator, Sequence, TypeVar @@ -81,7 +81,7 @@ def src_path(self) -> Path: def __iter__(self) -> Iterator[str]: """:return: an iterator that goes through existing environments""" - return chain(self._src.envs(self.core), self._extra_envs) + return chain(self._src.envs(self.core), chain.from_iterable(tee(self._extra_envs))) def sections(self) -> Iterator[Section]: yield from self._src.sections() diff --git a/src/tox/session/state.py b/src/tox/session/state.py index 6b63f39da..3f59ecd73 100644 --- a/src/tox/session/state.py +++ b/src/tox/session/state.py @@ -1,7 +1,7 @@ from __future__ import annotations import sys -from itertools import chain +from itertools import chain, tee from typing import TYPE_CHECKING, Sequence from tox.config.main import Config @@ -20,7 +20,7 @@ class State: """Runtime state holder.""" def __init__(self, options: Options, args: Sequence[str]) -> None: - extended_envs = chain.from_iterable(MANAGER.tox_extend_envs()) + (extended_envs,) = tee(chain.from_iterable(MANAGER.tox_extend_envs()), 1) self.conf = Config.make(options.parsed, options.pos_args, options.source, extended_envs) self.conf.core.add_constant( keys=["on_platform"], diff --git a/tests/plugin/test_inline.py b/tests/plugin/test_inline.py index 4fae64b27..5935801e7 100644 --- a/tests/plugin/test_inline.py +++ b/tests/plugin/test_inline.py @@ -43,6 +43,7 @@ def tox_extend_envs() -> tuple[str]: def tox_add_core_config(core_conf: ConfigSet, state: State) -> None: # noqa: ARG001 in_memory_config_loader = MemoryLoader( base=["sentinel-base"], + commands_pre=["sentinel-cmd"], description="sentinel-description", ) state.conf.memory_seed_loaders[env_name].append( @@ -59,3 +60,10 @@ def tox_add_core_config(core_conf: ConfigSet, state: State) -> None: # noqa: AR tox_config_result = project.run("config", "-e", "sentinel-env-name", "-qq") tox_config_result.assert_success() assert "base = sentinel-base" in tox_config_result.out + + tox_run_result = project.run("run", "-e", "sentinel-env-name", "-q") + tox_run_result.assert_failed() + expected_cmd_lookup_error_txt = ( + "sentinel-env-name: Exception running subprocess [Errno 2] No such file or directory: 'sentinel-cmd'\n" + ) + assert expected_cmd_lookup_error_txt in tox_run_result.out From 17ac9788e9513036c21e851fe3947fd9c3f25701 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 2 Sep 2025 16:14:18 +0200 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=93=9D=20Add=20a=20change=20note=20fo?= =?UTF-8?q?r=20PR=20#3598?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/changelog/3598.bugfix.rst | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/3598.bugfix.rst diff --git a/docs/changelog/3598.bugfix.rst b/docs/changelog/3598.bugfix.rst new file mode 100644 index 000000000..f1ee47ffa --- /dev/null +++ b/docs/changelog/3598.bugfix.rst @@ -0,0 +1,6 @@ +The :func:`tox_extend_envs() hook ` +recently added in :pull:`3591` turned out to not work well with +``tox run``. It was fixed internally, not to exhaust the underlying +iterator on the first use. + +-- by :user:`webknjaz` From 658da076ac509f480a38d1aa347fb38ed02e46fe Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 2 Sep 2025 17:36:29 +0200 Subject: [PATCH 3/4] Use the non-first `itertools.tee()` return value Before Python 3.11, the first value is the the underlying iterator. And we still exhaust it. So for backwards compatibility, we request two iterators and disregard the first one. --- src/tox/config/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tox/config/main.py b/src/tox/config/main.py index f699ede4b..9fbd9f4bc 100644 --- a/src/tox/config/main.py +++ b/src/tox/config/main.py @@ -81,7 +81,11 @@ def src_path(self) -> Path: def __iter__(self) -> Iterator[str]: """:return: an iterator that goes through existing environments""" - return chain(self._src.envs(self.core), chain.from_iterable(tee(self._extra_envs))) + # NOTE: `tee(self._extra_envs)[1]` is necessary for compatibility with + # NOTE: Python 3.11 and older versions. Once Python 3.12 is the lowest + # NOTE: supported version, it can be changed to + # NOTE: `chain.from_iterable(tee(self._extra_envs, 1))`. + return chain(self._src.envs(self.core), tee(self._extra_envs)[1]) def sections(self) -> Iterator[Section]: yield from self._src.sections() From ee5d5850cfb800655c59fd49b83cb114ba7ace3b Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 2 Sep 2025 18:17:45 +0200 Subject: [PATCH 4/4] Adjust `test_toxfile_py_w_ephemeral_envs` for Win --- tests/plugin/test_inline.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/plugin/test_inline.py b/tests/plugin/test_inline.py index 5935801e7..46af969a3 100644 --- a/tests/plugin/test_inline.py +++ b/tests/plugin/test_inline.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -63,7 +64,12 @@ def tox_add_core_config(core_conf: ConfigSet, state: State) -> None: # noqa: AR tox_run_result = project.run("run", "-e", "sentinel-env-name", "-q") tox_run_result.assert_failed() + underlying_expected_oserror_msg = ( + "[WinError 2] The system cannot find the file specified" + if sys.platform == "win32" + else "[Errno 2] No such file or directory: 'sentinel-cmd'" + ) expected_cmd_lookup_error_txt = ( - "sentinel-env-name: Exception running subprocess [Errno 2] No such file or directory: 'sentinel-cmd'\n" + f"sentinel-env-name: Exception running subprocess {underlying_expected_oserror_msg!s}\n" ) assert expected_cmd_lookup_error_txt in tox_run_result.out