From 2a4fa59e4cb3a4e436b117b7cb600dabf08873af Mon Sep 17 00:00:00 2001 From: Richard Si Date: Thu, 26 Jun 2025 19:09:46 -0400 Subject: [PATCH 01/11] Initial in-process build deps prototype --- src/pip/_internal/build_env.py | 125 +++++++++++++++++-- src/pip/_internal/cli/req_command.py | 1 + src/pip/_internal/distributions/base.py | 4 +- src/pip/_internal/distributions/installed.py | 8 +- src/pip/_internal/distributions/sdist.py | 23 ++-- src/pip/_internal/distributions/wheel.py | 4 +- src/pip/_internal/operations/prepare.py | 14 ++- 7 files changed, 151 insertions(+), 28 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 69a2e72ce42..9b5880f36de 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -10,8 +10,9 @@ import textwrap from collections import OrderedDict from collections.abc import Iterable +from optparse import Values from types import TracebackType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol from pip._vendor.packaging.version import Version @@ -26,6 +27,8 @@ if TYPE_CHECKING: from pip._internal.index.package_finder import PackageFinder + from pip._internal.operations.prepare import RequirementPreparer + from pip._internal.resolution.base import BaseResolver logger = logging.getLogger(__name__) @@ -43,6 +46,107 @@ def __init__(self, path: str) -> None: self.lib_dirs = _dedup(scheme.purelib, scheme.platlib) +class BuildEnvironmentInstaller(Protocol): + def install(self, requirements: Iterable[str], prefix: _Prefix) -> None: ... + + +class InprocessBuildEnvironmentInstaller: + """ + Build dependency installer that runs in the same pip process. + + Differences from the subprocess-based installer: + - Conflicts with already installed packages aren't detected + - + """ + + # TODO: ensure build tracking still works + # TODO: figure out what options are actually being inherited + + def __init__( + self, finder: PackageFinder, preparer: RequirementPreparer, options: Values + ) -> None: + from pip._internal.cache import WheelCache + from pip._internal.commands import create_command + + self.finder = finder + self.preparer = preparer + self.options = options + + self._install_command = create_command("install") + self._wheel_cache = WheelCache(options.cache_dir) + + def install(self, requirements: Iterable[str], prefix: _Prefix) -> None: + from pip._internal.req import install_given_reqs + from pip._internal.req.constructors import install_req_from_line + from pip._internal.wheel_builder import build, should_build_for_install_command + + ireqs = [] + for req in requirements: + ireq = install_req_from_line( + req, + comes_from=None, + # TODO: does --isolated matter here? + isolated=self.options.isolated_mode, + use_pep517=True, + user_supplied=True, + config_settings={}, + ) + ireqs.append(ireq) + + resolver = self._make_resolver() + requirement_set = resolver.resolve(ireqs, check_supported_wheels=True) + + reqs_to_build = filter( + should_build_for_install_command, requirement_set.requirements_to_install + ) + _, build_failures = build( + reqs_to_build, + wheel_cache=self._wheel_cache, + verify=True, + build_options=[], + global_options=[], + ) + + if build_failures: + raise InstallationError( + "Failed to build installable wheels for some " + "pyproject.toml based projects ({})".format( + ", ".join(r.name for r in build_failures) # type: ignore + ) + ) + + to_install = resolver.get_installation_order(requirement_set) + + install_given_reqs( + to_install, + global_options=[], + root=None, + home=None, + prefix=prefix.path, + warn_script_location=False, + use_user_site=False, + pycompile=False, + progress_bar="off", + ) + print(f"[in-process] installed {', '.join(requirements)}") + + def _make_resolver(self) -> BaseResolver: + # TODO: the old logic only uses the resolvelib resolver, this won't + return self._install_command.make_resolver( + preparer=self.preparer, + finder=self.finder, + options=self.options, + wheel_cache=self._wheel_cache, + use_user_site=False, + ignore_installed=True, + ignore_requires_python=self.options.ignore_requires_python, + force_reinstall=False, + upgrade_strategy="to-satisfy-only", + use_pep517=True, + py_version_info=self.options.python_version, + ) + + def get_runnable_pip() -> str: """Get a file to pass to a Python executable, to run the currently-running pip. @@ -205,7 +309,7 @@ def check_requirements( def install_requirements( self, - finder: PackageFinder, + installer: BuildEnvironmentInstaller, requirements: Iterable[str], prefix_as_string: str, *, @@ -216,13 +320,14 @@ def install_requirements( prefix.setup = True if not requirements: return - self._install_requirements( - get_runnable_pip(), - finder, - requirements, - prefix, - kind=kind, - ) + # self._install_requirements( + # get_runnable_pip(), + # finder, + # requirements, + # prefix, + # kind=kind, + # ) + installer.install(requirements, prefix) @staticmethod def _install_requirements( @@ -319,7 +424,7 @@ def cleanup(self) -> None: def install_requirements( self, - finder: PackageFinder, + finder: BuildEnvironmentInstaller, requirements: Iterable[str], prefix_as_string: str, *, diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 27255098830..631eeb69754 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -147,6 +147,7 @@ def make_requirement_preparer( verbosity=verbosity, legacy_resolver=legacy_resolver, resume_retries=options.resume_retries, + options=options, ) @classmethod diff --git a/src/pip/_internal/distributions/base.py b/src/pip/_internal/distributions/base.py index 6ebf49ad501..ea61f3501e7 100644 --- a/src/pip/_internal/distributions/base.py +++ b/src/pip/_internal/distributions/base.py @@ -7,7 +7,7 @@ from pip._internal.req import InstallRequirement if TYPE_CHECKING: - from pip._internal.index.package_finder import PackageFinder + from pip._internal.build_env import BuildEnvironmentInstaller class AbstractDistribution(metaclass=abc.ABCMeta): @@ -48,7 +48,7 @@ def get_metadata_distribution(self) -> BaseDistribution: @abc.abstractmethod def prepare_distribution_metadata( self, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> None: diff --git a/src/pip/_internal/distributions/installed.py b/src/pip/_internal/distributions/installed.py index 8ceed309446..b6a67df24f4 100644 --- a/src/pip/_internal/distributions/installed.py +++ b/src/pip/_internal/distributions/installed.py @@ -1,9 +1,13 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from pip._internal.distributions.base import AbstractDistribution -from pip._internal.index.package_finder import PackageFinder from pip._internal.metadata import BaseDistribution +if TYPE_CHECKING: + from pip._internal.build_env import BuildEnvironmentInstaller + class InstalledDistribution(AbstractDistribution): """Represents an installed package. @@ -22,7 +26,7 @@ def get_metadata_distribution(self) -> BaseDistribution: def prepare_distribution_metadata( self, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> None: diff --git a/src/pip/_internal/distributions/sdist.py b/src/pip/_internal/distributions/sdist.py index 67468955e4a..fecfebcfa3b 100644 --- a/src/pip/_internal/distributions/sdist.py +++ b/src/pip/_internal/distributions/sdist.py @@ -11,7 +11,7 @@ from pip._internal.utils.subprocess import runner_with_spinner_message if TYPE_CHECKING: - from pip._internal.index.package_finder import PackageFinder + from pip._internal.build_env import BuildEnvironmentInstaller logger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ def get_metadata_distribution(self) -> BaseDistribution: def prepare_distribution_metadata( self, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> None: @@ -46,7 +46,7 @@ def prepare_distribution_metadata( if should_isolate: # Setup an isolated environment and install the build backend static # requirements in it. - self._prepare_build_backend(finder) + self._prepare_build_backend(build_env_installer) # Check that if the requirement is editable, it either supports PEP 660 or # has a setup.py or a setup.cfg. This cannot be done earlier because we need # to setup the build backend to verify it supports build_editable, nor can @@ -56,7 +56,7 @@ def prepare_distribution_metadata( # without setup.py nor setup.cfg. self.req.isolated_editable_sanity_check() # Install the dynamic build requirements. - self._install_build_reqs(finder) + self._install_build_reqs(build_env_installer) # Check if the current environment provides build dependencies should_check_deps = self.req.use_pep517 and check_build_deps if should_check_deps: @@ -71,7 +71,9 @@ def prepare_distribution_metadata( self._raise_missing_reqs(missing) self.req.prepare_metadata() - def _prepare_build_backend(self, finder: PackageFinder) -> None: + def _prepare_build_backend( + self, build_env_installer: BuildEnvironmentInstaller + ) -> None: # Isolate in a BuildEnvironment and install the build-time # requirements. pyproject_requires = self.req.pyproject_requires @@ -79,7 +81,10 @@ def _prepare_build_backend(self, finder: PackageFinder) -> None: self.req.build_env = BuildEnvironment() self.req.build_env.install_requirements( - finder, pyproject_requires, "overlay", kind="build dependencies" + build_env_installer, + pyproject_requires, + "overlay", + kind="build dependencies", ) conflicting, missing = self.req.build_env.check_requirements( self.req.requirements_to_check @@ -115,7 +120,9 @@ def _get_build_requires_editable(self) -> Iterable[str]: with backend.subprocess_runner(runner): return backend.get_requires_for_build_editable() - def _install_build_reqs(self, finder: PackageFinder) -> None: + def _install_build_reqs( + self, build_env_installer: BuildEnvironmentInstaller + ) -> None: # Install any extra build dependencies that the backend requests. # This must be done in a second pass, as the pyproject.toml # dependencies must be installed before we can call the backend. @@ -131,7 +138,7 @@ def _install_build_reqs(self, finder: PackageFinder) -> None: if conflicting: self._raise_conflicts("the backend dependencies", conflicting) self.req.build_env.install_requirements( - finder, missing, "normal", kind="backend dependencies" + build_env_installer, missing, "normal", kind="backend dependencies" ) def _raise_conflicts( diff --git a/src/pip/_internal/distributions/wheel.py b/src/pip/_internal/distributions/wheel.py index 54e1e35f2c8..ee12bfadc2e 100644 --- a/src/pip/_internal/distributions/wheel.py +++ b/src/pip/_internal/distributions/wheel.py @@ -12,7 +12,7 @@ ) if TYPE_CHECKING: - from pip._internal.index.package_finder import PackageFinder + from pip._internal.build_env import BuildEnvironmentInstaller class WheelDistribution(AbstractDistribution): @@ -37,7 +37,7 @@ def get_metadata_distribution(self) -> BaseDistribution: def prepare_distribution_metadata( self, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> None: diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 908331eba29..f6db51ce8f3 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -13,6 +13,7 @@ from pip._vendor.packaging.utils import canonicalize_name +from pip._internal.build_env import InprocessBuildEnvironmentInstaller from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.distributions.installed import InstalledDistribution from pip._internal.exceptions import ( @@ -60,7 +61,7 @@ def _get_prepared_distribution( req: InstallRequirement, build_tracker: BuildTracker, - finder: PackageFinder, + build_env_installer: InprocessBuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> BaseDistribution: @@ -70,7 +71,7 @@ def _get_prepared_distribution( if tracker_id is not None: with build_tracker.track(req, tracker_id): abstract_dist.prepare_distribution_metadata( - finder, build_isolation, check_build_deps + build_env_installer, build_isolation, check_build_deps ) return abstract_dist.get_metadata_distribution() @@ -237,6 +238,8 @@ def __init__( verbosity: int, legacy_resolver: bool, resume_retries: int, + # TODO: handle this better + options, ) -> None: super().__init__() @@ -253,6 +256,9 @@ def __init__( # Is build isolation allowed? self.build_isolation = build_isolation + self.build_env_installer = InprocessBuildEnvironmentInstaller( + finder, self, options + ) # Should check build dependencies? self.check_build_deps = check_build_deps @@ -644,7 +650,7 @@ def _prepare_linked_requirement( dist = _get_prepared_distribution( req, self.build_tracker, - self.finder, + self.build_env_installer, self.build_isolation, self.check_build_deps, ) @@ -700,7 +706,7 @@ def prepare_editable_requirement( dist = _get_prepared_distribution( req, self.build_tracker, - self.finder, + self.build_env_installer, self.build_isolation, self.check_build_deps, ) From 56876d42b21e035690bff470e055b52cbedab53b Mon Sep 17 00:00:00 2001 From: Richard Si Date: Thu, 26 Jun 2025 19:10:05 -0400 Subject: [PATCH 02/11] Move legacy imp. to new inferface --- src/pip/_internal/build_env.py | 168 ++++++++++++------------ src/pip/_internal/cli/cmdoptions.py | 1 + src/pip/_internal/cli/req_command.py | 3 + src/pip/_internal/operations/prepare.py | 12 +- 4 files changed, 98 insertions(+), 86 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 9b5880f36de..122b21b5f6e 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -47,7 +47,88 @@ def __init__(self, path: str) -> None: class BuildEnvironmentInstaller(Protocol): - def install(self, requirements: Iterable[str], prefix: _Prefix) -> None: ... + def install( + self, requirements: Iterable[str], prefix: _Prefix, *, kind: str + ) -> None: ... + + +class SubprocessBuildEnvironmentInstaller: + """ + Install build dependencies by calling pip in a subprocess. + + XXX: this is the legacy installation method and will be removed at + some point once InprocessBuildEnvironmentInstaller is stable. + """ + + def __init__(self, finder: PackageFinder) -> None: + self.finder = finder + + def install( + self, requirements: Iterable[str], prefix: _Prefix, *, kind: str + ) -> None: + finder = self.finder + args: list[str] = [ + sys.executable, + get_runnable_pip(), + "install", + "--ignore-installed", + "--no-user", + "--prefix", + prefix.path, + "--no-warn-script-location", + "--disable-pip-version-check", + # As the build environment is ephemeral, it's wasteful to + # pre-compile everything, especially as not every Python + # module will be used/compiled in most cases. + "--no-compile", + # The prefix specified two lines above, thus + # target from config file or env var should be ignored + "--target", + "", + ] + if logger.getEffectiveLevel() <= logging.DEBUG: + args.append("-vv") + elif logger.getEffectiveLevel() <= VERBOSE: + args.append("-v") + for format_control in ("no_binary", "only_binary"): + formats = getattr(finder.format_control, format_control) + args.extend( + ( + "--" + format_control.replace("_", "-"), + ",".join(sorted(formats or {":none:"})), + ) + ) + + index_urls = finder.index_urls + if index_urls: + args.extend(["-i", index_urls[0]]) + for extra_index in index_urls[1:]: + args.extend(["--extra-index-url", extra_index]) + else: + args.append("--no-index") + for link in finder.find_links: + args.extend(["--find-links", link]) + + if finder.proxy: + args.extend(["--proxy", finder.proxy]) + for host in finder.trusted_hosts: + args.extend(["--trusted-host", host]) + if finder.custom_cert: + args.extend(["--cert", finder.custom_cert]) + if finder.client_cert: + args.extend(["--client-cert", finder.client_cert]) + if finder.allow_all_prereleases: + args.append("--pre") + if finder.prefer_binary: + args.append("--prefer-binary") + args.append("--") + args.extend(requirements) + with open_spinner(f"Installing {kind}") as spinner: + call_subprocess( + args, + command_desc=f"pip subprocess to install {kind}", + spinner=spinner, + ) class InprocessBuildEnvironmentInstaller: @@ -75,7 +156,9 @@ def __init__( self._install_command = create_command("install") self._wheel_cache = WheelCache(options.cache_dir) - def install(self, requirements: Iterable[str], prefix: _Prefix) -> None: + def install( + self, requirements: Iterable[str], prefix: _Prefix, *, kind: str + ) -> None: from pip._internal.req import install_given_reqs from pip._internal.req.constructors import install_req_from_line from pip._internal.wheel_builder import build, should_build_for_install_command @@ -320,86 +403,7 @@ def install_requirements( prefix.setup = True if not requirements: return - # self._install_requirements( - # get_runnable_pip(), - # finder, - # requirements, - # prefix, - # kind=kind, - # ) - installer.install(requirements, prefix) - - @staticmethod - def _install_requirements( - pip_runnable: str, - finder: PackageFinder, - requirements: Iterable[str], - prefix: _Prefix, - *, - kind: str, - ) -> None: - args: list[str] = [ - sys.executable, - pip_runnable, - "install", - "--ignore-installed", - "--no-user", - "--prefix", - prefix.path, - "--no-warn-script-location", - "--disable-pip-version-check", - # As the build environment is ephemeral, it's wasteful to - # pre-compile everything, especially as not every Python - # module will be used/compiled in most cases. - "--no-compile", - # The prefix specified two lines above, thus - # target from config file or env var should be ignored - "--target", - "", - ] - if logger.getEffectiveLevel() <= logging.DEBUG: - args.append("-vv") - elif logger.getEffectiveLevel() <= VERBOSE: - args.append("-v") - for format_control in ("no_binary", "only_binary"): - formats = getattr(finder.format_control, format_control) - args.extend( - ( - "--" + format_control.replace("_", "-"), - ",".join(sorted(formats or {":none:"})), - ) - ) - - index_urls = finder.index_urls - if index_urls: - args.extend(["-i", index_urls[0]]) - for extra_index in index_urls[1:]: - args.extend(["--extra-index-url", extra_index]) - else: - args.append("--no-index") - for link in finder.find_links: - args.extend(["--find-links", link]) - - if finder.proxy: - args.extend(["--proxy", finder.proxy]) - for host in finder.trusted_hosts: - args.extend(["--trusted-host", host]) - if finder.custom_cert: - args.extend(["--cert", finder.custom_cert]) - if finder.client_cert: - args.extend(["--client-cert", finder.client_cert]) - if finder.allow_all_prereleases: - args.append("--pre") - if finder.prefer_binary: - args.append("--prefer-binary") - args.append("--") - args.extend(requirements) - with open_spinner(f"Installing {kind}") as spinner: - call_subprocess( - args, - command_desc=f"pip subprocess to install {kind}", - spinner=spinner, - ) + installer.install(requirements, prefix, kind=kind) class NoOpBuildEnvironment(BuildEnvironment): diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 2476aca80d0..c7f35d859fd 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -1083,6 +1083,7 @@ def check_list_path_option(options: Values) -> None: choices=[ "legacy-resolver", "legacy-certs", + "legacy-build-deps-installer", ], help=("Enable deprecated functionality, that will be removed in the future."), ) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 631eeb69754..c03ed1cc720 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -147,6 +147,9 @@ def make_requirement_preparer( verbosity=verbosity, legacy_resolver=legacy_resolver, resume_retries=options.resume_retries, + inprocess_build_deps=( + "legacy-build-deps-installer" not in options.deprecated_features_enabled + ), options=options, ) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index f6db51ce8f3..9694b4e995e 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -13,7 +13,7 @@ from pip._vendor.packaging.utils import canonicalize_name -from pip._internal.build_env import InprocessBuildEnvironmentInstaller +from pip._internal.build_env import InprocessBuildEnvironmentInstaller, SubprocessBuildEnvironmentInstaller from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.distributions.installed import InstalledDistribution from pip._internal.exceptions import ( @@ -239,6 +239,7 @@ def __init__( legacy_resolver: bool, resume_retries: int, # TODO: handle this better + inprocess_build_deps: bool, options, ) -> None: super().__init__() @@ -256,9 +257,12 @@ def __init__( # Is build isolation allowed? self.build_isolation = build_isolation - self.build_env_installer = InprocessBuildEnvironmentInstaller( - finder, self, options - ) + if inprocess_build_deps: + self.build_env_installer = InprocessBuildEnvironmentInstaller( + finder, self, options + ) + else: + self.build_env_installer = SubprocessBuildEnvironmentInstaller(finder) # Should check build dependencies? self.check_build_deps = check_build_deps From 0d552ddfd8cafd2d22abac7fe6704e2e99135878 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Thu, 26 Jun 2025 21:24:18 -0400 Subject: [PATCH 03/11] Add custom rich spinner --- src/pip/_internal/cli/spinners.py | 77 +++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/cli/spinners.py b/src/pip/_internal/cli/spinners.py index 1d5c351d896..51041d47f48 100644 --- a/src/pip/_internal/cli/spinners.py +++ b/src/pip/_internal/cli/spinners.py @@ -6,13 +6,26 @@ import sys import time from collections.abc import Generator -from typing import IO +from typing import IO, Final + +from pip._vendor.rich.console import ( + Console, + ConsoleOptions, + RenderableType, + RenderResult, +) +from pip._vendor.rich.live import Live +from pip._vendor.rich.measure import Measurement +from pip._vendor.rich.text import Text from pip._internal.utils.compat import WINDOWS -from pip._internal.utils.logging import get_indentation +from pip._internal.utils.logging import get_console, get_indentation logger = logging.getLogger(__name__) +SPINNER_CHARS: Final = r"-\|/" +SPINS_PER_SECOND: Final = 8 + class SpinnerInterface: def spin(self) -> None: @@ -27,9 +40,9 @@ def __init__( self, message: str, file: IO[str] | None = None, - spin_chars: str = "-\\|/", + spin_chars: str = SPINNER_CHARS, # Empirically, 8 updates/second looks nice - min_update_interval_seconds: float = 0.125, + min_update_interval_seconds: float = 1 / SPINS_PER_SECOND, ): self._message = message if file is None: @@ -139,6 +152,62 @@ def open_spinner(message: str) -> Generator[SpinnerInterface, None, None]: spinner.finish("done") +class PipRichSpinner: + """ + Custom rich spinner that matches the style and API* of the legacy spinners. + + (*) Updates will be handled in a background thread by a rich live panel + which will call render() automatically at the appropriate time. + """ + + def __init__(self, label: str) -> None: + self.label = label + self._spin_cycle = itertools.cycle(SPINNER_CHARS) + self._spinner_text = "" + self._finished = False + self._indent = get_indentation() * " " + + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> RenderResult: + yield self.render() + + def __rich_measure__( + self, console: Console, options: ConsoleOptions + ) -> Measurement: + text = self.render() + return Measurement.get(console, options, text) + + def render(self) -> RenderableType: + # The wrapping rich Live instance will call this method at the + # appropriate interval. + if not self._finished: + self._spinner_text = next(self._spin_cycle) + + return Text.assemble(self._indent, self.label, " ... ", self._spinner_text) + + def finish(self, status: str) -> None: + """Stop spinning and set a final status message.""" + self._spinner_text = status + self._finished = True + + +@contextlib.contextmanager +def open_rich_spinner(label: str) -> Generator[None, None, None]: + spinner = PipRichSpinner(label) + with Live(spinner, refresh_per_second=SPINS_PER_SECOND, console=get_console()): + try: + yield + except KeyboardInterrupt: + spinner.finish("canceled") + raise + except Exception: + spinner.finish("error") + raise + else: + spinner.finish("done") + + HIDE_CURSOR = "\x1b[?25l" SHOW_CURSOR = "\x1b[?25h" From 51b6b4923a7f0e680bc785c8a6e577b0998bb67f Mon Sep 17 00:00:00 2001 From: Richard Si Date: Thu, 26 Jun 2025 22:32:21 -0400 Subject: [PATCH 04/11] Capture logs during build dep install --- src/pip/_internal/build_env.py | 44 +++++++++++---- src/pip/_internal/commands/install.py | 53 ++++++++++--------- src/pip/_internal/metadata/__init__.py | 3 +- src/pip/_internal/metadata/base.py | 4 +- src/pip/_internal/metadata/importlib/_envs.py | 2 +- src/pip/_internal/metadata/pkg_resources.py | 4 +- src/pip/_internal/operations/prepare.py | 5 +- src/pip/_internal/utils/logging.py | 32 ++++++++++- 8 files changed, 104 insertions(+), 43 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 122b21b5f6e..031a1b423da 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -10,6 +10,8 @@ import textwrap from collections import OrderedDict from collections.abc import Iterable +from contextlib import AbstractContextManager, nullcontext +from io import StringIO from optparse import Values from types import TracebackType from typing import TYPE_CHECKING, Protocol @@ -17,10 +19,10 @@ from pip._vendor.packaging.version import Version from pip import __file__ as pip_location -from pip._internal.cli.spinners import open_spinner +from pip._internal.cli.spinners import open_rich_spinner, open_spinner from pip._internal.locations import get_platlib, get_purelib, get_scheme from pip._internal.metadata import get_default_environment, get_environment -from pip._internal.utils.logging import VERBOSE +from pip._internal.utils.logging import VERBOSE, capture_logging from pip._internal.utils.packaging import get_requirement from pip._internal.utils.subprocess import call_subprocess from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds @@ -159,6 +161,26 @@ def __init__( def install( self, requirements: Iterable[str], prefix: _Prefix, *, kind: str ) -> None: + capture_ctx: AbstractContextManager[StringIO] + should_capture = not logger.isEnabledFor(VERBOSE) + if should_capture: + # Hide the logs from the installation of build dependencies. + # They will be shown only if an error occurs. + capture_ctx = capture_logging() + else: + # Otherwise, pass-through all logs (with a header). + capture_ctx = nullcontext(StringIO()) + logger.info("Installing %s ...", kind) + + # TODO: error handling + with open_rich_spinner(f"Installing {kind}"), capture_ctx as stream: + self._install_impl(requirements, prefix) + + # print("captured:", "\n---\n" + stream.getvalue().rstrip()) + # print("---") + + def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None: + from pip._internal.commands.install import installed_packages_summary from pip._internal.req import install_given_reqs from pip._internal.req.constructors import install_req_from_line from pip._internal.wheel_builder import build, should_build_for_install_command @@ -179,9 +201,11 @@ def install( resolver = self._make_resolver() requirement_set = resolver.resolve(ireqs, check_supported_wheels=True) - reqs_to_build = filter( - should_build_for_install_command, requirement_set.requirements_to_install - ) + reqs_to_build = [ + r + for r in requirement_set.requirements_to_install + if should_build_for_install_command(r) + ] _, build_failures = build( reqs_to_build, wheel_cache=self._wheel_cache, @@ -199,8 +223,7 @@ def install( ) to_install = resolver.get_installation_order(requirement_set) - - install_given_reqs( + installed = install_given_reqs( to_install, global_options=[], root=None, @@ -211,7 +234,10 @@ def install( pycompile=False, progress_bar="off", ) - print(f"[in-process] installed {', '.join(requirements)}") + + env = get_environment(prefix.lib_dirs) + if summary := installed_packages_summary(installed, env): + logger.info(summary) def _make_resolver(self) -> BaseResolver: # TODO: the old logic only uses the resolvelib resolver, this won't @@ -226,7 +252,7 @@ def _make_resolver(self) -> BaseResolver: force_reinstall=False, upgrade_strategy="to-satisfy-only", use_pep517=True, - py_version_info=self.options.python_version, + py_version_info=getattr(self.options, "python_version", None), ) diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 302bc63f67d..3614907594a 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -29,11 +29,11 @@ from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.exceptions import CommandError, InstallationError from pip._internal.locations import get_scheme -from pip._internal.metadata import get_environment +from pip._internal.metadata import BaseEnvironment, get_environment from pip._internal.models.installation_report import InstallationReport from pip._internal.operations.build.build_tracker import get_build_tracker from pip._internal.operations.check import ConflictDetails, check_install_conflicts -from pip._internal.req import install_given_reqs +from pip._internal.req import InstallationResult, install_given_reqs from pip._internal.req.req_install import ( InstallRequirement, check_legacy_setup_py_options, @@ -478,34 +478,13 @@ def run(self, options: Values, args: list[str]) -> int: ) env = get_environment(lib_locations) - # Display a summary of installed packages, with extra care to - # display a package name as it was requested by the user. - installed.sort(key=operator.attrgetter("name")) - summary = [] - installed_versions = {} - for distribution in env.iter_all_distributions(): - installed_versions[distribution.canonical_name] = distribution.version - for package in installed: - display_name = package.name - version = installed_versions.get(canonicalize_name(display_name), None) - if version: - text = f"{display_name}-{version}" - else: - text = display_name - summary.append(text) - if conflicts is not None: self._warn_about_conflicts( conflicts, resolver_variant=self.determine_resolver_variant(options), ) - - installed_desc = " ".join(summary) - if installed_desc: - write_output( - "Successfully installed %s", - installed_desc, - ) + if summary := installed_packages_summary(installed, env): + write_output(summary) except OSError as error: show_traceback = self.verbosity >= 1 @@ -644,6 +623,30 @@ def _warn_about_conflicts( logger.critical("\n".join(parts)) +def installed_packages_summary( + installed: list[InstallationResult], env: BaseEnvironment +) -> str: + # Format a summary of installed packages, with extra care to + # display a package name as it was requested by the user. + installed.sort(key=operator.attrgetter("name")) + summary = [] + installed_versions = {} + for distribution in env.iter_all_distributions(): + installed_versions[distribution.canonical_name] = distribution.version + for package in installed: + display_name = package.name + version = installed_versions.get(canonicalize_name(display_name), None) + if version: + text = f"{display_name}-{version}" + else: + text = display_name + summary.append(text) + + if not summary: + return "" + return f"Successfully installed {' '.join(summary)}" + + def get_lib_location_guesses( user: bool = False, home: str | None = None, diff --git a/src/pip/_internal/metadata/__init__.py b/src/pip/_internal/metadata/__init__.py index 927e375cad0..d96ffe00900 100644 --- a/src/pip/_internal/metadata/__init__.py +++ b/src/pip/_internal/metadata/__init__.py @@ -4,6 +4,7 @@ import functools import os import sys +from collections.abc import Sequence from typing import Literal, Protocol, cast from pip._internal.utils.deprecation import deprecated @@ -112,7 +113,7 @@ def get_default_environment() -> BaseEnvironment: return select_backend().Environment.default() -def get_environment(paths: list[str] | None) -> BaseEnvironment: +def get_environment(paths: Sequence[str] | None) -> BaseEnvironment: """Get a representation of the environment specified by ``paths``. This returns an Environment instance from the chosen backend based on the diff --git a/src/pip/_internal/metadata/base.py b/src/pip/_internal/metadata/base.py index 230e11473c6..78eafe076fc 100644 --- a/src/pip/_internal/metadata/base.py +++ b/src/pip/_internal/metadata/base.py @@ -8,7 +8,7 @@ import pathlib import re import zipfile -from collections.abc import Collection, Container, Iterable, Iterator +from collections.abc import Collection, Container, Iterable, Iterator, Sequence from typing import ( IO, Any, @@ -584,7 +584,7 @@ def default(cls) -> BaseEnvironment: raise NotImplementedError() @classmethod - def from_paths(cls, paths: list[str] | None) -> BaseEnvironment: + def from_paths(cls, paths: Sequence[str] | None) -> BaseEnvironment: raise NotImplementedError() def get_distribution(self, name: str) -> BaseDistribution | None: diff --git a/src/pip/_internal/metadata/importlib/_envs.py b/src/pip/_internal/metadata/importlib/_envs.py index 71a73b7311f..97eff39d2e0 100644 --- a/src/pip/_internal/metadata/importlib/_envs.py +++ b/src/pip/_internal/metadata/importlib/_envs.py @@ -122,7 +122,7 @@ def default(cls) -> BaseEnvironment: return cls(sys.path) @classmethod - def from_paths(cls, paths: list[str] | None) -> BaseEnvironment: + def from_paths(cls, paths: Sequence[str] | None) -> BaseEnvironment: if paths is None: return cls(sys.path) return cls(paths) diff --git a/src/pip/_internal/metadata/pkg_resources.py b/src/pip/_internal/metadata/pkg_resources.py index 89fce8b6e5d..d685c299dee 100644 --- a/src/pip/_internal/metadata/pkg_resources.py +++ b/src/pip/_internal/metadata/pkg_resources.py @@ -5,7 +5,7 @@ import logging import os import zipfile -from collections.abc import Collection, Iterable, Iterator, Mapping +from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence from typing import ( NamedTuple, ) @@ -256,7 +256,7 @@ def default(cls) -> BaseEnvironment: return cls(pkg_resources.working_set) @classmethod - def from_paths(cls, paths: list[str] | None) -> BaseEnvironment: + def from_paths(cls, paths: Sequence[str] | None) -> BaseEnvironment: return cls(pkg_resources.WorkingSet(paths)) def _iter_distributions(self) -> Iterator[BaseDistribution]: diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 9694b4e995e..a4cc04dae72 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -13,7 +13,10 @@ from pip._vendor.packaging.utils import canonicalize_name -from pip._internal.build_env import InprocessBuildEnvironmentInstaller, SubprocessBuildEnvironmentInstaller +from pip._internal.build_env import ( + InprocessBuildEnvironmentInstaller, + SubprocessBuildEnvironmentInstaller, +) from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.distributions.installed import InstalledDistribution from pip._internal.exceptions import ( diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index 5cdbeb7f753..1c867aec99c 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -9,7 +9,7 @@ import threading from collections.abc import Generator from dataclasses import dataclass -from io import TextIOWrapper +from io import StringIO, TextIOWrapper from logging import Filter from typing import Any, ClassVar @@ -29,7 +29,7 @@ from pip._internal.utils._log import VERBOSE, getLogger from pip._internal.utils.compat import WINDOWS from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX -from pip._internal.utils.misc import ensure_dir +from pip._internal.utils.misc import StreamWrapper, ensure_dir _log_state = threading.local() _stdout_console = None @@ -56,6 +56,34 @@ def _is_broken_pipe_error(exc_class: type[BaseException], exc: BaseException) -> return isinstance(exc, OSError) and exc.errno in (errno.EINVAL, errno.EPIPE) +@contextlib.contextmanager +def capture_logging() -> Generator[StringIO, None, None]: + """Capture all pip logs in a buffer temporarily.""" + # Patching sys.std(out|err) directly is not viable as the caller + # may want to emit non-logging output (e.g. a rich spinner). To + # avoid capturing that, temporarily patch the root logging handlers + # to use new rich consoles that write to a StringIO. + handlers = {} + for handler in logging.getLogger().handlers: + if isinstance(handler, RichPipStreamHandler): + # Also store the handler's original console so it can be + # restored on context exit. + handlers[handler] = handler.console + + # HACK: grab no_color attribute from a random handler console since + # it's a global option anyway. + no_color = next(iter(handlers.values())).no_color + fake_stream = StreamWrapper.from_stream(sys.stdout) + fake_console = PipConsole(file=fake_stream, no_color=no_color, soft_wrap=True) + try: + for handler in handlers: + handler.console = fake_console + yield fake_stream + finally: + for handler, original_console in handlers.items(): + handler.console = original_console + + @contextlib.contextmanager def indent_log(num: int = 2) -> Generator[None, None, None]: """ From 18b01832427b22068a0dde41ab347d305dddb4bc Mon Sep 17 00:00:00 2001 From: Richard Si Date: Thu, 26 Jun 2025 23:00:25 -0400 Subject: [PATCH 05/11] Move hacks from preparer API to req_command --- src/pip/_internal/build_env.py | 28 ++++++++++++++++--------- src/pip/_internal/cli/req_command.py | 23 +++++++++++++++----- src/pip/_internal/operations/prepare.py | 21 ++++++------------- 3 files changed, 42 insertions(+), 30 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 031a1b423da..2788c60fb70 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -11,6 +11,7 @@ from collections import OrderedDict from collections.abc import Iterable from contextlib import AbstractContextManager, nullcontext +from functools import partial from io import StringIO from optparse import Values from types import TracebackType @@ -145,22 +146,20 @@ class InprocessBuildEnvironmentInstaller: # TODO: ensure build tracking still works # TODO: figure out what options are actually being inherited - def __init__( - self, finder: PackageFinder, preparer: RequirementPreparer, options: Values - ) -> None: + def __init__(self, finder: PackageFinder, options: Values) -> None: from pip._internal.cache import WheelCache - from pip._internal.commands import create_command self.finder = finder - self.preparer = preparer self.options = options - self._install_command = create_command("install") + self.preparer: RequirementPreparer self._wheel_cache = WheelCache(options.cache_dir) def install( self, requirements: Iterable[str], prefix: _Prefix, *, kind: str ) -> None: + assert hasattr(self, "preparer"), "preparer must be available!" + capture_ctx: AbstractContextManager[StringIO] should_capture = not logger.isEnabledFor(VERBOSE) if should_capture: @@ -240,18 +239,27 @@ def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None: logger.info(summary) def _make_resolver(self) -> BaseResolver: - # TODO: the old logic only uses the resolvelib resolver, this won't - return self._install_command.make_resolver( + # Legacy installer never used the legacy resolver so create a + # resolvelib resolver directly. Yuck. + from pip._internal.req.constructors import install_req_from_req_string + from pip._internal.resolution.resolvelib.resolver import Resolver + + make_install_req = partial( + install_req_from_req_string, + isolated=self.options.isolated_mode, + use_pep517=True, + ) + return Resolver( preparer=self.preparer, finder=self.finder, - options=self.options, wheel_cache=self._wheel_cache, + make_install_req=make_install_req, use_user_site=False, + ignore_dependencies=False, ignore_installed=True, ignore_requires_python=self.options.ignore_requires_python, force_reinstall=False, upgrade_strategy="to-satisfy-only", - use_pep517=True, py_version_info=getattr(self.options, "python_version", None), ) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index c03ed1cc720..624014a9b7c 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -12,6 +12,11 @@ from optparse import Values from typing import Any +from pip._internal.build_env import ( + BuildEnvironmentInstaller, + InprocessBuildEnvironmentInstaller, + SubprocessBuildEnvironmentInstaller, +) from pip._internal.cache import WheelCache from pip._internal.cli import cmdoptions from pip._internal.cli.index_command import IndexGroupCommand @@ -131,11 +136,20 @@ def make_requirement_preparer( "fast-deps has no effect when used with the legacy resolver." ) - return RequirementPreparer( + build_isolation_installer: BuildEnvironmentInstaller + if "legacy-build-deps-installer" not in options.deprecated_features_enabled: + build_isolation_installer = InprocessBuildEnvironmentInstaller( + finder, options + ) + else: + build_isolation_installer = SubprocessBuildEnvironmentInstaller(finder) + + preparer = RequirementPreparer( build_dir=temp_build_dir_path, src_dir=options.src_dir, download_dir=download_dir, build_isolation=options.build_isolation, + build_isolation_installer=build_isolation_installer, check_build_deps=options.check_build_deps, build_tracker=build_tracker, session=session, @@ -147,11 +161,10 @@ def make_requirement_preparer( verbosity=verbosity, legacy_resolver=legacy_resolver, resume_retries=options.resume_retries, - inprocess_build_deps=( - "legacy-build-deps-installer" not in options.deprecated_features_enabled - ), - options=options, ) + if isinstance(build_isolation_installer, InprocessBuildEnvironmentInstaller): + build_isolation_installer.preparer = preparer + return preparer @classmethod def make_resolver( diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index a4cc04dae72..b408ce80550 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -13,10 +13,7 @@ from pip._vendor.packaging.utils import canonicalize_name -from pip._internal.build_env import ( - InprocessBuildEnvironmentInstaller, - SubprocessBuildEnvironmentInstaller, -) +from pip._internal.build_env import BuildEnvironmentInstaller from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.distributions.installed import InstalledDistribution from pip._internal.exceptions import ( @@ -64,7 +61,7 @@ def _get_prepared_distribution( req: InstallRequirement, build_tracker: BuildTracker, - build_env_installer: InprocessBuildEnvironmentInstaller, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> BaseDistribution: @@ -224,12 +221,14 @@ def _check_download_dir( class RequirementPreparer: """Prepares a Requirement""" - def __init__( + def __init__( # noqa: PLR0913 (too many parameters) self, + *, build_dir: str, download_dir: str | None, src_dir: str, build_isolation: bool, + build_isolation_installer: BuildEnvironmentInstaller, check_build_deps: bool, build_tracker: BuildTracker, session: PipSession, @@ -241,9 +240,6 @@ def __init__( verbosity: int, legacy_resolver: bool, resume_retries: int, - # TODO: handle this better - inprocess_build_deps: bool, - options, ) -> None: super().__init__() @@ -260,12 +256,7 @@ def __init__( # Is build isolation allowed? self.build_isolation = build_isolation - if inprocess_build_deps: - self.build_env_installer = InprocessBuildEnvironmentInstaller( - finder, self, options - ) - else: - self.build_env_installer = SubprocessBuildEnvironmentInstaller(finder) + self.build_env_installer = build_isolation_installer # Should check build dependencies? self.check_build_deps = check_build_deps From fa899dd4bba85be1cab13f9b93fb9c1dc4f4db41 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Fri, 27 Jun 2025 12:37:42 -0400 Subject: [PATCH 06/11] Add proper error handling, pt.1 --- src/pip/_internal/build_env.py | 52 +++++++++++++++++++----- src/pip/_internal/distributions/sdist.py | 7 +++- src/pip/_internal/exceptions.py | 40 +++++++++++++++++- 3 files changed, 86 insertions(+), 13 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 2788c60fb70..7276e0d389f 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -21,6 +21,7 @@ from pip import __file__ as pip_location from pip._internal.cli.spinners import open_rich_spinner, open_spinner +from pip._internal.exceptions import BuildDependencyInstallError, DiagnosticPipError from pip._internal.locations import get_platlib, get_purelib, get_scheme from pip._internal.metadata import get_default_environment, get_environment from pip._internal.utils.logging import VERBOSE, capture_logging @@ -31,6 +32,7 @@ if TYPE_CHECKING: from pip._internal.index.package_finder import PackageFinder from pip._internal.operations.prepare import RequirementPreparer + from pip._internal.req.req_install import InstallRequirement from pip._internal.resolution.base import BaseResolver logger = logging.getLogger(__name__) @@ -51,7 +53,12 @@ def __init__(self, path: str) -> None: class BuildEnvironmentInstaller(Protocol): def install( - self, requirements: Iterable[str], prefix: _Prefix, *, kind: str + self, + requirements: Iterable[str], + prefix: _Prefix, + *, + kind: str, + for_req: InstallRequirement, ) -> None: ... @@ -67,7 +74,12 @@ def __init__(self, finder: PackageFinder) -> None: self.finder = finder def install( - self, requirements: Iterable[str], prefix: _Prefix, *, kind: str + self, + requirements: Iterable[str], + prefix: _Prefix, + *, + kind: str, + for_req: InstallRequirement, ) -> None: finder = self.finder args: list[str] = [ @@ -156,27 +168,43 @@ def __init__(self, finder: PackageFinder, options: Values) -> None: self._wheel_cache = WheelCache(options.cache_dir) def install( - self, requirements: Iterable[str], prefix: _Prefix, *, kind: str + self, + requirements: Iterable[str], + prefix: _Prefix, + *, + kind: str, + for_req: InstallRequirement, ) -> None: assert hasattr(self, "preparer"), "preparer must be available!" capture_ctx: AbstractContextManager[StringIO] + spinner: AbstractContextManager[None] should_capture = not logger.isEnabledFor(VERBOSE) if should_capture: # Hide the logs from the installation of build dependencies. # They will be shown only if an error occurs. capture_ctx = capture_logging() + spinner = open_rich_spinner(f"Installing {kind}") else: # Otherwise, pass-through all logs (with a header). capture_ctx = nullcontext(StringIO()) + spinner = nullcontext() logger.info("Installing %s ...", kind) - # TODO: error handling - with open_rich_spinner(f"Installing {kind}"), capture_ctx as stream: - self._install_impl(requirements, prefix) - - # print("captured:", "\n---\n" + stream.getvalue().rstrip()) - # print("---") + try: + with spinner, capture_ctx as stream: + self._install_impl(requirements, prefix) + except Exception as exc: + if isinstance(exc, DiagnosticPipError): + logger.error("%s", exc, extra={"rich": True}) + logger.info("") + + raise BuildDependencyInstallError( + for_req, + requirements, + cause=exc, + log_lines=textwrap.dedent(stream.getvalue()).splitlines(), + ) def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None: from pip._internal.commands.install import installed_packages_summary @@ -431,13 +459,14 @@ def install_requirements( prefix_as_string: str, *, kind: str, + for_req: InstallRequirement, ) -> None: prefix = self._prefixes[prefix_as_string] assert not prefix.setup prefix.setup = True if not requirements: return - installer.install(requirements, prefix, kind=kind) + installer.install(requirements, prefix, kind=kind, for_req=for_req) class NoOpBuildEnvironment(BuildEnvironment): @@ -462,10 +491,11 @@ def cleanup(self) -> None: def install_requirements( self, - finder: BuildEnvironmentInstaller, + installer: BuildEnvironmentInstaller, requirements: Iterable[str], prefix_as_string: str, *, kind: str, + for_req: InstallRequirement, ) -> None: raise NotImplementedError() diff --git a/src/pip/_internal/distributions/sdist.py b/src/pip/_internal/distributions/sdist.py index fecfebcfa3b..261c4927df8 100644 --- a/src/pip/_internal/distributions/sdist.py +++ b/src/pip/_internal/distributions/sdist.py @@ -85,6 +85,7 @@ def _prepare_build_backend( pyproject_requires, "overlay", kind="build dependencies", + for_req=self.req, ) conflicting, missing = self.req.build_env.check_requirements( self.req.requirements_to_check @@ -138,7 +139,11 @@ def _install_build_reqs( if conflicting: self._raise_conflicts("the backend dependencies", conflicting) self.req.build_env.install_requirements( - build_env_installer, missing, "normal", kind="backend dependencies" + build_env_installer, + missing, + "normal", + kind="backend dependencies", + for_req=self.req, ) def _raise_conflicts( diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py index 0661897e944..c0f3398474c 100644 --- a/src/pip/_internal/exceptions.py +++ b/src/pip/_internal/exceptions.py @@ -14,7 +14,7 @@ import pathlib import re import sys -from collections.abc import Iterator +from collections.abc import Iterable, Iterator from itertools import chain, groupby, repeat from typing import TYPE_CHECKING, Literal @@ -865,3 +865,41 @@ def __init__(self) -> None: ), link="https://pip.pypa.io/en/stable/topics/dependency-resolution/#handling-resolution-too-deep-errors", ) + + +class BuildDependencyInstallError(DiagnosticPipError): + """Raised when build dependencies cannot be installed.""" + + reference = "failed-build-dependency-install" + + def __init__( + self, + req: InstallRequirement, + build_reqs: Iterable[str], + *, + cause: Exception, + log_lines: list[str], + ) -> None: + hint = None + + if not log_lines: + # No logs are available, they must have been printed earlier. + context = Text(f"ERROR: {cause}") + hint = "Look above for more details." + else: + if isinstance(cause, DiagnosticPipError): + hint = "Look above for the original error that caused this failure." + else: + log_lines.append(f"ERROR: {cause}") + + context = Text.assemble( + f"Installing {' '.join(build_reqs)}\n", + (f"[{len(log_lines)} lines of output]\n", "red"), + "\n".join(log_lines), + ("\n[end of output]", "red"), + ) + + message = Text.from_markup( + f"[green]Cannot install build dependencies[/] for {req!s}" + ) + super().__init__(message=message, context=context, hint_stmt=hint) From 870bef62ddbff252a5c61a026cab5f29732bec13 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Fri, 27 Jun 2025 12:53:38 -0400 Subject: [PATCH 07/11] Turn wheel build error into diagnostic (error handling, pt.2) --- src/pip/_internal/build_env.py | 15 ++++++--------- src/pip/_internal/commands/install.py | 13 ++++++------- src/pip/_internal/exceptions.py | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 7276e0d389f..8653f969ad0 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -21,7 +21,11 @@ from pip import __file__ as pip_location from pip._internal.cli.spinners import open_rich_spinner, open_spinner -from pip._internal.exceptions import BuildDependencyInstallError, DiagnosticPipError +from pip._internal.exceptions import ( + BuildDependencyInstallError, + DiagnosticPipError, + InstallWheelBuildError, +) from pip._internal.locations import get_platlib, get_purelib, get_scheme from pip._internal.metadata import get_default_environment, get_environment from pip._internal.utils.logging import VERBOSE, capture_logging @@ -155,7 +159,6 @@ class InprocessBuildEnvironmentInstaller: - """ - # TODO: ensure build tracking still works # TODO: figure out what options are actually being inherited def __init__(self, finder: PackageFinder, options: Values) -> None: @@ -240,14 +243,8 @@ def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None: build_options=[], global_options=[], ) - if build_failures: - raise InstallationError( - "Failed to build installable wheels for some " - "pyproject.toml based projects ({})".format( - ", ".join(r.name for r in build_failures) # type: ignore - ) - ) + raise InstallWheelBuildError(build_failures) to_install = resolver.get_installation_order(requirement_set) installed = install_given_reqs( diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 3614907594a..f91202cacaf 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -27,7 +27,11 @@ with_cleanup, ) from pip._internal.cli.status_codes import ERROR, SUCCESS -from pip._internal.exceptions import CommandError, InstallationError +from pip._internal.exceptions import ( + CommandError, + InstallationError, + InstallWheelBuildError, +) from pip._internal.locations import get_scheme from pip._internal.metadata import BaseEnvironment, get_environment from pip._internal.models.installation_report import InstallationReport @@ -434,12 +438,7 @@ def run(self, options: Values, args: list[str]) -> int: ) if build_failures: - raise InstallationError( - "Failed to build installable wheels for some " - "pyproject.toml based projects ({})".format( - ", ".join(r.name for r in build_failures) # type: ignore - ) - ) + raise InstallWheelBuildError(build_failures) to_install = resolver.get_installation_order(requirement_set) diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py index c0f3398474c..dcaba9d9c3d 100644 --- a/src/pip/_internal/exceptions.py +++ b/src/pip/_internal/exceptions.py @@ -903,3 +903,17 @@ def __init__( f"[green]Cannot install build dependencies[/] for {req!s}" ) super().__init__(message=message, context=context, hint_stmt=hint) + + +class InstallWheelBuildError(DiagnosticPipError): + reference = "failed-wheel-build-for-install" + + def __init__(self, failed: list[InstallRequirement]) -> None: + super().__init__( + message=( + "Failed to build installable wheels for some " + "pyproject.toml based projects" + ), + context=", ".join(r.name for r in failed), # type: ignore + hint_stmt=None, + ) From 7282e675ee8de9b55b8235814e039aba690ee157 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Fri, 27 Jun 2025 13:37:01 -0400 Subject: [PATCH 08/11] Create a new preparer for build deps --- src/pip/_internal/build_env.py | 40 +++++++++++++++++++++++----- src/pip/_internal/cli/req_command.py | 7 ++--- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 8653f969ad0..5ef1408a95a 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -35,7 +35,8 @@ if TYPE_CHECKING: from pip._internal.index.package_finder import PackageFinder - from pip._internal.operations.prepare import RequirementPreparer + from pip._internal.network.session import PipSession + from pip._internal.operations.build.build_tracker import BuildTracker from pip._internal.req.req_install import InstallRequirement from pip._internal.resolution.base import BaseResolver @@ -161,13 +162,42 @@ class InprocessBuildEnvironmentInstaller: # TODO: figure out what options are actually being inherited - def __init__(self, finder: PackageFinder, options: Values) -> None: + def __init__( + self, + finder: PackageFinder, + session: PipSession, + build_tracker: BuildTracker, + build_dir: str, + verbosity: int, + options: Values, + ) -> None: from pip._internal.cache import WheelCache + from pip._internal.operations.prepare import RequirementPreparer self.finder = finder self.options = options - self.preparer: RequirementPreparer + self._preparer = RequirementPreparer( + build_dir=build_dir, + # TODO: probably don't inherit --src or --download-dir? + src_dir=options.src_dir, + download_dir=None, + build_isolation=True, + build_isolation_installer=self, + check_build_deps=False, + build_tracker=build_tracker, + session=session, + progress_bar="off", + finder=finder, + # TODO: hash-checking should be extended to build deps, but that is + # deferred for later. + require_hashes=False, + use_user_site=False, + lazy_wheel=False, + verbosity=verbosity, + legacy_resolver=False, + resume_retries=options.resume_retries, + ) self._wheel_cache = WheelCache(options.cache_dir) def install( @@ -178,8 +208,6 @@ def install( kind: str, for_req: InstallRequirement, ) -> None: - assert hasattr(self, "preparer"), "preparer must be available!" - capture_ctx: AbstractContextManager[StringIO] spinner: AbstractContextManager[None] should_capture = not logger.isEnabledFor(VERBOSE) @@ -275,7 +303,7 @@ def _make_resolver(self) -> BaseResolver: use_pep517=True, ) return Resolver( - preparer=self.preparer, + preparer=self._preparer, finder=self.finder, wheel_cache=self._wheel_cache, make_install_req=make_install_req, diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 624014a9b7c..edda7ddacd0 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -139,12 +139,12 @@ def make_requirement_preparer( build_isolation_installer: BuildEnvironmentInstaller if "legacy-build-deps-installer" not in options.deprecated_features_enabled: build_isolation_installer = InprocessBuildEnvironmentInstaller( - finder, options + finder, session, build_tracker, temp_build_dir_path, verbosity, options ) else: build_isolation_installer = SubprocessBuildEnvironmentInstaller(finder) - preparer = RequirementPreparer( + return RequirementPreparer( build_dir=temp_build_dir_path, src_dir=options.src_dir, download_dir=download_dir, @@ -162,9 +162,6 @@ def make_requirement_preparer( legacy_resolver=legacy_resolver, resume_retries=options.resume_retries, ) - if isinstance(build_isolation_installer, InprocessBuildEnvironmentInstaller): - build_isolation_installer.preparer = preparer - return preparer @classmethod def make_resolver( From 671759fca9d45ce88f27e02b1be2e98bae481288 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Fri, 27 Jun 2025 14:15:07 -0400 Subject: [PATCH 09/11] Add more comments & document inherited options --- src/pip/_internal/build_env.py | 59 +++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 5ef1408a95a..a2257d5c6e5 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -155,12 +155,17 @@ class InprocessBuildEnvironmentInstaller: """ Build dependency installer that runs in the same pip process. - Differences from the subprocess-based installer: - - Conflicts with already installed packages aren't detected - - - """ + This contains a stripped down version of the install command with + only the logic necessary for installing build dependencies. The + finder, session, and build tracker are reused, but new instances + of everything else are created as needed. + + Options are inherited from the parent install command unless + they don't make sense for build dependencies (in which case, they + are hard-coded, see comments below). - # TODO: figure out what options are actually being inherited + NOTE: This can only be used with the resolvelib resolver. + """ def __init__( self, @@ -178,25 +183,28 @@ def __init__( self.options = options self._preparer = RequirementPreparer( + build_isolation_installer=self, + # Inherited options or state. + finder=finder, + session=session, build_dir=build_dir, - # TODO: probably don't inherit --src or --download-dir? + build_tracker=build_tracker, + verbosity=verbosity, + resume_retries=options.resume_retries, + # This probably shouldn't be inherited, but it won't be used + # anyway as it only applies to editable requirements. src_dir=options.src_dir, + # Hard-coded options (they should NOT be inherited). download_dir=None, build_isolation=True, - build_isolation_installer=self, check_build_deps=False, - build_tracker=build_tracker, - session=session, progress_bar="off", - finder=finder, # TODO: hash-checking should be extended to build deps, but that is - # deferred for later. + # deferred for later as it'd be a breaking change. require_hashes=False, use_user_site=False, lazy_wheel=False, - verbosity=verbosity, legacy_resolver=False, - resume_retries=options.resume_retries, ) self._wheel_cache = WheelCache(options.cache_dir) @@ -208,6 +216,7 @@ def install( kind: str, for_req: InstallRequirement, ) -> None: + """Install entrypoint. Manages output capturing and error handling.""" capture_ctx: AbstractContextManager[StringIO] spinner: AbstractContextManager[None] should_capture = not logger.isEnabledFor(VERBOSE) @@ -227,6 +236,8 @@ def install( self._install_impl(requirements, prefix) except Exception as exc: if isinstance(exc, DiagnosticPipError): + # Format similar to a nested subprocess error, where the + # causing error is shown first, followed by the build error. logger.error("%s", exc, extra={"rich": True}) logger.info("") @@ -238,6 +249,7 @@ def install( ) def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None: + """Core build dependency install logic.""" from pip._internal.commands.install import installed_packages_summary from pip._internal.req import install_given_reqs from pip._internal.req.constructors import install_req_from_line @@ -247,9 +259,10 @@ def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None: for req in requirements: ireq = install_req_from_line( req, - comes_from=None, - # TODO: does --isolated matter here? + # I have no idea whether inheriting this is useful, but whatever. isolated=self.options.isolated_mode, + # Hard-coded options (they should NOT be inherited). + comes_from=None, use_pep517=True, user_supplied=True, config_settings={}, @@ -268,6 +281,7 @@ def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None: reqs_to_build, wheel_cache=self._wheel_cache, verify=True, + # Hard-coded options (they should NOT be inherited). build_options=[], global_options=[], ) @@ -277,12 +291,16 @@ def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None: to_install = resolver.get_installation_order(requirement_set) installed = install_given_reqs( to_install, + prefix=prefix.path, + # Hard-coded options (they should NOT be inherited). global_options=[], root=None, home=None, - prefix=prefix.path, warn_script_location=False, use_user_site=False, + # As the build environment is ephemeral, it's wasteful to + # pre-compile everything, especially as not every Python + # module will be used/compiled in most cases. pycompile=False, progress_bar="off", ) @@ -292,6 +310,7 @@ def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None: logger.info(summary) def _make_resolver(self) -> BaseResolver: + """Create a new resolver for one time use.""" # Legacy installer never used the legacy resolver so create a # resolvelib resolver directly. Yuck. from pip._internal.req.constructors import install_req_from_req_string @@ -303,17 +322,19 @@ def _make_resolver(self) -> BaseResolver: use_pep517=True, ) return Resolver( + make_install_req=make_install_req, + # Inherited state. preparer=self._preparer, finder=self.finder, wheel_cache=self._wheel_cache, - make_install_req=make_install_req, + ignore_requires_python=self.options.ignore_requires_python, + # Hard-coded options (they should NOT be inherited). use_user_site=False, ignore_dependencies=False, ignore_installed=True, - ignore_requires_python=self.options.ignore_requires_python, force_reinstall=False, upgrade_strategy="to-satisfy-only", - py_version_info=getattr(self.options, "python_version", None), + py_version_info=None, ) From e25dc31d0c1f4b610f50d40e756a2905710abcbf Mon Sep 17 00:00:00 2001 From: Richard Si Date: Fri, 27 Jun 2025 14:24:45 -0400 Subject: [PATCH 10/11] Flip default to off (--use-feature opt-in) --- src/pip/_internal/cli/cmdoptions.py | 2 +- src/pip/_internal/cli/req_command.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index c7f35d859fd..bf366512388 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -1068,6 +1068,7 @@ def check_list_path_option(options: Values) -> None: default=[], choices=[ "fast-deps", + "inprocess-build-deps", ] + ALWAYS_ENABLED_FEATURES, help="Enable new functionality, that may be backward incompatible.", @@ -1083,7 +1084,6 @@ def check_list_path_option(options: Values) -> None: choices=[ "legacy-resolver", "legacy-certs", - "legacy-build-deps-installer", ], help=("Enable deprecated functionality, that will be removed in the future."), ) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index edda7ddacd0..884f3439f97 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -137,7 +137,11 @@ def make_requirement_preparer( ) build_isolation_installer: BuildEnvironmentInstaller - if "legacy-build-deps-installer" not in options.deprecated_features_enabled: + if "inprocess-build-deps" in options.features_enabled: + if resolver_variant == "legacy": + raise CommandError( + "inprocess-build-deps cannot be used with the legacy resolver." + ) build_isolation_installer = InprocessBuildEnvironmentInstaller( finder, session, build_tracker, temp_build_dir_path, verbosity, options ) From b55b0fd640e9a31cfbeebe2807fc51fd749b3f4d Mon Sep 17 00:00:00 2001 From: Richard Si Date: Fri, 27 Jun 2025 15:07:42 -0400 Subject: [PATCH 11/11] Fix test suite (w/ necessary refactors) --- src/pip/_internal/build_env.py | 17 +++++----- src/pip/_internal/distributions/sdist.py | 14 ++------ src/pip/_internal/exceptions.py | 8 ++--- tests/functional/test_build_env.py | 42 +++++++++++++----------- tests/functional/test_pep517.py | 15 +++++---- tests/unit/test_req.py | 3 ++ 6 files changed, 50 insertions(+), 49 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index a2257d5c6e5..45bf3b06450 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -63,7 +63,7 @@ def install( prefix: _Prefix, *, kind: str, - for_req: InstallRequirement, + for_req: InstallRequirement | None, ) -> None: ... @@ -84,7 +84,7 @@ def install( prefix: _Prefix, *, kind: str, - for_req: InstallRequirement, + for_req: InstallRequirement | None, ) -> None: finder = self.finder args: list[str] = [ @@ -214,7 +214,7 @@ def install( prefix: _Prefix, *, kind: str, - for_req: InstallRequirement, + for_req: InstallRequirement | None, ) -> None: """Install entrypoint. Manages output capturing and error handling.""" capture_ctx: AbstractContextManager[StringIO] @@ -377,7 +377,8 @@ def _get_system_sitepackages() -> set[str]: class BuildEnvironment: """Creates and manages an isolated environment to install build deps""" - def __init__(self) -> None: + def __init__(self, installer: BuildEnvironmentInstaller) -> None: + self.installer = installer temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True) self._prefixes = OrderedDict( @@ -500,19 +501,18 @@ def check_requirements( def install_requirements( self, - installer: BuildEnvironmentInstaller, requirements: Iterable[str], prefix_as_string: str, *, kind: str, - for_req: InstallRequirement, + for_req: InstallRequirement | None = None, ) -> None: prefix = self._prefixes[prefix_as_string] assert not prefix.setup prefix.setup = True if not requirements: return - installer.install(requirements, prefix, kind=kind, for_req=for_req) + self.installer.install(requirements, prefix, kind=kind, for_req=for_req) class NoOpBuildEnvironment(BuildEnvironment): @@ -537,11 +537,10 @@ def cleanup(self) -> None: def install_requirements( self, - installer: BuildEnvironmentInstaller, requirements: Iterable[str], prefix_as_string: str, *, kind: str, - for_req: InstallRequirement, + for_req: InstallRequirement | None = None, ) -> None: raise NotImplementedError() diff --git a/src/pip/_internal/distributions/sdist.py b/src/pip/_internal/distributions/sdist.py index 261c4927df8..e2821f89e00 100644 --- a/src/pip/_internal/distributions/sdist.py +++ b/src/pip/_internal/distributions/sdist.py @@ -79,13 +79,9 @@ def _prepare_build_backend( pyproject_requires = self.req.pyproject_requires assert pyproject_requires is not None - self.req.build_env = BuildEnvironment() + self.req.build_env = BuildEnvironment(build_env_installer) self.req.build_env.install_requirements( - build_env_installer, - pyproject_requires, - "overlay", - kind="build dependencies", - for_req=self.req, + pyproject_requires, "overlay", kind="build dependencies", for_req=self.req ) conflicting, missing = self.req.build_env.check_requirements( self.req.requirements_to_check @@ -139,11 +135,7 @@ def _install_build_reqs( if conflicting: self._raise_conflicts("the backend dependencies", conflicting) self.req.build_env.install_requirements( - build_env_installer, - missing, - "normal", - kind="backend dependencies", - for_req=self.req, + missing, "normal", kind="backend dependencies", for_req=self.req ) def _raise_conflicts( diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py index dcaba9d9c3d..09b7abae01d 100644 --- a/src/pip/_internal/exceptions.py +++ b/src/pip/_internal/exceptions.py @@ -874,7 +874,7 @@ class BuildDependencyInstallError(DiagnosticPipError): def __init__( self, - req: InstallRequirement, + req: InstallRequirement | None, build_reqs: Iterable[str], *, cause: Exception, @@ -899,9 +899,9 @@ def __init__( ("\n[end of output]", "red"), ) - message = Text.from_markup( - f"[green]Cannot install build dependencies[/] for {req!s}" - ) + message = Text("Cannot install build dependencies", "green") + if req: + message += Text(f" for {req}") super().__init__(message=message, context=context, hint_stmt=hint) diff --git a/tests/functional/test_build_env.py b/tests/functional/test_build_env.py index 1a1839b4027..ec1b59f068c 100644 --- a/tests/functional/test_build_env.py +++ b/tests/functional/test_build_env.py @@ -6,7 +6,11 @@ import pytest -from pip._internal.build_env import BuildEnvironment, _get_system_sitepackages +from pip._internal.build_env import ( + BuildEnvironment, + SubprocessBuildEnvironmentInstaller, + _get_system_sitepackages, +) from tests.lib import ( PipTestEnvironment, @@ -33,7 +37,10 @@ def run_with_build_env( import subprocess import sys - from pip._internal.build_env import BuildEnvironment + from pip._internal.build_env import ( + BuildEnvironment, + SubprocessBuildEnvironmentInstaller, + ) from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import PackageFinder from pip._internal.models.search_scope import SearchScope @@ -56,7 +63,9 @@ def run_with_build_env( ) with global_tempdir_manager(): - build_env = BuildEnvironment() + build_env = BuildEnvironment( + SubprocessBuildEnvironmentInstaller(finder) + ) """ ) + indent(dedent(setup_script_contents), " ") @@ -81,30 +90,26 @@ def run_with_build_env( def test_build_env_allow_empty_requirements_install() -> None: finder = make_test_finder() - build_env = BuildEnvironment() + build_env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) for prefix in ("normal", "overlay"): - build_env.install_requirements( - finder, [], prefix, kind="Installing build dependencies" - ) + build_env.install_requirements([], prefix, kind="Installing build dependencies") def test_build_env_allow_only_one_install(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "foo", "1.0") create_basic_wheel_for_package(script, "bar", "1.0") finder = make_test_finder(find_links=[os.fspath(script.scratch_path)]) - build_env = BuildEnvironment() + build_env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) for prefix in ("normal", "overlay"): build_env.install_requirements( - finder, ["foo"], prefix, kind=f"installing foo in {prefix}" + ["foo"], prefix, kind=f"installing foo in {prefix}" ) with pytest.raises(AssertionError): build_env.install_requirements( - finder, ["bar"], prefix, kind=f"installing bar in {prefix}" + ["bar"], prefix, kind=f"installing bar in {prefix}" ) with pytest.raises(AssertionError): - build_env.install_requirements( - finder, [], prefix, kind=f"installing in {prefix}" - ) + build_env.install_requirements([], prefix, kind=f"installing in {prefix}") def test_build_env_requirements_check(script: PipTestEnvironment) -> None: @@ -132,7 +137,7 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None: run_with_build_env( script, """ - build_env.install_requirements(finder, ['foo', 'bar==3.0'], 'normal', + build_env.install_requirements(['foo', 'bar==3.0'], 'normal', kind='installing foo in normal') r = build_env.check_requirements(['foo', 'bar', 'other']) @@ -149,9 +154,9 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None: run_with_build_env( script, """ - build_env.install_requirements(finder, ['foo', 'bar==3.0'], 'normal', + build_env.install_requirements(['foo', 'bar==3.0'], 'normal', kind='installing foo in normal') - build_env.install_requirements(finder, ['bar==1.0'], 'overlay', + build_env.install_requirements(['bar==1.0'], 'overlay', kind='installing foo in overlay') r = build_env.check_requirements(['foo', 'bar', 'other']) @@ -170,7 +175,6 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None: script, """ build_env.install_requirements( - finder, ["bar==3.0"], "normal", kind="installing bar in normal", @@ -193,9 +197,9 @@ def test_build_env_overlay_prefix_has_priority(script: PipTestEnvironment) -> No result = run_with_build_env( script, """ - build_env.install_requirements(finder, ['pkg==2.0'], 'overlay', + build_env.install_requirements(['pkg==2.0'], 'overlay', kind='installing pkg==2.0 in overlay') - build_env.install_requirements(finder, ['pkg==4.3'], 'normal', + build_env.install_requirements(['pkg==4.3'], 'normal', kind='installing pkg==4.3 in normal') """, """ diff --git a/tests/functional/test_pep517.py b/tests/functional/test_pep517.py index 9be3834241f..47dae96817f 100644 --- a/tests/functional/test_pep517.py +++ b/tests/functional/test_pep517.py @@ -7,7 +7,10 @@ import pytest import tomli_w -from pip._internal.build_env import BuildEnvironment +from pip._internal.build_env import ( + BuildEnvironment, + SubprocessBuildEnvironmentInstaller, +) from pip._internal.req import InstallRequirement from tests.lib import ( @@ -43,9 +46,9 @@ def test_backend(tmpdir: Path, data: TestData) -> None: req = InstallRequirement(None, None) req.source_dir = os.fspath(project_dir) # make req believe it has been unpacked req.load_pyproject_toml() - env = BuildEnvironment() finder = make_test_finder(find_links=[data.backends]) - env.install_requirements(finder, ["dummy_backend"], "normal", kind="Installing") + env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) + env.install_requirements(["dummy_backend"], "normal", kind="Installing") conflicting, missing = env.check_requirements(["dummy_backend"]) assert not conflicting assert not missing @@ -73,7 +76,7 @@ def test_backend_path(tmpdir: Path, data: TestData) -> None: req.source_dir = os.fspath(project_dir) # make req believe it has been unpacked req.load_pyproject_toml() - env = BuildEnvironment() + env = BuildEnvironment(object()) # type: ignore assert hasattr(req.pep517_backend, "build_wheel") with env: assert req.pep517_backend is not None @@ -91,9 +94,9 @@ def test_backend_path_and_dep(tmpdir: Path, data: TestData) -> None: req = InstallRequirement(None, None) req.source_dir = os.fspath(project_dir) # make req believe it has been unpacked req.load_pyproject_toml() - env = BuildEnvironment() finder = make_test_finder(find_links=[data.backends]) - env.install_requirements(finder, ["dummy_backend"], "normal", kind="Installing") + env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) + env.install_requirements(["dummy_backend"], "normal", kind="Installing") assert hasattr(req.pep517_backend, "build_wheel") with env: diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index ab9b8ea71c6..0547131134e 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -17,6 +17,7 @@ from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import Requirement +from pip._internal.build_env import SubprocessBuildEnvironmentInstaller from pip._internal.cache import WheelCache from pip._internal.commands import create_command from pip._internal.commands.install import InstallCommand @@ -97,11 +98,13 @@ def _basic_resolver( session = PipSession() with get_build_tracker() as tracker: + installer = SubprocessBuildEnvironmentInstaller(finder) preparer = RequirementPreparer( build_dir=os.path.join(self.tempdir, "build"), src_dir=os.path.join(self.tempdir, "src"), download_dir=None, build_isolation=True, + build_isolation_installer=installer, check_build_deps=False, build_tracker=tracker, session=session,