diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 69a2e72ce42..45bf3b06450 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -10,22 +10,35 @@ import textwrap from collections import OrderedDict from collections.abc import Iterable +from contextlib import AbstractContextManager, nullcontext +from functools import partial +from io import StringIO +from optparse import Values from types import TracebackType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol from pip._vendor.packaging.version import Version from pip import __file__ as pip_location -from pip._internal.cli.spinners import open_spinner +from pip._internal.cli.spinners import open_rich_spinner, open_spinner +from pip._internal.exceptions import ( + BuildDependencyInstallError, + DiagnosticPipError, + InstallWheelBuildError, +) from pip._internal.locations import get_platlib, get_purelib, get_scheme from pip._internal.metadata import get_default_environment, get_environment -from pip._internal.utils.logging import VERBOSE +from pip._internal.utils.logging import VERBOSE, capture_logging from pip._internal.utils.packaging import get_requirement from pip._internal.utils.subprocess import call_subprocess from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds if TYPE_CHECKING: from pip._internal.index.package_finder import PackageFinder + from pip._internal.network.session import PipSession + from pip._internal.operations.build.build_tracker import BuildTracker + from pip._internal.req.req_install import InstallRequirement + from pip._internal.resolution.base import BaseResolver logger = logging.getLogger(__name__) @@ -43,6 +56,288 @@ def __init__(self, path: str) -> None: self.lib_dirs = _dedup(scheme.purelib, scheme.platlib) +class BuildEnvironmentInstaller(Protocol): + def install( + self, + requirements: Iterable[str], + prefix: _Prefix, + *, + kind: str, + for_req: InstallRequirement | None, + ) -> None: ... + + +class SubprocessBuildEnvironmentInstaller: + """ + Install build dependencies by calling pip in a subprocess. + + XXX: this is the legacy installation method and will be removed at + some point once InprocessBuildEnvironmentInstaller is stable. + """ + + def __init__(self, finder: PackageFinder) -> None: + self.finder = finder + + def install( + self, + requirements: Iterable[str], + prefix: _Prefix, + *, + kind: str, + for_req: InstallRequirement | None, + ) -> None: + finder = self.finder + args: list[str] = [ + sys.executable, + get_runnable_pip(), + "install", + "--ignore-installed", + "--no-user", + "--prefix", + prefix.path, + "--no-warn-script-location", + "--disable-pip-version-check", + # As the build environment is ephemeral, it's wasteful to + # pre-compile everything, especially as not every Python + # module will be used/compiled in most cases. + "--no-compile", + # The prefix specified two lines above, thus + # target from config file or env var should be ignored + "--target", + "", + ] + if logger.getEffectiveLevel() <= logging.DEBUG: + args.append("-vv") + elif logger.getEffectiveLevel() <= VERBOSE: + args.append("-v") + for format_control in ("no_binary", "only_binary"): + formats = getattr(finder.format_control, format_control) + args.extend( + ( + "--" + format_control.replace("_", "-"), + ",".join(sorted(formats or {":none:"})), + ) + ) + + index_urls = finder.index_urls + if index_urls: + args.extend(["-i", index_urls[0]]) + for extra_index in index_urls[1:]: + args.extend(["--extra-index-url", extra_index]) + else: + args.append("--no-index") + for link in finder.find_links: + args.extend(["--find-links", link]) + + if finder.proxy: + args.extend(["--proxy", finder.proxy]) + for host in finder.trusted_hosts: + args.extend(["--trusted-host", host]) + if finder.custom_cert: + args.extend(["--cert", finder.custom_cert]) + if finder.client_cert: + args.extend(["--client-cert", finder.client_cert]) + if finder.allow_all_prereleases: + args.append("--pre") + if finder.prefer_binary: + args.append("--prefer-binary") + args.append("--") + args.extend(requirements) + with open_spinner(f"Installing {kind}") as spinner: + call_subprocess( + args, + command_desc=f"pip subprocess to install {kind}", + spinner=spinner, + ) + + +class InprocessBuildEnvironmentInstaller: + """ + Build dependency installer that runs in the same pip process. + + This contains a stripped down version of the install command with + only the logic necessary for installing build dependencies. The + finder, session, and build tracker are reused, but new instances + of everything else are created as needed. + + Options are inherited from the parent install command unless + they don't make sense for build dependencies (in which case, they + are hard-coded, see comments below). + + NOTE: This can only be used with the resolvelib resolver. + """ + + def __init__( + self, + finder: PackageFinder, + session: PipSession, + build_tracker: BuildTracker, + build_dir: str, + verbosity: int, + options: Values, + ) -> None: + from pip._internal.cache import WheelCache + from pip._internal.operations.prepare import RequirementPreparer + + self.finder = finder + self.options = options + + self._preparer = RequirementPreparer( + build_isolation_installer=self, + # Inherited options or state. + finder=finder, + session=session, + build_dir=build_dir, + build_tracker=build_tracker, + verbosity=verbosity, + resume_retries=options.resume_retries, + # This probably shouldn't be inherited, but it won't be used + # anyway as it only applies to editable requirements. + src_dir=options.src_dir, + # Hard-coded options (they should NOT be inherited). + download_dir=None, + build_isolation=True, + check_build_deps=False, + progress_bar="off", + # TODO: hash-checking should be extended to build deps, but that is + # deferred for later as it'd be a breaking change. + require_hashes=False, + use_user_site=False, + lazy_wheel=False, + legacy_resolver=False, + ) + self._wheel_cache = WheelCache(options.cache_dir) + + def install( + self, + requirements: Iterable[str], + prefix: _Prefix, + *, + kind: str, + for_req: InstallRequirement | None, + ) -> None: + """Install entrypoint. Manages output capturing and error handling.""" + capture_ctx: AbstractContextManager[StringIO] + spinner: AbstractContextManager[None] + should_capture = not logger.isEnabledFor(VERBOSE) + if should_capture: + # Hide the logs from the installation of build dependencies. + # They will be shown only if an error occurs. + capture_ctx = capture_logging() + spinner = open_rich_spinner(f"Installing {kind}") + else: + # Otherwise, pass-through all logs (with a header). + capture_ctx = nullcontext(StringIO()) + spinner = nullcontext() + logger.info("Installing %s ...", kind) + + try: + with spinner, capture_ctx as stream: + self._install_impl(requirements, prefix) + except Exception as exc: + if isinstance(exc, DiagnosticPipError): + # Format similar to a nested subprocess error, where the + # causing error is shown first, followed by the build error. + logger.error("%s", exc, extra={"rich": True}) + logger.info("") + + raise BuildDependencyInstallError( + for_req, + requirements, + cause=exc, + log_lines=textwrap.dedent(stream.getvalue()).splitlines(), + ) + + def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None: + """Core build dependency install logic.""" + from pip._internal.commands.install import installed_packages_summary + from pip._internal.req import install_given_reqs + from pip._internal.req.constructors import install_req_from_line + from pip._internal.wheel_builder import build, should_build_for_install_command + + ireqs = [] + for req in requirements: + ireq = install_req_from_line( + req, + # I have no idea whether inheriting this is useful, but whatever. + isolated=self.options.isolated_mode, + # Hard-coded options (they should NOT be inherited). + comes_from=None, + use_pep517=True, + user_supplied=True, + config_settings={}, + ) + ireqs.append(ireq) + + resolver = self._make_resolver() + requirement_set = resolver.resolve(ireqs, check_supported_wheels=True) + + reqs_to_build = [ + r + for r in requirement_set.requirements_to_install + if should_build_for_install_command(r) + ] + _, build_failures = build( + reqs_to_build, + wheel_cache=self._wheel_cache, + verify=True, + # Hard-coded options (they should NOT be inherited). + build_options=[], + global_options=[], + ) + if build_failures: + raise InstallWheelBuildError(build_failures) + + to_install = resolver.get_installation_order(requirement_set) + installed = install_given_reqs( + to_install, + prefix=prefix.path, + # Hard-coded options (they should NOT be inherited). + global_options=[], + root=None, + home=None, + warn_script_location=False, + use_user_site=False, + # As the build environment is ephemeral, it's wasteful to + # pre-compile everything, especially as not every Python + # module will be used/compiled in most cases. + pycompile=False, + progress_bar="off", + ) + + env = get_environment(prefix.lib_dirs) + if summary := installed_packages_summary(installed, env): + logger.info(summary) + + def _make_resolver(self) -> BaseResolver: + """Create a new resolver for one time use.""" + # Legacy installer never used the legacy resolver so create a + # resolvelib resolver directly. Yuck. + from pip._internal.req.constructors import install_req_from_req_string + from pip._internal.resolution.resolvelib.resolver import Resolver + + make_install_req = partial( + install_req_from_req_string, + isolated=self.options.isolated_mode, + use_pep517=True, + ) + return Resolver( + make_install_req=make_install_req, + # Inherited state. + preparer=self._preparer, + finder=self.finder, + wheel_cache=self._wheel_cache, + ignore_requires_python=self.options.ignore_requires_python, + # Hard-coded options (they should NOT be inherited). + use_user_site=False, + ignore_dependencies=False, + ignore_installed=True, + force_reinstall=False, + upgrade_strategy="to-satisfy-only", + py_version_info=None, + ) + + def get_runnable_pip() -> str: """Get a file to pass to a Python executable, to run the currently-running pip. @@ -82,7 +377,8 @@ def _get_system_sitepackages() -> set[str]: class BuildEnvironment: """Creates and manages an isolated environment to install build deps""" - def __init__(self) -> None: + def __init__(self, installer: BuildEnvironmentInstaller) -> None: + self.installer = installer temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True) self._prefixes = OrderedDict( @@ -205,96 +501,18 @@ def check_requirements( def install_requirements( self, - finder: PackageFinder, requirements: Iterable[str], prefix_as_string: str, *, kind: str, + for_req: InstallRequirement | None = None, ) -> None: prefix = self._prefixes[prefix_as_string] assert not prefix.setup prefix.setup = True if not requirements: return - self._install_requirements( - get_runnable_pip(), - finder, - requirements, - prefix, - kind=kind, - ) - - @staticmethod - def _install_requirements( - pip_runnable: str, - finder: PackageFinder, - requirements: Iterable[str], - prefix: _Prefix, - *, - kind: str, - ) -> None: - args: list[str] = [ - sys.executable, - pip_runnable, - "install", - "--ignore-installed", - "--no-user", - "--prefix", - prefix.path, - "--no-warn-script-location", - "--disable-pip-version-check", - # As the build environment is ephemeral, it's wasteful to - # pre-compile everything, especially as not every Python - # module will be used/compiled in most cases. - "--no-compile", - # The prefix specified two lines above, thus - # target from config file or env var should be ignored - "--target", - "", - ] - if logger.getEffectiveLevel() <= logging.DEBUG: - args.append("-vv") - elif logger.getEffectiveLevel() <= VERBOSE: - args.append("-v") - for format_control in ("no_binary", "only_binary"): - formats = getattr(finder.format_control, format_control) - args.extend( - ( - "--" + format_control.replace("_", "-"), - ",".join(sorted(formats or {":none:"})), - ) - ) - - index_urls = finder.index_urls - if index_urls: - args.extend(["-i", index_urls[0]]) - for extra_index in index_urls[1:]: - args.extend(["--extra-index-url", extra_index]) - else: - args.append("--no-index") - for link in finder.find_links: - args.extend(["--find-links", link]) - - if finder.proxy: - args.extend(["--proxy", finder.proxy]) - for host in finder.trusted_hosts: - args.extend(["--trusted-host", host]) - if finder.custom_cert: - args.extend(["--cert", finder.custom_cert]) - if finder.client_cert: - args.extend(["--client-cert", finder.client_cert]) - if finder.allow_all_prereleases: - args.append("--pre") - if finder.prefer_binary: - args.append("--prefer-binary") - args.append("--") - args.extend(requirements) - with open_spinner(f"Installing {kind}") as spinner: - call_subprocess( - args, - command_desc=f"pip subprocess to install {kind}", - spinner=spinner, - ) + self.installer.install(requirements, prefix, kind=kind, for_req=for_req) class NoOpBuildEnvironment(BuildEnvironment): @@ -319,10 +537,10 @@ def cleanup(self) -> None: def install_requirements( self, - finder: PackageFinder, requirements: Iterable[str], prefix_as_string: str, *, kind: str, + for_req: InstallRequirement | None = None, ) -> None: raise NotImplementedError() diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 2476aca80d0..bf366512388 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -1068,6 +1068,7 @@ def check_list_path_option(options: Values) -> None: default=[], choices=[ "fast-deps", + "inprocess-build-deps", ] + ALWAYS_ENABLED_FEATURES, help="Enable new functionality, that may be backward incompatible.", diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 27255098830..884f3439f97 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -12,6 +12,11 @@ from optparse import Values from typing import Any +from pip._internal.build_env import ( + BuildEnvironmentInstaller, + InprocessBuildEnvironmentInstaller, + SubprocessBuildEnvironmentInstaller, +) from pip._internal.cache import WheelCache from pip._internal.cli import cmdoptions from pip._internal.cli.index_command import IndexGroupCommand @@ -131,11 +136,24 @@ def make_requirement_preparer( "fast-deps has no effect when used with the legacy resolver." ) + build_isolation_installer: BuildEnvironmentInstaller + if "inprocess-build-deps" in options.features_enabled: + if resolver_variant == "legacy": + raise CommandError( + "inprocess-build-deps cannot be used with the legacy resolver." + ) + build_isolation_installer = InprocessBuildEnvironmentInstaller( + finder, session, build_tracker, temp_build_dir_path, verbosity, options + ) + else: + build_isolation_installer = SubprocessBuildEnvironmentInstaller(finder) + return RequirementPreparer( build_dir=temp_build_dir_path, src_dir=options.src_dir, download_dir=download_dir, build_isolation=options.build_isolation, + build_isolation_installer=build_isolation_installer, check_build_deps=options.check_build_deps, build_tracker=build_tracker, session=session, diff --git a/src/pip/_internal/cli/spinners.py b/src/pip/_internal/cli/spinners.py index 1d5c351d896..51041d47f48 100644 --- a/src/pip/_internal/cli/spinners.py +++ b/src/pip/_internal/cli/spinners.py @@ -6,13 +6,26 @@ import sys import time from collections.abc import Generator -from typing import IO +from typing import IO, Final + +from pip._vendor.rich.console import ( + Console, + ConsoleOptions, + RenderableType, + RenderResult, +) +from pip._vendor.rich.live import Live +from pip._vendor.rich.measure import Measurement +from pip._vendor.rich.text import Text from pip._internal.utils.compat import WINDOWS -from pip._internal.utils.logging import get_indentation +from pip._internal.utils.logging import get_console, get_indentation logger = logging.getLogger(__name__) +SPINNER_CHARS: Final = r"-\|/" +SPINS_PER_SECOND: Final = 8 + class SpinnerInterface: def spin(self) -> None: @@ -27,9 +40,9 @@ def __init__( self, message: str, file: IO[str] | None = None, - spin_chars: str = "-\\|/", + spin_chars: str = SPINNER_CHARS, # Empirically, 8 updates/second looks nice - min_update_interval_seconds: float = 0.125, + min_update_interval_seconds: float = 1 / SPINS_PER_SECOND, ): self._message = message if file is None: @@ -139,6 +152,62 @@ def open_spinner(message: str) -> Generator[SpinnerInterface, None, None]: spinner.finish("done") +class PipRichSpinner: + """ + Custom rich spinner that matches the style and API* of the legacy spinners. + + (*) Updates will be handled in a background thread by a rich live panel + which will call render() automatically at the appropriate time. + """ + + def __init__(self, label: str) -> None: + self.label = label + self._spin_cycle = itertools.cycle(SPINNER_CHARS) + self._spinner_text = "" + self._finished = False + self._indent = get_indentation() * " " + + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> RenderResult: + yield self.render() + + def __rich_measure__( + self, console: Console, options: ConsoleOptions + ) -> Measurement: + text = self.render() + return Measurement.get(console, options, text) + + def render(self) -> RenderableType: + # The wrapping rich Live instance will call this method at the + # appropriate interval. + if not self._finished: + self._spinner_text = next(self._spin_cycle) + + return Text.assemble(self._indent, self.label, " ... ", self._spinner_text) + + def finish(self, status: str) -> None: + """Stop spinning and set a final status message.""" + self._spinner_text = status + self._finished = True + + +@contextlib.contextmanager +def open_rich_spinner(label: str) -> Generator[None, None, None]: + spinner = PipRichSpinner(label) + with Live(spinner, refresh_per_second=SPINS_PER_SECOND, console=get_console()): + try: + yield + except KeyboardInterrupt: + spinner.finish("canceled") + raise + except Exception: + spinner.finish("error") + raise + else: + spinner.finish("done") + + HIDE_CURSOR = "\x1b[?25l" SHOW_CURSOR = "\x1b[?25h" diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 302bc63f67d..f91202cacaf 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -27,13 +27,17 @@ with_cleanup, ) from pip._internal.cli.status_codes import ERROR, SUCCESS -from pip._internal.exceptions import CommandError, InstallationError +from pip._internal.exceptions import ( + CommandError, + InstallationError, + InstallWheelBuildError, +) from pip._internal.locations import get_scheme -from pip._internal.metadata import get_environment +from pip._internal.metadata import BaseEnvironment, get_environment from pip._internal.models.installation_report import InstallationReport from pip._internal.operations.build.build_tracker import get_build_tracker from pip._internal.operations.check import ConflictDetails, check_install_conflicts -from pip._internal.req import install_given_reqs +from pip._internal.req import InstallationResult, install_given_reqs from pip._internal.req.req_install import ( InstallRequirement, check_legacy_setup_py_options, @@ -434,12 +438,7 @@ def run(self, options: Values, args: list[str]) -> int: ) if build_failures: - raise InstallationError( - "Failed to build installable wheels for some " - "pyproject.toml based projects ({})".format( - ", ".join(r.name for r in build_failures) # type: ignore - ) - ) + raise InstallWheelBuildError(build_failures) to_install = resolver.get_installation_order(requirement_set) @@ -478,34 +477,13 @@ def run(self, options: Values, args: list[str]) -> int: ) env = get_environment(lib_locations) - # Display a summary of installed packages, with extra care to - # display a package name as it was requested by the user. - installed.sort(key=operator.attrgetter("name")) - summary = [] - installed_versions = {} - for distribution in env.iter_all_distributions(): - installed_versions[distribution.canonical_name] = distribution.version - for package in installed: - display_name = package.name - version = installed_versions.get(canonicalize_name(display_name), None) - if version: - text = f"{display_name}-{version}" - else: - text = display_name - summary.append(text) - if conflicts is not None: self._warn_about_conflicts( conflicts, resolver_variant=self.determine_resolver_variant(options), ) - - installed_desc = " ".join(summary) - if installed_desc: - write_output( - "Successfully installed %s", - installed_desc, - ) + if summary := installed_packages_summary(installed, env): + write_output(summary) except OSError as error: show_traceback = self.verbosity >= 1 @@ -644,6 +622,30 @@ def _warn_about_conflicts( logger.critical("\n".join(parts)) +def installed_packages_summary( + installed: list[InstallationResult], env: BaseEnvironment +) -> str: + # Format a summary of installed packages, with extra care to + # display a package name as it was requested by the user. + installed.sort(key=operator.attrgetter("name")) + summary = [] + installed_versions = {} + for distribution in env.iter_all_distributions(): + installed_versions[distribution.canonical_name] = distribution.version + for package in installed: + display_name = package.name + version = installed_versions.get(canonicalize_name(display_name), None) + if version: + text = f"{display_name}-{version}" + else: + text = display_name + summary.append(text) + + if not summary: + return "" + return f"Successfully installed {' '.join(summary)}" + + def get_lib_location_guesses( user: bool = False, home: str | None = None, diff --git a/src/pip/_internal/distributions/base.py b/src/pip/_internal/distributions/base.py index 6ebf49ad501..ea61f3501e7 100644 --- a/src/pip/_internal/distributions/base.py +++ b/src/pip/_internal/distributions/base.py @@ -7,7 +7,7 @@ from pip._internal.req import InstallRequirement if TYPE_CHECKING: - from pip._internal.index.package_finder import PackageFinder + from pip._internal.build_env import BuildEnvironmentInstaller class AbstractDistribution(metaclass=abc.ABCMeta): @@ -48,7 +48,7 @@ def get_metadata_distribution(self) -> BaseDistribution: @abc.abstractmethod def prepare_distribution_metadata( self, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> None: diff --git a/src/pip/_internal/distributions/installed.py b/src/pip/_internal/distributions/installed.py index 8ceed309446..b6a67df24f4 100644 --- a/src/pip/_internal/distributions/installed.py +++ b/src/pip/_internal/distributions/installed.py @@ -1,9 +1,13 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from pip._internal.distributions.base import AbstractDistribution -from pip._internal.index.package_finder import PackageFinder from pip._internal.metadata import BaseDistribution +if TYPE_CHECKING: + from pip._internal.build_env import BuildEnvironmentInstaller + class InstalledDistribution(AbstractDistribution): """Represents an installed package. @@ -22,7 +26,7 @@ def get_metadata_distribution(self) -> BaseDistribution: def prepare_distribution_metadata( self, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> None: diff --git a/src/pip/_internal/distributions/sdist.py b/src/pip/_internal/distributions/sdist.py index 67468955e4a..e2821f89e00 100644 --- a/src/pip/_internal/distributions/sdist.py +++ b/src/pip/_internal/distributions/sdist.py @@ -11,7 +11,7 @@ from pip._internal.utils.subprocess import runner_with_spinner_message if TYPE_CHECKING: - from pip._internal.index.package_finder import PackageFinder + from pip._internal.build_env import BuildEnvironmentInstaller logger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ def get_metadata_distribution(self) -> BaseDistribution: def prepare_distribution_metadata( self, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> None: @@ -46,7 +46,7 @@ def prepare_distribution_metadata( if should_isolate: # Setup an isolated environment and install the build backend static # requirements in it. - self._prepare_build_backend(finder) + self._prepare_build_backend(build_env_installer) # Check that if the requirement is editable, it either supports PEP 660 or # has a setup.py or a setup.cfg. This cannot be done earlier because we need # to setup the build backend to verify it supports build_editable, nor can @@ -56,7 +56,7 @@ def prepare_distribution_metadata( # without setup.py nor setup.cfg. self.req.isolated_editable_sanity_check() # Install the dynamic build requirements. - self._install_build_reqs(finder) + self._install_build_reqs(build_env_installer) # Check if the current environment provides build dependencies should_check_deps = self.req.use_pep517 and check_build_deps if should_check_deps: @@ -71,15 +71,17 @@ def prepare_distribution_metadata( self._raise_missing_reqs(missing) self.req.prepare_metadata() - def _prepare_build_backend(self, finder: PackageFinder) -> None: + def _prepare_build_backend( + self, build_env_installer: BuildEnvironmentInstaller + ) -> None: # Isolate in a BuildEnvironment and install the build-time # requirements. pyproject_requires = self.req.pyproject_requires assert pyproject_requires is not None - self.req.build_env = BuildEnvironment() + self.req.build_env = BuildEnvironment(build_env_installer) self.req.build_env.install_requirements( - finder, pyproject_requires, "overlay", kind="build dependencies" + pyproject_requires, "overlay", kind="build dependencies", for_req=self.req ) conflicting, missing = self.req.build_env.check_requirements( self.req.requirements_to_check @@ -115,7 +117,9 @@ def _get_build_requires_editable(self) -> Iterable[str]: with backend.subprocess_runner(runner): return backend.get_requires_for_build_editable() - def _install_build_reqs(self, finder: PackageFinder) -> None: + def _install_build_reqs( + self, build_env_installer: BuildEnvironmentInstaller + ) -> None: # Install any extra build dependencies that the backend requests. # This must be done in a second pass, as the pyproject.toml # dependencies must be installed before we can call the backend. @@ -131,7 +135,7 @@ def _install_build_reqs(self, finder: PackageFinder) -> None: if conflicting: self._raise_conflicts("the backend dependencies", conflicting) self.req.build_env.install_requirements( - finder, missing, "normal", kind="backend dependencies" + missing, "normal", kind="backend dependencies", for_req=self.req ) def _raise_conflicts( diff --git a/src/pip/_internal/distributions/wheel.py b/src/pip/_internal/distributions/wheel.py index 54e1e35f2c8..ee12bfadc2e 100644 --- a/src/pip/_internal/distributions/wheel.py +++ b/src/pip/_internal/distributions/wheel.py @@ -12,7 +12,7 @@ ) if TYPE_CHECKING: - from pip._internal.index.package_finder import PackageFinder + from pip._internal.build_env import BuildEnvironmentInstaller class WheelDistribution(AbstractDistribution): @@ -37,7 +37,7 @@ def get_metadata_distribution(self) -> BaseDistribution: def prepare_distribution_metadata( self, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> None: diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py index 0661897e944..09b7abae01d 100644 --- a/src/pip/_internal/exceptions.py +++ b/src/pip/_internal/exceptions.py @@ -14,7 +14,7 @@ import pathlib import re import sys -from collections.abc import Iterator +from collections.abc import Iterable, Iterator from itertools import chain, groupby, repeat from typing import TYPE_CHECKING, Literal @@ -865,3 +865,55 @@ def __init__(self) -> None: ), link="https://pip.pypa.io/en/stable/topics/dependency-resolution/#handling-resolution-too-deep-errors", ) + + +class BuildDependencyInstallError(DiagnosticPipError): + """Raised when build dependencies cannot be installed.""" + + reference = "failed-build-dependency-install" + + def __init__( + self, + req: InstallRequirement | None, + build_reqs: Iterable[str], + *, + cause: Exception, + log_lines: list[str], + ) -> None: + hint = None + + if not log_lines: + # No logs are available, they must have been printed earlier. + context = Text(f"ERROR: {cause}") + hint = "Look above for more details." + else: + if isinstance(cause, DiagnosticPipError): + hint = "Look above for the original error that caused this failure." + else: + log_lines.append(f"ERROR: {cause}") + + context = Text.assemble( + f"Installing {' '.join(build_reqs)}\n", + (f"[{len(log_lines)} lines of output]\n", "red"), + "\n".join(log_lines), + ("\n[end of output]", "red"), + ) + + message = Text("Cannot install build dependencies", "green") + if req: + message += Text(f" for {req}") + super().__init__(message=message, context=context, hint_stmt=hint) + + +class InstallWheelBuildError(DiagnosticPipError): + reference = "failed-wheel-build-for-install" + + def __init__(self, failed: list[InstallRequirement]) -> None: + super().__init__( + message=( + "Failed to build installable wheels for some " + "pyproject.toml based projects" + ), + context=", ".join(r.name for r in failed), # type: ignore + hint_stmt=None, + ) diff --git a/src/pip/_internal/metadata/__init__.py b/src/pip/_internal/metadata/__init__.py index 927e375cad0..d96ffe00900 100644 --- a/src/pip/_internal/metadata/__init__.py +++ b/src/pip/_internal/metadata/__init__.py @@ -4,6 +4,7 @@ import functools import os import sys +from collections.abc import Sequence from typing import Literal, Protocol, cast from pip._internal.utils.deprecation import deprecated @@ -112,7 +113,7 @@ def get_default_environment() -> BaseEnvironment: return select_backend().Environment.default() -def get_environment(paths: list[str] | None) -> BaseEnvironment: +def get_environment(paths: Sequence[str] | None) -> BaseEnvironment: """Get a representation of the environment specified by ``paths``. This returns an Environment instance from the chosen backend based on the diff --git a/src/pip/_internal/metadata/base.py b/src/pip/_internal/metadata/base.py index 230e11473c6..78eafe076fc 100644 --- a/src/pip/_internal/metadata/base.py +++ b/src/pip/_internal/metadata/base.py @@ -8,7 +8,7 @@ import pathlib import re import zipfile -from collections.abc import Collection, Container, Iterable, Iterator +from collections.abc import Collection, Container, Iterable, Iterator, Sequence from typing import ( IO, Any, @@ -584,7 +584,7 @@ def default(cls) -> BaseEnvironment: raise NotImplementedError() @classmethod - def from_paths(cls, paths: list[str] | None) -> BaseEnvironment: + def from_paths(cls, paths: Sequence[str] | None) -> BaseEnvironment: raise NotImplementedError() def get_distribution(self, name: str) -> BaseDistribution | None: diff --git a/src/pip/_internal/metadata/importlib/_envs.py b/src/pip/_internal/metadata/importlib/_envs.py index 71a73b7311f..97eff39d2e0 100644 --- a/src/pip/_internal/metadata/importlib/_envs.py +++ b/src/pip/_internal/metadata/importlib/_envs.py @@ -122,7 +122,7 @@ def default(cls) -> BaseEnvironment: return cls(sys.path) @classmethod - def from_paths(cls, paths: list[str] | None) -> BaseEnvironment: + def from_paths(cls, paths: Sequence[str] | None) -> BaseEnvironment: if paths is None: return cls(sys.path) return cls(paths) diff --git a/src/pip/_internal/metadata/pkg_resources.py b/src/pip/_internal/metadata/pkg_resources.py index 89fce8b6e5d..d685c299dee 100644 --- a/src/pip/_internal/metadata/pkg_resources.py +++ b/src/pip/_internal/metadata/pkg_resources.py @@ -5,7 +5,7 @@ import logging import os import zipfile -from collections.abc import Collection, Iterable, Iterator, Mapping +from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence from typing import ( NamedTuple, ) @@ -256,7 +256,7 @@ def default(cls) -> BaseEnvironment: return cls(pkg_resources.working_set) @classmethod - def from_paths(cls, paths: list[str] | None) -> BaseEnvironment: + def from_paths(cls, paths: Sequence[str] | None) -> BaseEnvironment: return cls(pkg_resources.WorkingSet(paths)) def _iter_distributions(self) -> Iterator[BaseDistribution]: diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 908331eba29..b408ce80550 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -13,6 +13,7 @@ from pip._vendor.packaging.utils import canonicalize_name +from pip._internal.build_env import BuildEnvironmentInstaller from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.distributions.installed import InstalledDistribution from pip._internal.exceptions import ( @@ -60,7 +61,7 @@ def _get_prepared_distribution( req: InstallRequirement, build_tracker: BuildTracker, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> BaseDistribution: @@ -70,7 +71,7 @@ def _get_prepared_distribution( if tracker_id is not None: with build_tracker.track(req, tracker_id): abstract_dist.prepare_distribution_metadata( - finder, build_isolation, check_build_deps + build_env_installer, build_isolation, check_build_deps ) return abstract_dist.get_metadata_distribution() @@ -220,12 +221,14 @@ def _check_download_dir( class RequirementPreparer: """Prepares a Requirement""" - def __init__( + def __init__( # noqa: PLR0913 (too many parameters) self, + *, build_dir: str, download_dir: str | None, src_dir: str, build_isolation: bool, + build_isolation_installer: BuildEnvironmentInstaller, check_build_deps: bool, build_tracker: BuildTracker, session: PipSession, @@ -253,6 +256,7 @@ def __init__( # Is build isolation allowed? self.build_isolation = build_isolation + self.build_env_installer = build_isolation_installer # Should check build dependencies? self.check_build_deps = check_build_deps @@ -644,7 +648,7 @@ def _prepare_linked_requirement( dist = _get_prepared_distribution( req, self.build_tracker, - self.finder, + self.build_env_installer, self.build_isolation, self.check_build_deps, ) @@ -700,7 +704,7 @@ def prepare_editable_requirement( dist = _get_prepared_distribution( req, self.build_tracker, - self.finder, + self.build_env_installer, self.build_isolation, self.check_build_deps, ) diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index 5cdbeb7f753..1c867aec99c 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -9,7 +9,7 @@ import threading from collections.abc import Generator from dataclasses import dataclass -from io import TextIOWrapper +from io import StringIO, TextIOWrapper from logging import Filter from typing import Any, ClassVar @@ -29,7 +29,7 @@ from pip._internal.utils._log import VERBOSE, getLogger from pip._internal.utils.compat import WINDOWS from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX -from pip._internal.utils.misc import ensure_dir +from pip._internal.utils.misc import StreamWrapper, ensure_dir _log_state = threading.local() _stdout_console = None @@ -56,6 +56,34 @@ def _is_broken_pipe_error(exc_class: type[BaseException], exc: BaseException) -> return isinstance(exc, OSError) and exc.errno in (errno.EINVAL, errno.EPIPE) +@contextlib.contextmanager +def capture_logging() -> Generator[StringIO, None, None]: + """Capture all pip logs in a buffer temporarily.""" + # Patching sys.std(out|err) directly is not viable as the caller + # may want to emit non-logging output (e.g. a rich spinner). To + # avoid capturing that, temporarily patch the root logging handlers + # to use new rich consoles that write to a StringIO. + handlers = {} + for handler in logging.getLogger().handlers: + if isinstance(handler, RichPipStreamHandler): + # Also store the handler's original console so it can be + # restored on context exit. + handlers[handler] = handler.console + + # HACK: grab no_color attribute from a random handler console since + # it's a global option anyway. + no_color = next(iter(handlers.values())).no_color + fake_stream = StreamWrapper.from_stream(sys.stdout) + fake_console = PipConsole(file=fake_stream, no_color=no_color, soft_wrap=True) + try: + for handler in handlers: + handler.console = fake_console + yield fake_stream + finally: + for handler, original_console in handlers.items(): + handler.console = original_console + + @contextlib.contextmanager def indent_log(num: int = 2) -> Generator[None, None, None]: """ diff --git a/tests/functional/test_build_env.py b/tests/functional/test_build_env.py index 1a1839b4027..ec1b59f068c 100644 --- a/tests/functional/test_build_env.py +++ b/tests/functional/test_build_env.py @@ -6,7 +6,11 @@ import pytest -from pip._internal.build_env import BuildEnvironment, _get_system_sitepackages +from pip._internal.build_env import ( + BuildEnvironment, + SubprocessBuildEnvironmentInstaller, + _get_system_sitepackages, +) from tests.lib import ( PipTestEnvironment, @@ -33,7 +37,10 @@ def run_with_build_env( import subprocess import sys - from pip._internal.build_env import BuildEnvironment + from pip._internal.build_env import ( + BuildEnvironment, + SubprocessBuildEnvironmentInstaller, + ) from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import PackageFinder from pip._internal.models.search_scope import SearchScope @@ -56,7 +63,9 @@ def run_with_build_env( ) with global_tempdir_manager(): - build_env = BuildEnvironment() + build_env = BuildEnvironment( + SubprocessBuildEnvironmentInstaller(finder) + ) """ ) + indent(dedent(setup_script_contents), " ") @@ -81,30 +90,26 @@ def run_with_build_env( def test_build_env_allow_empty_requirements_install() -> None: finder = make_test_finder() - build_env = BuildEnvironment() + build_env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) for prefix in ("normal", "overlay"): - build_env.install_requirements( - finder, [], prefix, kind="Installing build dependencies" - ) + build_env.install_requirements([], prefix, kind="Installing build dependencies") def test_build_env_allow_only_one_install(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "foo", "1.0") create_basic_wheel_for_package(script, "bar", "1.0") finder = make_test_finder(find_links=[os.fspath(script.scratch_path)]) - build_env = BuildEnvironment() + build_env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) for prefix in ("normal", "overlay"): build_env.install_requirements( - finder, ["foo"], prefix, kind=f"installing foo in {prefix}" + ["foo"], prefix, kind=f"installing foo in {prefix}" ) with pytest.raises(AssertionError): build_env.install_requirements( - finder, ["bar"], prefix, kind=f"installing bar in {prefix}" + ["bar"], prefix, kind=f"installing bar in {prefix}" ) with pytest.raises(AssertionError): - build_env.install_requirements( - finder, [], prefix, kind=f"installing in {prefix}" - ) + build_env.install_requirements([], prefix, kind=f"installing in {prefix}") def test_build_env_requirements_check(script: PipTestEnvironment) -> None: @@ -132,7 +137,7 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None: run_with_build_env( script, """ - build_env.install_requirements(finder, ['foo', 'bar==3.0'], 'normal', + build_env.install_requirements(['foo', 'bar==3.0'], 'normal', kind='installing foo in normal') r = build_env.check_requirements(['foo', 'bar', 'other']) @@ -149,9 +154,9 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None: run_with_build_env( script, """ - build_env.install_requirements(finder, ['foo', 'bar==3.0'], 'normal', + build_env.install_requirements(['foo', 'bar==3.0'], 'normal', kind='installing foo in normal') - build_env.install_requirements(finder, ['bar==1.0'], 'overlay', + build_env.install_requirements(['bar==1.0'], 'overlay', kind='installing foo in overlay') r = build_env.check_requirements(['foo', 'bar', 'other']) @@ -170,7 +175,6 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None: script, """ build_env.install_requirements( - finder, ["bar==3.0"], "normal", kind="installing bar in normal", @@ -193,9 +197,9 @@ def test_build_env_overlay_prefix_has_priority(script: PipTestEnvironment) -> No result = run_with_build_env( script, """ - build_env.install_requirements(finder, ['pkg==2.0'], 'overlay', + build_env.install_requirements(['pkg==2.0'], 'overlay', kind='installing pkg==2.0 in overlay') - build_env.install_requirements(finder, ['pkg==4.3'], 'normal', + build_env.install_requirements(['pkg==4.3'], 'normal', kind='installing pkg==4.3 in normal') """, """ diff --git a/tests/functional/test_pep517.py b/tests/functional/test_pep517.py index 9be3834241f..47dae96817f 100644 --- a/tests/functional/test_pep517.py +++ b/tests/functional/test_pep517.py @@ -7,7 +7,10 @@ import pytest import tomli_w -from pip._internal.build_env import BuildEnvironment +from pip._internal.build_env import ( + BuildEnvironment, + SubprocessBuildEnvironmentInstaller, +) from pip._internal.req import InstallRequirement from tests.lib import ( @@ -43,9 +46,9 @@ def test_backend(tmpdir: Path, data: TestData) -> None: req = InstallRequirement(None, None) req.source_dir = os.fspath(project_dir) # make req believe it has been unpacked req.load_pyproject_toml() - env = BuildEnvironment() finder = make_test_finder(find_links=[data.backends]) - env.install_requirements(finder, ["dummy_backend"], "normal", kind="Installing") + env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) + env.install_requirements(["dummy_backend"], "normal", kind="Installing") conflicting, missing = env.check_requirements(["dummy_backend"]) assert not conflicting assert not missing @@ -73,7 +76,7 @@ def test_backend_path(tmpdir: Path, data: TestData) -> None: req.source_dir = os.fspath(project_dir) # make req believe it has been unpacked req.load_pyproject_toml() - env = BuildEnvironment() + env = BuildEnvironment(object()) # type: ignore assert hasattr(req.pep517_backend, "build_wheel") with env: assert req.pep517_backend is not None @@ -91,9 +94,9 @@ def test_backend_path_and_dep(tmpdir: Path, data: TestData) -> None: req = InstallRequirement(None, None) req.source_dir = os.fspath(project_dir) # make req believe it has been unpacked req.load_pyproject_toml() - env = BuildEnvironment() finder = make_test_finder(find_links=[data.backends]) - env.install_requirements(finder, ["dummy_backend"], "normal", kind="Installing") + env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) + env.install_requirements(["dummy_backend"], "normal", kind="Installing") assert hasattr(req.pep517_backend, "build_wheel") with env: diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index ab9b8ea71c6..0547131134e 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -17,6 +17,7 @@ from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import Requirement +from pip._internal.build_env import SubprocessBuildEnvironmentInstaller from pip._internal.cache import WheelCache from pip._internal.commands import create_command from pip._internal.commands.install import InstallCommand @@ -97,11 +98,13 @@ def _basic_resolver( session = PipSession() with get_build_tracker() as tracker: + installer = SubprocessBuildEnvironmentInstaller(finder) preparer = RequirementPreparer( build_dir=os.path.join(self.tempdir, "build"), src_dir=os.path.join(self.tempdir, "src"), download_dir=None, build_isolation=True, + build_isolation_installer=installer, check_build_deps=False, build_tracker=tracker, session=session,