diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 69a2e72ce42..3a246a1e349 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -11,7 +11,7 @@ from collections import OrderedDict from collections.abc import Iterable from types import TracebackType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol from pip._vendor.packaging.version import Version @@ -26,6 +26,7 @@ if TYPE_CHECKING: from pip._internal.index.package_finder import PackageFinder + from pip._internal.req.req_install import InstallRequirement logger = logging.getLogger(__name__) @@ -79,10 +80,108 @@ def _get_system_sitepackages() -> set[str]: return {os.path.normcase(path) for path in system_sites} +class BuildEnvironmentInstaller(Protocol): + """ + Interface for installing build dependencies into an isolated build + environment. + """ + + def install( + self, + requirements: Iterable[str], + prefix: _Prefix, + *, + kind: str, + for_req: InstallRequirement | None, + ) -> None: ... + + +class SubprocessBuildEnvironmentInstaller: + """ + Install build dependencies by calling pip in a subprocess. + """ + + def __init__(self, finder: PackageFinder) -> None: + self.finder = finder + + def install( + self, + requirements: Iterable[str], + prefix: _Prefix, + *, + kind: str, + for_req: InstallRequirement | None, + ) -> None: + finder = self.finder + args: list[str] = [ + sys.executable, + get_runnable_pip(), + "install", + "--ignore-installed", + "--no-user", + "--prefix", + prefix.path, + "--no-warn-script-location", + "--disable-pip-version-check", + # As the build environment is ephemeral, it's wasteful to + # pre-compile everything, especially as not every Python + # module will be used/compiled in most cases. + "--no-compile", + # The prefix specified two lines above, thus + # target from config file or env var should be ignored + "--target", + "", + ] + if logger.getEffectiveLevel() <= logging.DEBUG: + args.append("-vv") + elif logger.getEffectiveLevel() <= VERBOSE: + args.append("-v") + for format_control in ("no_binary", "only_binary"): + formats = getattr(finder.format_control, format_control) + args.extend( + ( + "--" + format_control.replace("_", "-"), + ",".join(sorted(formats or {":none:"})), + ) + ) + + index_urls = finder.index_urls + if index_urls: + args.extend(["-i", index_urls[0]]) + for extra_index in index_urls[1:]: + args.extend(["--extra-index-url", extra_index]) + else: + args.append("--no-index") + for link in finder.find_links: + args.extend(["--find-links", link]) + + if finder.proxy: + args.extend(["--proxy", finder.proxy]) + for host in finder.trusted_hosts: + args.extend(["--trusted-host", host]) + if finder.custom_cert: + args.extend(["--cert", finder.custom_cert]) + if finder.client_cert: + args.extend(["--client-cert", finder.client_cert]) + if finder.allow_all_prereleases: + args.append("--pre") + if finder.prefer_binary: + args.append("--prefer-binary") + args.append("--") + args.extend(requirements) + with open_spinner(f"Installing {kind}") as spinner: + call_subprocess( + args, + command_desc=f"pip subprocess to install {kind}", + spinner=spinner, + ) + + class BuildEnvironment: """Creates and manages an isolated environment to install build deps""" - def __init__(self) -> None: + def __init__(self, installer: BuildEnvironmentInstaller) -> None: + self.installer = installer temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True) self._prefixes = OrderedDict( @@ -205,96 +304,18 @@ def check_requirements( def install_requirements( self, - finder: PackageFinder, requirements: Iterable[str], prefix_as_string: str, *, kind: str, + for_req: InstallRequirement | None = None, ) -> None: prefix = self._prefixes[prefix_as_string] assert not prefix.setup prefix.setup = True if not requirements: return - self._install_requirements( - get_runnable_pip(), - finder, - requirements, - prefix, - kind=kind, - ) - - @staticmethod - def _install_requirements( - pip_runnable: str, - finder: PackageFinder, - requirements: Iterable[str], - prefix: _Prefix, - *, - kind: str, - ) -> None: - args: list[str] = [ - sys.executable, - pip_runnable, - "install", - "--ignore-installed", - "--no-user", - "--prefix", - prefix.path, - "--no-warn-script-location", - "--disable-pip-version-check", - # As the build environment is ephemeral, it's wasteful to - # pre-compile everything, especially as not every Python - # module will be used/compiled in most cases. - "--no-compile", - # The prefix specified two lines above, thus - # target from config file or env var should be ignored - "--target", - "", - ] - if logger.getEffectiveLevel() <= logging.DEBUG: - args.append("-vv") - elif logger.getEffectiveLevel() <= VERBOSE: - args.append("-v") - for format_control in ("no_binary", "only_binary"): - formats = getattr(finder.format_control, format_control) - args.extend( - ( - "--" + format_control.replace("_", "-"), - ",".join(sorted(formats or {":none:"})), - ) - ) - - index_urls = finder.index_urls - if index_urls: - args.extend(["-i", index_urls[0]]) - for extra_index in index_urls[1:]: - args.extend(["--extra-index-url", extra_index]) - else: - args.append("--no-index") - for link in finder.find_links: - args.extend(["--find-links", link]) - - if finder.proxy: - args.extend(["--proxy", finder.proxy]) - for host in finder.trusted_hosts: - args.extend(["--trusted-host", host]) - if finder.custom_cert: - args.extend(["--cert", finder.custom_cert]) - if finder.client_cert: - args.extend(["--client-cert", finder.client_cert]) - if finder.allow_all_prereleases: - args.append("--pre") - if finder.prefer_binary: - args.append("--prefer-binary") - args.append("--") - args.extend(requirements) - with open_spinner(f"Installing {kind}") as spinner: - call_subprocess( - args, - command_desc=f"pip subprocess to install {kind}", - spinner=spinner, - ) + self.installer.install(requirements, prefix, kind=kind, for_req=for_req) class NoOpBuildEnvironment(BuildEnvironment): @@ -319,10 +340,10 @@ def cleanup(self) -> None: def install_requirements( self, - finder: PackageFinder, requirements: Iterable[str], prefix_as_string: str, *, kind: str, + for_req: InstallRequirement | None = None, ) -> None: raise NotImplementedError() diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 27255098830..dc1328ff019 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -12,6 +12,7 @@ from optparse import Values from typing import Any +from pip._internal.build_env import SubprocessBuildEnvironmentInstaller from pip._internal.cache import WheelCache from pip._internal.cli import cmdoptions from pip._internal.cli.index_command import IndexGroupCommand @@ -136,6 +137,7 @@ def make_requirement_preparer( src_dir=options.src_dir, download_dir=download_dir, build_isolation=options.build_isolation, + build_isolation_installer=SubprocessBuildEnvironmentInstaller(finder), check_build_deps=options.check_build_deps, build_tracker=build_tracker, session=session, diff --git a/src/pip/_internal/distributions/base.py b/src/pip/_internal/distributions/base.py index 6ebf49ad501..ea61f3501e7 100644 --- a/src/pip/_internal/distributions/base.py +++ b/src/pip/_internal/distributions/base.py @@ -7,7 +7,7 @@ from pip._internal.req import InstallRequirement if TYPE_CHECKING: - from pip._internal.index.package_finder import PackageFinder + from pip._internal.build_env import BuildEnvironmentInstaller class AbstractDistribution(metaclass=abc.ABCMeta): @@ -48,7 +48,7 @@ def get_metadata_distribution(self) -> BaseDistribution: @abc.abstractmethod def prepare_distribution_metadata( self, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> None: diff --git a/src/pip/_internal/distributions/installed.py b/src/pip/_internal/distributions/installed.py index 8ceed309446..b6a67df24f4 100644 --- a/src/pip/_internal/distributions/installed.py +++ b/src/pip/_internal/distributions/installed.py @@ -1,9 +1,13 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from pip._internal.distributions.base import AbstractDistribution -from pip._internal.index.package_finder import PackageFinder from pip._internal.metadata import BaseDistribution +if TYPE_CHECKING: + from pip._internal.build_env import BuildEnvironmentInstaller + class InstalledDistribution(AbstractDistribution): """Represents an installed package. @@ -22,7 +26,7 @@ def get_metadata_distribution(self) -> BaseDistribution: def prepare_distribution_metadata( self, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> None: diff --git a/src/pip/_internal/distributions/sdist.py b/src/pip/_internal/distributions/sdist.py index 67468955e4a..e2821f89e00 100644 --- a/src/pip/_internal/distributions/sdist.py +++ b/src/pip/_internal/distributions/sdist.py @@ -11,7 +11,7 @@ from pip._internal.utils.subprocess import runner_with_spinner_message if TYPE_CHECKING: - from pip._internal.index.package_finder import PackageFinder + from pip._internal.build_env import BuildEnvironmentInstaller logger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ def get_metadata_distribution(self) -> BaseDistribution: def prepare_distribution_metadata( self, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> None: @@ -46,7 +46,7 @@ def prepare_distribution_metadata( if should_isolate: # Setup an isolated environment and install the build backend static # requirements in it. - self._prepare_build_backend(finder) + self._prepare_build_backend(build_env_installer) # Check that if the requirement is editable, it either supports PEP 660 or # has a setup.py or a setup.cfg. This cannot be done earlier because we need # to setup the build backend to verify it supports build_editable, nor can @@ -56,7 +56,7 @@ def prepare_distribution_metadata( # without setup.py nor setup.cfg. self.req.isolated_editable_sanity_check() # Install the dynamic build requirements. - self._install_build_reqs(finder) + self._install_build_reqs(build_env_installer) # Check if the current environment provides build dependencies should_check_deps = self.req.use_pep517 and check_build_deps if should_check_deps: @@ -71,15 +71,17 @@ def prepare_distribution_metadata( self._raise_missing_reqs(missing) self.req.prepare_metadata() - def _prepare_build_backend(self, finder: PackageFinder) -> None: + def _prepare_build_backend( + self, build_env_installer: BuildEnvironmentInstaller + ) -> None: # Isolate in a BuildEnvironment and install the build-time # requirements. pyproject_requires = self.req.pyproject_requires assert pyproject_requires is not None - self.req.build_env = BuildEnvironment() + self.req.build_env = BuildEnvironment(build_env_installer) self.req.build_env.install_requirements( - finder, pyproject_requires, "overlay", kind="build dependencies" + pyproject_requires, "overlay", kind="build dependencies", for_req=self.req ) conflicting, missing = self.req.build_env.check_requirements( self.req.requirements_to_check @@ -115,7 +117,9 @@ def _get_build_requires_editable(self) -> Iterable[str]: with backend.subprocess_runner(runner): return backend.get_requires_for_build_editable() - def _install_build_reqs(self, finder: PackageFinder) -> None: + def _install_build_reqs( + self, build_env_installer: BuildEnvironmentInstaller + ) -> None: # Install any extra build dependencies that the backend requests. # This must be done in a second pass, as the pyproject.toml # dependencies must be installed before we can call the backend. @@ -131,7 +135,7 @@ def _install_build_reqs(self, finder: PackageFinder) -> None: if conflicting: self._raise_conflicts("the backend dependencies", conflicting) self.req.build_env.install_requirements( - finder, missing, "normal", kind="backend dependencies" + missing, "normal", kind="backend dependencies", for_req=self.req ) def _raise_conflicts( diff --git a/src/pip/_internal/distributions/wheel.py b/src/pip/_internal/distributions/wheel.py index 54e1e35f2c8..ee12bfadc2e 100644 --- a/src/pip/_internal/distributions/wheel.py +++ b/src/pip/_internal/distributions/wheel.py @@ -12,7 +12,7 @@ ) if TYPE_CHECKING: - from pip._internal.index.package_finder import PackageFinder + from pip._internal.build_env import BuildEnvironmentInstaller class WheelDistribution(AbstractDistribution): @@ -37,7 +37,7 @@ def get_metadata_distribution(self) -> BaseDistribution: def prepare_distribution_metadata( self, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> None: diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 908331eba29..b408ce80550 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -13,6 +13,7 @@ from pip._vendor.packaging.utils import canonicalize_name +from pip._internal.build_env import BuildEnvironmentInstaller from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.distributions.installed import InstalledDistribution from pip._internal.exceptions import ( @@ -60,7 +61,7 @@ def _get_prepared_distribution( req: InstallRequirement, build_tracker: BuildTracker, - finder: PackageFinder, + build_env_installer: BuildEnvironmentInstaller, build_isolation: bool, check_build_deps: bool, ) -> BaseDistribution: @@ -70,7 +71,7 @@ def _get_prepared_distribution( if tracker_id is not None: with build_tracker.track(req, tracker_id): abstract_dist.prepare_distribution_metadata( - finder, build_isolation, check_build_deps + build_env_installer, build_isolation, check_build_deps ) return abstract_dist.get_metadata_distribution() @@ -220,12 +221,14 @@ def _check_download_dir( class RequirementPreparer: """Prepares a Requirement""" - def __init__( + def __init__( # noqa: PLR0913 (too many parameters) self, + *, build_dir: str, download_dir: str | None, src_dir: str, build_isolation: bool, + build_isolation_installer: BuildEnvironmentInstaller, check_build_deps: bool, build_tracker: BuildTracker, session: PipSession, @@ -253,6 +256,7 @@ def __init__( # Is build isolation allowed? self.build_isolation = build_isolation + self.build_env_installer = build_isolation_installer # Should check build dependencies? self.check_build_deps = check_build_deps @@ -644,7 +648,7 @@ def _prepare_linked_requirement( dist = _get_prepared_distribution( req, self.build_tracker, - self.finder, + self.build_env_installer, self.build_isolation, self.check_build_deps, ) @@ -700,7 +704,7 @@ def prepare_editable_requirement( dist = _get_prepared_distribution( req, self.build_tracker, - self.finder, + self.build_env_installer, self.build_isolation, self.check_build_deps, ) diff --git a/tests/functional/test_build_env.py b/tests/functional/test_build_env.py index 1a1839b4027..ec1b59f068c 100644 --- a/tests/functional/test_build_env.py +++ b/tests/functional/test_build_env.py @@ -6,7 +6,11 @@ import pytest -from pip._internal.build_env import BuildEnvironment, _get_system_sitepackages +from pip._internal.build_env import ( + BuildEnvironment, + SubprocessBuildEnvironmentInstaller, + _get_system_sitepackages, +) from tests.lib import ( PipTestEnvironment, @@ -33,7 +37,10 @@ def run_with_build_env( import subprocess import sys - from pip._internal.build_env import BuildEnvironment + from pip._internal.build_env import ( + BuildEnvironment, + SubprocessBuildEnvironmentInstaller, + ) from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import PackageFinder from pip._internal.models.search_scope import SearchScope @@ -56,7 +63,9 @@ def run_with_build_env( ) with global_tempdir_manager(): - build_env = BuildEnvironment() + build_env = BuildEnvironment( + SubprocessBuildEnvironmentInstaller(finder) + ) """ ) + indent(dedent(setup_script_contents), " ") @@ -81,30 +90,26 @@ def run_with_build_env( def test_build_env_allow_empty_requirements_install() -> None: finder = make_test_finder() - build_env = BuildEnvironment() + build_env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) for prefix in ("normal", "overlay"): - build_env.install_requirements( - finder, [], prefix, kind="Installing build dependencies" - ) + build_env.install_requirements([], prefix, kind="Installing build dependencies") def test_build_env_allow_only_one_install(script: PipTestEnvironment) -> None: create_basic_wheel_for_package(script, "foo", "1.0") create_basic_wheel_for_package(script, "bar", "1.0") finder = make_test_finder(find_links=[os.fspath(script.scratch_path)]) - build_env = BuildEnvironment() + build_env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) for prefix in ("normal", "overlay"): build_env.install_requirements( - finder, ["foo"], prefix, kind=f"installing foo in {prefix}" + ["foo"], prefix, kind=f"installing foo in {prefix}" ) with pytest.raises(AssertionError): build_env.install_requirements( - finder, ["bar"], prefix, kind=f"installing bar in {prefix}" + ["bar"], prefix, kind=f"installing bar in {prefix}" ) with pytest.raises(AssertionError): - build_env.install_requirements( - finder, [], prefix, kind=f"installing in {prefix}" - ) + build_env.install_requirements([], prefix, kind=f"installing in {prefix}") def test_build_env_requirements_check(script: PipTestEnvironment) -> None: @@ -132,7 +137,7 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None: run_with_build_env( script, """ - build_env.install_requirements(finder, ['foo', 'bar==3.0'], 'normal', + build_env.install_requirements(['foo', 'bar==3.0'], 'normal', kind='installing foo in normal') r = build_env.check_requirements(['foo', 'bar', 'other']) @@ -149,9 +154,9 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None: run_with_build_env( script, """ - build_env.install_requirements(finder, ['foo', 'bar==3.0'], 'normal', + build_env.install_requirements(['foo', 'bar==3.0'], 'normal', kind='installing foo in normal') - build_env.install_requirements(finder, ['bar==1.0'], 'overlay', + build_env.install_requirements(['bar==1.0'], 'overlay', kind='installing foo in overlay') r = build_env.check_requirements(['foo', 'bar', 'other']) @@ -170,7 +175,6 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None: script, """ build_env.install_requirements( - finder, ["bar==3.0"], "normal", kind="installing bar in normal", @@ -193,9 +197,9 @@ def test_build_env_overlay_prefix_has_priority(script: PipTestEnvironment) -> No result = run_with_build_env( script, """ - build_env.install_requirements(finder, ['pkg==2.0'], 'overlay', + build_env.install_requirements(['pkg==2.0'], 'overlay', kind='installing pkg==2.0 in overlay') - build_env.install_requirements(finder, ['pkg==4.3'], 'normal', + build_env.install_requirements(['pkg==4.3'], 'normal', kind='installing pkg==4.3 in normal') """, """ diff --git a/tests/functional/test_pep517.py b/tests/functional/test_pep517.py index 9be3834241f..47dae96817f 100644 --- a/tests/functional/test_pep517.py +++ b/tests/functional/test_pep517.py @@ -7,7 +7,10 @@ import pytest import tomli_w -from pip._internal.build_env import BuildEnvironment +from pip._internal.build_env import ( + BuildEnvironment, + SubprocessBuildEnvironmentInstaller, +) from pip._internal.req import InstallRequirement from tests.lib import ( @@ -43,9 +46,9 @@ def test_backend(tmpdir: Path, data: TestData) -> None: req = InstallRequirement(None, None) req.source_dir = os.fspath(project_dir) # make req believe it has been unpacked req.load_pyproject_toml() - env = BuildEnvironment() finder = make_test_finder(find_links=[data.backends]) - env.install_requirements(finder, ["dummy_backend"], "normal", kind="Installing") + env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) + env.install_requirements(["dummy_backend"], "normal", kind="Installing") conflicting, missing = env.check_requirements(["dummy_backend"]) assert not conflicting assert not missing @@ -73,7 +76,7 @@ def test_backend_path(tmpdir: Path, data: TestData) -> None: req.source_dir = os.fspath(project_dir) # make req believe it has been unpacked req.load_pyproject_toml() - env = BuildEnvironment() + env = BuildEnvironment(object()) # type: ignore assert hasattr(req.pep517_backend, "build_wheel") with env: assert req.pep517_backend is not None @@ -91,9 +94,9 @@ def test_backend_path_and_dep(tmpdir: Path, data: TestData) -> None: req = InstallRequirement(None, None) req.source_dir = os.fspath(project_dir) # make req believe it has been unpacked req.load_pyproject_toml() - env = BuildEnvironment() finder = make_test_finder(find_links=[data.backends]) - env.install_requirements(finder, ["dummy_backend"], "normal", kind="Installing") + env = BuildEnvironment(SubprocessBuildEnvironmentInstaller(finder)) + env.install_requirements(["dummy_backend"], "normal", kind="Installing") assert hasattr(req.pep517_backend, "build_wheel") with env: diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index ab9b8ea71c6..0547131134e 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -17,6 +17,7 @@ from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import Requirement +from pip._internal.build_env import SubprocessBuildEnvironmentInstaller from pip._internal.cache import WheelCache from pip._internal.commands import create_command from pip._internal.commands.install import InstallCommand @@ -97,11 +98,13 @@ def _basic_resolver( session = PipSession() with get_build_tracker() as tracker: + installer = SubprocessBuildEnvironmentInstaller(finder) preparer = RequirementPreparer( build_dir=os.path.join(self.tempdir, "build"), src_dir=os.path.join(self.tempdir, "src"), download_dir=None, build_isolation=True, + build_isolation_installer=installer, check_build_deps=False, build_tracker=tracker, session=session,