Skip to content

refactor: Add BuildEnvironmentInstaller protocol #13452

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 104 additions & 83 deletions src/pip/_internal/build_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from collections import OrderedDict
from collections.abc import Iterable
from types import TracebackType
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Protocol

from pip._vendor.packaging.version import Version

Expand All @@ -26,6 +26,7 @@

if TYPE_CHECKING:
from pip._internal.index.package_finder import PackageFinder
from pip._internal.req.req_install import InstallRequirement

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -79,10 +80,108 @@ def _get_system_sitepackages() -> set[str]:
return {os.path.normcase(path) for path in system_sites}


class BuildEnvironmentInstaller(Protocol):
"""
Interface for installing build dependencies into an isolated build
environment.
"""

def install(
self,
requirements: Iterable[str],
prefix: _Prefix,
*,
kind: str,
for_req: InstallRequirement | None,
) -> None: ...


class SubprocessBuildEnvironmentInstaller:
"""
Install build dependencies by calling pip in a subprocess.
"""

def __init__(self, finder: PackageFinder) -> None:
self.finder = finder

def install(
self,
requirements: Iterable[str],
prefix: _Prefix,
*,
kind: str,
for_req: InstallRequirement | None,
) -> None:
finder = self.finder
args: list[str] = [
sys.executable,
get_runnable_pip(),
"install",
"--ignore-installed",
"--no-user",
"--prefix",
prefix.path,
"--no-warn-script-location",
"--disable-pip-version-check",
# As the build environment is ephemeral, it's wasteful to
# pre-compile everything, especially as not every Python
# module will be used/compiled in most cases.
"--no-compile",
# The prefix specified two lines above, thus
# target from config file or env var should be ignored
"--target",
"",
]
if logger.getEffectiveLevel() <= logging.DEBUG:
args.append("-vv")
elif logger.getEffectiveLevel() <= VERBOSE:
args.append("-v")
for format_control in ("no_binary", "only_binary"):
formats = getattr(finder.format_control, format_control)
args.extend(
(
"--" + format_control.replace("_", "-"),
",".join(sorted(formats or {":none:"})),
)
)

index_urls = finder.index_urls
if index_urls:
args.extend(["-i", index_urls[0]])
for extra_index in index_urls[1:]:
args.extend(["--extra-index-url", extra_index])
else:
args.append("--no-index")
for link in finder.find_links:
args.extend(["--find-links", link])

if finder.proxy:
args.extend(["--proxy", finder.proxy])
for host in finder.trusted_hosts:
args.extend(["--trusted-host", host])
if finder.custom_cert:
args.extend(["--cert", finder.custom_cert])
if finder.client_cert:
args.extend(["--client-cert", finder.client_cert])
if finder.allow_all_prereleases:
args.append("--pre")
if finder.prefer_binary:
args.append("--prefer-binary")
args.append("--")
args.extend(requirements)
with open_spinner(f"Installing {kind}") as spinner:
call_subprocess(
args,
command_desc=f"pip subprocess to install {kind}",
spinner=spinner,
)


class BuildEnvironment:
"""Creates and manages an isolated environment to install build deps"""

def __init__(self) -> None:
def __init__(self, installer: BuildEnvironmentInstaller) -> None:
self.installer = installer
temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True)

self._prefixes = OrderedDict(
Expand Down Expand Up @@ -205,96 +304,18 @@ def check_requirements(

def install_requirements(
self,
finder: PackageFinder,
requirements: Iterable[str],
prefix_as_string: str,
*,
kind: str,
for_req: InstallRequirement | None = None,
) -> None:
prefix = self._prefixes[prefix_as_string]
assert not prefix.setup
prefix.setup = True
if not requirements:
return
self._install_requirements(
get_runnable_pip(),
finder,
requirements,
prefix,
kind=kind,
)

@staticmethod
def _install_requirements(
pip_runnable: str,
finder: PackageFinder,
requirements: Iterable[str],
prefix: _Prefix,
*,
kind: str,
) -> None:
args: list[str] = [
sys.executable,
pip_runnable,
"install",
"--ignore-installed",
"--no-user",
"--prefix",
prefix.path,
"--no-warn-script-location",
"--disable-pip-version-check",
# As the build environment is ephemeral, it's wasteful to
# pre-compile everything, especially as not every Python
# module will be used/compiled in most cases.
"--no-compile",
# The prefix specified two lines above, thus
# target from config file or env var should be ignored
"--target",
"",
]
if logger.getEffectiveLevel() <= logging.DEBUG:
args.append("-vv")
elif logger.getEffectiveLevel() <= VERBOSE:
args.append("-v")
for format_control in ("no_binary", "only_binary"):
formats = getattr(finder.format_control, format_control)
args.extend(
(
"--" + format_control.replace("_", "-"),
",".join(sorted(formats or {":none:"})),
)
)

index_urls = finder.index_urls
if index_urls:
args.extend(["-i", index_urls[0]])
for extra_index in index_urls[1:]:
args.extend(["--extra-index-url", extra_index])
else:
args.append("--no-index")
for link in finder.find_links:
args.extend(["--find-links", link])

if finder.proxy:
args.extend(["--proxy", finder.proxy])
for host in finder.trusted_hosts:
args.extend(["--trusted-host", host])
if finder.custom_cert:
args.extend(["--cert", finder.custom_cert])
if finder.client_cert:
args.extend(["--client-cert", finder.client_cert])
if finder.allow_all_prereleases:
args.append("--pre")
if finder.prefer_binary:
args.append("--prefer-binary")
args.append("--")
args.extend(requirements)
with open_spinner(f"Installing {kind}") as spinner:
call_subprocess(
args,
command_desc=f"pip subprocess to install {kind}",
spinner=spinner,
)
self.installer.install(requirements, prefix, kind=kind, for_req=for_req)


class NoOpBuildEnvironment(BuildEnvironment):
Expand All @@ -319,10 +340,10 @@ def cleanup(self) -> None:

def install_requirements(
self,
finder: PackageFinder,
requirements: Iterable[str],
prefix_as_string: str,
*,
kind: str,
for_req: InstallRequirement | None = None,
) -> None:
raise NotImplementedError()
2 changes: 2 additions & 0 deletions src/pip/_internal/cli/req_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from optparse import Values
from typing import Any

from pip._internal.build_env import SubprocessBuildEnvironmentInstaller
from pip._internal.cache import WheelCache
from pip._internal.cli import cmdoptions
from pip._internal.cli.index_command import IndexGroupCommand
Expand Down Expand Up @@ -136,6 +137,7 @@ def make_requirement_preparer(
src_dir=options.src_dir,
download_dir=download_dir,
build_isolation=options.build_isolation,
build_isolation_installer=SubprocessBuildEnvironmentInstaller(finder),
check_build_deps=options.check_build_deps,
build_tracker=build_tracker,
session=session,
Expand Down
4 changes: 2 additions & 2 deletions src/pip/_internal/distributions/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from pip._internal.req import InstallRequirement

if TYPE_CHECKING:
from pip._internal.index.package_finder import PackageFinder
from pip._internal.build_env import BuildEnvironmentInstaller


class AbstractDistribution(metaclass=abc.ABCMeta):
Expand Down Expand Up @@ -48,7 +48,7 @@ def get_metadata_distribution(self) -> BaseDistribution:
@abc.abstractmethod
def prepare_distribution_metadata(
self,
finder: PackageFinder,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
check_build_deps: bool,
) -> None:
Expand Down
8 changes: 6 additions & 2 deletions src/pip/_internal/distributions/installed.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
from __future__ import annotations

from typing import TYPE_CHECKING

from pip._internal.distributions.base import AbstractDistribution
from pip._internal.index.package_finder import PackageFinder
from pip._internal.metadata import BaseDistribution

if TYPE_CHECKING:
from pip._internal.build_env import BuildEnvironmentInstaller


class InstalledDistribution(AbstractDistribution):
"""Represents an installed package.
Expand All @@ -22,7 +26,7 @@ def get_metadata_distribution(self) -> BaseDistribution:

def prepare_distribution_metadata(
self,
finder: PackageFinder,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
check_build_deps: bool,
) -> None:
Expand Down
22 changes: 13 additions & 9 deletions src/pip/_internal/distributions/sdist.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from pip._internal.utils.subprocess import runner_with_spinner_message

if TYPE_CHECKING:
from pip._internal.index.package_finder import PackageFinder
from pip._internal.build_env import BuildEnvironmentInstaller

logger = logging.getLogger(__name__)

Expand All @@ -34,7 +34,7 @@ def get_metadata_distribution(self) -> BaseDistribution:

def prepare_distribution_metadata(
self,
finder: PackageFinder,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
check_build_deps: bool,
) -> None:
Expand All @@ -46,7 +46,7 @@ def prepare_distribution_metadata(
if should_isolate:
# Setup an isolated environment and install the build backend static
# requirements in it.
self._prepare_build_backend(finder)
self._prepare_build_backend(build_env_installer)
# Check that if the requirement is editable, it either supports PEP 660 or
# has a setup.py or a setup.cfg. This cannot be done earlier because we need
# to setup the build backend to verify it supports build_editable, nor can
Expand All @@ -56,7 +56,7 @@ def prepare_distribution_metadata(
# without setup.py nor setup.cfg.
self.req.isolated_editable_sanity_check()
# Install the dynamic build requirements.
self._install_build_reqs(finder)
self._install_build_reqs(build_env_installer)
# Check if the current environment provides build dependencies
should_check_deps = self.req.use_pep517 and check_build_deps
if should_check_deps:
Expand All @@ -71,15 +71,17 @@ def prepare_distribution_metadata(
self._raise_missing_reqs(missing)
self.req.prepare_metadata()

def _prepare_build_backend(self, finder: PackageFinder) -> None:
def _prepare_build_backend(
self, build_env_installer: BuildEnvironmentInstaller
) -> None:
# Isolate in a BuildEnvironment and install the build-time
# requirements.
pyproject_requires = self.req.pyproject_requires
assert pyproject_requires is not None

self.req.build_env = BuildEnvironment()
self.req.build_env = BuildEnvironment(build_env_installer)
self.req.build_env.install_requirements(
finder, pyproject_requires, "overlay", kind="build dependencies"
pyproject_requires, "overlay", kind="build dependencies", for_req=self.req
)
conflicting, missing = self.req.build_env.check_requirements(
self.req.requirements_to_check
Expand Down Expand Up @@ -115,7 +117,9 @@ def _get_build_requires_editable(self) -> Iterable[str]:
with backend.subprocess_runner(runner):
return backend.get_requires_for_build_editable()

def _install_build_reqs(self, finder: PackageFinder) -> None:
def _install_build_reqs(
self, build_env_installer: BuildEnvironmentInstaller
) -> None:
# Install any extra build dependencies that the backend requests.
# This must be done in a second pass, as the pyproject.toml
# dependencies must be installed before we can call the backend.
Expand All @@ -131,7 +135,7 @@ def _install_build_reqs(self, finder: PackageFinder) -> None:
if conflicting:
self._raise_conflicts("the backend dependencies", conflicting)
self.req.build_env.install_requirements(
finder, missing, "normal", kind="backend dependencies"
missing, "normal", kind="backend dependencies", for_req=self.req
)

def _raise_conflicts(
Expand Down
4 changes: 2 additions & 2 deletions src/pip/_internal/distributions/wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
)

if TYPE_CHECKING:
from pip._internal.index.package_finder import PackageFinder
from pip._internal.build_env import BuildEnvironmentInstaller


class WheelDistribution(AbstractDistribution):
Expand All @@ -37,7 +37,7 @@ def get_metadata_distribution(self) -> BaseDistribution:

def prepare_distribution_metadata(
self,
finder: PackageFinder,
build_env_installer: BuildEnvironmentInstaller,
build_isolation: bool,
check_build_deps: bool,
) -> None:
Expand Down
Loading