From 033576faa92bf0f43fa7017a7a6068b846fbb0ab Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 23 Aug 2021 04:08:33 +0800 Subject: [PATCH] Remove direct pkg_resources usages in req_install This also by extension converts req_uninstall since the two are very much coupled together. --- src/pip/_internal/commands/show.py | 63 +----- src/pip/_internal/distributions/sdist.py | 4 +- src/pip/_internal/distributions/wheel.py | 4 +- src/pip/_internal/index/package_finder.py | 2 +- src/pip/_internal/metadata/__init__.py | 13 +- src/pip/_internal/metadata/base.py | 91 +++++++- src/pip/_internal/metadata/pkg_resources.py | 37 +++ src/pip/_internal/network/lazy_wheel.py | 8 +- src/pip/_internal/operations/install/wheel.py | 4 +- src/pip/_internal/req/req_install.py | 74 ++---- src/pip/_internal/req/req_uninstall.py | 212 +++++++++--------- .../resolution/resolvelib/candidates.py | 4 +- src/pip/_internal/wheel_builder.py | 4 +- tests/lib/wheel.py | 8 +- tests/unit/test_finder.py | 25 +-- tests/unit/test_req.py | 4 +- tests/unit/test_req_uninstall.py | 13 +- 17 files changed, 295 insertions(+), 275 deletions(-) diff --git a/src/pip/_internal/commands/show.py b/src/pip/_internal/commands/show.py index 0bbe1209afe..629b24d1004 100644 --- a/src/pip/_internal/commands/show.py +++ b/src/pip/_internal/commands/show.py @@ -1,8 +1,6 @@ -import csv import logging -import pathlib from optparse import Values -from typing import Iterator, List, NamedTuple, Optional, Tuple +from typing import Iterator, List, NamedTuple, Optional from pip._vendor.packaging.utils import canonicalize_name @@ -69,33 +67,6 @@ class _PackageInfo(NamedTuple): files: Optional[List[str]] -def _convert_legacy_entry(entry: Tuple[str, ...], info: Tuple[str, ...]) -> str: - """Convert a legacy installed-files.txt path into modern RECORD path. - - The legacy format stores paths relative to the info directory, while the - modern format stores paths relative to the package root, e.g. the - site-packages directory. - - :param entry: Path parts of the installed-files.txt entry. - :param info: Path parts of the egg-info directory relative to package root. - :returns: The converted entry. - - For best compatibility with symlinks, this does not use ``abspath()`` or - ``Path.resolve()``, but tries to work with path parts: - - 1. While ``entry`` starts with ``..``, remove the equal amounts of parts - from ``info``; if ``info`` is empty, start appending ``..`` instead. - 2. Join the two directly. - """ - while entry and entry[0] == "..": - if not info or info[-1] == "..": - info += ("..",) - else: - info = info[:-1] - entry = entry[1:] - return str(pathlib.Path(*info, *entry)) - - def search_packages_info(query: List[str]) -> Iterator[_PackageInfo]: """ Gather details from installed distributions. Print distribution name, @@ -121,34 +92,6 @@ def _get_requiring_packages(current_dist: BaseDistribution) -> List[str]: in {canonicalize_name(d.name) for d in dist.iter_dependencies()} ] - def _files_from_record(dist: BaseDistribution) -> Optional[Iterator[str]]: - try: - text = dist.read_text("RECORD") - except FileNotFoundError: - return None - # This extra Path-str cast normalizes entries. - return (str(pathlib.Path(row[0])) for row in csv.reader(text.splitlines())) - - def _files_from_legacy(dist: BaseDistribution) -> Optional[Iterator[str]]: - try: - text = dist.read_text("installed-files.txt") - except FileNotFoundError: - return None - paths = (p for p in text.splitlines(keepends=False) if p) - root = dist.location - info = dist.info_directory - if root is None or info is None: - return paths - try: - info_rel = pathlib.Path(info).relative_to(root) - except ValueError: # info is not relative to root. - return paths - if not info_rel.parts: # info *is* root. - return paths - return ( - _convert_legacy_entry(pathlib.Path(p).parts, info_rel.parts) for p in paths - ) - for query_name in query_names: try: dist = installed[query_name] @@ -161,11 +104,11 @@ def _files_from_legacy(dist: BaseDistribution) -> Optional[Iterator[str]]: except FileNotFoundError: entry_points = [] - files_iter = _files_from_record(dist) or _files_from_legacy(dist) + files_iter = dist.iter_files() if files_iter is None: files: Optional[List[str]] = None else: - files = sorted(files_iter) + files = sorted(str(f) for f in files_iter) metadata = dist.metadata diff --git a/src/pip/_internal/distributions/sdist.py b/src/pip/_internal/distributions/sdist.py index 573fefae7ba..b347ecf04d8 100644 --- a/src/pip/_internal/distributions/sdist.py +++ b/src/pip/_internal/distributions/sdist.py @@ -19,9 +19,7 @@ class SourceDistribution(AbstractDistribution): """ def get_metadata_distribution(self) -> BaseDistribution: - from pip._internal.metadata.pkg_resources import Distribution as _Dist - - return _Dist(self.req.get_dist()) + return self.req.get_dist() def prepare_distribution_metadata( self, finder: PackageFinder, build_isolation: bool diff --git a/src/pip/_internal/distributions/wheel.py b/src/pip/_internal/distributions/wheel.py index 340b0f3c5c7..a8fe30cb3c9 100644 --- a/src/pip/_internal/distributions/wheel.py +++ b/src/pip/_internal/distributions/wheel.py @@ -5,7 +5,7 @@ from pip._internal.metadata import ( BaseDistribution, FilesystemWheel, - get_wheel_distribution, + get_distribution_for_wheel, ) @@ -23,7 +23,7 @@ def get_metadata_distribution(self) -> BaseDistribution: assert self.req.local_file_path, "Set as part of preparation during download" assert self.req.name, "Wheels are never unnamed" wheel = FilesystemWheel(self.req.local_file_path) - return get_wheel_distribution(wheel, canonicalize_name(self.req.name)) + return get_distribution_for_wheel(wheel, canonicalize_name(self.req.name)) def prepare_distribution_metadata( self, finder: PackageFinder, build_isolation: bool diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index a2702db7d35..82449b48839 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -881,7 +881,7 @@ def find_requirement( installed_version: Optional[_BaseVersion] = None if req.satisfied_by is not None: - installed_version = parse_version(req.satisfied_by.version) + installed_version = req.satisfied_by.version def _format_versions(cand_iter: Iterable[InstallationCandidate]) -> str: # This repeated parse_version and str() conversion is needed to diff --git a/src/pip/_internal/metadata/__init__.py b/src/pip/_internal/metadata/__init__.py index f4f2a4f6cdb..e2f67126187 100644 --- a/src/pip/_internal/metadata/__init__.py +++ b/src/pip/_internal/metadata/__init__.py @@ -38,7 +38,7 @@ def get_environment(paths: Optional[List[str]]) -> BaseEnvironment: return Environment.from_paths(paths) -def get_wheel_distribution(wheel: Wheel, canonical_name: str) -> BaseDistribution: +def get_distribution_for_wheel(wheel: Wheel, canonical_name: str) -> BaseDistribution: """Get the representation of the specified wheel's distribution metadata. This returns a Distribution instance from the chosen backend based on @@ -49,3 +49,14 @@ def get_wheel_distribution(wheel: Wheel, canonical_name: str) -> BaseDistributio from .pkg_resources import Distribution return Distribution.from_wheel(wheel, canonical_name) + + +def get_distribution_for_info_directory(directory_path: str) -> BaseDistribution: + """Get the specified info directory's distribution representation. + + The directory should be an on-disk ``NAME-VERSION.dist-info`` or + ``NAME.egg-info`` directory. + """ + from .pkg_resources import Distribution + + return Distribution.from_info_directory(directory_path) diff --git a/src/pip/_internal/metadata/base.py b/src/pip/_internal/metadata/base.py index e1b229d1ff4..86903bb7389 100644 --- a/src/pip/_internal/metadata/base.py +++ b/src/pip/_internal/metadata/base.py @@ -1,6 +1,8 @@ +import csv import email.message import json import logging +import pathlib import re import zipfile from typing import ( @@ -12,6 +14,7 @@ Iterator, List, Optional, + Tuple, Union, ) @@ -38,6 +41,33 @@ logger = logging.getLogger(__name__) +def _convert_legacy_file(entry: Tuple[str, ...], info: Tuple[str, ...]) -> pathlib.Path: + """Convert a legacy installed-files.txt path into modern RECORD path. + + The legacy format stores paths relative to the info directory, while the + modern format stores paths relative to the package root, e.g. the + site-packages directory. + + :param entry: Path parts of the installed-files.txt entry. + :param info: Path parts of the egg-info directory relative to package root. + :returns: The converted entry. + + For best compatibility with symlinks, this does not use ``abspath()`` or + ``Path.resolve()``, but tries to work with path parts: + + 1. While ``entry`` starts with ``..``, remove the equal amounts of parts + from ``info``; if ``info`` is empty, start appending ``..`` instead. + 2. Join the two directly. + """ + while entry and entry[0] == "..": + if not info or info[-1] == "..": + info += ("..",) + else: + info = info[:-1] + entry = entry[1:] + return pathlib.Path(*info, *entry) + + class BaseEntryPoint(Protocol): @property def name(self) -> str: @@ -127,6 +157,17 @@ def direct_url(self) -> Optional[DirectUrl]: def installer(self) -> str: raise NotImplementedError() + @property + def egg_link(self) -> Optional[str]: + """Location of the ``.egg-link`` for this distribution. + + If there's not a matching file, None is returned. Note that finding this + file does not necessarily mean the currently-installed distribution is + editable since the ``.egg-link`` can still be shadowed by a + non-editable installation located in front of it in ``sys.path``. + """ + raise NotImplementedError() + @property def editable(self) -> bool: raise NotImplementedError() @@ -146,11 +187,20 @@ def in_site_packages(self) -> bool: def read_text(self, name: str) -> str: """Read a file in the .dist-info (or .egg-info) directory. - Should raise ``FileNotFoundError`` if ``name`` does not exist in the - metadata directory. + :raises FileNotFoundError: ``name`` does not exist in the info directory. """ raise NotImplementedError() + def iterdir(self, name: str) -> Iterable[pathlib.PurePosixPath]: + """Iterate through a directory in the info directory. + + Each item is a path relative to the info directory. + + :raises FileNotFoundError: ``name`` does not exist in the info directory. + :raises NotADirectoryError: ``name`` exists in the info directory, but + is not a directory. + """ + def iter_entry_points(self) -> Iterable[BaseEntryPoint]: raise NotImplementedError() @@ -206,6 +256,43 @@ def iter_provided_extras(self) -> Iterable[str]: """ raise NotImplementedError() + def _iter_files_from_legacy(self) -> Optional[Iterator[pathlib.Path]]: + try: + text = self.read_text("installed-files.txt") + except FileNotFoundError: + return None + paths = (pathlib.Path(p) for p in text.splitlines(keepends=False) if p) + root = self.location + info = self.info_directory + if root is None or info is None: + return paths + try: + rel = pathlib.Path(info).relative_to(root) + except ValueError: # info is not relative to root. + return paths + if not rel.parts: # info *is* root. + return paths + return (_convert_legacy_file(p.parts, rel.parts) for p in paths) + + def _iter_files_from_record(self) -> Optional[Iterator[pathlib.Path]]: + try: + text = self.read_text("RECORD") + except FileNotFoundError: + return None + return (pathlib.Path(row[0]) for row in csv.reader(text.splitlines())) + + def iter_files(self) -> Optional[Iterator[pathlib.Path]]: + """Files in the distribution's record. + + For modern .dist-info distributions, this is the files listed in the + ``RECORD`` file. All entries are paths relative to this distribution's + ``location``. + + Note that this can be None for unmanagable distributions, e.g. an + installation performed by distutils or a foreign package manager. + """ + return self._iter_files_from_record() or self._iter_files_from_legacy() + class BaseEnvironment: """An environment containing distributions to introspect.""" diff --git a/src/pip/_internal/metadata/pkg_resources.py b/src/pip/_internal/metadata/pkg_resources.py index 75fd3518f2e..8cb9aa8cf08 100644 --- a/src/pip/_internal/metadata/pkg_resources.py +++ b/src/pip/_internal/metadata/pkg_resources.py @@ -1,5 +1,7 @@ import email.message import logging +import os +import pathlib from typing import ( TYPE_CHECKING, Collection, @@ -49,6 +51,25 @@ def from_wheel(cls, wheel: Wheel, name: str) -> "Distribution": dist = pkg_resources_distribution_for_wheel(zf, name, wheel.location) return cls(dist) + @classmethod + def from_info_directory(cls, path: str) -> "Distribution": + dist_dir = path.rstrip(os.sep) + + # Build a PathMetadata object, from path to metadata. :wink: + base_dir, dist_dir_name = os.path.split(dist_dir) + metadata = pkg_resources.PathMetadata(base_dir, dist_dir) + + # Determine the correct Distribution object type. + if dist_dir.endswith(".egg-info"): + dist_cls = pkg_resources.Distribution + dist_name = os.path.splitext(dist_dir_name)[0] + else: + assert dist_dir.endswith(".dist-info") + dist_cls = pkg_resources.DistInfoDistribution + dist_name = os.path.splitext(dist_dir_name)[0].split("-", 1)[0] + + return cls(dist_cls(base_dir, project_name=dist_name, metadata=metadata)) + @property def location(self) -> Optional[str]: return self._dist.location @@ -63,12 +84,19 @@ def canonical_name(self) -> "NormalizedName": @property def version(self) -> DistributionVersion: + # pkg_resouces may contain a different copy of packaging.version from + # pip in if the downstream distributor does a poor job debundling pip. + # We avoid parsed_version and use our vendored packaging instead. return parse_version(self._dist.version) @property def installer(self) -> str: return get_installer(self._dist) + @property + def egg_link(self) -> Optional[str]: + return misc.egg_link_path(self._dist) + @property def editable(self) -> bool: return misc.dist_is_editable(self._dist) @@ -90,6 +118,15 @@ def read_text(self, name: str) -> str: raise FileNotFoundError(name) return self._dist.get_metadata(name) + def iterdir(self, name: str) -> Iterable[pathlib.PurePosixPath]: + if not self._dist.has_metadata(name): + raise FileNotFoundError(name) + if not self._dist.metadata_isdir(name): + raise NotADirectoryError(name) + return ( + pathlib.PurePosixPath(name, n) for n in self._dist.metadata_listdir(name) + ) + def iter_entry_points(self) -> Iterable[BaseEntryPoint]: for group, entries in self._dist.get_entry_map().items(): for name, entry_point in entries.items(): diff --git a/src/pip/_internal/network/lazy_wheel.py b/src/pip/_internal/network/lazy_wheel.py index c9e44d5be58..a7ca1192ae0 100644 --- a/src/pip/_internal/network/lazy_wheel.py +++ b/src/pip/_internal/network/lazy_wheel.py @@ -11,7 +11,11 @@ from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response -from pip._internal.metadata import BaseDistribution, MemoryWheel, get_wheel_distribution +from pip._internal.metadata import ( + BaseDistribution, + MemoryWheel, + get_distribution_for_wheel, +) from pip._internal.network.session import PipSession from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks @@ -34,7 +38,7 @@ def dist_from_wheel_url(name: str, url: str, session: PipSession) -> BaseDistrib wheel = MemoryWheel(zf.name, zf) # type: ignore # After context manager exit, wheel.name # is an invalid file by intention. - return get_wheel_distribution(wheel, canonicalize_name(name)) + return get_distribution_for_wheel(wheel, canonicalize_name(name)) class LazyZipOverHTTP: diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index 075317622bc..98c58442148 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -45,7 +45,7 @@ from pip._internal.metadata import ( BaseDistribution, FilesystemWheel, - get_wheel_distribution, + get_distribution_for_wheel, ) from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, DirectUrl from pip._internal.models.scheme import SCHEME_KEYS, Scheme @@ -579,7 +579,7 @@ def is_script_scheme_path(path: RecordPath) -> bool: files = chain(files, other_scheme_files) # Get the defined entry points - distribution = get_wheel_distribution( + distribution = get_distribution_for_wheel( FilesystemWheel(wheel_path), canonicalize_name(name), ) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index add22b552cf..3a261fe7291 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -1,15 +1,15 @@ # The following comment should be removed at some point in the future. # mypy: strict-optional=False +import email.message import logging import os import shutil import sys import uuid import zipfile -from typing import Any, Dict, Iterable, List, Optional, Sequence, Union +from typing import Dict, Iterable, List, Optional, Sequence, Union -from pip._vendor import pkg_resources from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import Requirement from pip._vendor.packaging.specifiers import SpecifierSet @@ -17,11 +17,16 @@ from pip._vendor.packaging.version import Version from pip._vendor.packaging.version import parse as parse_version from pip._vendor.pep517.wrappers import Pep517HookCaller -from pip._vendor.pkg_resources import Distribution +from pip._vendor.pkg_resources import safe_extra # TODO: Put this in packaging? from pip._internal.build_env import BuildEnvironment, NoOpBuildEnvironment from pip._internal.exceptions import InstallationError from pip._internal.locations import get_scheme +from pip._internal.metadata import ( + BaseDistribution, + get_default_environment, + get_distribution_for_info_directory, +) from pip._internal.models.link import Link from pip._internal.operations.build.metadata import generate_metadata from pip._internal.operations.build.metadata_legacy import ( @@ -43,13 +48,9 @@ ask_path_exists, backup_dir, display_path, - dist_in_site_packages, - dist_in_usersite, - get_distribution, hide_url, redact_auth_from_url, ) -from pip._internal.utils.packaging import get_metadata from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds from pip._internal.utils.virtualenv import running_under_virtualenv from pip._internal.vcs import vcs @@ -57,32 +58,6 @@ logger = logging.getLogger(__name__) -def _get_dist(metadata_directory: str) -> Distribution: - """Return a pkg_resources.Distribution for the provided - metadata directory. - """ - dist_dir = metadata_directory.rstrip(os.sep) - - # Build a PathMetadata object, from path to metadata. :wink: - base_dir, dist_dir_name = os.path.split(dist_dir) - metadata = pkg_resources.PathMetadata(base_dir, dist_dir) - - # Determine the correct Distribution object type. - if dist_dir.endswith(".egg-info"): - dist_cls = pkg_resources.Distribution - dist_name = os.path.splitext(dist_dir_name)[0] - else: - assert dist_dir.endswith(".dist-info") - dist_cls = pkg_resources.DistInfoDistribution - dist_name = os.path.splitext(dist_dir_name)[0].split("-")[0] - - return dist_cls( - base_dir, - project_name=dist_name, - metadata=metadata, - ) - - class InstallRequirement: """ Represents something that may be installed later on, may have information @@ -138,16 +113,15 @@ def __init__( if extras: self.extras = extras elif req: - self.extras = {pkg_resources.safe_extra(extra) for extra in req.extras} + self.extras = {safe_extra(extra) for extra in req.extras} else: self.extras = set() if markers is None and req: markers = req.marker self.markers = markers - # This holds the pkg_resources.Distribution object if this requirement - # is already available: - self.satisfied_by: Optional[Distribution] = None + # This holds the distribution object if this requirement is already available: + self.satisfied_by: Optional[BaseDistribution] = None # Whether the installation process should try to uninstall an existing # distribution before installing this requirement. self.should_reinstall = False @@ -235,7 +209,7 @@ def format_debug(self) -> str: def name(self) -> Optional[str]: if self.req is None: return None - return pkg_resources.safe_name(self.req.name) + return canonicalize_name(self.req.name) @property def specifier(self) -> SpecifierSet: @@ -396,14 +370,10 @@ def check_if_exists(self, use_user_site: bool) -> None: """ if self.req is None: return - existing_dist = get_distribution(self.req.name) + existing_dist = get_default_environment().get_distribution(self.req.name) if not existing_dist: return - # pkg_resouces may contain a different copy of packaging.version from - # pip in if the downstream distributor does a poor job debundling pip. - # We avoid existing_dist.parsed_version and let SpecifierSet.contains - # parses the version instead. existing_version = existing_dist.version version_compatible = ( existing_version is not None @@ -412,15 +382,13 @@ def check_if_exists(self, use_user_site: bool) -> None: if not version_compatible: self.satisfied_by = None if use_user_site: - if dist_in_usersite(existing_dist): + if existing_dist.in_usersite: self.should_reinstall = True - elif running_under_virtualenv() and dist_in_site_packages( - existing_dist - ): + elif running_under_virtualenv() and existing_dist.in_site_packages: raise InstallationError( "Will not install to the user site because it will " "lack sys.path precedence to {} in {}".format( - existing_dist.project_name, existing_dist.location + existing_dist.canonical_name, existing_dist.location ) ) else: @@ -531,14 +499,14 @@ def prepare_metadata(self) -> None: self.assert_source_matches_version() @property - def metadata(self) -> Any: + def metadata(self) -> email.message.Message: if not hasattr(self, "_metadata"): - self._metadata = get_metadata(self.get_dist()) + self._metadata = self.get_dist().metadata return self._metadata - def get_dist(self) -> Distribution: - return _get_dist(self.metadata_directory) + def get_dist(self) -> BaseDistribution: + return get_distribution_for_info_directory(self.metadata_directory) def assert_source_matches_version(self) -> None: assert self.source_dir @@ -617,7 +585,7 @@ def uninstall( """ assert self.req - dist = get_distribution(self.req.name) + dist = get_default_environment().get_distribution(self.req.name) if not dist: logger.warning("Skipping %s as it is not installed.", self.name) return None diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index ef7352f7ba3..f4ac4b3d9d8 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -1,4 +1,3 @@ -import csv import functools import os import sys @@ -6,47 +5,36 @@ from importlib.util import cache_from_source from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple -from pip._vendor import pkg_resources -from pip._vendor.pkg_resources import Distribution - from pip._internal.exceptions import UninstallationError from pip._internal.locations import get_bin_prefix, get_bin_user +from pip._internal.metadata import BaseDistribution from pip._internal.utils.compat import WINDOWS from pip._internal.utils.logging import getLogger, indent_log -from pip._internal.utils.misc import ( - ask, - dist_in_usersite, - dist_is_local, - egg_link_path, - is_local, - normalize_path, - renames, - rmtree, -) +from pip._internal.utils.misc import ask, is_local, normalize_path, renames, rmtree from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory logger = getLogger(__name__) -def _script_names(dist: Distribution, script_name: str, is_gui: bool) -> List[str]: +def _script_names( + dist: BaseDistribution, + bin_dir: str, + script_name: str, + is_gui: bool, +) -> Iterator[str]: """Create the fully qualified name of the files created by {console,gui}_scripts for the given ``dist``. Returns the list of file names """ - if dist_in_usersite(dist): - bin_dir = get_bin_user() - else: - bin_dir = get_bin_prefix() exe_name = os.path.join(bin_dir, script_name) - paths_to_remove = [exe_name] + yield exe_name if WINDOWS: - paths_to_remove.append(exe_name + ".exe") - paths_to_remove.append(exe_name + ".exe.manifest") + yield f"{exe_name}.exe" + yield f"{exe_name}.exe.manifest" if is_gui: - paths_to_remove.append(exe_name + "-script.pyw") + yield f"{exe_name}-script.pyw" else: - paths_to_remove.append(exe_name + "-script.py") - return paths_to_remove + yield f"{exe_name}-script.py" def _unique(fn: Callable[..., Iterator[Any]]) -> Callable[..., Iterator[Any]]: @@ -62,7 +50,7 @@ def unique(*args: Any, **kw: Any) -> Iterator[Any]: @_unique -def uninstallation_paths(dist: Distribution) -> Iterator[str]: +def uninstallation_paths(dist: BaseDistribution) -> Iterator[str]: """ Yield all the uninstallation paths for dist based on RECORD-without-.py[co] @@ -76,33 +64,27 @@ def uninstallation_paths(dist: Distribution) -> Iterator[str]: https://packaging.python.org/specifications/recording-installed-packages/ """ - try: - r = csv.reader(dist.get_metadata_lines("RECORD")) - except FileNotFoundError as missing_record_exception: + dist_files = dist.iter_files() + if dist_files is None: msg = "Cannot uninstall {dist}, RECORD file not found.".format(dist=dist) try: - installer = next(dist.get_metadata_lines("INSTALLER")) + installer = dist.installer if not installer or installer == "pip": raise ValueError() except (OSError, StopIteration, ValueError): - dep = "{}=={}".format(dist.project_name, dist.version) + dep = "{}=={}".format(dist.canonical_name, dist.version) msg += ( " You might be able to recover from this via: " "'pip install --force-reinstall --no-deps {}'.".format(dep) ) else: msg += " Hint: The package was installed by {}.".format(installer) - raise UninstallationError(msg) from missing_record_exception - for row in r: - path = os.path.join(dist.location, row[0]) - yield path - if path.endswith(".py"): - dn, fn = os.path.split(path) - base = fn[:-3] - path = os.path.join(dn, base + ".pyc") - yield path - path = os.path.join(dn, base + ".pyo") - yield path + raise UninstallationError(msg) + for path in dist_files: + yield str(path) + if path.suffix == ".py": + yield str(path.with_suffix(".pyc")) + yield str(path.with_suffix(".pyo")) def compact(paths: Iterable[str]) -> Set[str]: @@ -317,7 +299,7 @@ class UninstallPathSet: """A set of file paths to be removed in the uninstallation of a requirement.""" - def __init__(self, dist: Distribution) -> None: + def __init__(self, dist: BaseDistribution) -> None: self.paths: Set[str] = set() self._refuse: Set[str] = set() self.pth: Dict[str, UninstallPthEntries] = {} @@ -367,11 +349,11 @@ def remove(self, auto_confirm: bool = False, verbose: bool = False) -> None: if not self.paths: logger.info( "Can't uninstall '%s'. No files were found to uninstall.", - self.dist.project_name, + self.dist.canonical_name, ) return - dist_name_version = self.dist.project_name + "-" + self.dist.version + dist_name_version = f"{self.dist.canonical_name}-{self.dist.version}" logger.info("Uninstalling %s:", dist_name_version) with indent_log(): @@ -422,10 +404,10 @@ def rollback(self) -> None: if not self._moved_paths.can_rollback: logger.error( "Can't roll back %s; was not uninstalled", - self.dist.project_name, + self.dist.canonical_name, ) return - logger.info("Rolling back uninstall of %s", self.dist.project_name) + logger.info("Rolling back uninstall of %s", self.dist.canonical_name) self._moved_paths.rollback() for pth in self.pth.values(): pth.rollback() @@ -435,12 +417,16 @@ def commit(self) -> None: self._moved_paths.commit() @classmethod - def from_dist(cls, dist: Distribution) -> "UninstallPathSet": - dist_path = normalize_path(dist.location) - if not dist_is_local(dist): + def from_dist(cls, dist: BaseDistribution) -> "UninstallPathSet": + dist_path = dist.location + if dist_path is None: + logger.info("Not uninstalling %s since it is not installed") + return cls(dist) + dist_path = normalize_path(dist_path) + if not dist.local: logger.info( "Not uninstalling %s at %s, outside environment %s", - dist.key, + dist.canonical_name, dist_path, sys.prefix, ) @@ -453,76 +439,74 @@ def from_dist(cls, dist: Distribution) -> "UninstallPathSet": }: logger.info( "Not uninstalling %s at %s, as it is in the standard library.", - dist.key, + dist, dist_path, ) return cls(dist) paths_to_remove = cls(dist) - develop_egg_link = egg_link_path(dist) + develop_egg_link = dist.egg_link develop_egg_link_egg_info = "{}.egg-info".format( - pkg_resources.to_filename(dist.project_name) + dist.raw_name.replace("-", "_") ) - egg_info_exists = dist.egg_info and os.path.exists(dist.egg_info) - # Special case for distutils installed package - distutils_egg_info = getattr(dist._provider, "path", None) + info_dir = dist.info_directory # Uninstall cases order do matter as in the case of 2 installs of the # same package, pip needs to uninstall the currently detected version if ( - egg_info_exists - and dist.egg_info.endswith(".egg-info") - and not dist.egg_info.endswith(develop_egg_link_egg_info) + info_dir + and os.path.exists(info_dir) + and info_dir.endswith(".egg-info") + and not info_dir.endswith(develop_egg_link_egg_info) ): # if dist.egg_info.endswith(develop_egg_link_egg_info), we # are in fact in the develop_egg_link case - paths_to_remove.add(dist.egg_info) - if dist.has_metadata("installed-files.txt"): - for installed_file in dist.get_metadata( - "installed-files.txt" - ).splitlines(): - path = os.path.normpath(os.path.join(dist.egg_info, installed_file)) - paths_to_remove.add(path) - # FIXME: need a test for this elif block - # occurs with --single-version-externally-managed/--record outside - # of pip - elif dist.has_metadata("top_level.txt"): - if dist.has_metadata("namespace_packages.txt"): - namespaces = dist.get_metadata("namespace_packages.txt") - else: - namespaces = [] + paths_to_remove.add(info_dir) + dist_files = dist.iter_files() + if dist_files is not None: + for dist_file in dist_files: + paths_to_remove.add(str(dist_file)) + else: + # FIXME: need a test for this block; occurs with + # --single-version-externally-managed/--record outside of pip. + try: + top_level = dist.read_text("top_level.txt") + except FileNotFoundError: + top_level = "" + try: + namespaces = dist.read_text("namespace_packages.txt") + except FileNotFoundError: + namespaces = "" for top_level_pkg in [ - p - for p in dist.get_metadata("top_level.txt").splitlines() - if p and p not in namespaces + p for p in top_level.splitlines() if p and p not in namespaces ]: - path = os.path.join(dist.location, top_level_pkg) + path = os.path.join(dist_path, top_level_pkg) paths_to_remove.add(path) - paths_to_remove.add(path + ".py") - paths_to_remove.add(path + ".pyc") - paths_to_remove.add(path + ".pyo") + paths_to_remove.add(f"{path}.py") + paths_to_remove.add(f"{path}.pyc") + paths_to_remove.add(f"{path}.pyo") - elif distutils_egg_info: + elif not info_dir or not os.path.isdir(info_dir): raise UninstallationError( "Cannot uninstall {!r}. It is a distutils installed project " "and thus we cannot accurately determine which files belong " "to it which would lead to only a partial uninstall.".format( - dist.project_name, + dist.canonical_name, ) ) - elif dist.location.endswith(".egg"): + elif dist_path.endswith(".egg"): # package installed by easy_install # We cannot match on dist.egg_name because it can slightly vary # i.e. setuptools-0.6c11-py2.6.egg vs setuptools-0.6rc11-py2.6.egg - paths_to_remove.add(dist.location) - easy_install_egg = os.path.split(dist.location)[1] + paths_to_remove.add(dist_path) + easy_install_egg = os.path.split(dist_path)[1] easy_install_pth = os.path.join( - os.path.dirname(dist.location), "easy-install.pth" + os.path.dirname(dist_path), "easy-install.pth" ) paths_to_remove.add_pth(easy_install_pth, "./" + easy_install_egg) - elif egg_info_exists and dist.egg_info.endswith(".dist-info"): + elif info_dir and os.path.exists(info_dir) and info_dir.endswith(".dist-info"): for path in uninstallation_paths(dist): paths_to_remove.add(path) @@ -531,46 +515,50 @@ def from_dist(cls, dist: Distribution) -> "UninstallPathSet": with open(develop_egg_link) as fh: link_pointer = os.path.normcase(fh.readline().strip()) assert ( - link_pointer == dist.location + link_pointer == dist_path ), "Egg-link {} does not match installed location of {} (at {})".format( - link_pointer, dist.project_name, dist.location + link_pointer, dist.raw_name, dist_path ) paths_to_remove.add(develop_egg_link) easy_install_pth = os.path.join( os.path.dirname(develop_egg_link), "easy-install.pth" ) - paths_to_remove.add_pth(easy_install_pth, dist.location) + paths_to_remove.add_pth(easy_install_pth, dist_path) else: logger.debug( "Not sure how to uninstall: %s - Check: %s", dist, - dist.location, + dist_path, ) + if dist.in_usersite: + bin_dir = get_bin_user() + else: + bin_dir = get_bin_prefix() + # find distutils scripts= scripts - if dist.has_metadata("scripts") and dist.metadata_isdir("scripts"): - for script in dist.metadata_listdir("scripts"): - if dist_in_usersite(dist): - bin_dir = get_bin_user() - else: - bin_dir = get_bin_prefix() - paths_to_remove.add(os.path.join(bin_dir, script)) + try: + scripts_iter = (p.name for p in dist.iterdir("scripts")) + except (FileNotFoundError, NotADirectoryError): + pass + else: + for name in scripts_iter: + paths_to_remove.add(os.path.join(bin_dir, name)) if WINDOWS: - paths_to_remove.add(os.path.join(bin_dir, script) + ".bat") + paths_to_remove.add(os.path.join(bin_dir, f"{name}.bat")) # find console_scripts - _scripts_to_remove = [] - console_scripts = dist.get_entry_map(group="console_scripts") - for name in console_scripts.keys(): - _scripts_to_remove.extend(_script_names(dist, name, False)) - # find gui_scripts - gui_scripts = dist.get_entry_map(group="gui_scripts") - for name in gui_scripts.keys(): - _scripts_to_remove.extend(_script_names(dist, name, True)) - - for s in _scripts_to_remove: - paths_to_remove.add(s) + for entry_point in dist.iter_entry_points(): + group = entry_point.group + if group == "console_scripts": + scripts = _script_names(dist, bin_dir, entry_point.name, is_gui=False) + elif group == "gui_scripts": + scripts = _script_names(dist, bin_dir, entry_point.name, is_gui=True) + else: + continue + for s in scripts: + paths_to_remove.add(s) return paths_to_remove diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 5c6f9a04a97..04cd1bf032c 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -93,8 +93,6 @@ def make_install_req_from_editable( def _make_install_req_from_dist( dist: BaseDistribution, template: InstallRequirement ) -> InstallRequirement: - from pip._internal.metadata.pkg_resources import Distribution as _Dist - if template.req: line = str(template.req) elif template.link: @@ -114,7 +112,7 @@ def _make_install_req_from_dist( hashes=template.hash_options, ), ) - ireq.satisfied_by = cast(_Dist, dist)._dist + ireq.satisfied_by = dist return ireq diff --git a/src/pip/_internal/wheel_builder.py b/src/pip/_internal/wheel_builder.py index cf039fe763b..e1255261a87 100644 --- a/src/pip/_internal/wheel_builder.py +++ b/src/pip/_internal/wheel_builder.py @@ -12,7 +12,7 @@ from pip._internal.cache import WheelCache from pip._internal.exceptions import InvalidWheelFilename, UnsupportedWheel -from pip._internal.metadata import FilesystemWheel, get_wheel_distribution +from pip._internal.metadata import FilesystemWheel, get_distribution_for_wheel from pip._internal.models.link import Link from pip._internal.models.wheel import Wheel from pip._internal.operations.build.wheel import build_wheel_pep517 @@ -166,7 +166,7 @@ def _verify_one(req: InstallRequirement, wheel_path: str) -> None: "Wheel has unexpected file name: expected {!r}, " "got {!r}".format(canonical_name, w.name), ) - dist = get_wheel_distribution(FilesystemWheel(wheel_path), canonical_name) + dist = get_distribution_for_wheel(FilesystemWheel(wheel_path), canonical_name) dist_verstr = str(dist.version) if canonicalize_version(dist_verstr) != canonicalize_version(w.version): raise InvalidWheelFilename( diff --git a/tests/lib/wheel.py b/tests/lib/wheel.py index aa4086941ba..8069cb7723b 100644 --- a/tests/lib/wheel.py +++ b/tests/lib/wheel.py @@ -25,7 +25,11 @@ from pip._vendor.requests.structures import CaseInsensitiveDict -from pip._internal.metadata import BaseDistribution, MemoryWheel, get_wheel_distribution +from pip._internal.metadata import ( + BaseDistribution, + MemoryWheel, + get_distribution_for_wheel, +) from tests.lib.path import Path # As would be used in metadata @@ -284,7 +288,7 @@ def as_zipfile(self) -> ZipFile: def as_distribution(self, name: str) -> BaseDistribution: stream = BytesIO(self.as_bytes()) - return get_wheel_distribution(MemoryWheel(self._name, stream), name) + return get_distribution_for_wheel(MemoryWheel(self._name, stream), name) def make_wheel( diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index b4fde11d99b..a3e455d2c7a 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -4,7 +4,7 @@ import pytest from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.tags import Tag -from pkg_resources import parse_version +from pip._vendor.packaging.version import Version import pip._internal.utils.compatibility_tags from pip._internal.exceptions import BestVersionAlreadyInstalled, DistributionNotFound @@ -80,13 +80,7 @@ def test_finder_detects_latest_already_satisfied_find_links(data): """Test PackageFinder detects latest already satisfied using find-links""" req = install_req_from_line("simple", None) # the latest simple in local pkgs is 3.0 - latest_version = "3.0" - satisfied_by = Mock( - location="/path", - parsed_version=parse_version(latest_version), - version=latest_version, - ) - req.satisfied_by = satisfied_by + req.satisfied_by = Mock(location="/path", version=Version("3.0")) finder = make_test_finder(find_links=[data.find_links]) with pytest.raises(BestVersionAlreadyInstalled): @@ -98,13 +92,7 @@ def test_finder_detects_latest_already_satisfied_pypi_links(): """Test PackageFinder detects latest already satisfied using pypi links""" req = install_req_from_line("initools", None) # the latest initools on PyPI is 0.3.1 - latest_version = "0.3.1" - satisfied_by = Mock( - location="/path", - parsed_version=parse_version(latest_version), - version=latest_version, - ) - req.satisfied_by = satisfied_by + req.satisfied_by = Mock(location="/path", version=Version("0.3.1")) finder = make_test_finder(index_urls=["http://pypi.org/simple/"]) with pytest.raises(BestVersionAlreadyInstalled): @@ -173,12 +161,7 @@ def test_existing_over_wheel_priority(self, data): `test_link_sorting` also covers this at a lower level """ req = install_req_from_line("priority", None) - latest_version = "1.0" - satisfied_by = Mock( - location="/path", - parsed_version=parse_version(latest_version), - version=latest_version, - ) + satisfied_by = Mock(location="/path", version=Version("1.0")) req.satisfied_by = satisfied_by finder = make_test_finder(find_links=[data.find_links]) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index f4c1d48d93f..bd9a0fba89a 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -7,7 +7,6 @@ from unittest.mock import patch import pytest -from pip._vendor import pkg_resources from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import Requirement @@ -435,8 +434,7 @@ def test_get_dist(self, path): req = install_req_from_line("foo") req.metadata_directory = path dist = req.get_dist() - assert isinstance(dist, pkg_resources.Distribution) - assert dist.project_name == "foo" + assert dist.canonical_name == "foo" assert dist.location == "/path/to".replace("/", os.path.sep) def test_markers(self): diff --git a/tests/unit/test_req_uninstall.py b/tests/unit/test_req_uninstall.py index 805a551aecd..fcb5ec2beb2 100644 --- a/tests/unit/test_req_uninstall.py +++ b/tests/unit/test_req_uninstall.py @@ -5,6 +5,7 @@ import pytest import pip._internal.req.req_uninstall +from pip._internal.metadata.base import BaseDistribution from pip._internal.req.req_uninstall import ( StashedUninstallPathSet, UninstallPathSet, @@ -24,13 +25,13 @@ def mock_is_local(path): def test_uninstallation_paths(): - class dist: - def get_metadata_lines(self, record): - return ["file.py,,", "file.pyc,,", "file.so,,", "nopyc.py"] + class FakeDist(BaseDistribution): + def read_text(self, name): + if name != "RECORD": + raise FileNotFoundError(name) + return "file.py,,\nfile.pyc,,\nfile.so,,\nnopyc.py" - location = "" - - d = dist() + d = FakeDist() paths = list(uninstallation_paths(d))