From 178cd3f2446632c779285f975991667a5c189f98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Sat, 9 Nov 2019 12:26:51 +0100 Subject: [PATCH 1/9] Better workaround for cache poisoning #3025 Make sure ``pip wheel`` never outputs pure python wheels with a python implementation tag. Better fix/workaround for `#3025 `_ by using a per-implementation wheel cache instead of caching pure python wheels with an implementation tag in their name. Fixes #7296 --- news/7296.bugfix | 5 +++++ src/pip/_internal/cache.py | 16 +++++++++++++++- src/pip/_internal/utils/misc.py | 11 +++++++++++ src/pip/_internal/wheel_builder.py | 10 +--------- tests/functional/test_install.py | 5 ++--- 5 files changed, 34 insertions(+), 13 deletions(-) create mode 100644 news/7296.bugfix diff --git a/news/7296.bugfix b/news/7296.bugfix new file mode 100644 index 00000000000..5d617bf75db --- /dev/null +++ b/news/7296.bugfix @@ -0,0 +1,5 @@ +Make sure ``pip wheel`` never outputs pure python wheels with a +python implementation tag. Better fix/workaround for +`#3025 `_ by +using a per-implementation wheel cache instead of caching pure python +wheels with an implementation tag in their name. diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 7a431f9a2ed..1ed7d584cb1 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -8,6 +8,7 @@ import hashlib import logging import os +import sys from pip._vendor.packaging.utils import canonicalize_name @@ -15,6 +16,7 @@ from pip._internal.models.link import Link from pip._internal.models.wheel import Wheel from pip._internal.utils.compat import expanduser +from pip._internal.utils.misc import interpreter_name from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.urls import path_to_url @@ -65,11 +67,23 @@ def _get_cache_path_parts(self, link): ) key_url = "#".join(key_parts) + # Include interpreter name, major and minor version in cache key + # to cope with ill-behaved sdists that build a different wheel + # depending on the python version their setup.py is being run on, + # and don't encode the difference in compatibility tags. + # https://github.com/pypa/pip/issues/7296 + key = "{}-{}.{} {}".format( + interpreter_name(), + sys.version_info[0], + sys.version_info[1], + key_url, + ) + # Encode our key url with sha224, we'll use this because it has similar # security properties to sha256, but with a shorter total output (and # thus less secure). However the differences don't make a lot of # difference for our use case here. - hashed = hashlib.sha224(key_url.encode()).hexdigest() + hashed = hashlib.sha224(key.encode()).hexdigest() # We want to nest the directories some to prevent having a ton of top # level directories where we might run out of sub directories on some diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 85acab856b0..9a802556269 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -11,6 +11,7 @@ import io import logging import os +import platform import posixpath import shutil import stat @@ -879,3 +880,13 @@ def hash_file(path, blocksize=1 << 20): length += len(block) h.update(block) return (h, length) # type: ignore + + +def interpreter_name(): + # type: () -> str + try: + name = sys.implementation.name # type: ignore + except AttributeError: # pragma: no cover + # Python 2.7 compatibility. + name = platform.python_implementation().lower() + return name diff --git a/src/pip/_internal/wheel_builder.py b/src/pip/_internal/wheel_builder.py index 994b2f1a3dc..9a408a3dbf8 100644 --- a/src/pip/_internal/wheel_builder.py +++ b/src/pip/_internal/wheel_builder.py @@ -9,7 +9,6 @@ import re import shutil -from pip._internal import pep425tags from pip._internal.models.link import Link from pip._internal.utils.logging import indent_log from pip._internal.utils.marker_files import has_delete_marker_file @@ -447,10 +446,6 @@ def build( ', '.join([req.name for (req, _) in buildset]), ) - python_tag = None - if should_unpack: - python_tag = pep425tags.implementation_tag - with indent_log(): build_success, build_failure = [], [] for req, output_dir in buildset: @@ -464,10 +459,7 @@ def build( build_failure.append(req) continue - wheel_file = self._build_one( - req, output_dir, - python_tag=python_tag, - ) + wheel_file = self._build_one(req, output_dir) if wheel_file: if should_unpack: # XXX: This is mildly duplicative with prepare_files, diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index ffc3736ba06..6b62d863545 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -12,7 +12,6 @@ from pip._vendor.six import PY2 from pip import __version__ as pip_current_version -from pip._internal import pep425tags from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.models.index import PyPI, TestPyPI from pip._internal.utils.misc import rmtree @@ -1297,9 +1296,9 @@ def test_install_builds_wheels(script, data, with_wheel): assert "Running setup.py install for requir" not in str(res), str(res) # wheelbroken has to run install assert "Running setup.py install for wheelb" in str(res), str(res) - # We want to make sure we used the correct implementation tag + # We want to make sure pure python wheels do not have an implementation tag assert wheels == [ - "Upper-2.0-{}-none-any.whl".format(pep425tags.implementation_tag), + "Upper-2.0-py{}-none-any.whl".format(sys.version_info[0]), ] From b14b37545a371afde6e09e02fbf940121853bd1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Sun, 10 Nov 2019 11:28:33 +0100 Subject: [PATCH 2/9] Remove unused pep425tags.implementation_tag --- src/pip/_internal/pep425tags.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/pip/_internal/pep425tags.py b/src/pip/_internal/pep425tags.py index 44b2d7ee0f8..b8aba326c71 100644 --- a/src/pip/_internal/pep425tags.py +++ b/src/pip/_internal/pep425tags.py @@ -83,14 +83,6 @@ def get_impl_version_info(): return sys.version_info[0], sys.version_info[1] -def get_impl_tag(): - # type: () -> str - """ - Returns the Tag for this specific implementation. - """ - return "{}{}".format(get_abbr_impl(), get_impl_ver()) - - def get_flag(var, fallback, expected=True, warn=True): # type: (str, Callable[..., bool], Union[bool, int], bool) -> bool """Use a fallback method for determining SOABI flags if the needed config @@ -459,6 +451,3 @@ def get_supported( supported.append(('py%s' % (version,), 'none', 'any')) return supported - - -implementation_tag = get_impl_tag() From e0165e7b300bd166cdf355221da90d0c28dcff52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Sun, 10 Nov 2019 11:33:41 +0100 Subject: [PATCH 3/9] Remove unused wheel_builder python_tag argument --- src/pip/_internal/utils/setuptools_build.py | 3 -- src/pip/_internal/wheel_builder.py | 32 ++------------------- tests/unit/test_wheel_builder.py | 16 ----------- 3 files changed, 2 insertions(+), 49 deletions(-) diff --git a/src/pip/_internal/utils/setuptools_build.py b/src/pip/_internal/utils/setuptools_build.py index 497b0eb4939..4147a650dca 100644 --- a/src/pip/_internal/utils/setuptools_build.py +++ b/src/pip/_internal/utils/setuptools_build.py @@ -52,7 +52,6 @@ def make_setuptools_bdist_wheel_args( global_options, # type: Sequence[str] build_options, # type: Sequence[str] destination_dir, # type: str - python_tag, # type: Optional[str] ): # type: (...) -> List[str] # NOTE: Eventually, we'd want to also -S to the flags here, when we're @@ -66,8 +65,6 @@ def make_setuptools_bdist_wheel_args( ) args += ["bdist_wheel", "-d", destination_dir] args += build_options - if python_tag is not None: - args += ["--python-tag", python_tag] return args diff --git a/src/pip/_internal/wheel_builder.py b/src/pip/_internal/wheel_builder.py index 9a408a3dbf8..e67b18781dd 100644 --- a/src/pip/_internal/wheel_builder.py +++ b/src/pip/_internal/wheel_builder.py @@ -46,14 +46,6 @@ logger = logging.getLogger(__name__) -def replace_python_tag(wheelname, new_tag): - # type: (str, str) -> str - """Replace the Python tag in a wheel file name with a new value.""" - parts = wheelname.split('-') - parts[-3] = new_tag - return '-'.join(parts) - - def _contains_egg_info( s, _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)): # type: (str, Pattern[str]) -> bool @@ -196,7 +188,6 @@ def _build_wheel_legacy( global_options, # type: List[str] build_options, # type: List[str] tempd, # type: str - python_tag=None, # type: Optional[str] ): # type: (...) -> Optional[str] """Build one unpacked package using the "legacy" build process. @@ -208,7 +199,6 @@ def _build_wheel_legacy( global_options=global_options, build_options=build_options, destination_dir=tempd, - python_tag=python_tag, ) spin_message = 'Building wheel for %s (setup.py)' % (name,) @@ -276,7 +266,6 @@ def _build_one( self, req, # type: InstallRequirement output_dir, # type: str - python_tag=None, # type: Optional[str] ): # type: (...) -> Optional[str] """Build one wheel. @@ -285,21 +274,17 @@ def _build_one( """ # Install build deps into temporary directory (PEP 518) with req.build_env: - return self._build_one_inside_env(req, output_dir, - python_tag=python_tag) + return self._build_one_inside_env(req, output_dir) def _build_one_inside_env( self, req, # type: InstallRequirement output_dir, # type: str - python_tag=None, # type: Optional[str] ): # type: (...) -> Optional[str] with TempDirectory(kind="wheel") as temp_dir: if req.use_pep517: - wheel_path = self._build_one_pep517( - req, temp_dir.path, python_tag=python_tag - ) + wheel_path = self._build_one_pep517(req, temp_dir.path) else: wheel_path = _build_wheel_legacy( name=req.name, @@ -308,7 +293,6 @@ def _build_one_inside_env( global_options=self.global_options, build_options=self.build_options, tempd=temp_dir.path, - python_tag=python_tag, ) if wheel_path is not None: @@ -333,7 +317,6 @@ def _build_one_pep517( self, req, # type: InstallRequirement tempd, # type: str - python_tag=None, # type: Optional[str] ): # type: (...) -> Optional[str] """Build one InstallRequirement using the PEP 517 build process. @@ -358,17 +341,6 @@ def _build_one_pep517( tempd, metadata_directory=req.metadata_directory, ) - if python_tag: - # General PEP 517 backends don't necessarily support - # a "--python-tag" option, so we rename the wheel - # file directly. - new_name = replace_python_tag(wheel_name, python_tag) - os.rename( - os.path.join(tempd, wheel_name), - os.path.join(tempd, new_name) - ) - # Reassign to simplify the return at the end of function - wheel_name = new_name except Exception: logger.error('Failed building wheel for %s', req.name) return None diff --git a/tests/unit/test_wheel_builder.py b/tests/unit/test_wheel_builder.py index 8db535dadc2..3e0fc0490ef 100644 --- a/tests/unit/test_wheel_builder.py +++ b/tests/unit/test_wheel_builder.py @@ -182,22 +182,6 @@ def test_format_command_result__empty_output(caplog, log_level): ] -def test_python_tag(): - wheelnames = [ - 'simplewheel-1.0-py2.py3-none-any.whl', - 'simplewheel-1.0-py27-none-any.whl', - 'simplewheel-2.0-1-py2.py3-none-any.whl', - ] - newnames = [ - 'simplewheel-1.0-py37-none-any.whl', - 'simplewheel-1.0-py37-none-any.whl', - 'simplewheel-2.0-1-py37-none-any.whl', - ] - for name, expected in zip(wheelnames, newnames): - result = wheel_builder.replace_python_tag(name, 'py37') - assert result == expected - - class TestWheelBuilder(object): def test_skip_building_wheels(self, caplog): From c4ef6163e547961aded5a894e770c0b191088572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Sat, 16 Nov 2019 11:58:53 +0100 Subject: [PATCH 4/9] New cache key generation algorithm Instead of building an URL-ish string that could be complex to describe and reproduce, generate a dictionary that is hashed with a simple algorithm. --- src/pip/_internal/cache.py | 28 ++++++++++++++++++---------- tests/unit/test_cache.py | 9 ++++++++- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 1ed7d584cb1..502de4d0bb3 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -22,13 +22,25 @@ from pip._internal.utils.urls import path_to_url if MYPY_CHECK_RUNNING: - from typing import Optional, Set, List, Any + from typing import Optional, Set, List, Any, Dict from pip._internal.models.format_control import FormatControl from pip._internal.pep425tags import Pep425Tag logger = logging.getLogger(__name__) +def _hash_dict(d): + # type: (Dict[str, str]) -> str + """Return a sha224 of a dictionary where keys and values are strings.""" + h = hashlib.new('sha224') + for k in sorted(d.keys()): + h.update(k.encode()) + h.update("=".encode()) + h.update(d[k].encode()) + h.update(b"\0") + return h.hexdigest() + + class Cache(object): """An abstract class - provides cache directories for data from links @@ -58,32 +70,28 @@ def _get_cache_path_parts(self, link): # We want to generate an url to use as our cache key, we don't want to # just re-use the URL because it might have other items in the fragment # and we don't care about those. - key_parts = [link.url_without_fragment] + key_parts = {"url": link.url_without_fragment} if link.hash_name is not None and link.hash is not None: - key_parts.append("=".join([link.hash_name, link.hash])) + key_parts[link.hash_name] = link.hash if link.subdirectory_fragment: - key_parts.append( - "=".join(["subdirectory", link.subdirectory_fragment]) - ) - key_url = "#".join(key_parts) + key_parts["subdirectory"] = link.subdirectory_fragment # Include interpreter name, major and minor version in cache key # to cope with ill-behaved sdists that build a different wheel # depending on the python version their setup.py is being run on, # and don't encode the difference in compatibility tags. # https://github.com/pypa/pip/issues/7296 - key = "{}-{}.{} {}".format( + key_parts["interpreter"] = "{}-{}.{}".format( interpreter_name(), sys.version_info[0], sys.version_info[1], - key_url, ) # Encode our key url with sha224, we'll use this because it has similar # security properties to sha256, but with a shorter total output (and # thus less secure). However the differences don't make a lot of # difference for our use case here. - hashed = hashlib.sha224(key.encode()).hexdigest() + hashed = _hash_dict(key_parts) # We want to nest the directories some to prevent having a ton of top # level directories where we might run out of sub directories on some diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index 79e4f624d19..e11b3e78a08 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -1,6 +1,6 @@ import os -from pip._internal.cache import WheelCache +from pip._internal.cache import WheelCache, _hash_dict from pip._internal.models.format_control import FormatControl from pip._internal.models.link import Link from pip._internal.utils.compat import expanduser @@ -42,3 +42,10 @@ def test_wheel_name_filter(tmpdir): assert wc.get(link, "package", [("py3", "none", "any")]) is not link # package2 does not match wheel name assert wc.get(link, "package2", [("py3", "none", "any")]) is link + + +def test_cache_hash(): + h = _hash_dict({"url": "https://g.c/o/r"}) + assert h == "c7d60d08b1079254d236e983501fa26c016d58d16010725b27ed0af2" + h = _hash_dict({"url": "https://g.c/o/r", "subdirectory": "sd"}) + assert h == "9cba35d4ccf04b7cde751b44db347fd0f21fa47d1276e32f9d47864c" From 66ba51ca7d2ae42b1ab8d17aa3a00e158baf6541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Sun, 17 Nov 2019 11:44:20 +0100 Subject: [PATCH 5/9] Use legacy cache entries when they exist. Pip 20 changes the cache key format to include the interpreter name. To avoid invalidating all existing caches, we continue using existing cache entries that were computed with the legacy algorithm. This should not regress issue #3025 because wheel cached in such legacy entries should have the python implementation tag set. --- src/pip/_internal/cache.py | 58 ++++++++++++++++++++++++++++++++------ tests/unit/test_cache.py | 20 +++++++++++++ 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 502de4d0bb3..e8495e82515 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -4,7 +4,6 @@ # The following comment should be removed at some point in the future. # mypy: strict-optional=False -import errno import hashlib import logging import os @@ -62,6 +61,34 @@ def __init__(self, cache_dir, format_control, allowed_formats): _valid_formats = {"source", "binary"} assert self.allowed_formats.union(_valid_formats) == _valid_formats + def _get_cache_path_parts_legacy(self, link): + # type: (Link) -> List[str] + """Get parts of part that must be os.path.joined with cache_dir + + Legacy cache key (pip < 20) for compatibility with older caches. + """ + + # We want to generate an url to use as our cache key, we don't want to + # just re-use the URL because it might have other items in the fragment + # and we don't care about those. + key_parts = [link.url_without_fragment] + if link.hash_name is not None and link.hash is not None: + key_parts.append("=".join([link.hash_name, link.hash])) + key_url = "#".join(key_parts) + + # Encode our key url with sha224, we'll use this because it has similar + # security properties to sha256, but with a shorter total output (and + # thus less secure). However the differences don't make a lot of + # difference for our use case here. + hashed = hashlib.sha224(key_url.encode()).hexdigest() + + # We want to nest the directories some to prevent having a ton of top + # level directories where we might run out of sub directories on some + # FS. + parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]] + + return parts + def _get_cache_path_parts(self, link): # type: (Link) -> List[str] """Get parts of part that must be os.path.joined with cache_dir @@ -116,13 +143,19 @@ def _get_candidates(self, link, canonical_package_name): if not self.allowed_formats.intersection(formats): return [] - root = self.get_path_for_link(link) - try: - return os.listdir(root) - except OSError as err: - if err.errno in {errno.ENOENT, errno.ENOTDIR}: - return [] - raise + candidates = [] + path = self.get_path_for_link(link) + if os.path.isdir(path): + candidates.extend(os.listdir(path)) + # TODO remove legacy path lookup in pip>=21 + legacy_path = self.get_path_for_link_legacy(link) + if os.path.isdir(legacy_path): + candidates.extend(os.listdir(legacy_path)) + return candidates + + def get_path_for_link_legacy(self, link): + # type: (Link) -> str + raise NotImplementedError() def get_path_for_link(self, link): # type: (Link) -> str @@ -164,6 +197,11 @@ def __init__(self, cache_dir, format_control): cache_dir, format_control, {"binary"} ) + def get_path_for_link_legacy(self, link): + # type: (Link) -> str + parts = self._get_cache_path_parts_legacy(link) + return os.path.join(self.cache_dir, "wheels", *parts) + def get_path_for_link(self, link): # type: (Link) -> str """Return a directory to store cached wheels for link @@ -256,6 +294,10 @@ def __init__(self, cache_dir, format_control): self._wheel_cache = SimpleWheelCache(cache_dir, format_control) self._ephem_cache = EphemWheelCache(format_control) + def get_path_for_link_legacy(self, link): + # type: (Link) -> str + return self._wheel_cache.get_path_for_link_legacy(link) + def get_path_for_link(self, link): # type: (Link) -> str return self._wheel_cache.get_path_for_link(link) diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index e11b3e78a08..a9baf7e95c3 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -49,3 +49,23 @@ def test_cache_hash(): assert h == "c7d60d08b1079254d236e983501fa26c016d58d16010725b27ed0af2" h = _hash_dict({"url": "https://g.c/o/r", "subdirectory": "sd"}) assert h == "9cba35d4ccf04b7cde751b44db347fd0f21fa47d1276e32f9d47864c" + + +def test_get_path_for_link_legacy(tmpdir): + """ + Test that an existing cache entry that was created with the legacy hashing + mechanism is used. + """ + wc = WheelCache(tmpdir, FormatControl()) + link = Link("https://g.c/o/r") + path = wc.get_path_for_link(link) + legacy_path = wc.get_path_for_link_legacy(link) + assert path != legacy_path + ensure_dir(path) + with open(os.path.join(path, "test-pyz-none-any.whl"), "w"): + pass + ensure_dir(legacy_path) + with open(os.path.join(legacy_path, "test-pyx-none-any.whl"), "w"): + pass + expected_candidates = {"test-pyx-none-any.whl", "test-pyz-none-any.whl"} + assert set(wc._get_candidates(link, "test")) == expected_candidates From bfb7db2f68dc707db8e4c4de219da09318ed017a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Mon, 18 Nov 2019 14:45:50 +0100 Subject: [PATCH 6/9] Advertise the new cache structure in a news file --- news/7296.removal | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 news/7296.removal diff --git a/news/7296.removal b/news/7296.removal new file mode 100644 index 00000000000..bd261c1f33f --- /dev/null +++ b/news/7296.removal @@ -0,0 +1,3 @@ +The pip>=20 wheel cache is not retro-compatible with previous versions. Until +version 21, pip will still be able to take advantage of existing legacy cache +entries. From 824dca1060027c2675fc442a98701469841f7aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Mon, 18 Nov 2019 17:26:28 +0100 Subject: [PATCH 7/9] Better support for unicode cache entries --- src/pip/_internal/cache.py | 12 ++++-------- tests/unit/test_cache.py | 6 ++++-- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index e8495e82515..930205f6461 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -5,6 +5,7 @@ # mypy: strict-optional=False import hashlib +import json import logging import os import sys @@ -30,14 +31,9 @@ def _hash_dict(d): # type: (Dict[str, str]) -> str - """Return a sha224 of a dictionary where keys and values are strings.""" - h = hashlib.new('sha224') - for k in sorted(d.keys()): - h.update(k.encode()) - h.update("=".encode()) - h.update(d[k].encode()) - h.update(b"\0") - return h.hexdigest() + """Return a stable sha224 of a dictionary.""" + s = json.dumps(d, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha224(s.encode("ascii")).hexdigest() class Cache(object): diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index a9baf7e95c3..8926f3af84e 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -46,9 +46,11 @@ def test_wheel_name_filter(tmpdir): def test_cache_hash(): h = _hash_dict({"url": "https://g.c/o/r"}) - assert h == "c7d60d08b1079254d236e983501fa26c016d58d16010725b27ed0af2" + assert h == "72aa79d3315c181d2cc23239d7109a782de663b6f89982624d8c1e86" h = _hash_dict({"url": "https://g.c/o/r", "subdirectory": "sd"}) - assert h == "9cba35d4ccf04b7cde751b44db347fd0f21fa47d1276e32f9d47864c" + assert h == "8b13391b6791bf7f3edeabb41ea4698d21bcbdbba7f9c7dc9339750d" + h = _hash_dict({"subdirectory": u"/\xe9e"}) + assert h == "f83b32dfa27a426dec08c21bf006065dd003d0aac78e7fc493d9014d" def test_get_path_for_link_legacy(tmpdir): From ab0593659df4e51748636e2e0068224f83a9b1e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Sun, 24 Nov 2019 19:08:51 +0100 Subject: [PATCH 8/9] Use interpreter_name and _version in cache keys --- src/pip/_internal/cache.py | 10 +++------- src/pip/_internal/pep425tags.py | 6 ++++++ src/pip/_internal/utils/misc.py | 11 ----------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 930205f6461..8c2041527c1 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -8,15 +8,14 @@ import json import logging import os -import sys from pip._vendor.packaging.utils import canonicalize_name from pip._internal.exceptions import InvalidWheelFilename from pip._internal.models.link import Link from pip._internal.models.wheel import Wheel +from pip._internal.pep425tags import interpreter_name, interpreter_version from pip._internal.utils.compat import expanduser -from pip._internal.utils.misc import interpreter_name from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.urls import path_to_url @@ -104,11 +103,8 @@ def _get_cache_path_parts(self, link): # depending on the python version their setup.py is being run on, # and don't encode the difference in compatibility tags. # https://github.com/pypa/pip/issues/7296 - key_parts["interpreter"] = "{}-{}.{}".format( - interpreter_name(), - sys.version_info[0], - sys.version_info[1], - ) + key_parts["interpreter_name"] = interpreter_name() + key_parts["interpreter_version"] = interpreter_version() # Encode our key url with sha224, we'll use this because it has similar # security properties to sha256, but with a shorter total output (and diff --git a/src/pip/_internal/pep425tags.py b/src/pip/_internal/pep425tags.py index b8aba326c71..3a291ebaeb1 100644 --- a/src/pip/_internal/pep425tags.py +++ b/src/pip/_internal/pep425tags.py @@ -54,6 +54,9 @@ def get_abbr_impl(): return pyimpl +interpreter_name = get_abbr_impl + + def version_info_to_nodot(version_info): # type: (Tuple[int, ...]) -> str # Only use up to the first two numbers. @@ -69,6 +72,9 @@ def get_impl_ver(): return impl_ver +interpreter_version = get_impl_ver + + def get_impl_version_info(): # type: () -> Tuple[int, ...] """Return sys.version_info-like tuple for use in decrementing the minor diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 9a802556269..85acab856b0 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -11,7 +11,6 @@ import io import logging import os -import platform import posixpath import shutil import stat @@ -880,13 +879,3 @@ def hash_file(path, blocksize=1 << 20): length += len(block) h.update(block) return (h, length) # type: ignore - - -def interpreter_name(): - # type: () -> str - try: - name = sys.implementation.name # type: ignore - except AttributeError: # pragma: no cover - # Python 2.7 compatibility. - name = platform.python_implementation().lower() - return name From e3c1ca137f2557f88e4bfb68d971f56531c7ff0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Mon, 2 Dec 2019 12:06:32 +0100 Subject: [PATCH 9/9] Update news/7296.removal Co-Authored-By: Pradyun Gedam --- news/7296.removal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/7296.removal b/news/7296.removal index bd261c1f33f..ef0e5f7495f 100644 --- a/news/7296.removal +++ b/news/7296.removal @@ -1,3 +1,3 @@ The pip>=20 wheel cache is not retro-compatible with previous versions. Until -version 21, pip will still be able to take advantage of existing legacy cache +pip 21.0, pip will continue to take advantage of existing legacy cache entries.