Merge pull request #9432 from uranusjr/new-resolver-dedup-on-backtracking

Avoid downloading multiple candidates of a same version

Author: pradyunsg

src/pip/_internal/resolution/resolvelib/factory.py

@@ -228,9 +228,12 @@ def iter_index_candidates():
             all_yanked = all(ican.link.is_yanked for ican in icans)
 
             # PackageFinder returns earlier versions first, so we reverse.
+            versions_found = set()  # type: Set[_BaseVersion]
             for ican in reversed(icans):
                 if not all_yanked and ican.link.is_yanked:
                     continue
+                if ican.version in versions_found:
+                    continue
                 candidate = self._make_candidate_from_link(
                     link=ican.link,
                     extras=extras,
@@ -241,6 +244,7 @@ def iter_index_candidates():
                 if candidate is None:
                     continue
                 yield candidate
+                versions_found.add(ican.version)
 
         return FoundCandidates(
             iter_index_candidates,

src/pip/_internal/resolution/resolvelib/found_candidates.py

@@ -16,23 +16,11 @@
 from pip._internal.utils.typing import MYPY_CHECK_RUNNING
 
 if MYPY_CHECK_RUNNING:
-    from typing import Callable, Iterator, Optional, Set
-
-    from pip._vendor.packaging.version import _BaseVersion
+    from typing import Callable, Iterator, Optional
 
     from .base import Candidate
 
 
-def _deduplicated_by_version(candidates):
-    # type: (Iterator[Candidate]) -> Iterator[Candidate]
-    returned = set()  # type: Set[_BaseVersion]
-    for candidate in candidates:
-        if candidate.version in returned:
-            continue
-        returned.add(candidate.version)
-        yield candidate
-
-
 def _insert_installed(installed, others):
     # type: (Candidate, Iterator[Candidate]) -> Iterator[Candidate]
     """Iterator for ``FoundCandidates``.
@@ -86,12 +74,15 @@ def __getitem__(self, index):
     def __iter__(self):
         # type: () -> Iterator[Candidate]
         if not self._installed:
-            candidates = self._get_others()
-        elif self._prefers_installed:
-            candidates = itertools.chain([self._installed], self._get_others())
-        else:
-            candidates = _insert_installed(self._installed, self._get_others())
-        return _deduplicated_by_version(candidates)
+            return self._get_others()
+        others = (
+            candidate
+            for candidate in self._get_others()
+            if candidate.version != self._installed.version
+        )
+        if self._prefers_installed:
+            return itertools.chain([self._installed], others)
+        return _insert_installed(self._installed, others)
 
     def __len__(self):
         # type: () -> int