execute batch downloads in parallel worker threads

news/12923.feature.rst

@@ -0,0 +1 @@
+Download concrete dists for metadata-only resolves in parallel using worker threads. Add ``--batch-download-parallelism`` CLI flag to limit parallelism.

news/12925.feature.rst

@@ -0,0 +1 @@
+Use very rich progress output for batch downloading. Use ``ProgressBarType`` enum class for ``--progress-bar`` choices.

src/pip/_internal/cli/base_command.py

@@ -17,6 +17,7 @@
 from pip._internal.cli import cmdoptions
 from pip._internal.cli.command_context import CommandContextMixIn
 from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter
+from pip._internal.cli.progress_bars import ProgressBarType
 from pip._internal.cli.status_codes import (
     ERROR,
     PREVIOUS_BUILD_DIR_ERROR,
@@ -176,8 +177,11 @@ def _main(self, args: list[str]) -> int:
         if options.debug_mode:
             self.verbosity = 2
 
-        if hasattr(options, "progress_bar") and options.progress_bar == "auto":
-            options.progress_bar = "on" if self.verbosity >= 0 else "off"
+        if getattr(options, "progress_bar", None) == ProgressBarType.AUTO.value:
+            if self.verbosity >= 0:
+                options.progress_bar = ProgressBarType.ON.value
+            else:
+                options.progress_bar = ProgressBarType.OFF.value
 
         reconfigure(no_color=options.no_color)
         level_number = setup_logging(

src/pip/_internal/cli/cmdoptions.py

@@ -24,6 +24,7 @@
 from pip._vendor.packaging.utils import canonicalize_name
 
 from pip._internal.cli.parser import ConfigOptionParser
+from pip._internal.cli.progress_bars import ProgressBarType
 from pip._internal.exceptions import CommandError
 from pip._internal.locations import USER_CACHE_DIR, get_src_prefix
 from pip._internal.models.format_control import FormatControl
@@ -228,15 +229,32 @@ class PipOption(Option):
     "--progress-bar",
     dest="progress_bar",
     type="choice",
-    choices=["auto", "on", "off", "raw"],
-    default="auto",
+    choices=ProgressBarType.choices(),
+    default=ProgressBarType.AUTO.value,
     help=(
         "Specify whether the progress bar should be used. In 'auto'"
         " mode, --quiet will suppress all progress bars."
-        " [auto, on, off, raw] (default: auto)"
+        f" {ProgressBarType.help_choices()} (default: %default)"
+    ),
+)
+
+batch_download_parallelism: Callable[..., Option] = partial(
+    Option,
+    "--batch-download-parallelism",
+    dest="batch_download_parallelism",
+    type="int",
+    default=10,
+    help=(
+        "Maximum parallelism employed for batch downloading of metadata-only dists"
+        " (default %default parallel requests)."
+        " Note that more than 10 downloads may overflow the requests connection pool,"
+        " which may affect performance."
+        " Note also that commands such as 'install --dry-run' should avoid downloads"
+        " entirely, and so will not be affected by this option."
     ),
 )
 
+
 log: Callable[..., Option] = partial(
     PipOption,
     "--log",