From 59972c75fa78eabb70f04abab56d9e3c0f36f5be Mon Sep 17 00:00:00 2001 From: Joey Ballentine Date: Wed, 27 Mar 2024 00:25:28 -0400 Subject: [PATCH 1/9] Show install progress when using `--progress-bar=raw` --- src/pip/_internal/cli/progress_bars.py | 20 +++++++++++++++++--- src/pip/_internal/commands/install.py | 1 + src/pip/_internal/req/__init__.py | 13 ++++++++++++- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/cli/progress_bars.py b/src/pip/_internal/cli/progress_bars.py index b842b1b316a..7802a11fea0 100644 --- a/src/pip/_internal/cli/progress_bars.py +++ b/src/pip/_internal/cli/progress_bars.py @@ -18,7 +18,7 @@ from pip._internal.cli.spinners import RateLimiter from pip._internal.utils.logging import get_indentation -DownloadProgressRenderer = Callable[[Iterable[bytes]], Iterator[bytes]] +ProgressRenderer = Callable[[Iterable[bytes]], Iterator[bytes]] def _rich_progress_bar( @@ -61,6 +61,7 @@ def _raw_progress_bar( iterable: Iterable[bytes], *, size: Optional[int], + chunk_size: Optional[int], ) -> Generator[bytes, None, None]: def write_progress(current: int, total: int) -> None: sys.stdout.write("Progress %d of %d\n" % (current, total)) @@ -72,7 +73,7 @@ def write_progress(current: int, total: int) -> None: write_progress(current, total) for chunk in iterable: - current += len(chunk) + current += chunk_size or len(chunk) if rate_limiter.ready() or current == total: write_progress(current, total) rate_limiter.reset() @@ -81,7 +82,7 @@ def write_progress(current: int, total: int) -> None: def get_download_progress_renderer( *, bar_type: str, size: Optional[int] = None -) -> DownloadProgressRenderer: +) -> ProgressRenderer: """Get an object that can be used to render the download progress. Returns a callable, that takes an iterable to "wrap". @@ -92,3 +93,16 @@ def get_download_progress_renderer( return functools.partial(_raw_progress_bar, size=size) else: return iter # no-op, when passed an iterator + + +def get_install_progress_renderer( + *, bar_type: str, total: Optional[int] = None +) -> ProgressRenderer: + """Get an object that can be used to render the install progress. + + Returns a callable, that takes an iterable to "wrap". + """ + if bar_type == "raw": + return functools.partial(_raw_progress_bar, size=total, chunk_size=1) + else: + return iter # no-op, when passed an iterator diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 6cf7571e4a6..dd218f51c80 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -458,6 +458,7 @@ def run(self, options: Values, args: List[str]) -> int: warn_script_location=warn_script_location, use_user_site=options.use_user_site, pycompile=options.compile, + progress_bar=options.progress_bar, ) lib_locations = get_lib_location_guesses( diff --git a/src/pip/_internal/req/__init__.py b/src/pip/_internal/req/__init__.py index 16de903a44c..c4b88b8d102 100644 --- a/src/pip/_internal/req/__init__.py +++ b/src/pip/_internal/req/__init__.py @@ -2,6 +2,7 @@ import logging from typing import Generator, List, Optional, Sequence, Tuple +from pip._internal.cli.progress_bars import get_install_progress_renderer from pip._internal.utils.logging import indent_log from .req_file import parse_requirements @@ -43,6 +44,7 @@ def install_given_reqs( warn_script_location: bool, use_user_site: bool, pycompile: bool, + progress_bar: str, ) -> List[InstallationResult]: """ Install everything in the given list. @@ -59,8 +61,17 @@ def install_given_reqs( installed = [] + show_progress = logger.getEffectiveLevel() <= logging.INFO + + items = to_install.items() + if show_progress: + renderer = get_install_progress_renderer( + bar_type=progress_bar, total=len(to_install) + ) + items = renderer(items) + with indent_log(): - for req_name, requirement in to_install.items(): + for req_name, requirement in items: if requirement.should_reinstall: logger.info("Attempting uninstall: %s", req_name) with indent_log(): From 9b5001e1c36c76301fb023698109b3dbb75d58a0 Mon Sep 17 00:00:00 2001 From: Joey Ballentine Date: Wed, 27 Mar 2024 00:37:16 -0400 Subject: [PATCH 2/9] Add changelog entry --- news/12601.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/12601.feature.rst diff --git a/news/12601.feature.rst b/news/12601.feature.rst new file mode 100644 index 00000000000..fa38a4aabac --- /dev/null +++ b/news/12601.feature.rst @@ -0,0 +1 @@ +Output install progress when 'raw' progress_bar type is in use From fa26d510b0e3c6813dda23454a0586aef75b4c86 Mon Sep 17 00:00:00 2001 From: Joey Ballentine Date: Wed, 27 Mar 2024 00:40:09 -0400 Subject: [PATCH 3/9] Fix types --- src/pip/_internal/cli/progress_bars.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/cli/progress_bars.py b/src/pip/_internal/cli/progress_bars.py index 7802a11fea0..9663af6f358 100644 --- a/src/pip/_internal/cli/progress_bars.py +++ b/src/pip/_internal/cli/progress_bars.py @@ -16,9 +16,13 @@ ) from pip._internal.cli.spinners import RateLimiter +from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.logging import get_indentation -ProgressRenderer = Callable[[Iterable[bytes]], Iterator[bytes]] +DownloadProgressRenderer = Callable[[Iterable[bytes]], Iterator[bytes]] +InstallProgressRenderer = Callable[ + [Iterable[str, InstallRequirement]], Iterator[str, InstallRequirement] +] def _rich_progress_bar( @@ -82,7 +86,7 @@ def write_progress(current: int, total: int) -> None: def get_download_progress_renderer( *, bar_type: str, size: Optional[int] = None -) -> ProgressRenderer: +) -> DownloadProgressRenderer: """Get an object that can be used to render the download progress. Returns a callable, that takes an iterable to "wrap". @@ -97,7 +101,7 @@ def get_download_progress_renderer( def get_install_progress_renderer( *, bar_type: str, total: Optional[int] = None -) -> ProgressRenderer: +) -> InstallProgressRenderer: """Get an object that can be used to render the install progress. Returns a callable, that takes an iterable to "wrap". From 9b8540efa904729b276a4c7246566fd7308a7d1b Mon Sep 17 00:00:00 2001 From: Joey Ballentine Date: Wed, 27 Mar 2024 00:41:56 -0400 Subject: [PATCH 4/9] Actually fix type --- src/pip/_internal/cli/progress_bars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/cli/progress_bars.py b/src/pip/_internal/cli/progress_bars.py index 9663af6f358..ee83b11dac4 100644 --- a/src/pip/_internal/cli/progress_bars.py +++ b/src/pip/_internal/cli/progress_bars.py @@ -21,7 +21,7 @@ DownloadProgressRenderer = Callable[[Iterable[bytes]], Iterator[bytes]] InstallProgressRenderer = Callable[ - [Iterable[str, InstallRequirement]], Iterator[str, InstallRequirement] + [Iterable[Tuple[str, InstallRequirement]]], Iterator[Tuple[str, InstallRequirement]] ] From fb1859fa95e5858a6456f6b196eba3f113b2c6bd Mon Sep 17 00:00:00 2001 From: Joey Ballentine Date: Wed, 27 Mar 2024 00:44:52 -0400 Subject: [PATCH 5/9] wrap with iter --- src/pip/_internal/req/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/req/__init__.py b/src/pip/_internal/req/__init__.py index c4b88b8d102..702a6fc0421 100644 --- a/src/pip/_internal/req/__init__.py +++ b/src/pip/_internal/req/__init__.py @@ -63,7 +63,7 @@ def install_given_reqs( show_progress = logger.getEffectiveLevel() <= logging.INFO - items = to_install.items() + items = iter(to_install.items()) if show_progress: renderer = get_install_progress_renderer( bar_type=progress_bar, total=len(to_install) From 6c95fb420457d2c814641c152346edf1c9e39c83 Mon Sep 17 00:00:00 2001 From: Joey Ballentine Date: Wed, 27 Mar 2024 00:47:52 -0400 Subject: [PATCH 6/9] use Any in raw progress return type yes i installed mypy locally to test this, so it should finally pass ci --- src/pip/_internal/cli/progress_bars.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/cli/progress_bars.py b/src/pip/_internal/cli/progress_bars.py index ee83b11dac4..3d74fcaa29a 100644 --- a/src/pip/_internal/cli/progress_bars.py +++ b/src/pip/_internal/cli/progress_bars.py @@ -1,6 +1,6 @@ import functools import sys -from typing import Callable, Generator, Iterable, Iterator, Optional, Tuple +from typing import Any, Callable, Generator, Iterable, Iterator, Optional, Tuple from pip._vendor.rich.progress import ( BarColumn, @@ -66,7 +66,7 @@ def _raw_progress_bar( *, size: Optional[int], chunk_size: Optional[int], -) -> Generator[bytes, None, None]: +) -> Generator[Any, None, None]: def write_progress(current: int, total: int) -> None: sys.stdout.write("Progress %d of %d\n" % (current, total)) sys.stdout.flush() From 3b42fa51e7de4578d7164f120edc8b8fab3184a4 Mon Sep 17 00:00:00 2001 From: Joey Ballentine Date: Wed, 27 Mar 2024 20:55:00 -0400 Subject: [PATCH 7/9] rename chunk_size to unit_size --- src/pip/_internal/cli/progress_bars.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/cli/progress_bars.py b/src/pip/_internal/cli/progress_bars.py index 3d74fcaa29a..b7fff83c57a 100644 --- a/src/pip/_internal/cli/progress_bars.py +++ b/src/pip/_internal/cli/progress_bars.py @@ -65,7 +65,7 @@ def _raw_progress_bar( iterable: Iterable[bytes], *, size: Optional[int], - chunk_size: Optional[int], + unit_size: Optional[int], ) -> Generator[Any, None, None]: def write_progress(current: int, total: int) -> None: sys.stdout.write("Progress %d of %d\n" % (current, total)) @@ -77,7 +77,7 @@ def write_progress(current: int, total: int) -> None: write_progress(current, total) for chunk in iterable: - current += chunk_size or len(chunk) + current += unit_size or len(chunk) if rate_limiter.ready() or current == total: write_progress(current, total) rate_limiter.reset() From 99de15437bae1a6485da971b8f0563caebe0c00c Mon Sep 17 00:00:00 2001 From: Joey Ballentine Date: Wed, 27 Mar 2024 22:23:27 -0400 Subject: [PATCH 8/9] use TypeVar --- src/pip/_internal/cli/progress_bars.py | 35 ++++++++++++++++++-------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/pip/_internal/cli/progress_bars.py b/src/pip/_internal/cli/progress_bars.py index b7fff83c57a..a098ea06808 100644 --- a/src/pip/_internal/cli/progress_bars.py +++ b/src/pip/_internal/cli/progress_bars.py @@ -1,6 +1,15 @@ import functools import sys -from typing import Any, Callable, Generator, Iterable, Iterator, Optional, Tuple +from typing import ( + Callable, + Generator, + Iterable, + Iterator, + Optional, + Sized, + Tuple, + TypeVar, +) from pip._vendor.rich.progress import ( BarColumn, @@ -19,10 +28,9 @@ from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.logging import get_indentation -DownloadProgressRenderer = Callable[[Iterable[bytes]], Iterator[bytes]] -InstallProgressRenderer = Callable[ - [Iterable[Tuple[str, InstallRequirement]]], Iterator[Tuple[str, InstallRequirement]] -] +P = TypeVar("P") + +ProgressRenderer = Callable[[Iterable[P]], Iterator[P]] def _rich_progress_bar( @@ -61,12 +69,15 @@ def _rich_progress_bar( progress.update(task_id, advance=len(chunk)) +T = TypeVar("T", bound=Sized) + + def _raw_progress_bar( - iterable: Iterable[bytes], + iterable: Iterable[T], *, size: Optional[int], unit_size: Optional[int], -) -> Generator[Any, None, None]: +) -> Generator[T, None, None]: def write_progress(current: int, total: int) -> None: sys.stdout.write("Progress %d of %d\n" % (current, total)) sys.stdout.flush() @@ -86,7 +97,7 @@ def write_progress(current: int, total: int) -> None: def get_download_progress_renderer( *, bar_type: str, size: Optional[int] = None -) -> DownloadProgressRenderer: +) -> ProgressRenderer[bytes]: """Get an object that can be used to render the download progress. Returns a callable, that takes an iterable to "wrap". @@ -94,19 +105,21 @@ def get_download_progress_renderer( if bar_type == "on": return functools.partial(_rich_progress_bar, bar_type=bar_type, size=size) elif bar_type == "raw": - return functools.partial(_raw_progress_bar, size=size) + return functools.partial[Iterator[bytes]](_raw_progress_bar, size=size) else: return iter # no-op, when passed an iterator def get_install_progress_renderer( *, bar_type: str, total: Optional[int] = None -) -> InstallProgressRenderer: +) -> ProgressRenderer[Tuple[str, InstallRequirement]]: """Get an object that can be used to render the install progress. Returns a callable, that takes an iterable to "wrap". """ if bar_type == "raw": - return functools.partial(_raw_progress_bar, size=total, chunk_size=1) + return functools.partial[Iterator[Tuple[str, InstallRequirement]]]( + _raw_progress_bar, size=total, unit_size=1 + ) else: return iter # no-op, when passed an iterator From 31d8dad2e975ee4defd7fff4ebcc7ea7879dc487 Mon Sep 17 00:00:00 2001 From: Joey Ballentine Date: Tue, 2 Apr 2024 00:15:56 -0400 Subject: [PATCH 9/9] default to 0 instead of None --- src/pip/_internal/cli/progress_bars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/cli/progress_bars.py b/src/pip/_internal/cli/progress_bars.py index a098ea06808..96e1653bcb3 100644 --- a/src/pip/_internal/cli/progress_bars.py +++ b/src/pip/_internal/cli/progress_bars.py @@ -76,7 +76,7 @@ def _raw_progress_bar( iterable: Iterable[T], *, size: Optional[int], - unit_size: Optional[int], + unit_size: int = 0, ) -> Generator[T, None, None]: def write_progress(current: int, total: int) -> None: sys.stdout.write("Progress %d of %d\n" % (current, total))