From 0a0fe938228d6952296f53ff03f40337677352f4 Mon Sep 17 00:00:00 2001 From: arena Date: Tue, 19 Mar 2024 17:31:36 +0800 Subject: [PATCH 1/8] raw progress bar mode --- docs/html/user_guide.rst | 6 ++++++ src/pip/_internal/cli/cmdoptions.py | 4 ++-- src/pip/_internal/cli/progress_bars.py | 25 +++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index f0cbded683d..a9021097490 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -856,6 +856,12 @@ We are using `freeze`_ here which outputs installed packages in requirements for reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']) +To programmatically monitor download progress use the ``--progress-bar=raw`` option. +This will print lines to stdout in the format ``Progress CURRENT of TOTAL``, where +``CURRENT`` and ``TOTAL`` are integers and the unit is bytes. +If the real total is unknown then ``TOTAL`` is set to ``0``. Be aware that the +specific formatting of pip's outputs are *not* guaranteed to be the same in future versions. + If you don't want to use pip's command line functionality, but are rather trying to implement code that works with Python packages, their metadata, or PyPI, then you should consider other, supported, packages that offer this type diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index d05e502f908..7becdd0cbe2 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -226,9 +226,9 @@ class PipOption(Option): "--progress-bar", dest="progress_bar", type="choice", - choices=["on", "off"], + choices=["on", "off", "raw"], default="on", - help="Specify whether the progress bar should be used [on, off] (default: on)", + help="Specify whether the progress bar should be used [on, off, raw] (default: on)", ) log: Callable[..., Option] = partial( diff --git a/src/pip/_internal/cli/progress_bars.py b/src/pip/_internal/cli/progress_bars.py index 0ad14031ca5..f1861636252 100644 --- a/src/pip/_internal/cli/progress_bars.py +++ b/src/pip/_internal/cli/progress_bars.py @@ -1,4 +1,5 @@ import functools +import sys from typing import Callable, Generator, Iterable, Iterator, Optional, Tuple from pip._vendor.rich.progress import ( @@ -14,6 +15,7 @@ TransferSpeedColumn, ) +from pip._internal.cli.spinners import RateLimiter from pip._internal.utils.logging import get_indentation DownloadProgressRenderer = Callable[[Iterable[bytes]], Iterator[bytes]] @@ -54,6 +56,27 @@ def _rich_progress_bar( yield chunk progress.update(task_id, advance=len(chunk)) +def _raw_progress_bar( + iterable: Iterable[bytes], + *, + size: Optional[int], +) -> Generator[bytes, None, None]: + + def write_progress(current, total): + sys.stdout.write("Progress %d of %d\n" % (current, total)) + sys.stdout.flush() + + current = 0 + total = size or 0 + rate_limiter = RateLimiter(0.25) + + write_progress(current, total) + for chunk in iterable: + current += len(chunk) + if rate_limiter.ready() or current == total: + write_progress(current, total) + rate_limiter.reset() + yield chunk def get_download_progress_renderer( *, bar_type: str, size: Optional[int] = None @@ -64,5 +87,7 @@ def get_download_progress_renderer( """ if bar_type == "on": return functools.partial(_rich_progress_bar, bar_type=bar_type, size=size) + elif bar_type == "raw": + return functools.partial(_raw_progress_bar, size=size) else: return iter # no-op, when passed an iterator From 4019197bdc195c84d04b2a78fcde0c1dc53bc008 Mon Sep 17 00:00:00 2001 From: arena Date: Tue, 19 Mar 2024 17:39:00 +0800 Subject: [PATCH 2/8] lint --- docs/html/user_guide.rst | 8 ++++---- src/pip/_internal/cli/progress_bars.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index a9021097490..2fa5552b1c5 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -856,10 +856,10 @@ We are using `freeze`_ here which outputs installed packages in requirements for reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']) -To programmatically monitor download progress use the ``--progress-bar=raw`` option. -This will print lines to stdout in the format ``Progress CURRENT of TOTAL``, where -``CURRENT`` and ``TOTAL`` are integers and the unit is bytes. -If the real total is unknown then ``TOTAL`` is set to ``0``. Be aware that the +To programmatically monitor download progress use the ``--progress-bar=raw`` option. +This will print lines to stdout in the format ``Progress CURRENT of TOTAL``, where +``CURRENT`` and ``TOTAL`` are integers and the unit is bytes. +If the real total is unknown then ``TOTAL`` is set to ``0``. Be aware that the specific formatting of pip's outputs are *not* guaranteed to be the same in future versions. If you don't want to use pip's command line functionality, but are rather diff --git a/src/pip/_internal/cli/progress_bars.py b/src/pip/_internal/cli/progress_bars.py index f1861636252..fc2f33eddd4 100644 --- a/src/pip/_internal/cli/progress_bars.py +++ b/src/pip/_internal/cli/progress_bars.py @@ -56,12 +56,12 @@ def _rich_progress_bar( yield chunk progress.update(task_id, advance=len(chunk)) + def _raw_progress_bar( iterable: Iterable[bytes], *, size: Optional[int], ) -> Generator[bytes, None, None]: - def write_progress(current, total): sys.stdout.write("Progress %d of %d\n" % (current, total)) sys.stdout.flush() @@ -78,6 +78,7 @@ def write_progress(current, total): rate_limiter.reset() yield chunk + def get_download_progress_renderer( *, bar_type: str, size: Optional[int] = None ) -> DownloadProgressRenderer: From 2f14e8f57d8c8f8416d0b9945546acc3c7127fde Mon Sep 17 00:00:00 2001 From: arena Date: Tue, 19 Mar 2024 17:40:07 +0800 Subject: [PATCH 3/8] type annotation --- src/pip/_internal/cli/progress_bars.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pip/_internal/cli/progress_bars.py b/src/pip/_internal/cli/progress_bars.py index fc2f33eddd4..b842b1b316a 100644 --- a/src/pip/_internal/cli/progress_bars.py +++ b/src/pip/_internal/cli/progress_bars.py @@ -62,7 +62,7 @@ def _raw_progress_bar( *, size: Optional[int], ) -> Generator[bytes, None, None]: - def write_progress(current, total): + def write_progress(current: int, total: int) -> None: sys.stdout.write("Progress %d of %d\n" % (current, total)) sys.stdout.flush() From eef2e23beb6b6cb38807496a96a8b5ed3f9467cf Mon Sep 17 00:00:00 2001 From: arena Date: Fri, 22 Mar 2024 08:05:49 +0800 Subject: [PATCH 4/8] news entry --- news/11508.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/11508.feature.rst diff --git a/news/11508.feature.rst b/news/11508.feature.rst new file mode 100644 index 00000000000..685426b1dc7 --- /dev/null +++ b/news/11508.feature.rst @@ -0,0 +1 @@ +Add a 'raw' progress_bar type for simple and parsable download progress reports \ No newline at end of file From ea93e2026575a7e9bb57e86e9af7f718b1754c8e Mon Sep 17 00:00:00 2001 From: Mark McLoughlin Date: Thu, 14 Mar 2024 17:34:33 +0000 Subject: [PATCH 5/8] Pass -vv to pip subprocess Fixes #12577 This looks like it was an oversight in #9450 - we should pass the correct verbosity level to build env install subprocesses. Tested with: ``` rm -rf ~/.cache/pip && rm -f *.whl && pip wheel --no-binary :all: hatchling ``` and all three verbosity levels, before and after this change, giving the following logs: ``` 33 patched-verbosity0.log 2549 patched-verbosity1.log 11938 patched-verbosity2.log 33 unpatched-verbosity0.log 99 unpatched-verbosity1.log 1030 unpatched-verbosity2.log ``` i.e. currently a lot of useful logs are being dropped from these install subprocesess even with -vvv --- news/12577.bugfix.rst | 1 + src/pip/_internal/build_env.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 news/12577.bugfix.rst diff --git a/news/12577.bugfix.rst b/news/12577.bugfix.rst new file mode 100644 index 00000000000..b408be6a8c9 --- /dev/null +++ b/news/12577.bugfix.rst @@ -0,0 +1 @@ +Ensure ``-vv`` gets passed to any ``pip install`` build environment subprocesses. diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 4f704a3547d..838de86474f 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -19,6 +19,7 @@ from pip._internal.cli.spinners import open_spinner from pip._internal.locations import get_platlib, get_purelib, get_scheme from pip._internal.metadata import get_default_environment, get_environment +from pip._internal.utils.logging import VERBOSE from pip._internal.utils.subprocess import call_subprocess from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds @@ -242,6 +243,8 @@ def _install_requirements( "--no-warn-script-location", ] if logger.getEffectiveLevel() <= logging.DEBUG: + args.append("-vv") + elif logger.getEffectiveLevel() <= VERBOSE: args.append("-v") for format_control in ("no_binary", "only_binary"): formats = getattr(finder.format_control, format_control) From e2b9b957d259f2c9e20fb21fc2f0688375fe07dd Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Thu, 21 Mar 2024 23:05:31 -0500 Subject: [PATCH 6/8] DOC: Clarify lack of index priority * Specifically mention the --index-url and --extra-index-url both provide locations for pip to look for packages in the 'Finding Packages' section. However, explicitly state that there is no priority ordering for the search locations. * This is a recurring point of confusion on the pip GitHub issue tracker as well as on https://github.com/pypa/packaging-problems/issues/, so additional clarification could help here. Co-authored-by: Henry Schreiner --- docs/html/cli/pip_install.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/html/cli/pip_install.rst b/docs/html/cli/pip_install.rst index 2664c75223d..d893fb7c8ef 100644 --- a/docs/html/cli/pip_install.rst +++ b/docs/html/cli/pip_install.rst @@ -212,11 +212,12 @@ and `there `_. pip offers a number of package index options for modifying how packages are found. -pip looks for packages in a number of places: on PyPI (if not disabled via -``--no-index``), in the local filesystem, and in any additional repositories -specified via ``--find-links`` or ``--index-url``. There is no ordering in -the locations that are searched. Rather they are all checked, and the "best" -match for the requirements (in terms of version number - see the +pip looks for packages in a number of places: on PyPI (or the index given as +``--index-url``, if not disabled via ``--no-index``), in the local filesystem, +and in any additional repositories specified via ``--find-links`` or +``--extra-index-url``. There is no priority in the locations that are searched. +Rather they are all checked, and the "best" match for the requirements (in +terms of version number - see the :ref:`specification ` for details) is selected. See the :ref:`pip install Examples`. From db6ccf9b546a710654fd1223d7362bd92eadb408 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Thu, 21 Mar 2024 23:21:13 -0500 Subject: [PATCH 7/8] Add trivial NEWS entry --- news/91d23d4d-a9cc-442f-a569-c46e0bdc3e64.trivial.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 news/91d23d4d-a9cc-442f-a569-c46e0bdc3e64.trivial.rst diff --git a/news/91d23d4d-a9cc-442f-a569-c46e0bdc3e64.trivial.rst b/news/91d23d4d-a9cc-442f-a569-c46e0bdc3e64.trivial.rst new file mode 100644 index 00000000000..e69de29bb2d From 72b6dfaa187df8d821d6b99ef9a3ee8ccb959a95 Mon Sep 17 00:00:00 2001 From: arenasys Date: Tue, 26 Mar 2024 10:24:24 +0800 Subject: [PATCH 8/8] news tweak --- news/11508.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/11508.feature.rst b/news/11508.feature.rst index 685426b1dc7..2f0d7e2d04d 100644 --- a/news/11508.feature.rst +++ b/news/11508.feature.rst @@ -1 +1 @@ -Add a 'raw' progress_bar type for simple and parsable download progress reports \ No newline at end of file +Add a 'raw' progress_bar type for simple and parsable download progress reports