From f4962cce702de226dd8b80030f057c60784203e5 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 30 Apr 2022 17:29:03 -0600 Subject: [PATCH 1/2] Optional truststore support This adds a --use-feature=truststore flag that, when specified on Python 3.10+ with truststore installed, switches pip to use truststore to provide HTTPS certificate validation, instead of certifi. This allows pip to verify certificates against custom certificates in the system store. truststore is deliberately NOT vendored because it is expected the library to be under active development in the short term, and this prevents users having to wait for a pip release to get potentially vital bug fixes needed to be made in truststore. Supplying the use-feature flag without installing truststore beforehand, or on Python versions prior to 3.10, results in a command error. --- news/11082.feature.rst | 3 ++ src/pip/_internal/cli/cmdoptions.py | 2 +- src/pip/_internal/cli/req_command.py | 55 +++++++++++++++++++--- src/pip/_internal/network/session.py | 70 ++++++++++++++++++++++++++-- 4 files changed, 119 insertions(+), 11 deletions(-) create mode 100644 news/11082.feature.rst diff --git a/news/11082.feature.rst b/news/11082.feature.rst new file mode 100644 index 00000000000..105fc44d655 --- /dev/null +++ b/news/11082.feature.rst @@ -0,0 +1,3 @@ +Add support to use `truststore `_ as an alternative SSL certificate verification backend. The backend can be enabled on Python 3.10 and later by installing ``truststore`` into the environment, and adding the ``--use-feature=truststore`` flag to various pip commands. + +``truststore`` differs from the current default verification backend (provided by ``certifi``) in it uses the operating system’s trust store, which can be better controlled and augmented to better support non-standard certificates. Depending on feedback, pip may switch to this as the default certificate verification backend in the future. diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index c84ecabd380..1d1360a3572 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -991,7 +991,7 @@ def check_list_path_option(options: Values) -> None: metavar="feature", action="append", default=[], - choices=["2020-resolver", "fast-deps"], + choices=["2020-resolver", "fast-deps", "truststore"], help="Enable new functionality, that may be backward incompatible.", ) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index aab177002d4..79cebb6a2a0 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -10,7 +10,7 @@ import sys from functools import partial from optparse import Values -from typing import Any, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, List, Optional, Tuple from pip._internal.cache import WheelCache from pip._internal.cli import cmdoptions @@ -42,9 +42,33 @@ ) from pip._internal.utils.virtualenv import running_under_virtualenv +if TYPE_CHECKING: + from ssl import SSLContext + logger = logging.getLogger(__name__) +def _create_truststore_ssl_context() -> Optional["SSLContext"]: + if sys.version_info < (3, 10): + raise CommandError("The truststore feature is only available for Python 3.10+") + + try: + import ssl + except ImportError: + logger.warning("Disabling truststore since ssl support is missing") + return None + + try: + import truststore + except ImportError: + raise CommandError( + "To use the truststore feature, 'truststore' must be installed into " + "pip's current environment." + ) + + return truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + + class SessionCommandMixin(CommandContextMixIn): """ @@ -84,15 +108,27 @@ def _build_session( options: Values, retries: Optional[int] = None, timeout: Optional[int] = None, + fallback_to_certifi: bool = False, ) -> PipSession: - assert not options.cache_dir or os.path.isabs(options.cache_dir) + cache_dir = options.cache_dir + assert not cache_dir or os.path.isabs(cache_dir) + + if "truststore" in options.features_enabled: + try: + ssl_context = _create_truststore_ssl_context() + except Exception: + if not fallback_to_certifi: + raise + ssl_context = None + else: + ssl_context = None + session = PipSession( - cache=( - os.path.join(options.cache_dir, "http") if options.cache_dir else None - ), + cache=os.path.join(cache_dir, "http") if cache_dir else None, retries=retries if retries is not None else options.retries, trusted_hosts=options.trusted_hosts, index_urls=self._get_index_urls(options), + ssl_context=ssl_context, ) # Handle custom ca-bundles from the user @@ -142,7 +178,14 @@ def handle_pip_version_check(self, options: Values) -> None: # Otherwise, check if we're using the latest version of pip available. session = self._build_session( - options, retries=0, timeout=min(5, options.timeout) + options, + retries=0, + timeout=min(5, options.timeout), + # This is set to ensure the function does not fail when truststore is + # specified in use-feature but cannot be loaded. This usually raises a + # CommandError and shows a nice user-facing error, but this function is not + # called in that try-except block. + fallback_to_certifi=True, ) with session: pip_self_version_check(session, options) diff --git a/src/pip/_internal/network/session.py b/src/pip/_internal/network/session.py index e2c8582e506..374d838e51a 100644 --- a/src/pip/_internal/network/session.py +++ b/src/pip/_internal/network/session.py @@ -15,11 +15,23 @@ import sys import urllib.parse import warnings -from typing import Any, Dict, Generator, List, Mapping, Optional, Sequence, Tuple, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generator, + List, + Mapping, + Optional, + Sequence, + Tuple, + Union, +) from pip._vendor import requests, urllib3 -from pip._vendor.cachecontrol import CacheControlAdapter -from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter +from pip._vendor.cachecontrol import CacheControlAdapter as _BaseCacheControlAdapter +from pip._vendor.requests.adapters import DEFAULT_POOLBLOCK, BaseAdapter +from pip._vendor.requests.adapters import HTTPAdapter as _BaseHTTPAdapter from pip._vendor.requests.models import PreparedRequest, Response from pip._vendor.requests.structures import CaseInsensitiveDict from pip._vendor.urllib3.connectionpool import ConnectionPool @@ -37,6 +49,12 @@ from pip._internal.utils.misc import build_url_from_netloc, parse_netloc from pip._internal.utils.urls import url_to_path +if TYPE_CHECKING: + from ssl import SSLContext + + from pip._vendor.urllib3.poolmanager import PoolManager + + logger = logging.getLogger(__name__) SecureOrigin = Tuple[str, str, Optional[Union[int, str]]] @@ -233,6 +251,48 @@ def close(self) -> None: pass +class _SSLContextAdapterMixin: + """Mixin to add the ``ssl_context`` contructor argument to HTTP adapters. + + The additional argument is forwarded directly to the pool manager. This allows us + to dynamically decide what SSL store to use at runtime, which is used to implement + the optional ``truststore`` backend. + """ + + def __init__( + self, + *, + ssl_context: Optional["SSLContext"] = None, + **kwargs: Any, + ) -> None: + self._ssl_context = ssl_context + super().__init__(**kwargs) + + def init_poolmanager( + self, + connections: int, + maxsize: int, + block: bool = DEFAULT_POOLBLOCK, + **pool_kwargs: Any, + ) -> "PoolManager": + if self._ssl_context is not None: + pool_kwargs.setdefault("ssl_context", self._ssl_context) + return super().init_poolmanager( # type: ignore[misc] + connections=connections, + maxsize=maxsize, + block=block, + **pool_kwargs, + ) + + +class HTTPAdapter(_SSLContextAdapterMixin, _BaseHTTPAdapter): + pass + + +class CacheControlAdapter(_SSLContextAdapterMixin, _BaseCacheControlAdapter): + pass + + class InsecureHTTPAdapter(HTTPAdapter): def cert_verify( self, @@ -266,6 +326,7 @@ def __init__( cache: Optional[str] = None, trusted_hosts: Sequence[str] = (), index_urls: Optional[List[str]] = None, + ssl_context: Optional["SSLContext"] = None, **kwargs: Any, ) -> None: """ @@ -318,13 +379,14 @@ def __init__( secure_adapter = CacheControlAdapter( cache=SafeFileCache(cache), max_retries=retries, + ssl_context=ssl_context, ) self._trusted_host_adapter = InsecureCacheControlAdapter( cache=SafeFileCache(cache), max_retries=retries, ) else: - secure_adapter = HTTPAdapter(max_retries=retries) + secure_adapter = HTTPAdapter(max_retries=retries, ssl_context=ssl_context) self._trusted_host_adapter = insecure_adapter self.mount("https://", secure_adapter) From a020e8c35c35a3e0ec16b42abd62ee689351b7f6 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 3 May 2022 10:42:17 -0600 Subject: [PATCH 2/2] Add very simple tests to ensure feature is enabled --- tests/functional/test_truststore.py | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/functional/test_truststore.py diff --git a/tests/functional/test_truststore.py b/tests/functional/test_truststore.py new file mode 100644 index 00000000000..33153d0fbf9 --- /dev/null +++ b/tests/functional/test_truststore.py @@ -0,0 +1,61 @@ +import sys +from typing import Any, Callable + +import pytest + +from tests.lib import PipTestEnvironment, TestPipResult + +PipRunner = Callable[..., TestPipResult] + + +@pytest.fixture() +def pip(script: PipTestEnvironment) -> PipRunner: + def pip(*args: str, **kwargs: Any) -> TestPipResult: + return script.pip(*args, "--use-feature=truststore", **kwargs) + + return pip + + +@pytest.mark.skipif(sys.version_info >= (3, 10), reason="3.10 can run truststore") +def test_truststore_error_on_old_python(pip: PipRunner) -> None: + result = pip( + "install", + "--no-index", + "does-not-matter", + expect_error=True, + ) + assert "The truststore feature is only available for Python 3.10+" in result.stderr + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason="3.10+ required for truststore") +def test_truststore_error_without_preinstalled(pip: PipRunner) -> None: + result = pip( + "install", + "--no-index", + "does-not-matter", + expect_error=True, + ) + assert ( + "To use the truststore feature, 'truststore' must be installed into " + "pip's current environment." + ) in result.stderr + + +@pytest.mark.skipif(sys.version_info < (3, 10), reason="3.10+ required for truststore") +@pytest.mark.network +@pytest.mark.parametrize( + "package", + [ + "INITools", + "https://github.com/pypa/pip-test-package/archive/refs/heads/master.zip", + ], + ids=["PyPI", "GitHub"], +) +def test_trustore_can_install( + script: PipTestEnvironment, + pip: PipRunner, + package: str, +) -> None: + script.pip("install", "truststore") + result = pip("install", package) + assert "Successfully installed" in result.stdout