Merge pull request #11647 from sethmlarson/truststore-by-default

Truststore by default

Author: pradyunsg

docs/html/topics/https-certificates.md

@@ -8,8 +8,7 @@
 
 By default, pip will perform SSL certificate verification for network
 connections it makes over HTTPS. These serve to prevent man-in-the-middle
-attacks against package downloads. This does not use the system certificate
-store but, instead, uses a bundled CA certificate store from {pypi}`certifi`.
+attacks against package downloads.
 
 ## Using a specific certificate store
 
@@ -20,43 +19,34 @@ variables.
 
 ## Using system certificate stores
 
-```{versionadded} 22.2
-Experimental support, behind `--use-feature=truststore`.
-As with any other CLI option, this can be enabled globally via config or environment variables.
-```
-
-It is possible to use the system trust store, instead of the bundled certifi
-certificates for verifying HTTPS certificates. This approach will typically
-support corporate proxy certificates without additional configuration.
-
-In order to use system trust stores, you need to use Python 3.10 or newer.
-
-  ```{pip-cli}
-  $ python -m pip install SomePackage --use-feature=truststore
-  [...]
-  Successfully installed SomePackage
-  ```
-
-### When to use
+```{versionadded} 24.2
 
-You should try using system trust stores when there is a custom certificate
-chain configured for your system that pip isn't aware of. Typically, this
-situation will manifest with an `SSLCertVerificationError` with the message
-"certificate verify failed: unable to get local issuer certificate":
+```
 
-```{pip-cli}
-$ pip install -U SomePackage
-[...]
-   SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (\_ssl.c:997)'))) - skipping
+```{note}
+Versions of pip prior to v24.2 did not use system certificates by default.
+To use system certificates with pip v22.2 or later, you must opt-in using the `--use-feature=truststore` CLI flag.
 ```
 
-This error means that OpenSSL wasn't able to find a trust anchor to verify the
-chain against. Using system trust stores instead of certifi will likely solve
-this issue.
+On Python 3.10 or later, by default
+system certificates are used in addition to certifi to verify HTTPS connections.
+This functionality is provided through the {pypi}`truststore` package.
 
 If you encounter a TLS/SSL error when using the `truststore` feature you should
 open an issue on the [truststore GitHub issue tracker] instead of pip's issue
 tracker. The maintainers of truststore will help diagnose and fix the issue.
 
+To opt-out of using system certificates you can pass the `--use-deprecated=legacy-certs`
+flag to pip.
+
+```{warning}
+On Python 3.9 or earlier, only certifi is used to verify HTTPS connections as
+`truststore` requires Python 3.10 or higher to function.
+
+The system certificate store won't be used in this case, so some situations like proxies
+with their own certificates may not work. Upgrading to at least Python 3.10 or later is
+the recommended method to resolve this issue.
+```
+
 [truststore github issue tracker]:
   https://github.com/sethmlarson/truststore/issues

news/11647.feature.rst

@@ -0,0 +1,4 @@
+Changed pip to use system certificates and certifi to verify HTTPS connections.
+This change only affects Python 3.10 or later, Python 3.9 and earlier only use certifi.
+
+To revert to previous behavior pass the flag ``--use-deprecated=legacy-certs``.

src/pip/_internal/cli/cmdoptions.py

@@ -996,6 +996,7 @@ def check_list_path_option(options: Values) -> None:
 
 # Features that are now always on. A warning is printed if they are used.
 ALWAYS_ENABLED_FEATURES = [
+    "truststore",  # always on since 24.2
     "no-binary-enable-wheel-cache",  # always on since 23.1
 ]
 
@@ -1008,7 +1009,6 @@ def check_list_path_option(options: Values) -> None:
     default=[],
     choices=[
         "fast-deps",
-        "truststore",
     ]
     + ALWAYS_ENABLED_FEATURES,
     help="Enable new functionality, that may be backward incompatible.",
@@ -1023,6 +1023,7 @@ def check_list_path_option(options: Values) -> None:
     default=[],
     choices=[
         "legacy-resolver",
+        "legacy-certs",
     ],
     help=("Enable deprecated functionality, that will be removed in the future."),
 )

src/pip/_internal/cli/index_command.py

@@ -12,9 +12,10 @@
 from optparse import Values
 from typing import TYPE_CHECKING, List, Optional
 
+from pip._vendor import certifi
+
 from pip._internal.cli.base_command import Command
 from pip._internal.cli.command_context import CommandContextMixIn
-from pip._internal.exceptions import CommandError
 
 if TYPE_CHECKING:
     from ssl import SSLContext
@@ -26,7 +27,8 @@
 
 def _create_truststore_ssl_context() -> Optional["SSLContext"]:
     if sys.version_info < (3, 10):
-        raise CommandError("The truststore feature is only available for Python 3.10+")
+        logger.debug("Disabling truststore because Python version isn't 3.10+")
+        return None
 
     try:
         import ssl
@@ -36,10 +38,13 @@ def _create_truststore_ssl_context() -> Optional["SSLContext"]:
 
     try:
         from pip._vendor import truststore
-    except ImportError as e:
-        raise CommandError(f"The truststore feature is unavailable: {e}")
+    except ImportError:
+        logger.warning("Disabling truststore because platform isn't supported")
+        return None
 
-    return truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    ctx = truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    ctx.load_verify_locations(certifi.where())
+    return ctx
 
 
 class SessionCommandMixin(CommandContextMixIn):
@@ -80,20 +85,14 @@ def _build_session(
         options: Values,
         retries: Optional[int] = None,
         timeout: Optional[int] = None,
-        fallback_to_certifi: bool = False,
     ) -> "PipSession":
         from pip._internal.network.session import PipSession
 
         cache_dir = options.cache_dir
         assert not cache_dir or os.path.isabs(cache_dir)
 
-        if "truststore" in options.features_enabled:
-            try:
-                ssl_context = _create_truststore_ssl_context()
-            except Exception:
-                if not fallback_to_certifi:
-                    raise
-                ssl_context = None
+        if "legacy-certs" not in options.deprecated_features_enabled:
+            ssl_context = _create_truststore_ssl_context()
         else:
             ssl_context = None
 
@@ -162,11 +161,6 @@ def handle_pip_version_check(self, options: Values) -> None:
             options,
             retries=0,
             timeout=min(5, options.timeout),
-            # This is set to ensure the function does not fail when truststore is
-            # specified in use-feature but cannot be loaded. This usually raises a
-            # CommandError and shows a nice user-facing error, but this function is not
-            # called in that try-except block.
-            fallback_to_certifi=True,
         )
         with session:
             _pip_self_version_check(session, options)

tests/functional/test_truststore.py

@@ -1,4 +1,3 @@
-import sys
 from typing import Any, Callable
 
 import pytest
@@ -9,25 +8,13 @@
 
 
 @pytest.fixture()
-def pip(script: PipTestEnvironment) -> PipRunner:
+def pip_no_truststore(script: PipTestEnvironment) -> PipRunner:
     def pip(*args: str, **kwargs: Any) -> TestPipResult:
-        return script.pip(*args, "--use-feature=truststore", **kwargs)
+        return script.pip(*args, "--use-deprecated=legacy-certs", **kwargs)
 
     return pip
 
 
-@pytest.mark.skipif(sys.version_info >= (3, 10), reason="3.10 can run truststore")
-def test_truststore_error_on_old_python(pip: PipRunner) -> None:
-    result = pip(
-        "install",
-        "--no-index",
-        "does-not-matter",
-        expect_error=True,
-    )
-    assert "The truststore feature is only available for Python 3.10+" in result.stderr
-
-
-@pytest.mark.skipif(sys.version_info < (3, 10), reason="3.10+ required for truststore")
 @pytest.mark.network
 @pytest.mark.parametrize(
     "package",
@@ -37,10 +24,10 @@ def test_truststore_error_on_old_python(pip: PipRunner) -> None:
     ],
     ids=["PyPI", "GitHub"],
 )
-def test_trustore_can_install(
+def test_no_truststore_can_install(
     script: PipTestEnvironment,
-    pip: PipRunner,
+    pip_no_truststore: PipRunner,
     package: str,
 ) -> None:
-    result = pip("install", package)
+    result = pip_no_truststore("install", package)
     assert "Successfully installed" in result.stdout

tests/lib/certs.py

@@ -5,7 +5,7 @@
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
-from cryptography.x509.oid import NameOID
+from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
 
 
 def make_tls_cert(hostname: str) -> Tuple[x509.Certificate, rsa.RSAPrivateKey]:
@@ -25,10 +25,23 @@ def make_tls_cert(hostname: str) -> Tuple[x509.Certificate, rsa.RSAPrivateKey]:
         .serial_number(x509.random_serial_number())
         .not_valid_before(datetime.now(timezone.utc))
         .not_valid_after(datetime.now(timezone.utc) + timedelta(days=10))
+        .add_extension(
+            x509.BasicConstraints(ca=True, path_length=9),
+            critical=True,
+        )
         .add_extension(
             x509.SubjectAlternativeName([x509.DNSName(hostname)]),
             critical=False,
         )
+        .add_extension(
+            x509.ExtendedKeyUsage(
+                [
+                    ExtendedKeyUsageOID.CLIENT_AUTH,
+                    ExtendedKeyUsageOID.SERVER_AUTH,
+                ]
+            ),
+            critical=True,
+        )
         .sign(key, hashes.SHA256(), default_backend())
     )
     return cert, key