Merge pull request #2224 from sirosen/fix-url-normalization

Bugfix: path normalization applied to URLs

Author: sirosen

changelog.d/2223.bugfix.md

@@ -0,0 +1,2 @@
+Fixed a bug which removed slashes from URLs in ``-r`` and ``-c`` in the output
+of ``pip-compile`` -- by {user}`sirosen`.

piptools/_compat/pip_compat.py

@@ -2,6 +2,7 @@
 
 import optparse
 import pathlib
+import urllib.parse
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Set, cast
 
@@ -140,6 +141,10 @@ def _relativize_comes_from_location(original_comes_from: str, /) -> str:
     # split on the space
     prefix, space_sep, suffix = original_comes_from.partition(" ")
 
+    # if the value part is a remote URI for pip, return the original
+    if _is_remote_pip_uri(suffix):
+        return original_comes_from
+
     file_path = pathlib.Path(suffix)
 
     # if the path was not absolute, normalize to posix-style and finish processing
@@ -169,11 +174,26 @@ def _normalize_comes_from_location(original_comes_from: str, /) -> str:
     # split on the space
     prefix, space_sep, suffix = original_comes_from.partition(" ")
 
+    # if the value part is a remote URI for pip, return the original
+    if _is_remote_pip_uri(suffix):
+        return original_comes_from
+
     # convert to a posix-style path
     suffix = pathlib.Path(suffix).as_posix()
     return f"{prefix}{space_sep}{suffix}"
 
 
+def _is_remote_pip_uri(value: str) -> bool:
+    """
+    Test a string to see if it is a URI treated as a remote file in ``pip``.
+    Specifically this means that it's a 'file', 'http', or 'https' URI.
+
+    The test is performed by trying a URL parse and reading the scheme.
+    """
+    scheme = urllib.parse.urlsplit(value).scheme
+    return scheme in {"file", "http", "https"}
+
+
 def create_wheel_cache(cache_dir: str, format_control: str | None = None) -> WheelCache:
     kwargs: dict[str, str | None] = {"cache_dir": cache_dir}
     if PIP_VERSION[:2] <= (23, 0):

tests/test_cli_compile.py

@@ -14,6 +14,7 @@
 from unittest.mock import MagicMock
 
 import pytest
+from pip._internal.network.session import PipSession
 from pip._internal.req.constructors import install_req_from_line
 from pip._internal.utils.hashes import FAVORITE_HASH
 from pip._internal.utils.urls import path_to_url
@@ -4044,3 +4045,79 @@ def test_second_order_requirements_relative_path_in_separate_dir(
             # via -r {output_path}
         """
     )
+
+
+@pytest.mark.parametrize(
+    "input_path_absolute", (True, False), ids=("absolute-input", "relative-input")
+)
+def test_url_constraints_are_not_treated_as_file_paths(
+    pip_conf,
+    make_package,
+    runner,
+    tmp_path,
+    monkeypatch,
+    input_path_absolute,
+):
+    """
+    Test normalization of ``-c`` constraints when the constraints are HTTPS URLs.
+    The constraints should be preserved verbatim.
+
+    This is a regression test for
+    https://github.com/jazzband/pip-tools/issues/2223
+    """
+    constraints_url = "https://example.com/files/common_constraints.txt"
+
+    reqs_in = tmp_path / "requirements.in"
+    reqs_in.write_text(
+        f"""
+        small-fake-a
+        -c {constraints_url}
+        """
+    )
+
+    input_dir_path = tmp_path if input_path_absolute else pathlib.Path(".")
+    input_path = (input_dir_path / "requirements.in").as_posix()
+
+    # TODO: find a better way of mocking the callout to get the constraints
+    #       file (use `responses`?)
+    #
+    # we need a mock response for `GET https://...` as fetched by pip
+    # although this is fragile, it can be adapted if pip changes
+    def fake_url_get(url):
+        response = mock.Mock()
+        response.reason = "Ok"
+        response.status_code = 200
+        response.url = url
+        response.text = "small-fake-a==0.2"
+        return response
+
+    mock_get = mock.Mock(side_effect=fake_url_get)
+
+    with monkeypatch.context() as revertable_ctx:
+        revertable_ctx.chdir(tmp_path)
+        revertable_ctx.setattr(PipSession, "get", mock_get)
+        out = runner.invoke(
+            cli,
+            [
+                "--output-file",
+                "-",
+                "--quiet",
+                "--no-header",
+                "--no-emit-options",
+                "-r",
+                input_path,
+            ],
+        )
+
+    # sanity check, pip should have tried to fetch the constraints
+    mock_get.assert_called_once_with(constraints_url)
+
+    assert out.exit_code == 0
+    assert out.stdout == dedent(
+        f"""\
+        small-fake-a==0.2
+            # via
+            #   -c {constraints_url}
+            #   -r {input_path}
+        """
+    )