Add a safer way to clear package finder cache

Rather than rebuilding the package finder, which seems to have a number
of unexpected and undesirable side-effects, implement a safe way to clear
the cache on a package finder.

A new regression test for #2220 validates the fix, demonstrating that we
preserve the `--extra-index-url` provided via `requirements.in`.

This also fixes #2294 , for similar reasons.

This change was also manually tested on the reproducers from #2220 and
 #2294, where it has the correct effect.

Author: sirosen

changelog.d/2220.bugfix.md

@@ -0,0 +1,3 @@
+Fixed a bug in which `pip-compile` lost any index URL options when looking up hashes -- by {user}`sirosen`.
+
+This caused errors when a package was only available from an extra index, and caused `pip-compile` to incorrectly drop index URL options from output, even when they were present in the input requirements.

changelog.d/2294.bugfix.md

@@ -0,0 +1 @@
+2220.bugfix.md

changelog.d/2305.bugfix.md

@@ -0,0 +1 @@
+2220.bugfix.md

piptools/repositories/pypi.py

@@ -116,6 +116,20 @@ def session(self) -> PipSession:
     def finder(self) -> PackageFinder:
         return self._finder
 
+    def _clear_finder_cache(self) -> None:
+        """Clear the cache of installation candidates."""
+        # `finder.find_all_candidates` is an lru_cache wrapped method on older `pip`
+        # versions, which can be cleared with `cache_clear()`
+        # but on newer versions, it's a simple method, and the underlying cache is a
+        # dict on the instance
+        # the same holds for `finder.find_best_candidate`
+        if _pip_api.PIP_VERSION_MAJOR_MINOR >= (25, 1):
+            self.finder._all_candidates.clear()
+            self.finder._best_candidates.clear()
+        else:
+            self.finder.find_all_candidates.cache_clear()
+            self.finder.find_best_candidate.cache_clear()
+
     @property
     def command(self) -> InstallCommand:
         """Return an install command instance."""
@@ -450,13 +464,10 @@ def _wheel_support_index_min(self: Wheel, tags: list[Tag]) -> int:
         Wheel.support_index_min = _wheel_support_index_min
         self._available_candidates_cache = {}
 
-        # Finder internally caches results, and there is no public method to
-        # clear the cache, so we re-create the object here. If we don't clear
-        # this cache then it can contain results from an earlier call when
-        # allow_all_wheels wasn't active. See GH-1532
-        self._finder = self.command._build_package_finder(
-            options=self.options, session=self.session
-        )
+        # Finder internally caches results. If we don't clear this cache then it can
+        # contain results from an earlier call when allow_all_wheels wasn't active.
+        # See GH-1532
+        self._clear_finder_cache()
 
         try:
             yield

tests/test_cli_compile.py

@@ -4203,3 +4203,46 @@ def test_that_self_referential_pyproject_toml_extra_can_be_compiled(
         small-fake-a==0.2
             # via foo
         """)
+
+
+def test_compile_with_generate_hashes_preserves_extra_index_url(
+    pip_with_index_conf,
+    minimal_wheels_path,
+    runner,
+    tmpdir_cwd,
+):
+    """
+    Regression test for
+    https://github.com/jazzband/pip-tools/issues/2220
+
+    Using ``--generate-hashes`` triggers the codepath which clears the package finder
+    cache (``allow_all_wheels()``), and that code incorrectly cleared more information
+    than desired, removing extra index URLs in addition to cached package info.
+    """
+    reqs_in = tmpdir_cwd / "requirements.in"
+    reqs_in.write_text(dedent("""\
+            --extra-index-url http://extraindex1.com
+
+            small-fake-a
+            """))
+
+    out = runner.invoke(
+        cli,
+        ["--output-file", "-", "--no-header", "--strip-extras", "--generate-hashes"],
+    )
+
+    # the output should contain
+    # - the `--index-url` from the pip config
+    # - the `--extra-index-url` from `requirements.in`
+    # - the `--find-links` option from pip config
+    #
+    # and then package resolution information
+    assert out.stdout == dedent(f"""\
+        --index-url http://example.com
+        --extra-index-url http://extraindex1.com
+        --find-links {minimal_wheels_path.as_posix()}
+
+        small-fake-a==0.2 \\
+            --hash=sha256:33e1acdca3b9162e002cedb0e58b350d731d1ed3f53a6b22e0a628bca7c7c6ed
+            # via -r requirements.in
+        """)