use .metadata distribution info when possible

When performing `install --dry-run` and PEP 658 .metadata files are
available to guide the resolve, do not download the associated wheels.

Rather use the distribution information directly from the .metadata
files when reporting the results on the CLI and in the --report file.

- describe the new --dry-run behavior
- finalize linked requirements immediately after resolve
- introduce is_concrete
- funnel InstalledDistribution through _get_prepared_distribution() too
- add test for new install --dry-run functionality (no downloading)

Author: cosmicexplorer

news/12186.bugfix.rst

@@ -0,0 +1 @@
+Avoid downloading any dists in ``install --dry-run`` if PEP 658 ``.metadata`` files or lazy wheels are available.

src/pip/_internal/commands/download.py

@@ -130,6 +130,9 @@ def run(self, options: Values, args: List[str]) -> int:
         self.trace_basic_info(finder)
 
         requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
+        preparer.finalize_linked_requirements(
+            requirement_set.requirements.values(), require_dist_files=True
+        )
 
         downloaded: List[str] = []
         for req in requirement_set.requirements.values():
@@ -138,8 +141,6 @@ def run(self, options: Values, args: List[str]) -> int:
                 preparer.save_linked_requirement(req)
                 downloaded.append(req.name)
 
-        preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
-
         if downloaded:
             write_output("Successfully downloaded %s", " ".join(downloaded))
 

src/pip/_internal/commands/install.py

@@ -85,7 +85,8 @@ def add_options(self) -> None:
             help=(
                 "Don't actually install anything, just print what would be. "
                 "Can be used in combination with --ignore-installed "
-                "to 'resolve' the requirements."
+                "to 'resolve' the requirements. If package metadata is available "
+                "or cached, --dry-run also avoids downloading the dependency at all."
             ),
         )
         self.cmd_opts.add_option(
@@ -379,6 +380,10 @@ def run(self, options: Values, args: List[str]) -> int:
             requirement_set = resolver.resolve(
                 reqs, check_supported_wheels=not options.target_dir
             )
+            preparer.finalize_linked_requirements(
+                requirement_set.requirements.values(),
+                require_dist_files=not options.dry_run,
+            )
 
             if options.json_report_file:
                 report = InstallationReport(requirement_set.requirements_to_install)

src/pip/_internal/commands/wheel.py

@@ -145,6 +145,9 @@ def run(self, options: Values, args: List[str]) -> int:
         self.trace_basic_info(finder)
 
         requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
+        preparer.finalize_linked_requirements(
+            requirement_set.requirements.values(), require_dist_files=True
+        )
 
         reqs_to_build: List[InstallRequirement] = []
         for req in requirement_set.requirements.values():
@@ -153,8 +156,6 @@ def run(self, options: Values, args: List[str]) -> int:
             elif should_build_for_wheel_command(req):
                 reqs_to_build.append(req)
 
-        preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
-
         # build wheels
         build_successes, build_failures = build(
             reqs_to_build,

src/pip/_internal/operations/check.py

@@ -23,7 +23,6 @@
 from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
 from pip._vendor.packaging.version import Version
 
-from pip._internal.distributions import make_distribution_for_install_requirement
 from pip._internal.metadata import get_default_environment
 from pip._internal.metadata.base import BaseDistribution
 from pip._internal.req.req_install import InstallRequirement
@@ -154,8 +153,8 @@ def _simulate_installation_of(
 
     # Modify it as installing requirement_set would (assuming no errors)
     for inst_req in to_install:
-        abstract_dist = make_distribution_for_install_requirement(inst_req)
-        dist = abstract_dist.get_metadata_distribution()
+        assert inst_req.is_concrete
+        dist = inst_req.get_dist()
         name = dist.canonical_name
         package_set[name] = PackageDetails(dist.version, list(dist.iter_dependencies()))
 

src/pip/_internal/operations/prepare.py

@@ -579,16 +579,71 @@ def prepare_linked_requirement(
             # None of the optimizations worked, fully prepare the requirement
             return self._prepare_linked_requirement(req, parallel_builds)
 
-    def prepare_linked_requirements_more(
+    def _extract_download_info(self, reqs: Iterable[InstallRequirement]) -> None:
+        """
+        `pip install --report` extracts the download info from each requirement for its
+        JSON output, so we need to make sure every requirement has this before finishing
+        the resolve. But .download_info will only be populated by the point this method
+        is called for requirements already found in the wheel cache, so we need to
+        synthesize it for uncached results. Luckily, a DirectUrl can be parsed directly
+        from a url without any other context. However, this also means the download info
+        will only contain a hash if the link itself declares the hash.
+        """
+        for req in reqs:
+            if req.download_info is None:
+                self._ensure_download_info(req)
+
+    def _force_fully_prepared(
+        self, reqs: Iterable[InstallRequirement], assert_has_dist_files: bool
+    ) -> None:
+        """
+        The legacy resolver seems to prepare requirements differently that can leave
+        them half-done in certain code paths. I'm not quite sure how it's doing things,
+        but at least we can do this to make sure they do things right.
+        """
+        for req in reqs:
+            req.prepared = True
+            if assert_has_dist_files:
+                assert req.is_concrete
+
+    def _ensure_dist_files(
         self, reqs: Iterable[InstallRequirement], parallel_builds: bool = False
     ) -> None:
-        """Prepare linked requirements more, if needed."""
+        """Download any metadata-only linked requirements."""
         metadata_only_reqs = [req for req in reqs if not req.is_concrete]
         self._complete_partial_requirements(
             metadata_only_reqs,
             parallel_builds=parallel_builds,
         )
 
+    def finalize_linked_requirements(
+        self,
+        reqs: Iterable[InstallRequirement],
+        require_dist_files: bool,
+        parallel_builds: bool = False,
+    ) -> None:
+        """Prepare linked requirements more, if needed.
+
+        Neighboring .metadata files as per PEP 658 or lazy wheels via fast-deps will be
+        preferred to extract metadata from any concrete requirement (one that has been
+        mapped to a Link) without downloading the underlying wheel or sdist. When ``pip
+        install --dry-run`` is called, we want to avoid ever downloading the underlying
+        dist, but we still need to provide all of the results that pip commands expect
+        from the typical resolve process.
+
+        Those expectations vary, but one distinction lies in whether the command needs
+        an actual physical dist somewhere on the filesystem, or just the metadata about
+        it from the resolver (as in ``pip install --report``). If the command requires
+        actual physical filesystem locations for the resolved dists, it must call this
+        method with ``require_dist_files=True`` to fully download anything
+        that remains.
+        """
+        if require_dist_files:
+            self._ensure_dist_files(reqs, parallel_builds=parallel_builds)
+        else:
+            self._extract_download_info(reqs)
+        self._force_fully_prepared(reqs, assert_has_dist_files=require_dist_files)
+
     def _ensure_local_file_path(
         self, req: InstallRequirement, hashes: Optional[Hashes]
     ) -> None:

src/pip/_internal/req/req_install.py

@@ -648,10 +648,7 @@ def cache_concrete_dist(self, dist: BaseDistribution) -> None:
             # If we set a dist twice for the same requirement, we must be hydrating
             # a concrete dist for what was previously virtual. This will occur in the
             # case of `install --dry-run` when PEP 658 metadata is available.
-
-            # TODO(#12186): avoid setting dist twice!
-            # assert not self._dist.is_concrete
-            pass
+            assert not self._dist.is_concrete
         assert dist.is_concrete
         self._dist = dist
 

src/pip/_internal/resolution/resolvelib/resolver.py

@@ -175,11 +175,6 @@ def resolve(
 
             req_set.add_named_requirement(ireq)
 
-        reqs = req_set.all_requirements
-        self.factory.preparer.prepare_linked_requirements_more(reqs)
-        for req in reqs:
-            req.prepared = True
-            assert req.is_concrete
         return req_set
 
     def get_installation_order(

tests/conftest.py

@@ -748,6 +748,9 @@ class FakePackage:
     requires_dist: Tuple[str, ...] = ()
     # This will override the Name specified in the actual dist's METADATA.
     metadata_name: Optional[str] = None
+    # Whether to delete the file this points to, which causes any attempt to fetch this
+    # package to fail unless it is processed as a metadata-only dist.
+    delete_linked_file: bool = False
 
     def metadata_filename(self) -> str:
         """This is specified by PEP 658."""
@@ -837,6 +840,27 @@ def fake_packages() -> Dict[str, List[FakePackage]]:
                 ("simple==1.0",),
             ),
         ],
+        "complex-dist": [
+            FakePackage(
+                "complex-dist",
+                "0.1",
+                "complex_dist-0.1-py2.py3-none-any.whl",
+                MetadataKind.Unhashed,
+                # Validate that the wheel isn't fetched if metadata is available and
+                # --dry-run is on, when the metadata presents no hash itself.
+                delete_linked_file=True,
+            ),
+        ],
+        "corruptwheel": [
+            FakePackage(
+                "corruptwheel",
+                "1.0",
+                "corruptwheel-1.0-py2.py3-none-any.whl",
+                # Validate that the wheel isn't fetched if metadata is available and
+                # --dry-run is on, when the metadata *does* present a hash.
+                MetadataKind.Sha256,
+            ),
+        ],
         "has-script": [
             # Ensure we check PEP 658 metadata hashing errors for wheel files.
             FakePackage(
@@ -922,10 +946,10 @@ def html_index_for_packages(
                 f'    <a href="{package_link.filename}" {package_link.generate_additional_tag()}>{package_link.filename}</a><br/>'  # noqa: E501
             )
             # (3.2) Copy over the corresponding file in `shared_data.packages`.
-            shutil.copy(
-                shared_data.packages / package_link.filename,
-                pkg_subdir / package_link.filename,
-            )
+            cached_file = shared_data.packages / package_link.filename
+            new_file = pkg_subdir / package_link.filename
+            if not package_link.delete_linked_file:
+                shutil.copy(cached_file, new_file)
             # (3.3) Write a metadata file, if applicable.
             if package_link.metadata != MetadataKind.NoFile:
                 with open(pkg_subdir / package_link.metadata_filename(), "wb") as f:
@@ -980,7 +1004,8 @@ def html_index_with_onetime_server(
     """Serve files from a generated pypi index, erroring if a file is downloaded more
     than once.
 
-    Provide `-i http://localhost:8000` to pip invocations to point them at this server.
+    Provide `-i http://localhost:<port>` to pip invocations to point them at
+    this server.
     """
 
     class InDirectoryServer(http.server.ThreadingHTTPServer):
@@ -995,7 +1020,7 @@ def finish_request(self: "Self", request: Any, client_address: Any) -> None:
     class Handler(OneTimeDownloadHandler):
         _seen_paths: ClassVar[Set[str]] = set()
 
-    with InDirectoryServer(("", 8000), Handler) as httpd:
+    with InDirectoryServer(("", 0), Handler) as httpd:
         server_thread = threading.Thread(target=httpd.serve_forever)
         server_thread.start()
 

tests/functional/test_download.py

@@ -1273,6 +1273,7 @@ def download_server_html_index(
 ) -> Callable[..., Tuple[TestPipResult, Path]]:
     """Execute `pip download` against a generated PyPI index."""
     download_dir = tmpdir / "download_dir"
+    _, server_port = html_index_with_onetime_server.server_address
 
     def run_for_generated_index(
         args: List[str],
@@ -1287,7 +1288,7 @@ def run_for_generated_index(
             "-d",
             str(download_dir),
             "-i",
-            "http://localhost:8000",
+            f"http://localhost:{server_port}",
             *args,
         ]
         result = script.pip(*pip_args, allow_error=allow_error)