use .metadata distribution info when possible

When performing `install --dry-run` and PEP 658 .metadata files are
available to guide the resolve, do not download the associated wheels.

Rather use the distribution information directly from the .metadata
files when reporting the results on the CLI and in the --report file.

- describe the new --dry-run behavior
- finalize linked requirements immediately after resolve
- introduce is_concrete
- funnel InstalledDistribution through _get_prepared_distribution() too

- Update src/pip/_internal/commands/install.py
Co-authored-by: Ed Morley <[email protected]>

- Update src/pip/_internal/commands/install.py
Co-authored-by: Pradyun Gedam <[email protected]>

Author: 2 people

news/12186.bugfix.rst

@@ -0,0 +1 @@
+Avoid downloading any dists in ``install --dry-run`` if PEP 658 ``.metadata`` files or lazy wheels are available.

src/pip/_internal/commands/download.py

@@ -130,6 +130,9 @@ def run(self, options: Values, args: List[str]) -> int:
         self.trace_basic_info(finder)
 
         requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
+        preparer.finalize_linked_requirements(
+            requirement_set.requirements.values(), hydrate_virtual_reqs=True
+        )
 
         downloaded: List[str] = []
         for req in requirement_set.requirements.values():
@@ -138,7 +141,6 @@ def run(self, options: Values, args: List[str]) -> int:
                 preparer.save_linked_requirement(req)
                 downloaded.append(req.name)
 
-        preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
         requirement_set.warn_legacy_versions_and_specifiers()
 
         if downloaded:

src/pip/_internal/commands/install.py

@@ -84,7 +84,8 @@ def add_options(self) -> None:
             help=(
                 "Don't actually install anything, just print what would be. "
                 "Can be used in combination with --ignore-installed "
-                "to 'resolve' the requirements."
+                "to 'resolve' the requirements. If PEP 658 or fast-deps metadata is "
+                "available, --dry-run also avoids downloading the dependency at all."
             ),
         )
         self.cmd_opts.add_option(
@@ -377,6 +378,10 @@ def run(self, options: Values, args: List[str]) -> int:
             requirement_set = resolver.resolve(
                 reqs, check_supported_wheels=not options.target_dir
             )
+            preparer.finalize_linked_requirements(
+                requirement_set.requirements.values(),
+                hydrate_virtual_reqs=not options.dry_run,
+            )
 
             if options.json_report_file:
                 report = InstallationReport(requirement_set.requirements_to_install)

src/pip/_internal/commands/wheel.py

@@ -145,6 +145,9 @@ def run(self, options: Values, args: List[str]) -> int:
         self.trace_basic_info(finder)
 
         requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
+        preparer.finalize_linked_requirements(
+            requirement_set.requirements.values(), hydrate_virtual_reqs=True
+        )
 
         reqs_to_build: List[InstallRequirement] = []
         for req in requirement_set.requirements.values():
@@ -153,7 +156,6 @@ def run(self, options: Values, args: List[str]) -> int:
             elif should_build_for_wheel_command(req):
                 reqs_to_build.append(req)
 
-        preparer.prepare_linked_requirements_more(requirement_set.requirements.values())
         requirement_set.warn_legacy_versions_and_specifiers()
 
         # build wheels

src/pip/_internal/distributions/__init__.py

@@ -1,4 +1,5 @@
 from pip._internal.distributions.base import AbstractDistribution
+from pip._internal.distributions.installed import InstalledDistribution
 from pip._internal.distributions.sdist import SourceDistribution
 from pip._internal.distributions.wheel import WheelDistribution
 from pip._internal.req.req_install import InstallRequirement
@@ -8,6 +9,10 @@ def make_distribution_for_install_requirement(
     install_req: InstallRequirement,
 ) -> AbstractDistribution:
     """Returns a Distribution for the given InstallRequirement"""
+    # Only pre-installed requirements will have a .satisfied_by dist.
+    if install_req.satisfied_by:
+        return InstalledDistribution(install_req)
+
     # Editable requirements will always be source distributions. They use the
     # legacy logic until we create a modern standard for them.
     if install_req.editable:

src/pip/_internal/distributions/base.py

@@ -25,6 +25,10 @@ def __init__(self, req: InstallRequirement) -> None:
         super().__init__()
         self.req = req
 
+    @abc.abstractproperty
+    def add_to_build_tracker(self) -> bool:
+        raise NotImplementedError()
+
     @abc.abstractmethod
     def get_metadata_distribution(self) -> BaseDistribution:
         raise NotImplementedError()

src/pip/_internal/distributions/installed.py

@@ -10,9 +10,15 @@ class InstalledDistribution(AbstractDistribution):
     been computed.
     """
 
+    @property
+    def add_to_build_tracker(self) -> bool:
+        return False
+
     def get_metadata_distribution(self) -> BaseDistribution:
-        assert self.req.satisfied_by is not None, "not actually installed"
-        return self.req.satisfied_by
+        dist = self.req.satisfied_by
+        assert dist is not None, "not actually installed"
+        self.req.cache_concrete_dist(dist)
+        return dist
 
     def prepare_distribution_metadata(
         self,

src/pip/_internal/distributions/sdist.py

@@ -5,7 +5,7 @@
 from pip._internal.distributions.base import AbstractDistribution
 from pip._internal.exceptions import InstallationError
 from pip._internal.index.package_finder import PackageFinder
-from pip._internal.metadata import BaseDistribution
+from pip._internal.metadata import BaseDistribution, get_directory_distribution
 from pip._internal.utils.subprocess import runner_with_spinner_message
 
 logger = logging.getLogger(__name__)
@@ -18,8 +18,18 @@ class SourceDistribution(AbstractDistribution):
     generated, either using PEP 517 or using the legacy `setup.py egg_info`.
     """
 
+    @property
+    def add_to_build_tracker(self) -> bool:
+        return True
+
     def get_metadata_distribution(self) -> BaseDistribution:
-        return self.req.get_dist()
+        assert (
+            self.req.metadata_directory
+        ), "Set as part of .prepare_distribution_metadata()"
+        dist = get_directory_distribution(self.req.metadata_directory)
+        self.req.cache_concrete_dist(dist)
+        self.req.validate_sdist_metadata()
+        return dist
 
     def prepare_distribution_metadata(
         self,
@@ -58,7 +68,11 @@ def prepare_distribution_metadata(
                 self._raise_conflicts("the backend dependencies", conflicting)
             if missing:
                 self._raise_missing_reqs(missing)
-        self.req.prepare_metadata()
+
+        # NB: we must still call .cache_concrete_dist() and .validate_sdist_metadata()
+        # before the InstallRequirement itself has been updated with the metadata from
+        # this directory!
+        self.req.prepare_metadata_directory()
 
     def _prepare_build_backend(self, finder: PackageFinder) -> None:
         # Isolate in a BuildEnvironment and install the build-time

src/pip/_internal/distributions/wheel.py

@@ -15,6 +15,10 @@ class WheelDistribution(AbstractDistribution):
     This does not need any preparation as wheels can be directly unpacked.
     """
 
+    @property
+    def add_to_build_tracker(self) -> bool:
+        return True
+
     def get_metadata_distribution(self) -> BaseDistribution:
         """Loads the metadata from the wheel file into memory and returns a
         Distribution that uses it, not relying on the wheel file or
@@ -23,7 +27,9 @@ def get_metadata_distribution(self) -> BaseDistribution:
         assert self.req.local_file_path, "Set as part of preparation during download"
         assert self.req.name, "Wheels are never unnamed"
         wheel = FilesystemWheel(self.req.local_file_path)
-        return get_wheel_distribution(wheel, canonicalize_name(self.req.name))
+        dist = get_wheel_distribution(wheel, canonicalize_name(self.req.name))
+        self.req.cache_concrete_dist(dist)
+        return dist
 
     def prepare_distribution_metadata(
         self,

src/pip/_internal/metadata/base.py

@@ -105,6 +105,14 @@ class RequiresEntry(NamedTuple):
 
 
 class BaseDistribution(Protocol):
+    @property
+    def is_concrete(self) -> bool:
+        """Whether the distribution really exists somewhere on disk.
+
+        If this is false, it has been synthesized from metadata via
+        ``.from_metadata_file_contents()``."""
+        raise NotImplementedError()
+
     @classmethod
     def from_directory(cls, directory: str) -> "BaseDistribution":
         """Load the distribution from a metadata directory.
@@ -667,6 +675,10 @@ def iter_installed_distributions(
 class Wheel(Protocol):
     location: str
 
+    @property
+    def is_concrete(self) -> bool:
+        raise NotImplementedError()
+
     def as_zipfile(self) -> zipfile.ZipFile:
         raise NotImplementedError()
 
@@ -675,6 +687,10 @@ class FilesystemWheel(Wheel):
     def __init__(self, location: str) -> None:
         self.location = location
 
+    @property
+    def is_concrete(self) -> bool:
+        return True
+
     def as_zipfile(self) -> zipfile.ZipFile:
         return zipfile.ZipFile(self.location, allowZip64=True)
 
@@ -684,5 +700,9 @@ def __init__(self, location: str, stream: IO[bytes]) -> None:
         self.location = location
         self.stream = stream
 
+    @property
+    def is_concrete(self) -> bool:
+        return False
+
     def as_zipfile(self) -> zipfile.ZipFile:
         return zipfile.ZipFile(self.stream, allowZip64=True)

src/pip/_internal/metadata/importlib/_dists.py

@@ -99,16 +99,22 @@ def __init__(
         dist: importlib.metadata.Distribution,
         info_location: Optional[BasePath],
         installed_location: Optional[BasePath],
+        concrete: bool,
     ) -> None:
         self._dist = dist
         self._info_location = info_location
         self._installed_location = installed_location
+        self._concrete = concrete
+
+    @property
+    def is_concrete(self) -> bool:
+        return self._concrete
 
     @classmethod
     def from_directory(cls, directory: str) -> BaseDistribution:
         info_location = pathlib.Path(directory)
         dist = importlib.metadata.Distribution.at(info_location)
-        return cls(dist, info_location, info_location.parent)
+        return cls(dist, info_location, info_location.parent, concrete=True)
 
     @classmethod
     def from_metadata_file_contents(
@@ -125,7 +131,7 @@ def from_metadata_file_contents(
         metadata_path.write_bytes(metadata_contents)
         # Construct dist pointing to the newly created directory.
         dist = importlib.metadata.Distribution.at(metadata_path.parent)
-        return cls(dist, metadata_path.parent, None)
+        return cls(dist, metadata_path.parent, None, concrete=False)
 
     @classmethod
     def from_wheel(cls, wheel: Wheel, name: str) -> BaseDistribution:
@@ -136,7 +142,12 @@ def from_wheel(cls, wheel: Wheel, name: str) -> BaseDistribution:
             raise InvalidWheel(wheel.location, name) from e
         except UnsupportedWheel as e:
             raise UnsupportedWheel(f"{name} has an invalid wheel, {e}")
-        return cls(dist, dist.info_location, pathlib.PurePosixPath(wheel.location))
+        return cls(
+            dist,
+            dist.info_location,
+            pathlib.PurePosixPath(wheel.location),
+            concrete=wheel.is_concrete,
+        )
 
     @property
     def location(self) -> Optional[str]:

src/pip/_internal/metadata/importlib/_envs.py

@@ -81,7 +81,7 @@ def find(self, location: str) -> Iterator[BaseDistribution]:
                 installed_location: Optional[BasePath] = None
             else:
                 installed_location = info_location.parent
-            yield Distribution(dist, info_location, installed_location)
+            yield Distribution(dist, info_location, installed_location, concrete=True)
 
     def find_linked(self, location: str) -> Iterator[BaseDistribution]:
         """Read location in egg-link files and return distributions in there.
@@ -105,7 +105,7 @@ def find_linked(self, location: str) -> Iterator[BaseDistribution]:
                 continue
             target_location = str(path.joinpath(target_rel))
             for dist, info_location in self._find_impl(target_location):
-                yield Distribution(dist, info_location, path)
+                yield Distribution(dist, info_location, path, concrete=True)
 
     def _find_eggs_in_dir(self, location: str) -> Iterator[BaseDistribution]:
         from pip._vendor.pkg_resources import find_distributions
@@ -117,7 +117,7 @@ def _find_eggs_in_dir(self, location: str) -> Iterator[BaseDistribution]:
                 if not entry.name.endswith(".egg"):
                     continue
                 for dist in find_distributions(entry.path):
-                    yield legacy.Distribution(dist)
+                    yield legacy.Distribution(dist, concrete=True)
 
     def _find_eggs_in_zip(self, location: str) -> Iterator[BaseDistribution]:
         from pip._vendor.pkg_resources import find_eggs_in_zip
@@ -129,7 +129,7 @@ def _find_eggs_in_zip(self, location: str) -> Iterator[BaseDistribution]:
         except zipimport.ZipImportError:
             return
         for dist in find_eggs_in_zip(importer, location):
-            yield legacy.Distribution(dist)
+            yield legacy.Distribution(dist, concrete=True)
 
     def find_eggs(self, location: str) -> Iterator[BaseDistribution]:
         """Find eggs in a location.

src/pip/_internal/metadata/pkg_resources.py

@@ -69,8 +69,13 @@ def run_script(self, script_name: str, namespace: str) -> None:
 
 
 class Distribution(BaseDistribution):
-    def __init__(self, dist: pkg_resources.Distribution) -> None:
+    def __init__(self, dist: pkg_resources.Distribution, concrete: bool) -> None:
         self._dist = dist
+        self._concrete = concrete
+
+    @property
+    def is_concrete(self) -> bool:
+        return self._concrete
 
     @classmethod
     def from_directory(cls, directory: str) -> BaseDistribution:
@@ -90,7 +95,7 @@ def from_directory(cls, directory: str) -> BaseDistribution:
             dist_name = os.path.splitext(dist_dir_name)[0].split("-")[0]
 
         dist = dist_cls(base_dir, project_name=dist_name, metadata=metadata)
-        return cls(dist)
+        return cls(dist, concrete=True)
 
     @classmethod
     def from_metadata_file_contents(
@@ -107,7 +112,7 @@ def from_metadata_file_contents(
             metadata=InMemoryMetadata(metadata_dict, filename),
             project_name=project_name,
         )
-        return cls(dist)
+        return cls(dist, concrete=False)
 
     @classmethod
     def from_wheel(cls, wheel: Wheel, name: str) -> BaseDistribution:
@@ -128,7 +133,7 @@ def from_wheel(cls, wheel: Wheel, name: str) -> BaseDistribution:
             metadata=InMemoryMetadata(metadata_dict, wheel.location),
             project_name=name,
         )
-        return cls(dist)
+        return cls(dist, concrete=wheel.is_concrete)
 
     @property
     def location(self) -> Optional[str]:
@@ -233,7 +238,7 @@ def from_paths(cls, paths: Optional[List[str]]) -> BaseEnvironment:
 
     def _iter_distributions(self) -> Iterator[BaseDistribution]:
         for dist in self._ws:
-            yield Distribution(dist)
+            yield Distribution(dist, concrete=True)
 
     def _search_distribution(self, name: str) -> Optional[BaseDistribution]:
         """Find a distribution matching the ``name`` in the environment.

src/pip/_internal/models/installation_report.py

@@ -29,7 +29,7 @@ def _install_req_to_dict(cls, ireq: InstallRequirement) -> Dict[str, Any]:
             "requested": ireq.user_supplied,
             # PEP 566 json encoding for metadata
             # https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata
-            "metadata": ireq.get_dist().metadata_dict,
+            "metadata": ireq.cached_dist.metadata_dict,
         }
         if ireq.user_supplied and ireq.extras:
             # For top level requirements, the list of requested extras, if any.

src/pip/_internal/operations/check.py

@@ -9,7 +9,6 @@
 from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
 from pip._vendor.packaging.version import LegacyVersion
 
-from pip._internal.distributions import make_distribution_for_install_requirement
 from pip._internal.metadata import get_default_environment
 from pip._internal.metadata.base import DistributionVersion
 from pip._internal.req.req_install import InstallRequirement
@@ -127,8 +126,8 @@ def _simulate_installation_of(
 
     # Modify it as installing requirement_set would (assuming no errors)
     for inst_req in to_install:
-        abstract_dist = make_distribution_for_install_requirement(inst_req)
-        dist = abstract_dist.get_metadata_distribution()
+        assert inst_req.is_concrete
+        dist = inst_req.cached_dist
         name = dist.canonical_name
         package_set[name] = PackageDetails(dist.version, list(dist.iter_dependencies()))