Add --report option to pip install

Author: sbidoul

news/53.feature.rst

@@ -0,0 +1,3 @@
+Add ``--report`` to the install command to generate a json report of what was installed.
+In combination with ``--dry-run`` and ``--ignore-installed`` it can be used to resolve
+the requirements.

src/pip/_internal/commands/install.py

@@ -1,4 +1,5 @@
 import errno
+import json
 import operator
 import os
 import shutil
@@ -21,6 +22,7 @@
 from pip._internal.locations import get_scheme
 from pip._internal.metadata import get_environment
 from pip._internal.models.format_control import FormatControl
+from pip._internal.models.installation_report import InstallationReport
 from pip._internal.operations.build.build_tracker import get_build_tracker
 from pip._internal.operations.check import ConflictDetails, check_install_conflicts
 from pip._internal.req import install_given_reqs
@@ -250,6 +252,19 @@ def add_options(self) -> None:
         self.parser.insert_option_group(0, index_opts)
         self.parser.insert_option_group(0, self.cmd_opts)
 
+        self.cmd_opts.add_option(
+            "--report",
+            dest="json_report_file",
+            metavar="file",
+            default=None,
+            help=(
+                "Generate a JSON file describing what pip did to install "
+                "the provided requirements. "
+                "Can be used in combination with --dry-run and --ignore-installed "
+                "to 'resolve' the requirements."
+            ),
+        )
+
     @with_cleanup
     def run(self, options: Values, args: List[str]) -> int:
         if options.use_user_site and options.target_dir is not None:
@@ -353,6 +368,11 @@ def run(self, options: Values, args: List[str]) -> int:
                 reqs, check_supported_wheels=not options.target_dir
             )
 
+            if options.json_report_file:
+                report = InstallationReport(requirement_set.requirements_to_install)
+                with open(options.json_report_file, "w") as f:
+                    json.dump(report.to_dict(), f)
+
             if options.dry_run:
                 would_install_items = sorted(
                     (r.metadata["name"], r.metadata["version"])

src/pip/_internal/models/installation_report.py

@@ -0,0 +1,39 @@
+from typing import Any, Dict, Sequence
+
+from pip._internal.req.req_install import InstallRequirement
+
+
+class InstallationReport:
+    def __init__(self, install_requirements: Sequence[InstallRequirement]):
+        self._install_requirements = install_requirements
+
+    @classmethod
+    def _install_req_to_dict(cls, ireq: InstallRequirement) -> Dict[str, Any]:
+        assert ireq.download_info, f"No download_info for {ireq}"
+        res = {
+            # PEP 610 json for the download URL. download_info.archive_info.hash may
+            # be absent when the requirement was installed from the wheel cache
+            # and the cache entry was populated by an older pip version that did not
+            # record origin.json.
+            "download_info": ireq.download_info.to_dict(),
+            # is_direct is true if the requirement was a direct URL reference (which
+            # includes editable requirements), and false if the requirement was
+            # downloaded from a PEP 503 index or --find-links.
+            "is_direct": bool(ireq.original_link),
+            # requested is true if the requirement was specified by the user (aka
+            # top level requirement), and false if it was installed as a dependency of a
+            # requirement. https://peps.python.org/pep-0376/#requested
+            "requested": ireq.user_supplied,
+            # PEP 566 json encoding for metadata
+            # https://www.python.org/dev/peps/pep-0566/#json-compatible-metadata
+            "metadata": ireq.get_dist().json_metadata,
+        }
+        return res
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "install": {
+                ireq.get_dist().metadata["Name"]: self._install_req_to_dict(ireq)
+                for ireq in self._install_requirements
+            }
+        }

tests/functional/test_install_report.py

@@ -0,0 +1,175 @@
+import json
+from pathlib import Path
+
+import pytest
+
+from ..lib import PipTestEnvironment, TestData
+
+
+@pytest.mark.usefixtures("with_wheel")
+def test_install_report_basic(
+    script: PipTestEnvironment, shared_data: TestData, tmp_path: Path
+) -> None:
+    report_path = tmp_path / "report.json"
+    script.pip(
+        "install",
+        "simplewheel",
+        "--dry-run",
+        "--no-index",
+        "--find-links",
+        str(shared_data.root / "packages/"),
+        "--report",
+        str(report_path),
+    )
+    report = json.loads(report_path.read_text())
+    assert "install" in report
+    assert len(report["install"]) == 1
+    assert "simplewheel" in report["install"]
+    simplewheel_report = report["install"]["simplewheel"]
+    assert simplewheel_report["metadata"]["name"] == "simplewheel"
+    assert simplewheel_report["requested"] is True
+    assert simplewheel_report["is_direct"] is False
+    url = simplewheel_report["download_info"]["url"]
+    assert url.startswith("file://")
+    assert url.endswith("/packages/simplewheel-2.0-1-py2.py3-none-any.whl")
+    assert (
+        simplewheel_report["download_info"]["archive_info"]["hash"]
+        == "sha256=191d6520d0570b13580bf7642c97ddfbb46dd04da5dd2cf7bef9f32391dfe716"
+    )
+
+
+@pytest.mark.usefixtures("with_wheel")
+def test_install_report_dep(
+    script: PipTestEnvironment, shared_data: TestData, tmp_path: Path
+) -> None:
+    """Test dependencies are present in the install report with requested=False."""
+    report_path = tmp_path / "report.json"
+    script.pip(
+        "install",
+        "require_simple",
+        "--dry-run",
+        "--no-index",
+        "--find-links",
+        str(shared_data.root / "packages/"),
+        "--report",
+        str(report_path),
+    )
+    report = json.loads(report_path.read_text())
+    assert len(report["install"]) == 2
+    assert report["install"]["require-simple"]["requested"] is True
+    assert report["install"]["simple"]["requested"] is False
+
+
+@pytest.mark.network
+@pytest.mark.usefixtures("with_wheel")
+def test_install_report_index(script: PipTestEnvironment, tmp_path: Path) -> None:
+    """Test report for wheels obtained from index."""
+    report_path = tmp_path / "report.json"
+    script.pip(
+        "install",
+        "--dry-run",
+        "Paste[openid]==1.7.5.1",
+        "--report",
+        str(report_path),
+    )
+    report = json.loads(report_path.read_text())
+    assert len(report["install"]) == 2
+    assert report["install"]["paste"]["requested"] is True
+    assert report["install"]["python-openid"]["requested"] is False
+    paste_report = report["install"]["paste"]
+    assert paste_report["download_info"]["url"].startswith(
+        "https://files.pythonhosted.org/"
+    )
+    assert paste_report["download_info"]["url"].endswith("/Paste-1.7.5.1.tar.gz")
+    assert (
+        paste_report["download_info"]["archive_info"]["hash"]
+        == "sha256=11645842ba8ec986ae8cfbe4c6cacff5c35f0f4527abf4f5581ae8b4ad49c0b6"
+    )
+
+
+@pytest.mark.network
+@pytest.mark.usefixtures("with_wheel")
+def test_install_report_vcs_and_wheel_cache(
+    script: PipTestEnvironment, tmp_path: Path
+) -> None:
+    """Test report for wheels obtained from index."""
+    cache_dir = tmp_path / "cache"
+    report_path = tmp_path / "report.json"
+    script.pip(
+        "install",
+        "git+https://github.com/pypa/pip-test-package"
+        "@5547fa909e83df8bd743d3978d6667497983a4b7",
+        "--cache-dir",
+        str(cache_dir),
+        "--report",
+        str(report_path),
+    )
+    report = json.loads(report_path.read_text())
+    assert len(report["install"]) == 1
+    pip_test_package_report = report["install"]["pip-test-package"]
+    assert pip_test_package_report["is_direct"] is True
+    assert pip_test_package_report["requested"] is True
+    assert (
+        pip_test_package_report["download_info"]["url"]
+        == "https://github.com/pypa/pip-test-package"
+    )
+    assert pip_test_package_report["download_info"]["vcs_info"]["vcs"] == "git"
+    assert (
+        pip_test_package_report["download_info"]["vcs_info"]["commit_id"]
+        == "5547fa909e83df8bd743d3978d6667497983a4b7"
+    )
+    # Now do it again to make sure the cache is used and that the report still contains
+    # the original VCS url.
+    report_path.unlink()
+    result = script.pip(
+        "install",
+        "pip-test-package @ git+https://github.com/pypa/pip-test-package"
+        "@5547fa909e83df8bd743d3978d6667497983a4b7",
+        "--ignore-installed",
+        "--cache-dir",
+        str(cache_dir),
+        "--report",
+        str(report_path),
+    )
+    assert "Using cached pip_test_package" in result.stdout
+    report = json.loads(report_path.read_text())
+    assert len(report["install"]) == 1
+    pip_test_package_report = report["install"]["pip-test-package"]
+    assert pip_test_package_report["is_direct"] is True
+    assert pip_test_package_report["requested"] is True
+    assert (
+        pip_test_package_report["download_info"]["url"]
+        == "https://github.com/pypa/pip-test-package"
+    )
+    assert pip_test_package_report["download_info"]["vcs_info"]["vcs"] == "git"
+    assert (
+        pip_test_package_report["download_info"]["vcs_info"]["commit_id"]
+        == "5547fa909e83df8bd743d3978d6667497983a4b7"
+    )
+
+
+@pytest.mark.network
+@pytest.mark.usefixtures("with_wheel")
+def test_install_report_vcs_editable(
+    script: PipTestEnvironment, tmp_path: Path
+) -> None:
+    """Test report for wheels obtained from index."""
+    report_path = tmp_path / "report.json"
+    script.pip(
+        "install",
+        "--editable",
+        "git+https://github.com/pypa/pip-test-package"
+        "@5547fa909e83df8bd743d3978d6667497983a4b7"
+        "#egg=pip-test-package",
+        "--report",
+        str(report_path),
+    )
+    report = json.loads(report_path.read_text())
+    assert len(report["install"]) == 1
+    pip_test_package_report = report["install"]["pip-test-package"]
+    assert pip_test_package_report["is_direct"] is True
+    assert pip_test_package_report["download_info"]["url"].startswith("file://")
+    assert pip_test_package_report["download_info"]["url"].endswith(
+        "/src/pip-test-package"
+    )
+    assert pip_test_package_report["download_info"]["dir_info"]["editable"] is True