Support PEP-658

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 default_language_version:
-  python: "3.11"
+  python: "3.12"
 
 exclude: (^micropip/externals|^tests/vendored|^tests/test_data)
 repos:

.readthedocs.yml

@@ -12,6 +12,6 @@ python:
       path: .
 
 build:
-  os: ubuntu-20.04
+  os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.12"

CHANGELOG.md

@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Added support for PEP-658.
+  [#139](https://github.com/pyodide/micropip/pull/139)
+
 ## [0.6.0] - 2024/01/31
 
 ### Fixed

micropip/package_index.py

@@ -13,6 +13,7 @@
 from ._compat import HttpStatusError, fetch_string_and_headers
 from ._utils import is_package_compatible, parse_version
 from .externals.mousebender.simple import from_project_details_html
+from .types import DistributionMetadata
 from .wheelinfo import WheelInfo
 
 DEFAULT_INDEX_URLS = ["https://pypi.org/simple"]
@@ -150,6 +151,11 @@ def _compatible_wheels(
                 hashes = file["digests"] if "digests" in file else file["hashes"]
                 sha256 = hashes.get("sha256")
 
+                # Check if the metadata file is available (PEP 658)
+                data_dist_info_metadata: DistributionMetadata = file.get(
+                    "data-dist-info-metadata"
+                )
+
                 # Size of the file in bytes, if available (PEP 700)
                 # This key is not available in the Simple API HTML response, so this field may be None
                 size = file.get("size")
@@ -161,6 +167,7 @@ def _compatible_wheels(
                     version=version,
                     sha256=sha256,
                     size=size,
+                    data_dist_info_metadata=data_dist_info_metadata,
                 )
 
         releases_compatible = {

micropip/transaction.py

@@ -231,8 +231,16 @@ async def add_wheel(
         logger.info("Collecting %s%s", wheel.name, specifier)
         logger.info("  Downloading %s", wheel.url.split("/")[-1])
 
-        await wheel.download(self.fetch_kwargs)
+        wheel_download_task = asyncio.create_task(wheel.download(self.fetch_kwargs))
         if self.deps:
+            try:
+                await wheel.download_pep658_metadata(self.fetch_kwargs)
+            except OSError:
+                # If something goes wrong while downloading the metadata
+                # we just log the error and continue, as the metadata can be fetched extracted from the wheel.
+                logger.debug("Metadata not available for %s", wheel.name)
+
+            await wheel_download_task
             await self.gather_requirements(wheel.requires(extras))
 
         self.wheels.append(wheel)

micropip/types.py

@@ -0,0 +1,5 @@
+# Distribution Metadata type (PEP 658)
+# None = metadata not available
+# bool = metadata available, but no checksum
+# dict[str, str] = metadata available with checksum
+DistributionMetadata = bool | dict[str, str] | None

micropip/wheelinfo.py

@@ -19,6 +19,7 @@
 )
 from ._utils import parse_wheel_filename
 from .metadata import Metadata, safe_name, wheel_dist_info_dir
+from .types import DistributionMetadata
 
 
 @dataclass
@@ -43,6 +44,7 @@ class WheelInfo:
     parsed_url: ParseResult
     sha256: str | None = None
     size: int | None = None  # Size in bytes, if available (PEP 700)
+    data_dist_info_metadata: DistributionMetadata = None  # Wheel's metadata (PEP 658)
 
     # Fields below are only available after downloading the wheel, i.e. after calling `download()`.
 
@@ -55,6 +57,7 @@ class WheelInfo:
 
     def __post_init__(self):
         self._project_name = safe_name(self.name)
+        self.metadata_url = self.url + ".metadata"
 
     @classmethod
     def from_url(cls, url: str) -> "WheelInfo":
@@ -84,6 +87,7 @@ def from_package_index(
         version: Version,
         sha256: str | None,
         size: int | None,
+        data_dist_info_metadata: DistributionMetadata = None,
     ) -> "WheelInfo":
         """Extract available metadata from response received from package index"""
         parsed_url = urlparse(url)
@@ -99,6 +103,7 @@ def from_package_index(
             parsed_url=parsed_url,
             sha256=sha256,
             size=size,
+            data_dist_info_metadata=data_dist_info_metadata,
         )
 
     async def install(self, target: Path) -> None:
@@ -125,10 +130,36 @@ async def download(self, fetch_kwargs: dict[str, Any]):
         if self._data is not None:
             return
 
-        self._data = await self._fetch_bytes(fetch_kwargs)
-        with zipfile.ZipFile(io.BytesIO(self._data)) as zf:
-            metadata_path = wheel_dist_info_dir(zf, self.name) + "/" + Metadata.PKG_INFO
-            self._metadata = Metadata(zipfile.Path(zf, metadata_path))
+        self._data = await self._fetch_bytes(self.url, fetch_kwargs)
+
+        # The wheel's metadata might be downloaded separately from the wheel itself.
+        # If it is not downloaded yet or if the metadata is not available, extract it from the wheel.
+        if self._metadata is None:
+            with zipfile.ZipFile(io.BytesIO(self._data)) as zf:
+                metadata_path = (
+                    wheel_dist_info_dir(zf, self.name) + "/" + Metadata.PKG_INFO
+                )
+                self._metadata = Metadata(zipfile.Path(zf, metadata_path))
+
+    async def download_pep658_metadata(
+        self,
+        fetch_kwargs: dict[str, Any],
+    ) -> None:
+        """
+        Download the wheel's metadata. If the metadata is not available, return None.
+        """
+        if self.data_dist_info_metadata is None:
+            return None
+
+        data = await self._fetch_bytes(self.metadata_url, fetch_kwargs)
+
+        match self.data_dist_info_metadata:
+            case {"sha256": checksum}:  # sha256 checksum available
+                _validate_sha256_checksum(data, checksum)
+            case _:  # no checksum available
+                pass
+
+        self._metadata = Metadata(data)
 
     def requires(self, extras: set[str]) -> list[Requirement]:
         """
@@ -143,9 +174,9 @@ def requires(self, extras: set[str]) -> list[Requirement]:
         self._requires = requires
         return requires
 
-    async def _fetch_bytes(self, fetch_kwargs: dict[str, Any]):
+    async def _fetch_bytes(self, url: str, fetch_kwargs: dict[str, Any]):
         try:
-            return await fetch_bytes(self.url, fetch_kwargs)
+            return await fetch_bytes(url, fetch_kwargs)
         except OSError as e:
             if self.parsed_url.hostname in [
                 "files.pythonhosted.org",
@@ -154,7 +185,7 @@ async def _fetch_bytes(self, fetch_kwargs: dict[str, Any]):
                 raise e
             else:
                 raise ValueError(
-                    f"Can't fetch wheel from '{self.url}'. "
+                    f"Can't fetch wheel from '{url}'. "
                     "One common reason for this is when the server blocks "
                     "Cross-Origin Resource Sharing (CORS). "
                     "Check if the server is sending the correct 'Access-Control-Allow-Origin' header."

pyproject.toml

@@ -6,7 +6,7 @@ authors = [
 description = "A lightweight Python package installer for the web"
 readme = "README.md"
 license = { file="LICENSE" }
-requires-python = ">=3.10"
+requires-python = ">=3.12"
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
@@ -64,7 +64,7 @@ known-first-party = [
 ]
 
 [tool.mypy]
-python_version = "3.11"
+python_version = "3.12"
 show_error_codes = true
 warn_unreachable = true
 ignore_missing_imports = true

tests/conftest.py

@@ -97,6 +97,7 @@ class Wheel:
         _path: Path
 
         name: str
+        version: str
         filename: str
         top_level: str
         url: str
@@ -118,24 +119,30 @@ def __enter__(self):
     def __exit__(self, *args: Any):
         self._httpserver.__exit__(*args)
 
-    def _register_handler(self, path: Path) -> str:
-        self._httpserver.expect_request(f"/{path.name}").respond_with_data(
-            path.read_bytes(),
+    def _register_handler(self, endpoint: str, data: bytes) -> str:
+        self._httpserver.expect_request(f"/{endpoint}").respond_with_data(
+            data,
             content_type="application/zip",
             headers={"Access-Control-Allow-Origin": "*"},
         )
 
-        return self._httpserver.url_for(f"/{path.name}")
+        return self._httpserver.url_for(f"/{endpoint}")
 
     def add_wheel(self, path: Path, replace: bool = True):
-        name = parse_wheel_filename(path.name)[0]
-        url = self._register_handler(path)
+        name, version = parse_wheel_filename(path.name)[:2]
+        url = self._register_handler(path.name, path.read_bytes())
+
+        metadata_file_endpoint = path.with_suffix(".whl.metadata")
+        metadata_file_gzipped = path.with_suffix(".whl.metadata.gz")
+        if metadata_file_gzipped.exists():
+            data = _read_gzipped_testfile(metadata_file_gzipped)
+            self._register_handler(metadata_file_endpoint.name, data)
 
         if name in self._wheels and not replace:
             return
 
         self._wheels[name] = self.Wheel(
-            path, name, path.name, name.replace("-", "_"), url
+            path, name, str(version), path.name, name.replace("-", "_"), url
         )
 
     def get(self, name: str) -> Wheel:

tests/test_wheelinfo.py

@@ -22,15 +22,19 @@ def test_from_package_index():
     version = "0.0.1"
     sha256 = "dummy-sha256"
     size = 1234
+    data_dist_info_metadata = True
 
-    wheel = WheelInfo.from_package_index(name, filename, url, version, sha256, size)
+    wheel = WheelInfo.from_package_index(
+        name, filename, url, version, sha256, size, data_dist_info_metadata
+    )
 
     assert wheel.name == name
     assert str(wheel.version) == version
     assert wheel.url == url
     assert wheel.filename == filename
     assert wheel.size == size
     assert wheel.sha256 == sha256
+    assert wheel.data_dist_info_metadata == data_dist_info_metadata
 
 
 def test_extract(wheel_catalog, tmp_path):
@@ -89,3 +93,98 @@ async def test_requires(wheel_catalog, tmp_path):
     requirements_extra_testing = [str(r.name) for r in wheel.requires({"testing"})]
     assert "pluggy" in requirements_extra_testing
     assert "hypothesis" in requirements_extra_testing
+
+
+@pytest.mark.asyncio
+async def test_download_pep658_metadata(wheel_catalog):
+    pytest_wheel = wheel_catalog.get("pytest")
+    sha256 = "dummy-sha256"
+    size = 1234
+
+    # 1) metadata available
+    wheel_with_metadata = WheelInfo.from_package_index(
+        pytest_wheel.name,
+        pytest_wheel.filename,
+        pytest_wheel.url,
+        pytest_wheel.version,
+        sha256,
+        size,
+        data_dist_info_metadata=True,
+    )
+
+    assert wheel_with_metadata._metadata is None
+    await wheel_with_metadata.download_pep658_metadata({})
+    assert wheel_with_metadata._metadata is not None
+
+    # metadata should be calculated from the metadata file
+    deps = wheel_with_metadata._metadata.deps
+    assert None in deps
+    assert "testing" in deps
+
+    # 2) metadata not available
+    wheel_without_metadata = WheelInfo.from_package_index(
+        pytest_wheel.name,
+        pytest_wheel.filename,
+        pytest_wheel.url,
+        pytest_wheel.version,
+        sha256,
+        size,
+        data_dist_info_metadata=None,
+    )
+
+    assert wheel_without_metadata._metadata is None
+    await wheel_without_metadata.download_pep658_metadata({})
+    assert wheel_without_metadata._metadata is None
+
+    # 3) the metadata extracted from the wheel should be the same
+    wheel = WheelInfo.from_package_index(
+        pytest_wheel.name,
+        pytest_wheel.filename,
+        pytest_wheel.url,
+        pytest_wheel.version,
+        sha256,
+        size,
+        data_dist_info_metadata=None,
+    )
+
+    assert wheel._metadata is None
+    await wheel.download({})
+    assert wheel._metadata is not None
+
+    assert wheel._metadata.deps == wheel_with_metadata._metadata.deps
+
+
+@pytest.mark.asyncio
+async def test_download_pep658_metadata_checksum(wheel_catalog):
+    pytest_wheel = wheel_catalog.get("pytest")
+    sha256 = "dummy-sha256"
+    size = 1234
+
+    wheel = WheelInfo.from_package_index(
+        pytest_wheel.name,
+        pytest_wheel.filename,
+        pytest_wheel.url,
+        pytest_wheel.version,
+        sha256,
+        size,
+        data_dist_info_metadata={"sha256": "dummy-sha256"},
+    )
+
+    assert wheel._metadata is None
+    with pytest.raises(RuntimeError, match="Invalid checksum: expected dummy-sha256"):
+        await wheel.download_pep658_metadata({})
+
+    checksum = "62eb95408ccec185e7a3b8f354a1df1721cd8f463922f5a900c7bf4b69c5a4e8"  # TODO: calculate this from the file
+    wheel = WheelInfo.from_package_index(
+        pytest_wheel.name,
+        pytest_wheel.filename,
+        pytest_wheel.url,
+        pytest_wheel.version,
+        sha256,
+        size,
+        data_dist_info_metadata={"sha256": checksum},
+    )
+
+    assert wheel._metadata is None
+    await wheel.download_pep658_metadata({})
+    assert wheel._metadata is not None