BLD Use metadata file when installing cross build env (#4830)

Author: ryanking13

pyodide_build/build_env.py

@@ -5,13 +5,15 @@
 import os
 import re
 import subprocess
+import sys
 from collections.abc import Iterator
 from contextlib import nullcontext, redirect_stdout
 from io import StringIO
 from pathlib import Path
 
 from packaging.tags import Tag, compatible_tags, cpython_tags
 
+from . import __version__
 from .common import search_pyproject_toml, xbuildenv_dirname
 from .config import ConfigManager
 from .recipe import load_all_recipes
@@ -257,3 +259,15 @@ def check_emscripten_version() -> None:
         raise RuntimeError(
             f"Incorrect Emscripten version {installed_version}. Need Emscripten version {needed_version}"
         )
+
+
+def local_versions() -> dict[str, str]:
+    """
+    Returns the versions of the local Python interpreter and the pyodide-build.
+    This information is used for checking compatibility with the cross-build environment.
+    """
+    return {
+        "python": f"{sys.version_info.major}.{sys.version_info.minor}",
+        "pyodide-build": __version__,
+        # "emscripten": "TODO"
+    }

pyodide_build/cli/xbuildenv.py

@@ -2,8 +2,13 @@
 
 import typer
 
+from ..build_env import local_versions
 from ..common import xbuildenv_dirname
 from ..xbuildenv import CrossBuildEnvManager
+from ..xbuildenv_releases import (
+    cross_build_env_metadata_url,
+    load_cross_build_env_metadata,
+)
 
 DIRNAME = xbuildenv_dirname()
 
@@ -32,6 +37,12 @@ def _install(
         DIRNAME, help="path to cross-build environment directory"
     ),
     url: str = typer.Option(None, help="URL to download cross-build environment from"),
+    force_install: bool = typer.Option(
+        False,
+        "--force",
+        "-f",
+        help="force installation even if the version is not compatible",
+    ),
 ) -> None:
     """
     Install cross-build environment.
@@ -44,9 +55,9 @@ def _install(
     manager = CrossBuildEnvManager(path)
 
     if url:
-        manager.install(url=url)
+        manager.install(url=url, force_install=force_install)
     else:
-        manager.install(version=version)
+        manager.install(version=version, force_install=force_install)
 
     typer.echo(f"Pyodide cross-build environment installed at {path.resolve()}")
 
@@ -125,3 +136,79 @@ def _use(
     manager = CrossBuildEnvManager(path)
     manager.use_version(version)
     typer.echo(f"Pyodide cross-build environment {version} is now in use")
+
+
+@app.command("search")
+def _search(
+    metadata_path: str = typer.Option(
+        None,
+        "--metadata",
+        help="path to cross-build environment metadata file. It can be a URL or a local file. If not given, the default metadata file is used.",
+    ),
+    show_all: bool = typer.Option(
+        False,
+        "--all",
+        "-a",
+        help="search all versions, without filtering out incompatible ones",
+    ),
+) -> None:
+    """
+    Search for available versions of cross-build environment.
+    """
+
+    # TODO: cache the metadata file somewhere to avoid downloading it every time
+    metadata_path = metadata_path or cross_build_env_metadata_url()
+    metadata = load_cross_build_env_metadata(metadata_path)
+    local = local_versions()
+
+    if show_all:
+        releases = metadata.list_compatible_releases()
+    else:
+        releases = metadata.list_compatible_releases(
+            python_version=local["python"],
+            pyodide_build_version=local["pyodide-build"],
+        )
+
+    if not releases:
+        typer.echo(
+            "No compatible cross-build environment found for your system. Try using --all to see all versions."
+        )
+        raise typer.Exit(1)
+
+    table = []
+    columns = [
+        # column name, width
+        ("Version", 10),
+        ("Python", 10),
+        ("Emscripten", 10),
+        ("pyodide-build", 25),
+        ("Compatible", 10),
+    ]
+    header = [f"{name:{width}}" for name, width in columns]
+    divider = ["-" * width for _, width in columns]
+
+    table.append("\t".join(header))
+    table.append("\t".join(divider))
+
+    for release in releases:
+        compatible = (
+            "Yes"
+            if release.is_compatible(
+                python_version=local["python"],
+                pyodide_build_version=local["pyodide-build"],
+            )
+            else "No"
+        )
+        pyodide_build_range = f"{release.min_pyodide_build_version or ''} - {release.max_pyodide_build_version or ''}"
+
+        row = [
+            f"{release.version:{columns[0][1]}}",
+            f"{release.python_version:{columns[1][1]}}",
+            f"{release.emscripten_version:{columns[2][1]}}",
+            f"{pyodide_build_range:{columns[3][1]}}",
+            f"{compatible:{columns[4][1]}}",
+        ]
+
+        table.append("\t".join(row))
+
+    print("\n".join(table))

pyodide_build/tests/fixture.py

@@ -1,3 +1,4 @@
+import json
 import os
 from pathlib import Path
 
@@ -162,3 +163,57 @@ def mock_emscripten(tmp_path, dummy_xbuildenv, reset_env_vars, reset_cache):
     }
 
     os.environ["PATH"] = original_path
+
+
+@pytest.fixture(scope="function")
+def fake_xbuildenv_releases_compatible(tmp_path, dummy_xbuildenv_url):
+    """
+    Create a fake metadata file with a single release that is compatible with the local environment.
+    """
+    local = build_env.local_versions()
+    fake_releases = {
+        "releases": {
+            "0.1.0": {
+                "version": "0.1.0",
+                "url": dummy_xbuildenv_url,
+                "sha256": "1234567890abcdef",
+                "python_version": f"{local['python']}.0",
+                "emscripten_version": "1.39.8",
+            },
+            "0.2.0": {
+                "version": "0.2.0",
+                "url": dummy_xbuildenv_url,
+                "sha256": "1234567890abcdef",
+                "python_version": f"{local['python']}.0",
+                "emscripten_version": "2.39.8",
+            },
+        },
+    }
+
+    metadata_path = Path(tmp_path) / "metadata-compat.json"
+    metadata_path.write_text(json.dumps(fake_releases))
+
+    yield metadata_path
+
+
+@pytest.fixture(scope="function")
+def fake_xbuildenv_releases_incompatible(tmp_path, dummy_xbuildenv_url):
+    """
+    Create a fake metadata file with a single release that is incompatible with the local environment.
+    """
+    fake_releases = {
+        "releases": {
+            "0.1.0": {
+                "version": "0.1.0",
+                "url": dummy_xbuildenv_url,
+                "sha256": "1234567890abcdef",
+                "python_version": "4.5.6",
+                "emscripten_version": "1.39.8",
+            },
+        },
+    }
+
+    metadata_path = Path(tmp_path) / "metadata-incompat.json"
+    metadata_path.write_text(json.dumps(fake_releases))
+
+    yield metadata_path

pyodide_build/tests/test_cli_xbuildenv.py

@@ -1,3 +1,6 @@
+# flake8: noqa
+
+import os
 import shutil
 from pathlib import Path
 
@@ -9,6 +12,13 @@
     xbuildenv,
 )
 from pyodide_build.common import chdir
+from pyodide_build.xbuildenv_releases import CROSS_BUILD_ENV_METADATA_URL_ENV_VAR
+
+from .fixture import (
+    dummy_xbuildenv_url,
+    fake_xbuildenv_releases_compatible,
+    fake_xbuildenv_releases_incompatible,
+)
 
 
 def mock_pyodide_lock() -> PyodideLockSpec:
@@ -89,6 +99,84 @@ def test_xbuildenv_install(tmp_path, mock_xbuildenv_url):
     assert (concrete_path / ".installed").exists()
 
 
+def test_xbuildenv_install_version(tmp_path, fake_xbuildenv_releases_compatible):
+    envpath = Path(tmp_path) / ".xbuildenv"
+
+    os.environ.pop(CROSS_BUILD_ENV_METADATA_URL_ENV_VAR, None)
+    os.environ[CROSS_BUILD_ENV_METADATA_URL_ENV_VAR] = str(
+        fake_xbuildenv_releases_compatible
+    )
+
+    result = runner.invoke(
+        xbuildenv.app,
+        [
+            "install",
+            "0.1.0",
+            "--path",
+            str(envpath),
+        ],
+    )
+
+    os.environ.pop(CROSS_BUILD_ENV_METADATA_URL_ENV_VAR, None)
+
+    assert result.exit_code == 0, result.stdout
+    assert "Downloading Pyodide cross-build environment" in result.stdout, result.stdout
+    assert "Installing Pyodide cross-build environment" in result.stdout, result.stdout
+    assert (envpath / "xbuildenv").is_symlink()
+    assert (envpath / "xbuildenv").resolve().exists()
+    assert (envpath / "0.1.0").exists()
+
+    concrete_path = (envpath / "xbuildenv").resolve()
+    assert (concrete_path / ".installed").exists()
+
+
+def test_xbuildenv_install_force_install(
+    tmp_path, fake_xbuildenv_releases_incompatible
+):
+    envpath = Path(tmp_path) / ".xbuildenv"
+
+    os.environ.pop(CROSS_BUILD_ENV_METADATA_URL_ENV_VAR, None)
+    os.environ[CROSS_BUILD_ENV_METADATA_URL_ENV_VAR] = str(
+        fake_xbuildenv_releases_incompatible
+    )
+
+    result = runner.invoke(
+        xbuildenv.app,
+        [
+            "install",
+            "0.1.0",
+            "--path",
+            str(envpath),
+        ],
+    )
+
+    # should fail if no force option is given
+    assert result.exit_code != 0, result.stdout
+
+    result = runner.invoke(
+        xbuildenv.app,
+        [
+            "install",
+            "0.1.0",
+            "--path",
+            str(envpath),
+            "--force",
+        ],
+    )
+
+    assert result.exit_code == 0, result.stdout
+    assert "Downloading Pyodide cross-build environment" in result.stdout, result.stdout
+    assert "Installing Pyodide cross-build environment" in result.stdout, result.stdout
+    assert (envpath / "xbuildenv").is_symlink()
+    assert (envpath / "xbuildenv").resolve().exists()
+    assert (envpath / "0.1.0").exists()
+
+    concrete_path = (envpath / "xbuildenv").resolve()
+    assert (concrete_path / ".installed").exists()
+
+    os.environ.pop(CROSS_BUILD_ENV_METADATA_URL_ENV_VAR, None)
+
+
 def test_xbuildenv_version(tmp_path):
     envpath = Path(tmp_path) / ".xbuildenv"
 
@@ -207,3 +295,58 @@ def test_xbuildenv_uninstall(tmp_path):
 
     assert result.exit_code != 0, result.stdout
     assert isinstance(result.exception, ValueError), result.exception
+
+
+def test_xbuildenv_search(
+    tmp_path, fake_xbuildenv_releases_compatible, fake_xbuildenv_releases_incompatible
+):
+    result = runner.invoke(
+        xbuildenv.app,
+        [
+            "search",
+            "--metadata",
+            str(fake_xbuildenv_releases_compatible),
+        ],
+    )
+
+    assert result.exit_code == 0, result.stdout
+    assert "0.1.0" in result.stdout, result.stdout
+
+    result = runner.invoke(
+        xbuildenv.app,
+        [
+            "search",
+            "--metadata",
+            str(fake_xbuildenv_releases_incompatible),
+        ],
+    )
+
+    assert result.exit_code != 0, result.stdout
+    assert (
+        "No compatible cross-build environment found for your system" in result.stdout
+    )
+    assert "0.1.0" not in result.stdout, result.stdout
+
+    result = runner.invoke(
+        xbuildenv.app,
+        [
+            "search",
+            "--metadata",
+            str(fake_xbuildenv_releases_incompatible),
+            "--all",
+        ],
+    )
+
+    assert result.exit_code == 0, result.stdout
+
+    header = result.stdout.splitlines()[0]
+    assert header.split() == [
+        "Version",
+        "Python",
+        "Emscripten",
+        "pyodide-build",
+        "Compatible",
+    ]
+
+    row1 = result.stdout.splitlines()[2]
+    assert row1.split() == ["0.1.0", "4.5.6", "1.39.8", "-", "No"]