Install python.exe into Windows venvs for console scripts

.github/workflows/main.yml

@@ -321,6 +321,32 @@ jobs:
             ./scripts/pyodide_venv_test.sh
           fi
 
+      # Pyodide's python.exe launcher mangles the path to python.bat when it
+      # holds a space, so for now this runs somewhere without one.
+      # TODO: move this back to "C:\Program Files\pyodide venv test" when
+      # https://github.com/pyodide/pyodide/pull/6381 is released.
+      - name: Create a venv outside the workspace
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          $root = "C:\pyodide-venv-test"
+          New-Item -ItemType Directory -Force -Path $root | Out-Null
+
+          Set-Location $root
+
+          pyodide venv .venv-pyodide
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+
+          .\.venv-pyodide\Scripts\pip.bat install pygments
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+
+          Get-ChildItem .\.venv-pyodide\Scripts
+          $script = Get-ChildItem .\.venv-pyodide\Scripts -Filter "pygmentize*" |
+            Select-Object -First 1
+          if (-not $script) { throw "pygmentize console script was not created" }
+
+          & $script.FullName -V
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
 
   coverage:
     name: Collect and upload coverage

CHANGELOG.md

@@ -21,6 +21,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `uv`-managed Pythons on macOS, which live under `Library/Application Support`.
   [#409](https://github.com/pyodide/pyodide-build/pull/409)
 
+- Pyodide virtual environments on Windows now include the `python.exe` shim that
+  Pyodide ships, and console scripts installed into them point at it. The venv
+  previously had no `.exe` interpreter at all, and the launcher that pip puts in
+  front of a console script refuses to invoke anything else, so no installed
+  script could be run. This needs a cross-build environment from Pyodide 0.29.3
+  or later, which is when the shim was first shipped.
+  [#409](https://github.com/pyodide/pyodide-build/pull/409)
+
 ### Changed
 
 - Replaced the `pydantic` dependency with `attrs` + `cattrs` for recipe

integration_tests/Makefile

@@ -83,6 +83,9 @@ clean:
 	rm -rf recipes/*/build
 	rm -rf test_venv
 	rm -rf "test cmdline runner"
+# TODO: drop the line below, the Windows script uses the name above again, when
+# https://github.com/pyodide/pyodide/pull/6381 is released
+	rm -rf test-cmdline-runner
 	rm -rf src/numpy-*
 	rm -rf src/numpy-*.tar.gz
 	rm -rf numpy-*

integration_tests/scripts/pyodide_venv_test.bat

@@ -1,11 +1,14 @@
 @echo on
 setlocal enabledelayedexpansion
 
-REM Clean up and create test directory. The space in the name is
-REM deliberate, as it lets us test venv paths that need quoting.
-if exist "test cmdline runner" rmdir /s /q "test cmdline runner"
-mkdir "test cmdline runner"
-cd "test cmdline runner"
+REM Clean up and create test directory. This has no space in it, unlike the
+REM POSIX script, because Pyodide's python.exe launcher mangles the path to
+REM python.bat when it contains one.
+REM TODO: rename this back to "test cmdline runner", matching the POSIX script,
+REM when https://github.com/pyodide/pyodide/pull/6381 is released.
+if exist test-cmdline-runner rmdir /s /q test-cmdline-runner
+mkdir test-cmdline-runner
+cd test-cmdline-runner
 if errorlevel 1 exit /b 1
 
 REM Create host virtual environment

pyodide_build/out_of_tree/pip_wrapper.py

@@ -16,6 +16,7 @@ class Config:
     pip_wrapper_name: str
     platform_data: tuple[str, str, str, str, str]
     pyodide_platform: str
+    script_interpreter_suffix: str
     sysconfigdata_dir: str
 
 
@@ -37,7 +38,10 @@ def get_executable():
         raise RuntimeError(
             f'Internal Pyodide error: expected sys.executable="{sys.executable}" to end with "{CONFIG.executable_symlink_suffix}"'
         )
-    return sys.executable.removesuffix(CONFIG.executable_symlink_suffix)
+    return (
+        sys.executable.removesuffix(CONFIG.executable_symlink_suffix)
+        + CONFIG.script_interpreter_suffix
+    )
 
 
 scripts.get_executable = get_executable  # type: ignore[attr-defined]

pyodide_build/out_of_tree/venv.py

@@ -187,6 +187,21 @@ def interpreter_symlink_path(self) -> Path:
         """Get the path to the Pyodide Python interpreter symlink."""
         return self.venv_bin / self.python_exe_name
 
+    @property
+    def script_interpreter_suffix(self) -> str:
+        """Get the extension of the interpreter that console scripts invoke.
+
+        This is the ``+ suffix`` half of the calculation that ``pip_wrapper``
+        does to turn ``sys.executable`` into the Pyodide interpreter, the other
+        half being ``host_python_symlink_suffix``.
+        """
+        return ""
+
+    @property
+    def script_interpreter_path(self) -> Path:
+        """Get the path to the interpreter that installed console scripts invoke."""
+        return self.venv_bin / f"python{self.script_interpreter_suffix}"
+
     @property
     def pyodide_cli_path(self) -> Path:
         """Get the path to the pyodide CLI script in the virtualenv."""
@@ -348,6 +363,7 @@ def _get_pyodide_pip_config(self) -> dict[str, Any]:
         return dict(
             executable_symlink_suffix=self.host_python_symlink_suffix,
             exe_suffix=self.exe_suffix,
+            script_interpreter_suffix=self.script_interpreter_suffix,
             pip_patched_name=self.pip_patched_path.name,
             pip_wrapper_name=self.pip_wrapper_path.name,
             platform_data=ast.literal_eval(result.stdout),
@@ -585,6 +601,21 @@ def host_python_symlink_suffix(self) -> str:
         """Get the host python symlink name."""
         return "-host-link.exe"
 
+    @property
+    def script_interpreter_suffix(self) -> str:
+        """Get the extension of the interpreter that console scripts invoke.
+
+        The launcher that pip puts in front of a console script can only
+        invoke an ``.exe``, so these point at the ``python.exe`` shim rather
+        than at ``python.bat``.
+        """
+        return ".exe"
+
+    @property
+    def interpreter_launcher_path(self) -> Path:
+        """Get the path to the python.exe shim in the Pyodide dist directory."""
+        return pyodide_dist_dir() / "python.exe"
+
     @property
     def host_python_wrapper(self) -> str:
         """Get the content of the host python wrapper script.
@@ -645,6 +676,31 @@ def _create_python_symlink(self) -> None:
                 python_bat.unlink(missing_ok=True)
                 python_bat.symlink_to(self.interpreter_path)
 
+        self._install_interpreter_launcher()
+
+    def _install_interpreter_launcher(self) -> None:
+        """Put the python.exe shim from the Pyodide dist next to python.bat.
+
+        Pyodide ships a small launcher that forwards to python.bat, so that the
+        venv has an ``.exe`` interpreter like an ordinary one does. Without it
+        the launchers pip generates for console scripts have nothing they can
+        invoke, since they refuse any interpreter that is not an ``.exe``.
+        """
+        if not self.interpreter_launcher_path.exists():
+            # Pyodide only started shipping the shim in 0.29.3
+            logger.warning(
+                "%s not found in the cross-build environment. Console scripts "
+                "installed into this virtualenv will not be runnable.",
+                self.interpreter_launcher_path.name,
+            )
+            return
+
+        # Copy rather than symlink. The shim resolves its own path before
+        # looking for python.bat beside it, so a symlink would send it to the
+        # dist directory and it would lose track of the virtualenv.
+        self.script_interpreter_path.unlink(missing_ok=True)
+        shutil.copy(self.interpreter_launcher_path, self.script_interpreter_path)
+
     def _create_pyodide_script(self) -> None:
         """Write pyodide cli script into the virtualenv bin folder."""
         PATH = os.environ["PATH"]

pyodide_build/tests/test_venv.py

@@ -33,7 +33,7 @@ def base_test_dir(tmp_path_factory):
 
     manager = CrossBuildEnvManager(xbuildenv_test_name)
     manager.install(
-        url="https://github.com/pyodide/pyodide/releases/download/0.27.3/xbuildenv-0.27.3.tar.bz2"
+        url="https://github.com/pyodide/pyodide/releases/download/0.29.4/xbuildenv-0.29.4.tar.bz2"
     )
 
     os.chdir(cwd)
@@ -630,6 +630,66 @@ def test_windows_pyodide_cli_script_runs_with_spaces_in_paths(tmp_path, monkeypa
     assert result.stdout.splitlines() == [str(pyodide_root), "build ."]
 
 
+def test_windows_interpreter_launcher_is_copied(tmp_path, monkeypatch):
+    """``python.exe`` has to be a copy of the shim, not a symlink to it.
+
+    The shim resolves its own path before looking for ``python.bat`` beside it,
+    so a symlink would resolve into the dist directory and run the interpreter
+    outside the virtualenv.
+    """
+    dist_dir = tmp_path / "dist"
+    dist_dir.mkdir()
+    (dist_dir / "python.exe").write_bytes(b"pyodide launcher shim")
+    monkeypatch.setattr(venv, "pyodide_dist_dir", lambda: dist_dir)
+
+    pyodide_venv = _windows_venv_with_spaces(tmp_path)
+    pyodide_venv._install_interpreter_launcher()
+
+    launcher = pyodide_venv.script_interpreter_path
+    assert launcher.name == "python.exe"
+    assert not launcher.is_symlink()
+    assert launcher.read_bytes() == b"pyodide launcher shim"
+
+
+def test_windows_interpreter_launcher_may_be_absent(tmp_path, monkeypatch):
+    """Cross-build environments older than Pyodide 0.29.3 ship no python.exe."""
+    dist_dir = tmp_path / "dist"
+    dist_dir.mkdir()
+    monkeypatch.setattr(venv, "pyodide_dist_dir", lambda: dist_dir)
+
+    pyodide_venv = _windows_venv_with_spaces(tmp_path)
+    pyodide_venv._install_interpreter_launcher()
+
+    assert not pyodide_venv.script_interpreter_path.exists()
+
+
+@pytest.mark.parametrize(
+    "make_venv",
+    [_unix_venv_with_spaces, _windows_venv_with_spaces],
+    ids=["unix", "windows"],
+)
+def test_pip_wrapper_executable_is_the_interpreter(tmp_path, make_venv):
+    """The path pip bakes into console scripts must be the venv interpreter.
+
+    ``pip_wrapper.get_executable`` rewrites ``sys.executable`` (the host python
+    symlink) into the interpreter a console script should invoke. The two names
+    differ by more than the symlink suffix on Windows, where the host symlink is
+    ``python-host-link.exe`` and the target is ``python.exe``, so dropping the
+    suffix alone used to leave a path that does not exist.
+    """
+    pyodide_venv = make_venv(tmp_path)
+
+    # What pip_wrapper.get_executable() computes. It cannot import from
+    # pyodide-build, so the calculation is repeated rather than shared.
+    sys_executable = str(pyodide_venv.host_python_symlink_path)
+    executable = (
+        sys_executable.removesuffix(pyodide_venv.host_python_symlink_suffix)
+        + pyodide_venv.script_interpreter_suffix
+    )
+
+    assert executable == str(pyodide_venv.script_interpreter_path)
+
+
 def test_cleanup_skips_preexisting_directory(monkeypatch, tmp_path):
     """A failed venv creation must not delete a pre-existing destination."""
     dest = tmp_path / "existing-venv"