Fix --sh-boot for PEX_TOOLS=1 and broken venvs. (#74)

Author: jsirois

CHANGES.md

@@ -1,5 +1,10 @@
 # Release Notes
 
+## 0.7.1
+
+This release fixes injected `--sh-boot` PEXes to honor `PEX_TOOLS=1` and be robust to underlying
+venv breaks due to system Python upgrades or uninstalls.
+
 ## 0.7.0
 
 This release adds support for PEX_TOOLS when `pexrc` is built with the `tools` feature; e.g.:

Cargo.lock

@@ -1,9 +1,11 @@
 # Copyright 2026 Pex project contributors.
 # SPDX-License-Identifier: Apache-2.0
 
+cargo-features = ["profile-rustflags"]
+
 [package]
 name = "pexrc"
-version = "0.7.0"
+version = "0.7.1"
 edition = "2024"
 publish = false
 
@@ -112,6 +114,11 @@ fingerprint = {
     algorithm = "sha256", hash = "26497108bbd3a71cada865d712878bc3957fcaebf2bc431357f560f0524e6273"
 }
 
+[profile.dev]
+# N.B.: This is needed to get a debug build for the Windows clib dll. It turns out COFF only
+# supports 2^16 symbols and we have edged over that.
+rustflags = ["-Zshare-generics=off"]
+
 [profile.release]
 codegen-units = 1
 lto = true

Cargo.toml

@@ -12,6 +12,7 @@ cache = { path = "../cache" }
 const_format = { workspace = true }
 fs-err = { workspace = true }
 interpreter = { path = "../interpreter" }
+itertools = { workspace = true }
 pex = { path = "../pex" }
 pexrs = { path = "../pexrs" }
 platform = { path = "../platform" }

crates/boot/Cargo.toml

@@ -13,6 +13,7 @@ set -eu
 RAW_DEFAULT_PEXRC_ROOT="{pexrc_root}"
 VENV_RELPATH="{venv_relpath}"
 PYTHONS="{pythons}"
+PYTHON_ARGS="{python_args}"
 # --- split --- #
 
 # N.B.: The SC2116 warning suppressions below are in place to ensure tilde-expansion of the
@@ -38,21 +39,18 @@ on_fast_path() {
     [ -z "${PEX_IGNORE_RCFILES:-}" ] \
       && [ -z "${PEX_PYTHON:-}" ] \
       && [ -z "${PEX_PYTHON_PATH:-}" ] \
-      && [ -z "${PEX_PATH:-}" ]
+      && [ -z "${PEX_PATH:-}" ] \
+      && [ -z "${PEX_TOOLS:-}" ]
 }
 
 if on_fast_path; then
   for python in ${PYTHONS} ; do
-      if [ -x "${VENV}/sh-boot/${python}" ]; then
+      if [ -x "${VENV}/sh-boot/base-${python}" ] && [ -x "${VENV}/sh-boot/pex-${python}" ]; then
           # The fast path: We're installed under the PEXRC_ROOT and the venv interpreter to use is
           # embedded in the shebang of our venv pex script; so just execute that script directly.
           export PEX="$0"
 
-          # TODO: XXX: Instead of linking the venv pex script to the sh-boot python, link the
-          # venv python. This will ensure when a base interpreter gets uninstalled, the venv will
-          # automatically invalidate. As it stands the venv pex script we link to can exist but have
-          # a shebang whose python is gone.
-          exec "${VENV}/sh-boot/${python}" "$@"
+          exec "${VENV}/sh-boot/pex-${python}" "$@"
       fi
   done
 fi
@@ -71,11 +69,14 @@ python_exe="$(find_python)"
 if [ -n "${python_exe}" ]; then
     if [ -n "${PEX_VERBOSE:-}" ]; then
         echo >&2 "$0 used /bin/sh boot to select python: ${python_exe} for re-exec..."
-        echo >&2 "Running pex to lay itself out under PEXRC_ROOT."
+        if [ -n "${PEX_VERBOSE:-}" ]; then
+          echo >&2 "Running pex to invoke PEX_TOOLS."
+        else
+          echo >&2 "Running pex to lay itself out under PEXRC_ROOT."
+        fi
     fi
     export _PEXRC_SH_BOOT_SEED_DIR="${VENV}/sh-boot"
-    # TODO: XXX: Inject -sE or -I if hermetic?
-    exec "${python_exe}" "$0" "$@"
+    exec "${python_exe}" "${PYTHON_ARGS}" "$0" "$@"
 fi
 
 echo >&2 "Failed to find any of these python binaries on the PATH:"

crates/boot/src/boot.sh

@@ -11,7 +11,8 @@ use cache::CacheDir;
 use const_format::str_split;
 use fs_err as fs;
 use fs_err::File;
-use interpreter::{InterpreterConstraints, SearchPath, SelectionStrategy};
+use interpreter::{InterpreterConstraints, SearchPath, SelectionStrategy, VersionSpec};
+use itertools::Itertools;
 use pex::{Pex, PexPath};
 use pexrs::venv_dir;
 use platform::path_as_str;
@@ -21,7 +22,11 @@ use zip::write::{FileOptionExtension, FileOptions};
 const SH_BOOT_SHEBANG: &[u8] = b"#!/bin/sh\n";
 const SH_BOOT_PARTS: [&str; 4] = str_split!(include_str!("boot.sh"), "# --- split --- #\n");
 
-pub fn sh_boot_shebang(pex: &Path, escaped: bool) -> anyhow::Result<Option<String>> {
+pub fn sh_boot_shebang(
+    pex: &Path,
+    hermetic: bool,
+    escaped: bool,
+) -> anyhow::Result<Option<String>> {
     let pex = Pex::load(pex)?;
 
     let mut sh_boot_shebang_buffer: [_; SH_BOOT_SHEBANG.len()] = [0; SH_BOOT_SHEBANG.len()];
@@ -54,13 +59,28 @@ pub fn sh_boot_shebang(pex: &Path, escaped: bool) -> anyhow::Result<Option<Strin
     let pythons = interpreter_constraints
         .calculate_compatible_binary_names(selection_strategy)
         .into_iter()
-        .map(|binary_name| {
+        .map(|(binary_name, version_spec)| {
             binary_name
                 .into_string()
+                .map(|binary_name| (binary_name, version_spec))
                 .map_err(|err| anyhow!("{err}", err = err.display()))
         })
         .collect::<anyhow::Result<Vec<_>>>()?;
-
+    let python_args = if hermetic {
+        if pythons.iter().any(|(_, version_spec)| {
+            matches!(version_spec, None | Some(VersionSpec::Major(_)))
+                || matches!(
+                    version_spec,
+                    Some(VersionSpec::MajorMinor(major, minor)) if (*major, *minor) < (3, 4)
+                )
+        }) {
+            "-sE"
+        } else {
+            "-I"
+        }
+    } else {
+        ""
+    };
     Ok(Some(format!(
         "{shebang}{start_escape}{header}{vars}{body}{end_escape}\n",
         shebang = SH_BOOT_PARTS[0], // N.B.: SH_BOOT_SHEBANG
@@ -72,7 +92,14 @@ pub fn sh_boot_shebang(pex: &Path, escaped: bool) -> anyhow::Result<Option<Strin
                 pex.info.pex_root.as_deref().unwrap_or_default(),
             )
             .replace("{venv_relpath}", path_as_str(venv_relpath)?)
-            .replace("{pythons}", &pythons.join("\n")),
+            .replace(
+                "{pythons}",
+                &pythons
+                    .iter()
+                    .map(|(binary_name, _)| binary_name)
+                    .join("\n")
+            )
+            .replace("{python_args}", python_args),
         body = SH_BOOT_PARTS[3].trim_end(),
         end_escape = if escaped { "\n'''\n" } else { "\n" },
     )))

crates/boot/src/lib.rs

@@ -9,7 +9,7 @@ use std::str::FromStr;
 use std::sync::LazyLock;
 
 use anyhow::bail;
-use indexmap::{IndexSet, indexset};
+use indexmap::{IndexMap, IndexSet};
 use log::{debug, warn};
 use pep440_rs::{Operator, Version, VersionSpecifier, VersionSpecifiers};
 use pep508_rs::{ExtraName, MarkerTree, PackageName, Requirement, VersionOrUrl};
@@ -276,6 +276,11 @@ struct PythonBinarySpec {
     suffix: Option<&'static str>,
 }
 
+pub enum VersionSpec {
+    MajorMinor(u8, u8),
+    Major(u8),
+}
+
 pub struct InterpreterConstraints(Vec<InterpreterConstraint>);
 
 impl InterpreterConstraints {
@@ -384,9 +389,9 @@ impl InterpreterConstraints {
     pub fn calculate_compatible_binary_names(
         &self,
         selection_strategy: SelectionStrategy,
-    ) -> IndexSet<OsString> {
+    ) -> IndexMap<OsString, Option<VersionSpec>> {
         let binary_specs = self.calculate_compatible_binary_specs(selection_strategy);
-        let mut binary_names: IndexSet<OsString> = IndexSet::new();
+        let mut binary_names: IndexMap<OsString, Option<VersionSpec>> = IndexMap::new();
         for binary_spec in &binary_specs {
             binary_names.insert(
                 format!(
@@ -397,6 +402,10 @@ impl InterpreterConstraints {
                     suffix = binary_spec.suffix.unwrap_or("")
                 )
                 .into(),
+                Some(VersionSpec::MajorMinor(
+                    binary_spec.major,
+                    binary_spec.minor,
+                )),
             );
         }
         for binary_spec in &binary_specs {
@@ -407,10 +416,11 @@ impl InterpreterConstraints {
                     major = binary_spec.major
                 )
                 .into(),
+                Some(VersionSpec::Major(binary_spec.major)),
             );
         }
         for binary_spec in &binary_specs {
-            binary_names.insert(binary_spec.name.into());
+            binary_names.insert(binary_spec.name.into(), None);
         }
         binary_names
     }
@@ -422,9 +432,11 @@ impl InterpreterConstraints {
     ) -> anyhow::Result<impl Iterator<Item = PathBuf>> {
         let (python, path, known_paths) = search_path.into_parts()?;
         let binary_names = if let Some(python) = python {
-            indexset! {python}
+            vec![python]
         } else {
             self.calculate_compatible_binary_names(selection_strategy)
+                .into_keys()
+                .collect()
         };
         Ok(PythonExeIter {
             known_paths,
@@ -522,6 +534,7 @@ mod tests {
     use std::ffi::OsStr;
     use std::str::FromStr;
 
+    use indexmap::IndexSet;
     use pep440_rs::VersionSpecifiers;
 
     use crate::constraints::{InterpreterConstraint, InterpreterImplementation};
@@ -602,7 +615,10 @@ mod tests {
     #[test]
     fn test_interpreter_constraints_binary_names_all_default_order() {
         let ics = InterpreterConstraints::try_from::<&str>(&[]).unwrap();
-        let binary_names = ics.calculate_compatible_binary_names(SelectionStrategy::Oldest);
+        let binary_names = ics
+            .calculate_compatible_binary_names(SelectionStrategy::Oldest)
+            .into_keys()
+            .collect::<IndexSet<_>>();
 
         assert_eq!(&["python2.7", "pypy2.7"], &binary_names[0..2]);
 
@@ -646,7 +662,10 @@ mod tests {
     #[test]
     fn test_interpreter_constraints_binary_names_all_newest_first() {
         let ics = InterpreterConstraints::try_from::<&str>(&[]).unwrap();
-        let binary_names = ics.calculate_compatible_binary_names(SelectionStrategy::Newest);
+        let binary_names = ics
+            .calculate_compatible_binary_names(SelectionStrategy::Newest)
+            .into_keys()
+            .collect::<IndexSet<_>>();
 
         assert!(
             binary_names.get_index_of(os_str("python3.15"))
@@ -705,6 +724,8 @@ mod tests {
                 "pypy",
             ],
             ics.calculate_compatible_binary_names(SelectionStrategy::Newest)
+                .into_keys()
+                .collect::<Vec<_>>()
                 .as_slice()
         );
         assert_eq!(
@@ -722,6 +743,8 @@ mod tests {
                 "python",
             ],
             ics.calculate_compatible_binary_names(SelectionStrategy::Oldest)
+                .into_keys()
+                .collect::<Vec<_>>()
                 .as_slice()
         );
     }

crates/interpreter/src/constraints.rs

@@ -13,7 +13,12 @@ mod platform;
 mod search_path;
 mod tag;
 
-pub use constraints::{InterpreterConstraint, InterpreterConstraints, SelectionStrategy};
+pub use constraints::{
+    InterpreterConstraint,
+    InterpreterConstraints,
+    SelectionStrategy,
+    VersionSpec,
+};
 pub use interpreter::Interpreter;
 pub use platform::Platform;
 pub use search_path::SearchPath;

crates/interpreter/src/lib.rs

@@ -237,9 +237,22 @@ fn prepare_venv<'a>(
         #[cfg(unix)]
         if let Some(sh_boot_seed_dir) = sh_boot_seed_dir {
             fs_err::create_dir_all(&sh_boot_seed_dir)?;
+            let python = venv_interpreter.most_specific_exe_name();
+            // N.B.: This symlink is probed by the --sh-boot script to confirm the venv is still
+            // linked to an existing base Python (no uninstalls or upgrades).
+            platform::unix::symlink(
+                venv_interpreter
+                    .clone()
+                    .resolve_base_interpreter(&mut pex.scripts()?)?
+                    .path,
+                sh_boot_seed_dir.join(format!("base-{python}")),
+                false,
+            )?;
+            // N.B.: This is what the --sh-boot script executes after the probe for venv viability
+            // succeeds.
             platform::unix::symlink(
                 venv_dir.join("pex"),
-                sh_boot_seed_dir.join(venv_interpreter.most_specific_exe_name()),
+                sh_boot_seed_dir.join(format!("pex-{python}")),
                 true,
             )?;
         }

crates/pexrs/src/lib.rs

@@ -57,7 +57,9 @@ fn inject_pex_dir(
 ) -> anyhow::Result<()> {
     // Make sure we have a shebang early. This partially validates the pex to inject is a valid one
     // before expending too much effort copying files below.
-    let shebang = if let Some(sh_boot_shebang) = sh_boot_shebang(pex.path, true)? {
+    let shebang = if let Some(sh_boot_shebang) =
+        sh_boot_shebang(pex.path, pex.info.venv_hermetic_scripts, true)?
+    {
         sh_boot_shebang
     } else {
         let original_main = pex.path.join("__main__.py");
@@ -192,7 +194,9 @@ fn inject_pex_zip(
 ) -> anyhow::Result<()> {
     let zip_read_fp = File::open(pex.path)?;
     let mut src_zip = ZipArchive::new(&zip_read_fp)?;
-    let prefix = if let Some(sh_boot_shebang) = sh_boot_shebang(pex.path, false)? {
+    let prefix = if let Some(sh_boot_shebang) =
+        sh_boot_shebang(pex.path, pex.info.venv_hermetic_scripts, false)?
+    {
         Some(sh_boot_shebang.into_bytes())
     } else {
         let first_entry = src_zip.by_index(0)?;