Fix env cache churn from defaulted exclude-newer and race in concurrent env creation

src/groundhog_hpc/app/init.py

@@ -15,6 +15,7 @@
     get_endpoint_schema_comments,
     parse_endpoint_spec,
 )
+from groundhog_hpc.configuration.models import _default_exclude_newer
 from groundhog_hpc.configuration.pep723 import (
     Pep723Metadata,
     add_endpoint_to_script,
@@ -81,8 +82,10 @@ def init(
     else:
         python = default_meta.requires_python
 
-    assert default_meta.tool and default_meta.tool.uv
-    exclude_newer = default_meta.tool.uv.exclude_newer
+    # exclude-newer is unset by default on the model (so it never churns env
+    # hashes or gets baked in by rewrites); new scripts still get an explicit
+    # timestamp for reproducibility.
+    exclude_newer = _default_exclude_newer()
 
     # Parse endpoint specs if provided
     endpoint_specs = []

src/groundhog_hpc/configuration/models.py

@@ -76,17 +76,18 @@ class UvMetadata(BaseModel, extra="allow", serialize_by_alias=True):
     See uv documentation for full precedence hierarchy.
 
     Attributes:
-        exclude_newer: Limit packages to versions uploaded before cutoff (ISO 8601 timestamp)
+        exclude_newer: Limit packages to versions uploaded before cutoff (ISO 8601
+            timestamp). Defaults to None (unset); when unset, an effective default
+            (the build time) is injected at command-templating time rather than here,
+            so the volatile value never leaks into env hashes or file rewrites.
         python_preference: Control system vs managed Python ("managed" | "only-managed" | "system" | "only-system")
         index_url: Primary package index URL (default: PyPI)
         extra_index_url: Additional package indexes (searched after index_url)
         python_downloads: Control automatic Python downloads ("automatic" | "manual" | "never")
         offline: Disable all network access (use only cache and local files)
     """
 
-    exclude_newer: str | None = Field(
-        default_factory=_default_exclude_newer, alias="exclude-newer"
-    )
+    exclude_newer: str | None = Field(default=None, alias="exclude-newer")
     python_preference: str | None = Field(
         default="only-managed", alias="python-preference"
     )

src/groundhog_hpc/templates/shell_command.sh.jinja

@@ -1,7 +1,13 @@
 set -euo pipefail
 
 TASK_DIR=$(mktemp -d)
-trap 'rm -rf "$TASK_DIR"' EXIT
+# Clean up the task dir plus any in-progress temp build dirs if we exit
+# before publishing (the :+ expansions are no-ops while unset, and after a
+# successful publish the mv has renamed the temp path away, so the trap
+# never touches published state).
+{% raw %}
+trap 'rm -rf "$TASK_DIR" ${{ENV_TMP:+"$ENV_TMP" "$ENV_TMP.uv.toml"}} ${{UV_BOOT_TMP:+"$UV_BOOT_TMP"}}' EXIT
+{% endraw %}
 
 if command -v uv &> /dev/null; then
     UV_BIN=$(command -v uv)
@@ -79,39 +85,67 @@ else
     fi
     {% endraw %}
 
+    # Build into a unique temp dir next to the final location, then rename
+    # into place at the end, so concurrent tasks that share an env hash
+    # never see (or reuse) a half-built environment.
+    # $RANDOM guards against pid collisions across PID namespaces (e.g.
+    # containerized tasks sharing a hostname and bind-mounted scratch).
+    ENV_TMP="$ENV_DIR.tmp.$(hostname).$$.$RANDOM"
+    mkdir -p "$(dirname "$ENV_DIR")"
+
     {% if uv_config_toml %}
 
     # Both `uv venv` and `uv pip install` will receive --config-file, then
-    # the file is moved into $ENV_DIR as a human-readable audit trail.
-    {% raw %}UV_TOML="${{ENV_DIR}}.uv.toml"{% endraw %}
-    mkdir -p "$(dirname "$UV_TOML")"
+    # the file is moved into the environment as a human-readable audit trail.
+    UV_TOML="$ENV_TMP.uv.toml"
 
     cat > "$UV_TOML" << 'UV_CONFIG_EOF'
 {{ uv_config_toml | escape_braces }}
 UV_CONFIG_EOF
     {% endif %}
 
-    "$UV_BIN" venv "$ENV_DIR"{% if requires_python %} --python "{{ requires_python }}"{% endif %}{% if uv_config_toml %} --config-file "$UV_TOML"{% endif %}
+    # --relocatable keeps entry-point shebangs valid after the rename below,
+    # but only exists in uv >= 0.2.31; probe --help so older endpoint uv
+    # installs degrade gracefully instead of hard-failing on an unknown flag.
+    if "$UV_BIN" venv --help 2>/dev/null | grep -q -- --relocatable; then
+        UV_VENV_RELOCATABLE="--relocatable"
+    else
+        UV_VENV_RELOCATABLE=""
+    fi
+
+    # unquoted $UV_VENV_RELOCATABLE is intentional: it must expand to zero
+    # arguments when empty
+    "$UV_BIN" venv $UV_VENV_RELOCATABLE "$ENV_TMP"{% if requires_python %} --python "{{ requires_python }}"{% endif %}{% if uv_config_toml %} --config-file "$UV_TOML"{% endif %}
 
     {% if uv_config_toml %}
-    mv "$UV_TOML" "$ENV_DIR/uv.toml"
+    mv "$UV_TOML" "$ENV_TMP/uv.toml"
     {% endif %}
 
     # Install dependencies
-    "$UV_BIN" pip install --python "$ENV_DIR/bin/python" \
-        {% if uv_config_toml %}--config-file "$ENV_DIR/uv.toml" {% endif %}\
+    "$UV_BIN" pip install --python "$ENV_TMP/bin/python" \
+        {% if uv_config_toml %}--config-file "$ENV_TMP/uv.toml" {% endif %}\
         --exclude-newer-package groundhog-hpc={{ groundhog_timestamp }} \
         {% for dep in dependencies %}"{{ dep }}" {% endfor %}{{ version_spec }}
 
     # Write metadata for posterity
-    cat > "$ENV_DIR/groundhog-meta.json" << 'META_EOF'
+    cat > "$ENV_TMP/groundhog-meta.json" << 'META_EOF'
 {{ '{{' }}
     "created_at": "{{ groundhog_timestamp }}",
     "requires_python": "{{ requires_python }}",
     "dependencies": {{ dependencies | tojson }},
     "groundhog_version": "{{ groundhog_version }}"
 {{ '}}' }}
 META_EOF
+
+    # Publish atomically; if another task published the same env first,
+    # discard our build and use theirs.
+    if [ -d "$ENV_DIR" ] || ! mv "$ENV_TMP" "$ENV_DIR" 2>/dev/null; then
+        rm -rf "$ENV_TMP"
+    elif [ -d "$ENV_DIR/$(basename "$ENV_TMP")" ]; then
+        # mv raced with another publisher and nested our build inside the
+        # winner's env dir; remove the stray copy.
+        rm -rf "$ENV_DIR/$(basename "$ENV_TMP")"
+    fi
 fi
 
 # Run using the cached environment's Python directly (bypasses uv resolution)

src/groundhog_hpc/templating.py

@@ -42,6 +42,12 @@ def compute_env_hash(metadata: Pep723Metadata) -> str:
     a script can have many endpoints and worker_init content is not
     always environment-affecting.
 
+    An unset exclude-newer never affects the hash: the model defaults it to
+    None (excluded from the dump below), and the effective default — the
+    build-time clock — is injected only when templating the shell command.
+    Only an exclude-newer the user actually pinned in the script header
+    affects the hash.
+
     Args:
         metadata: PEP 723 metadata from the user script
 
@@ -134,7 +140,7 @@ def template_shell_command(script_path: str, function_name: str) -> str:
         local_log_level = local_log_level.upper()
         logger.debug(f"Propagating log level to remote: {local_log_level}")
 
-    uv_config_toml = _serialize_uv_toml(metadata)
+    uv_config_toml = _serialize_uv_toml(metadata, groundhog_timestamp)
 
     shell_template = jinja_env.get_template("shell_command.sh.jinja")
     shell_command_string = shell_template.render(
@@ -155,18 +161,25 @@ def template_shell_command(script_path: str, function_name: str) -> str:
     return shell_command_string
 
 
-def _serialize_uv_toml(metadata: Pep723Metadata | None) -> str:
+def _serialize_uv_toml(
+    metadata: Pep723Metadata | None, default_exclude_newer: str
+) -> str:
     """Serialize [tool.uv] settings to uv.toml format for uv pip install.
 
     Returns a TOML string containing all non-None settings from the user's
-    [tool.uv] block, or an empty string if there are no settings.
+    [tool.uv] block, or an empty string if there are no settings (or no
+    metadata at all).
+
+    If the user did not pin exclude-newer in their script, the provided
+    default (the build-time clock) is injected here so fresh environment
+    builds still resolve against a fixed cutoff — without the volatile
+    value ever entering the env hash or the user's script file.
     """
     if not metadata or not metadata.tool or not metadata.tool.uv:
         return ""
 
     uv_dict = metadata.tool.uv.model_dump(by_alias=True, exclude_none=True)
-    if not uv_dict:
-        return ""
+    uv_dict.setdefault("exclude-newer", default_exclude_newer)
 
     return tomlkit.dumps(uv_dict).strip()
 

tests/test_pep723.py

@@ -107,7 +107,9 @@ def test_create_with_defaults(self):
         assert metadata.dependencies == []
         assert metadata.tool is not None
         assert metadata.tool.uv is not None
-        assert metadata.tool.uv.exclude_newer is not None
+        # exclude-newer stays unset by default; the effective default (build
+        # time) is injected at command-templating time, not on the model
+        assert metadata.tool.uv.exclude_newer is None
 
     def test_create_with_explicit_values(self):
         """Test creating metadata with explicit values."""