update piplock-renewal to skip pr it changes are only timestamps

Author: mtchoum1

.github/workflows/piplock-renewal.yaml

@@ -95,13 +95,33 @@ jobs:
           ./uv run manifests/tools/generate_kustomization.py
 
       - name: Refresh ImageStream notebook dependency annotations from pylocks
+        env:
+          # Used to resolve abbreviated SHAs → full OID before ``git fetch`` (GitHub API); avoids rate limits.
+          GITHUB_TOKEN: ${{ github.token }}
         run: |
           ./uv run python manifests/tools/update_imagestream_annotations_from_pylock.py --variant odh
           ./uv run python manifests/tools/update_imagestream_annotations_from_pylock.py --variant rhoai
 
       - name: Validate manifests (make test)
         run: make test
 
+      # ``uv pip compile`` rewrites the header ``--exclude-newer=…`` timestamp every run.
+      # When the solver output is unchanged, drop the working tree so we do not open a
+      # multi-file PR that only churns metadata (and avoids rebase conflicts on real PRs).
+      - name: Discard ephemeral-only lockfile renewal diff
+        run: |
+          set -euo pipefail
+          if git diff --quiet HEAD; then
+            echo "No changes vs HEAD."
+            exit 0
+          fi
+          if ./uv run python scripts/ci/lockfile_renewal_diff_is_metadata_only.py; then
+            echo "Only ephemeral lockfile metadata changed; discarding working tree changes."
+            git restore --source=HEAD --staged --worktree .
+            exit 0
+          fi
+          echo "Substantive changes present; proceeding to PR creation."
+
       - name: Create Pull Request
         env:
           GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}

manifests/tools/update_imagestream_annotations_from_pylock.py

@@ -7,10 +7,16 @@
   ``pylock.toml`` pins. If no file exists on disk, the SHA from ``commit-latest.env`` is used with
   ``git show`` (fetching from the canonical repo if needed): ``https://github.com/opendatahub-io/notebooks.git``
   (``--variant odh``) or ``https://github.com/red-hat-data-services/notebooks.git`` (``--variant rhoai``).
-- Older tags (e.g. ``-2025-2``): SHA from ``commit.env`` (``<base>-commit-2025-2``).
+- Older tags (e.g. ``-2025-2``): SHA from ``commit.env``. Those commits may exist only on the canonical
+  upstream (especially RHDS for ``--variant rhoai``); the script runs ``git fetch`` before ``git show``
+  when the object is not already in the clone (fork CI often lacks RHDS-only SHAs until fetched).
 
 Those SHAs match ``manifests/tools/generate_kustomization.py`` / ConfigMap keys.
 
+Abbreviated SHAs (from ``commit*.env``) are expanded to full 40-character OIDs via the GitHub REST API
+when the canonical remote is ``github.com`` — ``git fetch … <short>`` treats short hex as a ref name and
+fails with ``couldn't find remote ref``. Set ``GITHUB_TOKEN`` / ``GH_TOKEN`` for API auth (rate limits).
+
 Dependency *names* and ordering are taken from the existing manifest; versions are updated from
 the resolved lockfile at each ref (same translation rules as ``tests/test_main.py``). Older commits may only have ``requirements.txt`` or flavor files such as ``requirements.cpu.txt``
 instead of ``pylock.toml`` / ``uv.lock.d/pylock.*.toml``; those are parsed as pinned PEP 508 requirements.
@@ -31,10 +37,14 @@
 import dataclasses
 import json
 import logging
+import os
 import re
+import shlex
 import subprocess
 import sys
 import tomllib
+import urllib.error
+import urllib.request
 from pathlib import Path
 from typing import Any
 
@@ -84,7 +94,7 @@
     }
 )
 
-# Canonical Git URLs for ``git fetch`` when the ``-n`` tag commit is not already in the local object DB.
+# Canonical Git URLs for ``git fetch`` when a pinned commit is not already in the local object DB.
 _CANONICAL_REPO_URL: dict[str, str] = {
     "odh": "https://github.com/opendatahub-io/notebooks.git",
     "rhoai": "https://github.com/red-hat-data-services/notebooks.git",
@@ -94,6 +104,64 @@
 # Repo env files use short SHAs (7+ chars); full 40-char hashes are also accepted.
 _GIT_HEX_OBJECT_ID = re.compile(r"^[0-9a-f]{7,40}$")
 
+_GITHUB_REPO_RE = re.compile(
+    r"github\.com[:/](?P<owner>[^/]+)/(?P<repo>[^/?.#]+)(?:\.git)?(?:/|$)",
+    re.IGNORECASE,
+)
+
+
+def _parse_github_owner_repo(git_url: str) -> tuple[str, str] | None:
+    m = _GITHUB_REPO_RE.search(git_url)
+    if not m:
+        return None
+    return m.group("owner"), m.group("repo")
+
+
+def _resolve_github_commit_full_sha(git_url: str, rev: str) -> str | None:
+    """Resolve abbreviated git SHA to full 40-char lowercase via GitHub REST API.
+
+    ``git fetch https://…/repo.git <short>`` treats ``<short>`` as a *ref name*, which yields
+    ``couldn't find remote ref``. Passing the full OID fixes fetch against github.com.
+
+    Uses ``GITHUB_TOKEN`` / ``GH_TOKEN`` when set (CI rate limits and private forks).
+    """
+    rev_clean = rev.strip().lower()
+    if not _GIT_HEX_OBJECT_ID.fullmatch(rev_clean):
+        return None
+    if len(rev_clean) == 40:
+        return rev_clean
+    parsed = _parse_github_owner_repo(git_url)
+    if parsed is None:
+        return None
+    owner, repo = parsed
+    api = f"https://api.github.com/repos/{owner}/{repo}/commits/{rev_clean}"
+    token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "User-Agent": "opendatahub-notebooks-manifest-tools",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    if token:
+        headers["Authorization"] = f"Bearer {token.strip()}"
+    req = urllib.request.Request(api, headers=headers)
+    try:
+        with urllib.request.urlopen(req, timeout=120) as resp:
+            body = json.loads(resp.read().decode())
+        sha = body.get("sha")
+        if isinstance(sha, str):
+            sha = sha.lower()
+            if len(sha) == 40 and sha.startswith(rev_clean):
+                return sha
+    except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, json.JSONDecodeError, KeyError):
+        return None
+    return None
+
+
+def _rev_for_git_fetch(git_url: str, rev: str) -> str:
+    """Return a revision string suitable for ``git fetch <url> <rev>`` (full OID when possible)."""
+    expanded = _resolve_github_commit_full_sha(git_url, rev)
+    return expanded if expanded is not None else rev.strip()
+
 
 def _is_under_jupyter_rocm_tree(directory: Path) -> bool:
     parts = directory.parts
@@ -345,22 +413,31 @@ def _git_commit_exists(rev: str) -> bool:
     return p.returncode == 0
 
 
-def _git_fetch_commit_from(url: str, rev: str) -> bool:
-    p = subprocess.run(
-        ["git", "-C", str(ROOT), "fetch", "--quiet", "--no-tags", url, rev],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    return p.returncode == 0
-
-
-def _ensure_n_tag_commit_from_canonical_upstream(variant: str, rev: str) -> bool:
-    """Ensure ``rev`` is available for ``git show`` using the ODH or RHDS canonical ``.git`` URL for ``-n`` tags."""
+def _git_fetch_commit_from(url: str, rev: str) -> tuple[bool, str]:
+    """Fetch ``rev`` from ``url`` into the local object database. Returns (ok, detail on failure)."""
+    fetch_rev = _rev_for_git_fetch(url, rev)
+    cmd = ["git", "-C", str(ROOT), "fetch", "--quiet", "--no-tags", url, fetch_rev]
+    p = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    if p.returncode == 0:
+        return True, ""
+    err = (p.stderr or "").strip()
+    if not err and (p.stdout or "").strip():
+        err = (p.stdout or "").strip()
+    if not err:
+        err = f"exit code {p.returncode}"
+    quoted = " ".join(shlex.quote(c) for c in cmd)
+    return False, f"{err}\n  (command: {quoted})"
+
+
+def _ensure_commit_from_canonical_upstream(variant: str, rev: str) -> tuple[bool, str | None]:
+    """Ensure ``rev`` is available for ``git show`` via ``git fetch`` from the variant canonical repo."""
     url = _CANONICAL_REPO_URL[variant]
     if _git_commit_exists(rev):
-        return True
-    return _git_fetch_commit_from(url, rev)
+        return True, None
+    ok, detail = _git_fetch_commit_from(url, rev)
+    if ok:
+        return True, None
+    return False, detail
 
 
 def _git_show_text(rev: str, rel_path: str) -> str | None:
@@ -622,6 +699,8 @@ def run_variant(variant: str, dry_run: bool) -> int:
             if idx >= len(tags):
                 break
             sha = _sha_for_tag(base_key, suffix, latest, released)
+            sha_desc = sha if sha else "<missing>"
+            print(f"check {path.name} tag {idx}: {base_key}{suffix} (commit={sha_desc})", file=sys.stderr)
             nb_dir = resolve_notebook_directory(candidates, base_key)
             if nb_dir is None:
                 want = notebook_dirname_from_base_key(base_key)
@@ -639,19 +718,36 @@ def run_variant(variant: str, dry_run: bool) -> int:
                 shown = _worktree_read_first_existing(rel_paths)
             else:
                 shown = None
+            if shown is not None:
+                rel_used, _text = shown
+                print(
+                    f"ok {path.name} tag {idx}: using worktree lockfile {rel_used}",
+                    file=sys.stderr,
+                )
 
             if shown is None:
                 if not sha:
                     print(f"skip {path.name} tag {idx}: no SHA for {base_key}{suffix}", file=sys.stderr)
                     continue
-                if suffix == "-n" and not _ensure_n_tag_commit_from_canonical_upstream(variant, sha):
+                # Released tags (e.g. ``-2025-2``) use ``commit.env`` SHAs that may live only on RHDS/ODH;
+                # a fresh checkout (e.g. GitHub Actions) does not contain them until we fetch.
+                ok_upstream, fetch_err = _ensure_commit_from_canonical_upstream(variant, sha)
+                if not ok_upstream:
+                    url = _CANONICAL_REPO_URL[variant]
                     print(
-                        f"skip {path.name} tag {idx}: could not resolve commit {sha} via "
-                        f"{_CANONICAL_REPO_URL[variant]}",
+                        f"skip {path.name} tag {idx}: git fetch failed for commit {sha} from {url}\n"
+                        f"  {fetch_err}",
                         file=sys.stderr,
                     )
                     continue
+                print(f"ok {path.name} tag {idx}: commit {sha} is available locally", file=sys.stderr)
                 shown = _git_show_first_existing(sha, rel_paths)
+                if shown is not None:
+                    rel_used, _text = shown
+                    print(
+                        f"ok {path.name} tag {idx}: got lockfile {rel_used} from {sha}",
+                        file=sys.stderr,
+                    )
             if shown is None:
                 print(
                     f"skip {path.name} tag {idx}: no lockfile (tried worktree/git {'; '.join(rel_paths)})",

scripts/ci/lockfile_renewal_diff_is_metadata_only.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""Return whether ``git diff HEAD`` is only ephemeral UV / pylock metadata.
+
+``make refresh-lock-files`` rewrites the UV header comment on every run with a
+fresh ``--exclude-newer=…`` timestamp.  When the solver output is otherwise
+unchanged, the renewal workflow would open a noisy PR touching many files.
+
+Exit codes (for use from bash ``if ./uv run python …; then``):
+
+- ``0`` — every changed file differs from ``HEAD`` only after normalizing
+  ephemeral fields (skip opening a PR; caller may ``git restore``).
+- ``1`` — at least one changed file has substantive edits, or git/diff error.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+
+# UV embeds ``--exclude-newer`` in the second-line ``uv pip compile`` header comment.
+_EXCLUDE_NEWER_RE = re.compile(r"--exclude-newer(?:=\s*|\s+)(\S+)")
+# Future / alternate pylock headers (e.g. PEP 751 style metadata).
+_CREATED_AT_RE = re.compile(
+    r"^created-at\s*=\s*(?:\"[^\"]*\"|'[^']*'|[^\s#]+)\s*$",
+    flags=re.MULTILINE,
+)
+
+_PLACEHOLDER = "<ephemeral-lockfile-metadata>"
+
+
+def normalize_ephemeral_lockfile_text(content: str) -> str:
+    """Strip volatile timestamps from lockfile text for equality checks."""
+    s = _EXCLUDE_NEWER_RE.sub(f"--exclude-newer={_PLACEHOLDER}", content)
+    s = _CREATED_AT_RE.sub(f'created-at = "{_PLACEHOLDER}"', s)
+    return s
+
+
+def _git_output(args: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["git", *args],
+        cwd=cwd,
+        check=False,
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+    )
+
+
+def _changed_paths_vs_head(cwd: Path) -> list[str] | None:
+    p = _git_output(["diff", "--name-only", "-z", "HEAD"], cwd=cwd)
+    if p.returncode != 0:
+        return None
+    names = [n for n in p.stdout.split("\0") if n]
+    return names
+
+
+def _is_binary_change(cwd: Path, path: str) -> bool:
+    p = _git_output(["diff", "--numstat", "HEAD", "--", path], cwd=cwd)
+    if p.returncode != 0 or not p.stdout.strip():
+        return False
+    first = p.stdout.splitlines()[0].split("\t", 1)[0]
+    return first == "-"
+
+
+def _blob_at_head(cwd: Path, path: str) -> bytes | None:
+    p = subprocess.run(
+        ["git", "show", f"HEAD:{path}"],
+        cwd=cwd,
+        check=False,
+        capture_output=True,
+    )
+    if p.returncode != 0:
+        return None
+    return p.stdout
+
+
+def diff_vs_head_is_metadata_only(*, repo_root: Path) -> bool | None:
+    """``True`` if every worktree change vs ``HEAD`` is metadata-only, ``False`` if substantive.
+
+    Returns ``None`` if git failed or a path could not be read.
+    """
+    paths = _changed_paths_vs_head(repo_root)
+    if paths is None:
+        return None
+    if not paths:
+        return True
+
+    for path in paths:
+        if _is_binary_change(repo_root, path):
+            return False
+        head_bytes = _blob_at_head(repo_root, path)
+        if head_bytes is None:
+            return False
+        work_path = repo_root / path
+        if not work_path.is_file():
+            return False
+        try:
+            head_text = head_bytes.decode("utf-8")
+            work_text = work_path.read_text(encoding="utf-8")
+        except UnicodeDecodeError:
+            return False
+
+        if normalize_ephemeral_lockfile_text(head_text) != normalize_ephemeral_lockfile_text(work_text):
+            return False
+    return True
+
+
+def main() -> int:
+    result = diff_vs_head_is_metadata_only(repo_root=ROOT)
+    if result is None:
+        print("lockfile_renewal_diff_is_metadata_only: git diff failed", file=sys.stderr)
+        return 1
+    if result:
+        print("lockfile_renewal_diff_is_metadata_only: only ephemeral metadata changed")
+        return 0
+    print("lockfile_renewal_diff_is_metadata_only: substantive changes detected")
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())