Skip to content

Commit 36a1419

Browse files
banesullivanbanesullivan
authored
Add merge-queue-gated PyVista parity suite (#112)
1 parent c0af2ba commit 36a1419

6 files changed

Lines changed: 470 additions & 0 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@ on:
6767
# environment). See the `publish` job at the bottom.
6868
release:
6969
types: [published]
70+
# The merge queue runs the full gate — including the heavy `pyvista` job — on
71+
# the queued commit BEFORE the squash lands, so a red run blocks the merge.
72+
# Jobs gated `if: github.event_name != 'pull_request'` (build-lto + *-lto) also
73+
# run here, so the merge queue validates the SHIPPED config too.
74+
merge_group:
7075
workflow_dispatch:
7176

7277
concurrency:
@@ -312,6 +317,87 @@ jobs:
312317
path: /tmp/rx-out
313318
if-no-files-found: ignore
314319

320+
# Cheap detector so a PR that BUMPS the pin (tests/pyvista/PYVISTA_REF) runs the
321+
# PyVista gate without needing the `pyvista-full` label — the pin must never
322+
# advance unvalidated. Plain git diff (no third-party action). Only meaningful
323+
# on pull_request; merge_group / dispatch satisfy the `pyvista` if on their own.
324+
changes:
325+
name: detect PyVista pin change
326+
runs-on: ${{ vars.FVTK_RUNNER_LINUX_SMALL || 'ubuntu-latest' }}
327+
outputs:
328+
pyvista_ref: ${{ steps.f.outputs.changed }}
329+
steps:
330+
- uses: actions/checkout@v5
331+
with:
332+
fetch-depth: 0 # need the PR base commit in history for the diff
333+
- id: f
334+
env:
335+
EVENT: ${{ github.event_name }}
336+
BASE: ${{ github.event.pull_request.base.sha }}
337+
HEAD: ${{ github.event.pull_request.head.sha }}
338+
run: |
339+
changed=false
340+
if [ "$EVENT" = "pull_request" ] && \
341+
git diff --name-only "$BASE" "$HEAD" | grep -qx 'tests/pyvista/PYVISTA_REF'; then
342+
changed=true
343+
fi
344+
echo "PYVISTA_REF changed on this PR: $changed"
345+
echo "changed=$changed" >> "$GITHUB_OUTPUT"
346+
347+
pyvista:
348+
name: PyVista suite (built fvtk wheel, py3.13)
349+
needs: [build, changes]
350+
# PyVista's OWN full test suite driving fvtk (vtkmodules->fvtk shim) — the
351+
# downstream contract gate, sister to bitexact/renderexact. HEAVY, so it is
352+
# kept OFF the PR fast path: it runs in the merge queue (the blocking gate
353+
# before a squash lands), on manual dispatch, and on any PR that either carries
354+
# the `pyvista-full` label OR bumps the pin (tests/pyvista/PYVISTA_REF — so the
355+
# pin can never advance unvalidated). Uses the FAST O2 `build` wheel, NOT
356+
# build-lto: optimization level is parity-invariant (proven maxULP=0 by
357+
# bitexact + bitexact-lto), so PyVista returns the same pass/fail on either
358+
# wheel — and the O2 wheel is already built, so this never waits on the slow
359+
# LTO link. Mark this check REQUIRED for the merge queue in branch protection
360+
# so a red run blocks the merge.
361+
if: >-
362+
github.event_name == 'merge_group' ||
363+
github.event_name == 'workflow_dispatch' ||
364+
(github.event_name == 'pull_request' &&
365+
(contains(github.event.pull_request.labels.*.name, 'pyvista-full') ||
366+
needs.changes.outputs.pyvista_ref == 'true'))
367+
runs-on: ${{ vars.FVTK_RUNNER_LINUX_SMALL || 'ubuntu-latest' }}
368+
steps:
369+
- uses: actions/checkout@v5
370+
with:
371+
fetch-depth: 0
372+
- uses: actions/setup-python@v5
373+
with:
374+
python-version: "3.13"
375+
- name: Install Xvfb + Mesa software GL (llvmpipe)
376+
# Xvfb gives PyVista a real DISPLAY (its plotting suite warns + segfault-
377+
# guards without one, under filterwarnings=error); Mesa llvmpipe does the
378+
# software GLX rendering. Mirrors PyVista's own CI render environment.
379+
run: |
380+
sudo apt-get update
381+
sudo apt-get install -y --no-install-recommends \
382+
xvfb libgl1-mesa-dri libglx-mesa0 libegl1 libgles2 libosmesa6 libxcb1
383+
- uses: actions/download-artifact@v4
384+
with:
385+
# The FAST O2 Linux wheel (fvtk-wheels-linux-*); pip --find-links picks
386+
# the cp312-abi3 wheel (tag-compatible on 3.13). macOS/Windows artifacts
387+
# are a disjoint prefix and never reach this Linux gate.
388+
pattern: fvtk-wheels-linux-*
389+
merge-multiple: true
390+
path: dist
391+
- name: PyVista suite (pinned ref, vtkmodules->fvtk shim)
392+
run: ci/run-pyvista.sh dist python
393+
- name: Upload pytest logs + junit
394+
if: always()
395+
uses: actions/upload-artifact@v4
396+
with:
397+
name: pyvista-output
398+
path: /tmp/pv-out
399+
if-no-files-found: ignore
400+
315401
# === LTO-on-main legs (push-to-main / workflow_dispatch ONLY) ==============
316402
# The PR gate above builds the FAST Linux wheel (FVTK_LTO=0 + -O2) so PRs stay
317403
# cheap — but that means the configuration we actually SHIP (the tag-gated

‎.github/workflows/pyvista-bump.yml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# Self-bumping pin for the PyVista parity gate.
2+
#
3+
# Weekly, resolve PyVista's current `main` HEAD and, if it differs from the
4+
# pinned SHA in tests/pyvista/PYVISTA_REF, open a PR bumping it with the
5+
# `pyvista-full` label. That label runs the full PyVista suite on the PR (and the
6+
# merge queue runs it on enqueue regardless), so the pin only advances through a
7+
# gate that proves fvtk still matches that PyVista revision.
8+
#
9+
# NOTE: the default GITHUB_TOKEN cannot trigger CI from a bot-opened PR. Set repo
10+
# secret PYVISTA_BUMP_TOKEN (a PAT with contents:write + pull-requests:write) to
11+
# get the suite running on the bump PR's own checks. Without it the merge queue
12+
# still validates the bump before it lands.
13+
name: Bump pinned PyVista
14+
15+
on:
16+
schedule:
17+
- cron: "0 6 * * 1" # Mondays 06:00 UTC
18+
workflow_dispatch:
19+
20+
permissions:
21+
contents: write
22+
pull-requests: write
23+
24+
concurrency:
25+
group: pyvista-bump
26+
cancel-in-progress: true
27+
28+
jobs:
29+
bump:
30+
runs-on: ${{ vars.FVTK_RUNNER_LINUX_SMALL || 'ubuntu-latest' }}
31+
steps:
32+
- uses: actions/checkout@v5
33+
34+
- name: Resolve PyVista main HEAD
35+
id: head
36+
run: |
37+
sha=$(git ls-remote https://github.com/pyvista/pyvista refs/heads/main | cut -f1)
38+
[ -n "$sha" ] || { echo "::error::could not resolve pyvista main HEAD"; exit 1; }
39+
echo "sha=$sha" >> "$GITHUB_OUTPUT"
40+
echo "short=${sha:0:12}" >> "$GITHUB_OUTPUT"
41+
42+
- name: Update PYVISTA_REF if changed
43+
id: bump
44+
run: |
45+
cur=$(tr -d '[:space:]' < tests/pyvista/PYVISTA_REF)
46+
new="${{ steps.head.outputs.sha }}"
47+
if [ "$cur" = "$new" ]; then
48+
echo "pin already at $new — nothing to do"
49+
echo "changed=false" >> "$GITHUB_OUTPUT"
50+
else
51+
printf '%s\n' "$new" > tests/pyvista/PYVISTA_REF
52+
echo "bumped $cur -> $new"
53+
echo "changed=true" >> "$GITHUB_OUTPUT"
54+
fi
55+
56+
- name: Ensure pyvista-full label exists
57+
if: steps.bump.outputs.changed == 'true'
58+
env:
59+
GH_TOKEN: ${{ github.token }}
60+
run: |
61+
gh label create pyvista-full \
62+
--color 1d76db \
63+
--description "Run the full PyVista test suite on this PR" \
64+
--force
65+
66+
- name: Open / update bump PR
67+
if: steps.bump.outputs.changed == 'true'
68+
uses: peter-evans/create-pull-request@v7
69+
with:
70+
# A PAT (PYVISTA_BUMP_TOKEN) makes the PR able to trigger the `pyvista`
71+
# job's checks; falls back to GITHUB_TOKEN (merge queue still gates).
72+
token: ${{ secrets.PYVISTA_BUMP_TOKEN || github.token }}
73+
branch: chore/pyvista-bump
74+
base: main
75+
delete-branch: true
76+
labels: pyvista-full
77+
commit-message: "test(pyvista): bump pinned PyVista to ${{ steps.head.outputs.short }}"
78+
title: "test(pyvista): bump pinned PyVista to ${{ steps.head.outputs.short }}"
79+
body: |
80+
Automated bump of the PyVista revision pinned by the PyVista parity gate.
81+
82+
`tests/pyvista/PYVISTA_REF` → `${{ steps.head.outputs.sha }}`
83+
84+
The `pyvista-full` label runs the full PyVista suite against the built
85+
fvtk wheel on this PR, so the bump is validated before it lands. If the
86+
suite goes red, an upstream change broke parity — fix fvtk, or triage
87+
into `tests/pyvista/deselect.txt` with a one-line WHY.

‎ci/run-pyvista.sh‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
#!/usr/bin/env bash
2+
#
3+
# PyVista parity gate: run PyVista's OWN full test suite against the built fvtk
4+
# wheel, with `vtkmodules.*` redirected to `fvtk.*`. This proves the contract
5+
# downstream actually depends on — not just our internal bit-exact / pixel-exact
6+
# gates, but PyVista's thousands of behavioral + plotting assertions driving the
7+
# fvtk graphics + data stack.
8+
#
9+
# Sister to ci/run-bitexact.sh / ci/run-renderexact.sh, but instead of our own
10+
# vtkmodules-only scenes it installs PyVista (pinned to tests/pyvista/PYVISTA_REF)
11+
# and runs its suite. ONE venv: stock `vtk` is NEVER installed, so any import
12+
# that slips past the fvtk shim fails loud (ModuleNotFoundError) rather than
13+
# silently testing stock VTK — never a false green.
14+
#
15+
# Heavy by design (full suite, software GL). NOT on the PR fast path — see the
16+
# `pyvista` job in .github/workflows/ci.yml (merge queue / dispatch / labeled PR).
17+
#
18+
# Usage: ci/run-pyvista.sh <wheel-dir> [base-python] (base-python default python3)
19+
set -euxo pipefail
20+
21+
WHEELDIR="${1:?usage: ci/run-pyvista.sh <wheel-dir> [base-python]}"
22+
BASE_PY="${2:-python3}"
23+
SRC="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
24+
HERE="$SRC/tests/pyvista"
25+
26+
REF="$(tr -d '[:space:]' < "$HERE/PYVISTA_REF")"
27+
[ -n "$REF" ] || { echo "ERROR: tests/pyvista/PYVISTA_REF is empty"; exit 1; }
28+
29+
OUT="${PYVISTA_OUTDIR:-/tmp/pv-out}"
30+
rm -rf "$OUT"; mkdir -p "$OUT"
31+
32+
# Headless software GL via Xvfb — the SAME path PyVista's own CI uses to render
33+
# (and to generate its committed image_cache): an Xvfb virtual display + Mesa
34+
# llvmpipe over GLX. We deliberately do NOT force surfaceless EGL here (unlike
35+
# run-renderexact.sh, which drives raw vtkmodules): PyVista warns + segfault-
36+
# guards when no DISPLAY is present, runs filterwarnings=error, and its image
37+
# cache is GLX/llvmpipe — so matching that environment is what keeps the suite
38+
# green. We wrap each pytest invocation in xvfb-run below; here we only pin
39+
# software rendering and keep every Plotter offscreen.
40+
export LIBGL_ALWAYS_SOFTWARE=1
41+
export GALLIUM_DRIVER=llvmpipe
42+
export PYVISTA_OFF_SCREEN=true
43+
# Provides a real DISPLAY (silences _warn_xserver) + a fresh server per call.
44+
XVFB=(xvfb-run -a -s "-screen 0 1280x1024x24")
45+
46+
# --- fetch PyVista at the pinned SHA (shallow, single commit) ----------------
47+
# Pinning (not floating main) keeps the gate deterministic; .github/workflows/
48+
# pyvista-bump.yml advances the pin through a self-validating PR. The committed
49+
# image_cache that PyVista's plotting tests diff against rides along with the
50+
# checkout.
51+
PVDIR="${PYVISTA_SRC:-/tmp/pyvista-src}"
52+
if [ ! -e "$PVDIR/pyproject.toml" ]; then
53+
rm -rf "$PVDIR"; mkdir -p "$PVDIR"
54+
git -C "$PVDIR" init -q
55+
git -C "$PVDIR" remote add origin https://github.com/pyvista/pyvista
56+
git -C "$PVDIR" fetch -q --depth 1 origin "$REF"
57+
git -C "$PVDIR" checkout -q FETCH_HEAD
58+
fi
59+
echo ">>> PyVista @ $(git -C "$PVDIR" rev-parse --short HEAD) (pinned $REF)"
60+
61+
# --- venv: PyVista (no deps) + test group + fvtk wheel + redirect shim -------
62+
"$BASE_PY" -m venv /tmp/pv
63+
/tmp/pv/bin/pip -q install --upgrade pip
64+
# Non-editable: the clone dir is named `pyvista`, which an editable install
65+
# mistakes for a namespace package. --no-deps so PyVista does NOT drag stock
66+
# `vtk` into the venv (the shim must be the ONLY vtkmodules provider).
67+
/tmp/pv/bin/pip -q install --no-deps "$PVDIR"
68+
# PyVista's `test` dependency-group (pytest, pytest-xdist, pytest-pyvista, ...).
69+
# pytest-timeout is NOT in that group, so add it explicitly for --timeout.
70+
/tmp/pv/bin/pip -q install --group "$PVDIR/pyproject.toml:test" pytest-timeout
71+
# Built fvtk wheel. --find-links resolves `fvtk` from the local dir while PyPI
72+
# stays available for fvtk's own deps (matplotlib/numpy/...) — so NO --no-index.
73+
/tmp/pv/bin/pip -q install --find-links "$WHEELDIR" fvtk
74+
# vtkmodules.* -> fvtk.* redirect, active at interpreter startup via sitecustomize
75+
# (.pth). Same shim the bit-exact / PGO harnesses use.
76+
SP=$(/tmp/pv/bin/python -c 'import sysconfig;print(sysconfig.get_paths()["purelib"])')
77+
cp "$SRC/tools/fvtk_shim.py" "$SP/_fvtk_shim.py"
78+
echo "import _fvtk_shim" > "$SP/_fvtk_shim.pth"
79+
# Put the venv's bin on PATH: we invoke python by absolute path (no `activate`),
80+
# so without this the CLI tests that shell out to the bare `pyvista`/`pytest`
81+
# console-scripts get FileNotFoundError.
82+
export PATH="/tmp/pv/bin:$PATH"
83+
84+
# --- identity: prove PyVista is driving fvtk through the shim ----------------
85+
/tmp/pv/bin/python - <<'PY'
86+
import vtkmodules
87+
assert "fvtk" in (vtkmodules.__file__ or ""), \
88+
f"vtkmodules NOT redirected to fvtk (got {vtkmodules.__file__!r}) -- shim failed"
89+
import fvtk, pyvista
90+
from vtkmodules.vtkCommonCore import vtkVersion
91+
print("vtkmodules ->", vtkmodules.__file__)
92+
print("fvtk ->", fvtk.__file__)
93+
print("VTK version->", vtkVersion.GetVTKVersion())
94+
print("pyvista ->", pyvista.__version__, pyvista.__file__)
95+
PY
96+
97+
# --- deselect list -----------------------------------------------------------
98+
# Static triage entries from tests/pyvista/deselect.txt (one nodeid per line).
99+
DESELECT=()
100+
while IFS= read -r raw || [ -n "$raw" ]; do
101+
line="${raw%%#*}" # strip comment
102+
line="${line#"${line%%[![:space:]]*}"}" # ltrim
103+
line="${line%"${line##*[![:space:]]}"}" # rtrim
104+
[ -n "$line" ] && DESELECT+=(--deselect "$line")
105+
done < "$HERE/deselect.txt"
106+
107+
# snake_case opt-out: if this fvtk wheel was built with VTK_DISABLE_PYTHON_
108+
# PROPERTIES, VTK emits no `obj.snake_case` descriptors and PyVista's snake_case
109+
# tests fail BY DESIGN, not by regression. Probe at runtime and deselect only
110+
# then, so the gate still exercises them against a wheel built WITH properties.
111+
SNAKE_DISABLED="$(/tmp/pv/bin/python -c 'import vtkmodules.vtkCommonCore as m; print("0" if hasattr(m.vtkObject(), "global_warning_display") else "1")')"
112+
if [ "$SNAKE_DISABLED" = "1" ]; then
113+
echo ">>> snake_case API disabled in this wheel: deselecting PyVista snake_case tests"
114+
DESELECT+=(
115+
--deselect "tests/test_attributes.py::test_vtk_snake_case_api_is_disabled"
116+
--deselect "tests/test_attributes.py::test_dir_snake_case_visible_when_allowed"
117+
--deselect "tests/core/test_utilities.py::test_vtk_snake_case"
118+
--deselect "tests/core/test_utilities.py::test_is_vtk_attribute"
119+
# Same root cause: with no snake_case property descriptors, PyVista cannot
120+
# tell which CamelCase names are VTK-inherited, so its dir() filtering
121+
# (hide-inherited / opt-in-show) can't engage. Fails by design, not by
122+
# regression — bare nodeids deselect every parametrization.
123+
--deselect "tests/test_attributes.py::test_dir_hides_vtk_inherited_attributes"
124+
--deselect "tests/test_attributes.py::test_dir_show_vtk_api_opt_in"
125+
)
126+
else
127+
echo ">>> snake_case API present in this wheel: running PyVista snake_case tests"
128+
fi
129+
130+
cd "$PVDIR"
131+
# Two SEPARATE pytest invocations matching PyVista's tox layout (test-core,
132+
# test-plotting). One xdist run over the whole tree cross-pollutes core fixtures
133+
# into plotting workers and triggers spurious check_gc teardown errors.
134+
# -n auto : full xdist parallelism within each env
135+
# -m "not needs_download" : skip network-flaky tests (array element, not a
136+
# string — a quoted string splits into broken tokens)
137+
# --timeout=120 : hang guard per test
138+
SHARED=(-n auto --tb=short --no-header -ra --color=no -m "not needs_download" --timeout=120)
139+
140+
set +e
141+
echo "=== core tests ==="
142+
"${XVFB[@]}" /tmp/pv/bin/python -m pytest "${SHARED[@]}" "${DESELECT[@]+"${DESELECT[@]}"}" \
143+
--ignore=tests/plotting \
144+
--ignore=tests/typing \
145+
--test_downloads \
146+
--junitxml="$OUT/junit-core.xml" \
147+
tests/ 2>&1 | tee "$OUT/pytest-core.log"
148+
STATUS_CORE=${PIPESTATUS[0]}
149+
150+
echo "=== plotting tests ==="
151+
"${XVFB[@]}" /tmp/pv/bin/python -m pytest "${SHARED[@]}" "${DESELECT[@]+"${DESELECT[@]}"}" \
152+
--disallow_unused_cache \
153+
--junitxml="$OUT/junit-plotting.xml" \
154+
tests/plotting 2>&1 | tee "$OUT/pytest-plotting.log"
155+
STATUS_PLOT=${PIPESTATUS[0]}
156+
set -e
157+
158+
echo ">>> logs: $OUT/pytest-{core,plotting}.log"
159+
echo ">>> junit: $OUT/junit-{core,plotting}.xml"
160+
exit $(( STATUS_CORE | STATUS_PLOT ))

‎tests/pyvista/PYVISTA_REF‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
e8d6f05987fedf085f4629be428b78e84fc59d65

0 commit comments

Comments
 (0)