Nightly Health #162

Workflow file for this run

.github/workflows/nightly-health.yml at 955d2f8

	name: Nightly Health

	# Run the full test suite (including tutorials) on the GPU runner every night.
	#
	# Schedule: 07:00 UTC daily ≈ 02:00 EST / 03:00 EDT
	#
	# Outputs:
	# - GitHub Actions job summary (visible in the Actions UI after each run)
	# - Artifact "health-dashboard": index.html + status.json (90-day retention)
	# - Artifact "health-test-results": JUnit XML + coverage reports (90-day retention)
	#
	# The suite runs as two pytest invocations, core and tutorials, so that an
	# overrunning tutorial cannot destroy the core suite's results.
	#
	# The dashboard is NOT deployed to GitHub Pages to avoid overwriting the
	# documentation site published by docs.yml. Use the workflow status badge
	# for a live pass/fail indicator in README:
	#
	# [![Nightly Health](https://github.com/Project-MONAI/monai-physio/actions/workflows/nightly-health.yml/badge.svg)](https://github.com/Project-MONAI/monai-physio/actions/workflows/nightly-health.yml)

	on:
	schedule:
	- cron: '0 7 * * *' # 07:00 UTC = ~02:00 EST / 03:00 EDT
	workflow_dispatch:
	inputs:
	reason:
	description: 'Reason for manual trigger'
	required: false
	default: 'Manual health check'
	# Branch selection is handled by the standard GitHub UI branch picker -
	# no free-text ref input, so arbitrary non-default refs cannot be used
	# to produce a misleading public artifact or dashboard.

	permissions:
	contents: read

	jobs:
	# ──────────────────────────────────────────────────────────────────────────
	# 1. Run the full test suite on the GPU Windows runner
	# ──────────────────────────────────────────────────────────────────────────
	health-tests:
	name: Health Tests (GPU)
	runs-on: [self-hosted, Windows, X64, gpu]
	timeout-minutes: 360

	outputs:
	# Captures the pytest steps' combined outcome (success / failure) for
	# the dashboard. The job itself is failed by the gate step at the end,
	# after the artifacts have been uploaded.
	test-outcome: ${{ (steps.run-core-tests.outcome == 'success' && steps.run-tutorial-tests.outcome == 'success') && 'success' \|\| 'failure' }}

	steps:
	- name: Checkout code
	uses: actions/checkout@v6
	with:
	lfs: true

	- name: Check GPU availability
	run: nvidia-smi

	- name: Check nvcc availability
	# torch-scatter compiles CUDA kernels from source when no matching
	# pre-built wheel exists. Without nvcc on PATH the build silently falls
	# back to a CPU-only extension that only fails later, at test time, with
	# an opaque scatter error. Fail here instead.
	run: \|
	$nvcc = Get-Command nvcc -ErrorAction SilentlyContinue
	if (-not $nvcc) {
	Write-Error "nvcc not found on PATH; torch-scatter would build CPU-only"
	exit 1
	}
	Write-Output "nvcc found at $($nvcc.Source)"
	nvcc --version
	if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

	- name: Create venv in RUNNER_TEMP
	# The newest interpreter the project supports, so this job is the one
	# that catches a dependency shipping no wheel for it. ci.yml runs the
	# floor (3.11) for the opposite reason. Both are on the runner.
	run: \|
	& "C:\actions-runner\python\WPy64-3.13.12.0\python\python.exe" -m venv "$env:RUNNER_TEMP\monai-physio-venv"
	echo "$env:RUNNER_TEMP\monai-physio-venv\Scripts" >> $env:GITHUB_PATH

	- name: Cache uv packages
	uses: actions/cache@v4
	with:
	path: ~\AppData\Local\uv\cache
	key: ${{ runner.os }}-uv-${{ hashFiles('pyproject.toml') }}
	restore-keys: \|
	${{ runner.os }}-uv-

	- name: Cache test data
	uses: actions/cache@v4
	with:
	path: \|
	tests/data/
	tests/results/
	key: test-data-${{ hashFiles('tests/test_*.py') }}-v2
	restore-keys: \|
	test-data-

	- name: Install uv and package
	# Invoke via python -m uv so uv targets the active venv interpreter.
	# nvidia-physicsnemo/torch-geometric/torch-scatter are base
	# dependencies (installed unconditionally), so --run-physicsnemo
	# (Tutorial 9 and any tests marked requires_physicsnemo) works without
	# a separate extra.
	#
	# torch-scatter compiles against torch and imports it at build time, but
	# declares neither torch nor setuptools as a build dependency. Its build
	# therefore runs with isolation disabled (--no-build-isolation-package,
	# passed explicitly rather than relying on uv discovering
	# no-build-isolation-package in pyproject.toml), which makes the venv
	# itself the build environment: setuptools, wheel, and torch must all be
	# installed before ANY monai-physio extra is resolved, since
	# torch-scatter is now a base dependency and is pulled in immediately.
	#
	# torch, torchvision, and torchaudio are all pinned to the pytorch-cu130
	# index here because the uv pip interface ignores tool.uv.sources.
	# Pinning torch alone is not enough: the [cuda13] extra leaves the other
	# two unconstrained, so they would resolve from PyPI and be built
	# against a different CUDA runtime than torch. Installing all three up
	# front means [cuda13] finds them already satisfied. --index-url (not
	# --index) replaces PyPI for this command and matches the flag used in
	# .readthedocs.yaml; the pytorch index also serves the transitive deps.
	#
	# PowerShell does not abort a multi-line run block when a native command
	# exits non-zero, so each install is checked explicitly. Otherwise a
	# failed prerequisite install is masked by the error it causes later.
	run: \|
	python -m pip install --upgrade pip uv
	if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
	python -m uv pip install setuptools wheel
	if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
	python -m uv pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu130
	if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
	python -c "import torch; print(f'build-time torch {torch.__version__} (CUDA {torch.version.cuda})')"
	if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
	python -m uv pip install -e ".[dev_cuda13]" --no-build-isolation-package torch-scatter
	if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

	- name: Assert CUDA is accessible
	run: \|
	python -c "
	import sys, torch, cupy
	print(f'PyTorch {torch.__version__} \| CUDA toolkit {torch.version.cuda} \| CuPy {cupy.__version__}')
	if not torch.cuda.is_available():
	print('ERROR: torch.cuda.is_available() returned False', file=sys.stderr)
	sys.exit(1)
	n = torch.cuda.device_count()
	if n == 0:
	print('ERROR: torch.cuda.device_count() == 0', file=sys.stderr)
	sys.exit(1)
	cn = cupy.cuda.runtime.getDeviceCount()
	if cn == 0:
	print('ERROR: cupy.cuda.runtime.getDeviceCount() == 0', file=sys.stderr)
	sys.exit(1)
	print(f'OK: {n} GPU(s) visible to PyTorch and CuPy')
	"

	# The suite runs in two invocations rather than one. pytest-timeout on
	# Windows can only kill the whole process, so a single overrunning
	# tutorial used to take the JUnit XML, the coverage and every other
	# test's result with it, leaving the dashboard nothing to render.
	# Splitting the runs bounds that blast radius to one of them, and the
	# tutorial run additionally isolates each test in an xdist worker.
	#
	# --max-test-seconds fails a test that finishes but took too long, so the
	# overrun is reported with its duration and the rest of the suite still
	# runs. --timeout is the pytest-timeout backstop above it, left to catch
	# a genuine hang and nothing else. Both steps are continue-on-error so
	# the artifacts are always uploaded; the gate step at the end fails the
	# job.
	- name: Run core test suite
	id: run-core-tests
	continue-on-error: true
	run: \|
	pytest tests/ --ignore=tests/test_tutorials.py -v `
	--run-all --require-tutorial-data `
	--max-test-seconds=400 `
	--timeout=900 `
	--cov=monai_physio `
	--junitxml=test-results-core.xml
	env:
	CUDA_VISIBLE_DEVICES: 0
	# The datasets, the results and the trained networks live on the
	# runner's own disk rather than in the checkout, which
	# actions/checkout wipes every run. Each root has a "test" subtree
	# that this suite reads and writes, so a nightly run never touches a
	# full run's files, and the downsampled subsets the fixtures build
	# under <input>/test survive between runs instead of being rebuilt.
	# Unset, each falls back to its in-repo default; see data/README.md.
	MONAI_PHYSIO_INPUT_DATA_DIR: D:\MONAI-Physio\nightly-runner\data
	MONAI_PHYSIO_OUTPUT_DATA_DIR: D:\MONAI-Physio\nightly-runner\output
	MONAI_PHYSIO_WEIGHTS_DIR: D:\MONAI-Physio\nightly-runner\network_weights

	- name: Run tutorial test suite
	id: run-tutorial-tests
	continue-on-error: true
	# -n 1 keeps the tutorials strictly serial -- they share one GPU, one
	# "test" output subtree, and several bootstrap their prerequisite
	# tutorial inline -- while running them in an xdist worker process.
	# pytest-timeout then kills the worker rather than the session: xdist
	# reports that test as crashed, starts a fresh worker and carries on,
	# so an overrun costs one tutorial instead of the whole run.
	#
	# --cov-append adds to the core run's data, and the reports are written
	# here so that they cover both runs.
	#
	# --timeout sits well above --max-test-seconds on purpose. Killing a
	# worker discards everything the tutorial had printed, which is exactly
	# the record needed to see where its time went, so the backstop must
	# only catch a tutorial that will never finish. An overrun that does
	# finish is reported by --max-test-seconds instead, with its duration
	# and its output intact.
	run: \|
	pytest tests/test_tutorials.py -v `
	--run-all --require-tutorial-data `
	-n 1 --max-worker-restart=40 `
	--max-test-seconds=600 `
	--timeout=2400 `
	--cov=monai_physio `
	--cov-append `
	--cov-report=xml `
	--cov-report=json `
	--junitxml=test-results-tutorials.xml
	env:
	CUDA_VISIBLE_DEVICES: 0
	# The datasets, the results and the trained networks live on the
	# runner's own disk rather than in the checkout, which
	# actions/checkout wipes every run. Each root has a "test" subtree
	# that this suite reads and writes, so a nightly run never touches a
	# full run's files, and the downsampled subsets the fixtures build
	# under <input>/test survive between runs instead of being rebuilt.
	# Unset, each falls back to its in-repo default; see data/README.md.
	MONAI_PHYSIO_INPUT_DATA_DIR: D:\MONAI-Physio\nightly-runner\data
	MONAI_PHYSIO_OUTPUT_DATA_DIR: D:\MONAI-Physio\nightly-runner\output
	MONAI_PHYSIO_WEIGHTS_DIR: D:\MONAI-Physio\nightly-runner\network_weights

	- name: Upload test results
	uses: actions/upload-artifact@v6
	if: always()
	with:
	name: health-test-results
	path: \|
	test-results-core.xml
	test-results-tutorials.xml
	coverage.xml
	coverage.json
	retention-days: 90

	- name: Fail the job if the test suite did not pass
	# The pytest step is continue-on-error so that the artifacts and the
	# dashboard are produced whatever happened. Without this gate the job
	# would report success even when tests failed or the step timed out,
	# and the workflow status badge would say green while nothing passed.
	# build-dashboard runs on if: always(), so it still gets its inputs.
	if: steps.run-core-tests.outcome != 'success' \|\| steps.run-tutorial-tests.outcome != 'success'
	run: \|
	Write-Output "Core suite outcome: ${{ steps.run-core-tests.outcome }}"
	Write-Output "Tutorial suite outcome: ${{ steps.run-tutorial-tests.outcome }}"
	Write-Output "See the health-test-results artifact for the JUnit XML."
	exit 1

	# ──────────────────────────────────────────────────────────────────────────
	# 2. Build the HTML dashboard from test results (runs even if tests failed)
	# ──────────────────────────────────────────────────────────────────────────
	build-dashboard:
	name: Build Dashboard
	runs-on: ubuntu-latest
	needs: health-tests
	if: always()
	permissions:
	contents: write

	steps:
	- name: Checkout code
	uses: actions/checkout@v6

	- name: Download test results
	uses: actions/download-artifact@v6
	with:
	name: health-test-results
	path: results/
	# Artifact may be absent if health-tests was cancelled before upload.
	continue-on-error: true

	- name: Set up Python 3.13
	uses: actions/setup-python@v6
	with:
	python-version: '3.13'

	- name: Build dashboard
	run: \|
	python .github/scripts/build_dashboard.py \
	--results-dir results/ \
	--output-dir dashboard/ \
	--run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
	--timestamp "$(python -c 'from datetime import datetime, timezone; print(datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))')" \
	--health-outcome "${{ needs.health-tests.outputs.test-outcome }}"

	- name: Write GitHub Actions job summary
	run: cat dashboard/summary.md >> "$GITHUB_STEP_SUMMARY"

	- name: Upload dashboard artifact
	uses: actions/upload-artifact@v6
	with:
	name: health-dashboard
	path: dashboard/
	retention-days: 90

	- name: Push status.json to nightly-status branch
	# Force-push a single-file orphan branch so docs.yml can fetch
	# status.json via the GitHub API and include it in the Pages bundle.
	run: \|
	git config user.name "github-actions[bot]"
	git config user.email "github-actions[bot]@users.noreply.github.com"
	cp dashboard/status.json /tmp/status.json
	git checkout --orphan nightly-status-tmp
	git rm -rf . --quiet \|\| true
	cp /tmp/status.json status.json
	git add status.json
	git commit -m "chore: update nightly status [skip ci]"
	git push origin HEAD:nightly-status --force
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Navigation Menu

Uh oh!

Nightly Health #162

Workflow file

Nightly Health #162

Uh oh!

Workflow file for this run