nightly-health.yml

name: Nightly Health

# Run the full test suite (including tutorials) on the GPU runner every night.
#
# Schedule: 07:00 UTC daily ≈ 02:00 EST / 03:00 EDT
#
# Outputs:
#   - GitHub Actions job summary (visible in the Actions UI after each run)
#   - Artifact "health-dashboard": index.html + status.json (90-day retention)
#   - Artifact "health-test-results": JUnit XML + coverage reports (90-day retention)
#
# The suite runs as two pytest invocations, core and tutorials, so that an
# overrunning tutorial cannot destroy the core suite's results.
#
# The dashboard is NOT deployed to GitHub Pages to avoid overwriting the
# documentation site published by docs.yml.  Use the workflow status badge
# for a live pass/fail indicator in README:
#
#   [![Nightly Health](https://github.com/Project-MONAI/physiotwin4d/actions/workflows/nightly-health.yml/badge.svg)](https://github.com/Project-MONAI/physiotwin4d/actions/workflows/nightly-health.yml)

on:
  schedule:
    - cron: '0 7 * * *'   # 07:00 UTC = ~02:00 EST / 03:00 EDT
  workflow_dispatch:
    inputs:
      reason:
        description: 'Reason for manual trigger'
        required: false
        default: 'Manual health check'
        # Branch selection is handled by the standard GitHub UI branch picker —
        # no free-text ref input, so arbitrary non-default refs cannot be used
        # to produce a misleading public artifact or dashboard.

permissions:
  contents: read

jobs:
  # ──────────────────────────────────────────────────────────────────────────
  # 1. Run the full test suite on the GPU Windows runner
  # ──────────────────────────────────────────────────────────────────────────
  health-tests:
    name: Health Tests (GPU)
    runs-on: [self-hosted, Windows, X64, gpu]
    timeout-minutes: 360

    outputs:
      # Captures the pytest steps' combined outcome (success / failure) for
      # the dashboard. The job itself is failed by the gate step at the end,
      # after the artifacts have been uploaded.
      test-outcome: ${{ (steps.run-core-tests.outcome == 'success' && steps.run-tutorial-tests.outcome == 'success') && 'success' || 'failure' }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v6
        with:
          lfs: true

      - name: Check GPU availability
        run: nvidia-smi

      - name: Check nvcc availability
        # torch-scatter compiles CUDA kernels from source when no matching
        # pre-built wheel exists. Without nvcc on PATH the build silently falls
        # back to a CPU-only extension that only fails later, at test time, with
        # an opaque scatter error. Fail here instead.
        run: |
          $nvcc = Get-Command nvcc -ErrorAction SilentlyContinue
          if (-not $nvcc) {
              Write-Error "nvcc not found on PATH; torch-scatter would build CPU-only"
              exit 1
          }
          Write-Output "nvcc found at $($nvcc.Source)"
          nvcc --version
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

      - name: Create venv in RUNNER_TEMP
        # The newest interpreter the project supports, so this job is the one
        # that catches a dependency shipping no wheel for it. ci.yml runs the
        # floor (3.11) for the opposite reason. Both are on the runner.
        run: |
          & "C:\actions-runner\python\WPy64-3.13.12.0\python\python.exe" -m venv "$env:RUNNER_TEMP\physiotwin4d-venv"
          echo "$env:RUNNER_TEMP\physiotwin4d-venv\Scripts" >> $env:GITHUB_PATH

      - name: Cache uv packages
        uses: actions/cache@v4
        with:
          path: ~\AppData\Local\uv\cache
          key: ${{ runner.os }}-uv-${{ hashFiles('pyproject.toml') }}
          restore-keys: |
            ${{ runner.os }}-uv-

      - name: Cache test data
        uses: actions/cache@v4
        with:
          path: |
            tests/data/
            tests/results/
          key: test-data-${{ hashFiles('tests/test_*.py') }}-v2
          restore-keys: |
            test-data-

      - name: Install uv and package
        # Invoke via python -m uv so uv targets the active venv interpreter.
        # The [physicsnemo] extra is required for --run-physicsnemo (Tutorial 9
        # and any tests marked requires_physicsnemo).
        #
        # torch-scatter compiles against torch and imports it at build time, but
        # declares neither torch nor setuptools as a build dependency. Its build
        # therefore runs with isolation disabled (--no-build-isolation-package,
        # passed explicitly rather than relying on uv discovering
        # no-build-isolation-package in pyproject.toml), which makes the venv
        # itself the build environment: setuptools, wheel, and torch must all be
        # installed before the physicsnemo extra is resolved.
        #
        # torch, torchvision, and torchaudio are all pinned to the pytorch-cu130
        # index here because the uv pip interface ignores tool.uv.sources.
        # Pinning torch alone is not enough: the [cuda13] extra leaves the other
        # two unconstrained, so they would resolve from PyPI and be built
        # against a different CUDA runtime than torch. Installing all three up
        # front means [cuda13] finds them already satisfied. --index-url (not
        # --index) replaces PyPI for this command and matches the flag used in
        # .readthedocs.yaml; the pytorch index also serves the transitive deps.
        #
        # PowerShell does not abort a multi-line run block when a native command
        # exits non-zero, so each install is checked explicitly. Otherwise a
        # failed prerequisite install is masked by the error it causes later.
        run: |
          python -m pip install --upgrade pip uv
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          python -m uv pip install setuptools wheel
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          python -m uv pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu130
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          python -c "import torch; print(f'build-time torch {torch.__version__} (CUDA {torch.version.cuda})')"
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          python -m uv pip install -e ".[cuda13]"
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          python -m uv pip install -e ".[test,docs,cuda13,dev,physicsnemo]" --no-build-isolation-package torch-scatter
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

      - name: Assert CUDA is accessible
        run: |
          python -c "
          import sys, torch, cupy
          print(f'PyTorch {torch.__version__}  |  CUDA toolkit {torch.version.cuda}  |  CuPy {cupy.__version__}')
          if not torch.cuda.is_available():
              print('ERROR: torch.cuda.is_available() returned False', file=sys.stderr)
              sys.exit(1)
          n = torch.cuda.device_count()
          if n == 0:
              print('ERROR: torch.cuda.device_count() == 0', file=sys.stderr)
              sys.exit(1)
          cn = cupy.cuda.runtime.getDeviceCount()
          if cn == 0:
              print('ERROR: cupy.cuda.runtime.getDeviceCount() == 0', file=sys.stderr)
              sys.exit(1)
          print(f'OK: {n} GPU(s) visible to PyTorch and CuPy')
          "

      # The suite runs in two invocations rather than one. pytest-timeout on
      # Windows can only kill the whole process, so a single overrunning
      # tutorial used to take the JUnit XML, the coverage and every other
      # test's result with it, leaving the dashboard nothing to render.
      # Splitting the runs bounds that blast radius to one of them, and the
      # tutorial run additionally isolates each test in an xdist worker.
      #
      # --max-test-seconds fails a test that finishes but took too long, so the
      # overrun is reported with its duration and the rest of the suite still
      # runs. --timeout is the pytest-timeout backstop above it, left to catch
      # a genuine hang and nothing else. Both steps are continue-on-error so
      # the artifacts are always uploaded; the gate step at the end fails the
      # job.
      - name: Run core test suite
        id: run-core-tests
        continue-on-error: true
        run: |
          pytest tests/ --ignore=tests/test_tutorials.py -v `
            --run-all --require-tutorial-data `
            --max-test-seconds=400 `
            --timeout=900 `
            --cov=physiotwin4d `
            --junitxml=test-results-core.xml
        env:
          CUDA_VISIBLE_DEVICES: 0
          # The datasets, the results and the trained networks live on the
          # runner's own disk rather than in the checkout, which
          # actions/checkout wipes every run. Each root has a "test" subtree
          # that this suite reads and writes, so a nightly run never touches a
          # full run's files, and the downsampled subsets the fixtures build
          # under <input>/test survive between runs instead of being rebuilt.
          # Unset, each falls back to its in-repo default; see data/README.md.
          PHYSIOTWIN_INPUT_DATA_DIR: D:\PhysioTwin4D\nightly-runner\data
          PHYSIOTWIN_OUTPUT_DATA_DIR: D:\PhysioTwin4D\nightly-runner\output
          PHYSIOTWIN_WEIGHTS_DIR: D:\PhysioTwin4D\nightly-runner\network_weights

      - name: Run tutorial test suite
        id: run-tutorial-tests
        continue-on-error: true
        # -n 1 keeps the tutorials strictly serial -- they share one GPU, one
        # "test" output subtree, and several bootstrap their prerequisite
        # tutorial inline -- while running them in an xdist worker process.
        # pytest-timeout then kills the worker rather than the session: xdist
        # reports that test as crashed, starts a fresh worker and carries on,
        # so an overrun costs one tutorial instead of the whole run.
        #
        # --cov-append adds to the core run's data, and the reports are written
        # here so that they cover both runs.
        #
        # --timeout sits well above --max-test-seconds on purpose. Killing a
        # worker discards everything the tutorial had printed, which is exactly
        # the record needed to see where its time went, so the backstop must
        # only catch a tutorial that will never finish. An overrun that does
        # finish is reported by --max-test-seconds instead, with its duration
        # and its output intact.
        run: |
          pytest tests/test_tutorials.py -v `
            --run-all --require-tutorial-data `
            -n 1 --max-worker-restart=40 `
            --max-test-seconds=600 `
            --timeout=2400 `
            --cov=physiotwin4d `
            --cov-append `
            --cov-report=xml `
            --cov-report=json `
            --junitxml=test-results-tutorials.xml
        env:
          CUDA_VISIBLE_DEVICES: 0
          # The datasets, the results and the trained networks live on the
          # runner's own disk rather than in the checkout, which
          # actions/checkout wipes every run. Each root has a "test" subtree
          # that this suite reads and writes, so a nightly run never touches a
          # full run's files, and the downsampled subsets the fixtures build
          # under <input>/test survive between runs instead of being rebuilt.
          # Unset, each falls back to its in-repo default; see data/README.md.
          PHYSIOTWIN_INPUT_DATA_DIR: D:\PhysioTwin4D\nightly-runner\data
          PHYSIOTWIN_OUTPUT_DATA_DIR: D:\PhysioTwin4D\nightly-runner\output
          PHYSIOTWIN_WEIGHTS_DIR: D:\PhysioTwin4D\nightly-runner\network_weights

      - name: Upload test results
        uses: actions/upload-artifact@v6
        if: always()
        with:
          name: health-test-results
          path: |
            test-results-core.xml
            test-results-tutorials.xml
            coverage.xml
            coverage.json
          retention-days: 90

      - name: Fail the job if the test suite did not pass
        # The pytest step is continue-on-error so that the artifacts and the
        # dashboard are produced whatever happened. Without this gate the job
        # would report success even when tests failed or the step timed out,
        # and the workflow status badge would say green while nothing passed.
        # build-dashboard runs on if: always(), so it still gets its inputs.
        if: steps.run-core-tests.outcome != 'success' || steps.run-tutorial-tests.outcome != 'success'
        run: |
          Write-Output "Core suite outcome: ${{ steps.run-core-tests.outcome }}"
          Write-Output "Tutorial suite outcome: ${{ steps.run-tutorial-tests.outcome }}"
          Write-Output "See the health-test-results artifact for the JUnit XML."
          exit 1

  # ──────────────────────────────────────────────────────────────────────────
  # 2. Build the HTML dashboard from test results (runs even if tests failed)
  # ──────────────────────────────────────────────────────────────────────────
  build-dashboard:
    name: Build Dashboard
    runs-on: ubuntu-latest
    needs: health-tests
    if: always()
    permissions:
      contents: write

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Download test results
        uses: actions/download-artifact@v6
        with:
          name: health-test-results
          path: results/
        # Artifact may be absent if health-tests was cancelled before upload.
        continue-on-error: true

      - name: Set up Python 3.13
        uses: actions/setup-python@v6
        with:
          python-version: '3.13'

      - name: Build dashboard
        run: |
          python .github/scripts/build_dashboard.py \
            --results-dir results/ \
            --output-dir dashboard/ \
            --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
            --timestamp "$(python -c 'from datetime import datetime, timezone; print(datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))')" \
            --health-outcome "${{ needs.health-tests.outputs.test-outcome }}"

      - name: Write GitHub Actions job summary
        run: cat dashboard/summary.md >> "$GITHUB_STEP_SUMMARY"

      - name: Upload dashboard artifact
        uses: actions/upload-artifact@v6
        with:
          name: health-dashboard
          path: dashboard/
          retention-days: 90

      - name: Push status.json to nightly-status branch
        # Force-push a single-file orphan branch so docs.yml can fetch
        # status.json via the GitHub API and include it in the Pages bundle.
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          cp dashboard/status.json /tmp/status.json
          git checkout --orphan nightly-status-tmp
          git rm -rf . --quiet || true
          cp /tmp/status.json status.json
          git add status.json
          git commit -m "chore: update nightly status [skip ci]"
          git push origin HEAD:nightly-status --force
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}