ENH: address long run-time of GPU tests (#127)

Author: aylward

.github/scripts/build_dashboard.py

@@ -85,6 +85,28 @@ def parse_junit(xml_path: Path) -> dict:
         }
 
 
+def parse_junit_dir(results_dir: Path) -> dict:
+    """Merge every ``test-results*.xml`` in ``results_dir`` into one summary.
+
+    The nightly suite runs as more than one pytest invocation, so that an
+    overrun in one cannot take the other's results with it.  Each writes its
+    own JUnit XML and the dashboard reports their sum.
+    """
+    reports = [
+        parse_junit(path) for path in sorted(results_dir.glob("test-results*.xml"))
+    ]
+    available = [report for report in reports if report["available"]]
+    if not available:
+        return parse_junit(results_dir / "test-results.xml")
+
+    merged = {
+        key: sum(report[key] for report in available)
+        for key in ("tests", "passed", "failed", "errors", "skipped")
+    }
+    merged["available"] = True
+    return merged
+
+
 def parse_coverage(json_path: Path) -> dict:
     """Return coverage percentage from a coverage.py JSON report."""
     if not json_path.exists():
@@ -357,7 +379,7 @@ def main() -> None:
     parser.add_argument(
         "--results-dir",
         default="results/",
-        help="Directory containing test-results.xml and coverage.json",
+        help="Directory containing the test-results*.xml files and coverage.json",
     )
     parser.add_argument(
         "--output-dir",
@@ -384,7 +406,7 @@ def main() -> None:
     )
 
     data = {
-        "junit": parse_junit(results_dir / "test-results.xml"),
+        "junit": parse_junit_dir(results_dir),
         "coverage": parse_coverage(results_dir / "coverage.json"),
         "run_url": args.run_url,
         "timestamp": timestamp,

.github/workflows/nightly-health.yml

@@ -9,6 +9,9 @@ name: Nightly Health
 #   - Artifact "health-dashboard": index.html + status.json (90-day retention)
 #   - Artifact "health-test-results": JUnit XML + coverage reports (90-day retention)
 #
+# The suite runs as two pytest invocations, core and tutorials, so that an
+# overrunning tutorial cannot destroy the core suite's results.
+#
 # The dashboard is NOT deployed to GitHub Pages to avoid overwriting the
 # documentation site published by docs.yml.  Use the workflow status badge
 # for a live pass/fail indicator in README:
@@ -41,10 +44,10 @@ jobs:
     timeout-minutes: 360
 
     outputs:
-      # Captures the pytest step's actual outcome (success / failure / skipped)
-      # for the dashboard. The job itself is failed by the gate step at the end,
+      # Captures the pytest steps' combined outcome (success / failure) for
+      # the dashboard. The job itself is failed by the gate step at the end,
       # after the artifacts have been uploaded.
-      test-outcome: ${{ steps.run-tests.outcome }}
+      test-outcome: ${{ (steps.run-core-tests.outcome == 'success' && steps.run-tutorial-tests.outcome == 'success') && 'success' || 'failure' }}
 
     steps:
       - name: Checkout code
@@ -154,26 +157,65 @@ jobs:
           print(f'OK: {n} GPU(s) visible to PyTorch and CuPy')
           "
 
-      - name: Run health test suite
-        id: run-tests
-        # continue-on-error keeps the job running so artifacts are always uploaded.
-        # The step outcome (success/failure) is still captured and passed downstream.
+      # The suite runs in two invocations rather than one. pytest-timeout on
+      # Windows can only kill the whole process, so a single overrunning
+      # tutorial used to take the JUnit XML, the coverage and every other
+      # test's result with it, leaving the dashboard nothing to render.
+      # Splitting the runs bounds that blast radius to one of them, and the
+      # tutorial run additionally isolates each test in an xdist worker.
+      #
+      # --max-test-seconds fails a test that finishes but took too long, so the
+      # overrun is reported with its duration and the rest of the suite still
+      # runs. --timeout is the pytest-timeout backstop above it, left to catch
+      # a genuine hang and nothing else. Both steps are continue-on-error so
+      # the artifacts are always uploaded; the gate step at the end fails the
+      # job.
+      - name: Run core test suite
+        id: run-core-tests
+        continue-on-error: true
+        run: |
+          pytest tests/ --ignore=tests/test_tutorials.py -v `
+            --run-all --require-tutorial-data `
+            --max-test-seconds=400 `
+            --timeout=900 `
+            --cov=physiotwin4d `
+            --junitxml=test-results-core.xml
+        env:
+          CUDA_VISIBLE_DEVICES: 0
+          # The datasets, the results and the trained networks live on the
+          # runner's own disk rather than in the checkout, which
+          # actions/checkout wipes every run. Each root has a "test" subtree
+          # that this suite reads and writes, so a nightly run never touches a
+          # full run's files, and the downsampled subsets the fixtures build
+          # under <input>/test survive between runs instead of being rebuilt.
+          # Unset, each falls back to its in-repo default; see data/README.md.
+          PHYSIOTWIN_INPUT_DATA_DIR: D:\PhysioTwin4D\nightly-runner\data
+          PHYSIOTWIN_OUTPUT_DATA_DIR: D:\PhysioTwin4D\nightly-runner\output
+          PHYSIOTWIN_WEIGHTS_DIR: D:\PhysioTwin4D\nightly-runner\network_weights
+
+      - name: Run tutorial test suite
+        id: run-tutorial-tests
         continue-on-error: true
-        # --max-test-seconds fails a test that finishes but took too long, so the
-        # overrun is reported with its duration and the rest of the suite still
-        # runs. --timeout raises the pytest-timeout backstop above it: on
-        # Windows that backstop can only kill the whole process, which would
-        # take the JUnit XML and every other result with it, so it is left to
-        # catch a genuine hang and nothing else. Both numbers are provisional
-        # until the per-tutorial timings are measured.
+        # -n 1 keeps the tutorials strictly serial -- they share one GPU, one
+        # "test" output subtree, and several bootstrap their prerequisite
+        # tutorial inline -- while running them in an xdist worker process.
+        # pytest-timeout then kills the worker rather than the session: xdist
+        # reports that test as crashed, starts a fresh worker and carries on,
+        # so an overrun costs one tutorial instead of the whole run.
+        #
+        # --cov-append adds to the core run's data, and the reports are written
+        # here so that they cover both runs.
         run: |
-          pytest tests/ -v --run-all --require-tutorial-data `
-            --max-test-seconds=900 `
-            --timeout=3600 `
+          pytest tests/test_tutorials.py -v `
+            --run-all --require-tutorial-data `
+            -n 1 --max-worker-restart=40 `
+            --max-test-seconds=600 `
+            --timeout=900 `
             --cov=physiotwin4d `
+            --cov-append `
             --cov-report=xml `
             --cov-report=json `
-            --junitxml=test-results.xml
+            --junitxml=test-results-tutorials.xml
         env:
           CUDA_VISIBLE_DEVICES: 0
           # The datasets, the results and the trained networks live on the
@@ -193,7 +235,8 @@ jobs:
         with:
           name: health-test-results
           path: |
-            test-results.xml
+            test-results-core.xml
+            test-results-tutorials.xml
             coverage.xml
             coverage.json
           retention-days: 90
@@ -204,9 +247,10 @@ jobs:
         # would report success even when tests failed or the step timed out,
         # and the workflow status badge would say green while nothing passed.
         # build-dashboard runs on if: always(), so it still gets its inputs.
-        if: steps.run-tests.outcome != 'success'
+        if: steps.run-core-tests.outcome != 'success' || steps.run-tutorial-tests.outcome != 'success'
         run: |
-          Write-Output "Health test suite outcome: ${{ steps.run-tests.outcome }}"
+          Write-Output "Core suite outcome: ${{ steps.run-core-tests.outcome }}"
+          Write-Output "Tutorial suite outcome: ${{ steps.run-tutorial-tests.outcome }}"
           Write-Output "See the health-test-results artifact for the JUnit XML."
           exit 1
 

tutorials/parameters_heart_ct_kcl.py

@@ -46,6 +46,11 @@ class ParametersHeartCTKCL(ParametersBase):
             is resampled to before it is meshed into tetrahedra, which is the
             resulting element size.  Below the thinnest wall of the heart, so
             that the myocardium survives the coarsening.
+        model_points: Points kept per surface when building the shape model.
+            ``0`` keeps every point, which is what a full run does.
+        model_points_test: Same, under ``TestTools.running_as_test``, where the
+            KCL meshes are read at full resolution because that dataset has no
+            downsampled test subset.
         number_of_pca_components: PCA components retained when building the
             heart statistical model, and used when fitting it to a patient.
         number_of_pca_components_test: Same, under ``TestTools.running_as_test``.
@@ -85,6 +90,9 @@ class ParametersHeartCTKCL(ParametersBase):
     surface_reduction_rate: float = 0.5
     mesh_element_size_mm: float = 1.5
 
+    model_points: int = 0
+    model_points_test: int = 20000
+
     number_of_pca_components: int = 10
     number_of_pca_components_test: int = 5
 
@@ -133,6 +141,10 @@ def pca_components(self, test_mode: bool) -> int:
             else (self.number_of_pca_components)
         )
 
+    def points_per_model(self, test_mode: bool) -> int:
+        """Return the per-surface point budget for this run mode."""
+        return self.model_points_test if test_mode else self.model_points
+
     def greedy_iterations(self, test_mode: bool) -> list[int]:
         """Return the Greedy iteration schedule for this run mode."""
         return list(

tutorials/tutorial_06_heart_create_statistical_model.py

@@ -26,6 +26,7 @@
 from parameters_heart_ct_kcl import HEART_CT_KCL
 
 from physiotwin4d import (
+    ContourTools,
     TestTools,
     WorkflowCreateStatisticalModel,
 )
@@ -53,6 +54,11 @@
     data_dir = HEART_CT_KCL.input_directory(test_mode)
     number_of_pca_components = HEART_CT_KCL.pca_components(test_mode)
 
+    # Points kept per surface; 0 keeps every point.  The KCL meshes are the one
+    # dataset with no downsampled test subset, so test mode reduces them here
+    # instead, as tutorial_06_duke_heart_create_statistical_model.py does.
+    model_points = HEART_CT_KCL.points_per_model(test_mode)
+
     # Distance-map weights finetuned by
     # tutorial_02_duke_heart_distancemap_finetune_icon.py.  Stock uniGradICON weights
     # are out of distribution for distance maps, so without these the
@@ -93,21 +99,42 @@
     sample_files = [
         path for path in sample_files if HEART_CT_KCL.hold_out_case not in path.name
     ]
+    if test_mode:
+        sample_files = sample_files[:3]
     if len(sample_files) < 3:
         raise FileNotFoundError(
             f"Need at least 3 sample meshes under {sample_dir} or {data_dir}.\n"
             "See data/README.md for download instructions."
         )
 
-    reference_mesh = cast(pv.DataSet, pv.read(str(reference_file)))
-    sample_meshes = [cast(pv.DataSet, pv.read(str(path))) for path in sample_files]
+    contour_tools = ContourTools(log_level=log_level)
+
+    def read_model_surface(path: Path) -> pv.DataSet:
+        """Read a mesh, reduced to ``model_points`` when a budget is set."""
+        mesh = cast(pv.DataSet, pv.read(str(path)))
+        if not model_points:
+            return mesh
+        surface = contour_tools.extract_surface(mesh)
+        return cast(
+            pv.DataSet,
+            contour_tools.remesh_and_smooth_surface(
+                surface, 1.0 - model_points / surface.n_points, 0
+            ),
+        )
+
+    reference_mesh = read_model_surface(reference_file)
+    sample_meshes = [read_model_surface(path) for path in sample_files]
 
     # Workflow initialization
 
     workflow = WorkflowCreateStatisticalModel(
         sample_meshes=sample_meshes,
         reference_mesh=reference_mesh,
         number_of_pca_components=number_of_pca_components,
+        # The distance maps step 3 registers are rasterized at this resolution,
+        # and generating, dilating and affinely registering them is what the
+        # step costs.  2 mm is an eighth of the voxels of the 1 mm default.
+        reference_spatial_resolution=2.0 if test_mode else 1.0,
         icp_transform_type=HEART_CT_KCL.icp_transform_type,
         mask_dilation_mm=HEART_CT_KCL.mask_dilation_mm,
         distance_squared_max=HEART_CT_KCL.distancemap_squared_max,

tutorials/tutorial_06_lung_create_statistical_model.py

@@ -77,7 +77,7 @@
 
     # Atlas iterations used to build the reference surface; 1 is a single
     # template-biased pass.
-    mean_surface_iterations = 3
+    mean_surface_iterations = 1 if test_mode else 3
 
     # Distance-map weights finetuned by
     # tutorial_02_lung_distancemap_finetune_icon.py.  Stock uniGradICON weights