|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Run numerical validation executables and evaluate convergence criteria.""" |
| 3 | + |
| 4 | +from __future__ import annotations |
| 5 | + |
| 6 | +import argparse |
| 7 | +import datetime as dt |
| 8 | +import json |
| 9 | +import os |
| 10 | +import platform |
| 11 | +import shlex |
| 12 | +import socket |
| 13 | +import subprocess |
| 14 | +import sys |
| 15 | +from pathlib import Path |
| 16 | +from typing import Dict, List, Optional, Tuple |
| 17 | + |
| 18 | + |
| 19 | +VALIDATION_TARGET = "validation_thermal_mass" |
| 20 | + |
| 21 | + |
| 22 | +def run_cmd(cmd: List[str], cwd: Path) -> subprocess.CompletedProcess[str]: |
| 23 | + return subprocess.run( |
| 24 | + cmd, |
| 25 | + cwd=str(cwd), |
| 26 | + text=True, |
| 27 | + capture_output=True, |
| 28 | + check=False, |
| 29 | + ) |
| 30 | + |
| 31 | + |
| 32 | +def cmake_is_multi_config(preset: str, repo_root: Path) -> bool: |
| 33 | + result = run_cmd(["cmake", "--preset", preset, "-N", "-LA"], cwd=repo_root) |
| 34 | + blob = (result.stdout or "") + "\n" + (result.stderr or "") |
| 35 | + return "CMAKE_CONFIGURATION_TYPES" in blob |
| 36 | + |
| 37 | + |
| 38 | +def resolve_build_dir(repo_root: Path, preset: str, override: Optional[Path]) -> Path: |
| 39 | + if override: |
| 40 | + return override.resolve() |
| 41 | + |
| 42 | + special = { |
| 43 | + "ci-linux-release-server": repo_root / "build-server", |
| 44 | + "dev-windows-server": repo_root / "build-windows-server", |
| 45 | + "ci-windows-release-server": repo_root / "build-windows-server-release", |
| 46 | + } |
| 47 | + if preset in special: |
| 48 | + return special[preset] |
| 49 | + return (repo_root / "build" / preset).resolve() |
| 50 | + |
| 51 | + |
| 52 | +def pick_executable(build_dir: Path, target: str, config: Optional[str]) -> Optional[Path]: |
| 53 | + ext = ".exe" if os.name == "nt" else "" |
| 54 | + |
| 55 | + candidates = [ |
| 56 | + build_dir / "tests" / "validation" / f"{target}{ext}", |
| 57 | + build_dir / "tests" / f"{target}{ext}", |
| 58 | + ] |
| 59 | + if config: |
| 60 | + candidates.insert(0, build_dir / "tests" / "validation" / config / f"{target}{ext}") |
| 61 | + candidates.insert(1, build_dir / "tests" / config / f"{target}{ext}") |
| 62 | + |
| 63 | + for path in candidates: |
| 64 | + if path.is_file() and os.access(path, os.X_OK): |
| 65 | + return path |
| 66 | + |
| 67 | + globbed = [ |
| 68 | + p |
| 69 | + for p in build_dir.glob(f"**/{target}{ext}") |
| 70 | + if p.is_file() and os.access(p, os.X_OK) and "CMakeFiles" not in p.parts |
| 71 | + ] |
| 72 | + if not globbed: |
| 73 | + return None |
| 74 | + |
| 75 | + def score(path: Path) -> Tuple[int, int, int]: |
| 76 | + in_validation = 1 if "validation" in path.parts else 0 |
| 77 | + in_tests = 1 if "tests" in path.parts else 0 |
| 78 | + in_config = 1 if config and config in path.parts else 0 |
| 79 | + shorter = -len(str(path)) |
| 80 | + return (in_validation + in_tests, in_config, shorter) |
| 81 | + |
| 82 | + globbed.sort(key=score, reverse=True) |
| 83 | + return globbed[0] |
| 84 | + |
| 85 | + |
| 86 | +def ensure_dir(path: Path) -> None: |
| 87 | + path.mkdir(parents=True, exist_ok=True) |
| 88 | + |
| 89 | + |
| 90 | +def default_output_dir(repo_root: Path, preset: str) -> Path: |
| 91 | + ts = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") |
| 92 | + return repo_root / "artifacts" / "validation" / f"{ts}_{preset}" |
| 93 | + |
| 94 | + |
| 95 | +def git_metadata(repo_root: Path) -> Dict[str, object]: |
| 96 | + commit = run_cmd(["git", "rev-parse", "HEAD"], cwd=repo_root) |
| 97 | + short = run_cmd(["git", "rev-parse", "--short", "HEAD"], cwd=repo_root) |
| 98 | + dirty = run_cmd(["git", "status", "--porcelain"], cwd=repo_root) |
| 99 | + return { |
| 100 | + "commit": (commit.stdout or "").strip(), |
| 101 | + "commit_short": (short.stdout or "").strip(), |
| 102 | + "dirty": bool((dirty.stdout or "").strip()), |
| 103 | + } |
| 104 | + |
| 105 | + |
| 106 | +def as_number(value: object) -> Optional[float]: |
| 107 | + if isinstance(value, bool): |
| 108 | + return None |
| 109 | + if isinstance(value, (int, float)): |
| 110 | + return float(value) |
| 111 | + return None |
| 112 | + |
| 113 | + |
| 114 | +def extract_min_orders(results_doc: Dict[str, object]) -> Tuple[float, float]: |
| 115 | + summary_obj = results_doc.get("summary") |
| 116 | + if not isinstance(summary_obj, dict): |
| 117 | + raise ValueError("Validation results missing summary object") |
| 118 | + |
| 119 | + min_orders_obj = summary_obj.get("min_observed_order_linf") |
| 120 | + if not isinstance(min_orders_obj, dict): |
| 121 | + raise ValueError("Validation results missing summary.min_observed_order_linf") |
| 122 | + |
| 123 | + euler_raw = min_orders_obj.get("forward_euler") |
| 124 | + rk4_raw = min_orders_obj.get("rk4") |
| 125 | + euler = as_number(euler_raw) |
| 126 | + rk4 = as_number(rk4_raw) |
| 127 | + if euler is None or rk4 is None: |
| 128 | + raise ValueError("Validation results contain non-numeric order values") |
| 129 | + return (euler, rk4) |
| 130 | + |
| 131 | + |
| 132 | +def main() -> int: |
| 133 | + parser = argparse.ArgumentParser(description="Run numerical validation and evaluate orders") |
| 134 | + parser.add_argument("--preset", default=("dev-windows-release" if os.name == "nt" else "dev-release")) |
| 135 | + parser.add_argument("--config", default=None, help="Build configuration for multi-config generators") |
| 136 | + parser.add_argument("--build-dir", default=None, help="Override build directory") |
| 137 | + parser.add_argument("--output-dir", default=None, help="Artifact output directory") |
| 138 | + parser.add_argument("--no-build", action="store_true", help="Skip configure/build and run existing executable") |
| 139 | + parser.add_argument("--duration-s", type=float, default=10.0, help="Simulation duration in seconds") |
| 140 | + parser.add_argument( |
| 141 | + "--dt-values", |
| 142 | + default="0.4,0.2,0.1,0.05,0.025", |
| 143 | + help="Comma-separated dt values used by validation executable", |
| 144 | + ) |
| 145 | + parser.add_argument("--min-order-forward-euler", type=float, default=0.9) |
| 146 | + parser.add_argument("--min-order-rk4", type=float, default=3.5) |
| 147 | + parser.add_argument( |
| 148 | + "--enforce-order", |
| 149 | + action="store_true", |
| 150 | + help="Return non-zero when observed order is below threshold", |
| 151 | + ) |
| 152 | + args = parser.parse_args() |
| 153 | + |
| 154 | + repo_root = Path(__file__).resolve().parent.parent |
| 155 | + build_dir = resolve_build_dir(repo_root, args.preset, Path(args.build_dir) if args.build_dir else None) |
| 156 | + output_dir = Path(args.output_dir).resolve() if args.output_dir else default_output_dir(repo_root, args.preset) |
| 157 | + ensure_dir(output_dir) |
| 158 | + |
| 159 | + multi_config = cmake_is_multi_config(args.preset, repo_root) |
| 160 | + config = args.config |
| 161 | + if multi_config and not config: |
| 162 | + config = "Release" |
| 163 | + |
| 164 | + configure_cmd = ["cmake", "--preset", args.preset] |
| 165 | + build_cmd = ["cmake", "--build", "--preset", args.preset] |
| 166 | + if config: |
| 167 | + build_cmd.extend(["--config", config]) |
| 168 | + build_cmd.extend(["--target", VALIDATION_TARGET]) |
| 169 | + |
| 170 | + if not args.no_build: |
| 171 | + cfg = run_cmd(configure_cmd, cwd=repo_root) |
| 172 | + (output_dir / "configure.stdout.log").write_text(cfg.stdout or "", encoding="utf-8") |
| 173 | + (output_dir / "configure.stderr.log").write_text(cfg.stderr or "", encoding="utf-8") |
| 174 | + if cfg.returncode != 0: |
| 175 | + print("Configure failed. See configure logs in", output_dir) |
| 176 | + return cfg.returncode |
| 177 | + |
| 178 | + bld = run_cmd(build_cmd, cwd=repo_root) |
| 179 | + (output_dir / "build.stdout.log").write_text(bld.stdout or "", encoding="utf-8") |
| 180 | + (output_dir / "build.stderr.log").write_text(bld.stderr or "", encoding="utf-8") |
| 181 | + if bld.returncode != 0: |
| 182 | + print("Build failed. See build logs in", output_dir) |
| 183 | + return bld.returncode |
| 184 | + |
| 185 | + exe = pick_executable(build_dir, VALIDATION_TARGET, config) |
| 186 | + if exe is None: |
| 187 | + print(f"Validation executable '{VALIDATION_TARGET}' was not found in {build_dir}") |
| 188 | + return 2 |
| 189 | + |
| 190 | + results_json_path = output_dir / "validation_results.json" |
| 191 | + results_csv_path = output_dir / "validation_results.csv" |
| 192 | + cmd = [ |
| 193 | + str(exe), |
| 194 | + "--output-json", |
| 195 | + str(results_json_path), |
| 196 | + "--output-csv", |
| 197 | + str(results_csv_path), |
| 198 | + "--duration-s", |
| 199 | + str(args.duration_s), |
| 200 | + "--dt-values", |
| 201 | + args.dt_values, |
| 202 | + ] |
| 203 | + run = run_cmd(cmd, cwd=repo_root) |
| 204 | + (output_dir / "validation.stdout.log").write_text(run.stdout or "", encoding="utf-8") |
| 205 | + (output_dir / "validation.stderr.log").write_text(run.stderr or "", encoding="utf-8") |
| 206 | + if run.returncode != 0: |
| 207 | + print("Validation executable failed. See validation logs in", output_dir) |
| 208 | + return run.returncode |
| 209 | + |
| 210 | + raw_doc = json.loads(results_json_path.read_text(encoding="utf-8")) |
| 211 | + if not isinstance(raw_doc, dict): |
| 212 | + raise ValueError("Validation results JSON must be an object") |
| 213 | + observed_euler, observed_rk4 = extract_min_orders(raw_doc) |
| 214 | + |
| 215 | + euler_ok = observed_euler >= args.min_order_forward_euler |
| 216 | + rk4_ok = observed_rk4 >= args.min_order_rk4 |
| 217 | + overall_ok = euler_ok and rk4_ok |
| 218 | + |
| 219 | + evaluation = { |
| 220 | + "thresholds": { |
| 221 | + "forward_euler_min_order_linf": args.min_order_forward_euler, |
| 222 | + "rk4_min_order_linf": args.min_order_rk4, |
| 223 | + "enforce_order": args.enforce_order, |
| 224 | + }, |
| 225 | + "observed": { |
| 226 | + "forward_euler_min_order_linf": observed_euler, |
| 227 | + "rk4_min_order_linf": observed_rk4, |
| 228 | + }, |
| 229 | + "checks": { |
| 230 | + "forward_euler": euler_ok, |
| 231 | + "rk4": rk4_ok, |
| 232 | + "overall": overall_ok, |
| 233 | + }, |
| 234 | + "run": { |
| 235 | + "preset": args.preset, |
| 236 | + "config": config, |
| 237 | + "multi_config": multi_config, |
| 238 | + "build_dir": str(build_dir), |
| 239 | + "output_dir": str(output_dir), |
| 240 | + "platform": platform.platform(), |
| 241 | + "python": platform.python_version(), |
| 242 | + "hostname": socket.gethostname(), |
| 243 | + "git": git_metadata(repo_root), |
| 244 | + "command": " ".join(shlex.quote(part) for part in cmd), |
| 245 | + }, |
| 246 | + } |
| 247 | + (output_dir / "validation_evaluation.json").write_text( |
| 248 | + json.dumps(evaluation, indent=2) + "\n", |
| 249 | + encoding="utf-8", |
| 250 | + ) |
| 251 | + |
| 252 | + print(f"Validation artifacts written to: {output_dir}") |
| 253 | + print(f"Results manifest: {results_json_path}") |
| 254 | + print(f"Observed min order (forward_euler, linf): {observed_euler:.6f}") |
| 255 | + print(f"Observed min order (rk4, linf): {observed_rk4:.6f}") |
| 256 | + |
| 257 | + if args.enforce_order and not overall_ok: |
| 258 | + print("Convergence order thresholds failed.") |
| 259 | + return 1 |
| 260 | + |
| 261 | + if not overall_ok: |
| 262 | + print("Convergence thresholds not met (non-fatal without --enforce-order).") |
| 263 | + else: |
| 264 | + print("Convergence thresholds satisfied.") |
| 265 | + return 0 |
| 266 | + |
| 267 | + |
| 268 | +if __name__ == "__main__": |
| 269 | + sys.exit(main()) |
0 commit comments