Add performance testing scripts for DTypeFront

scripts/python/dtypefront_perf_table.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+"""Generate a DTypeFront performance table from benchmark logs."""
+
+from __future__ import annotations
+
+import argparse
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+
+METHOD_ORDER = {
+    "ROS2S": 0,
+    "Rodas3P": 1,
+    "Rodas4P": 2,
+    "VODE": 3,
+    "Rodas5P": 4,
+}
+
+
+@dataclass
+class Metrics:
+    method: str
+    fom_us: float | None = None
+    mupdates: float | None = None
+    tp_wall_s: float | None = None
+    chem_s: float | None = None
+    chem_pct: float | None = None
+    adv_s: float | None = None
+    adv_pct: float | None = None
+    hydro_s: float | None = None
+    hydro_pct: float | None = None
+    subcycles: float | None = None
+    failed: str | None = None
+
+    @property
+    def rad_no_ode_pct(self) -> float | None:
+        if self.adv_pct is None or self.hydro_pct is None or self.chem_pct is None:
+            return None
+        return self.adv_pct - self.hydro_pct - self.chem_pct
+
+    @property
+    def hydro_only_speedup(self) -> float | None:
+        if self.hydro_pct is None or self.hydro_pct == 0.0:
+            return None
+        return 100.0 / self.hydro_pct
+
+    @property
+    def rad_substep_cost(self) -> float | None:
+        if self.adv_s is None or self.hydro_s is None or self.subcycles is None or self.hydro_s == 0.0:
+            return None
+        return (self.adv_s - self.hydro_s) / (self.hydro_s * self.subcycles)
+
+
+def discover_logs(paths: list[Path]) -> list[Path]:
+    logs: list[Path] = []
+    for path in paths:
+        if path.is_dir():
+            logs.extend(sorted(path.glob("*.log")))
+        else:
+            logs.append(path)
+    return logs
+
+
+def method_from_log(text: str, path: Path) -> str:
+    match = re.search(r"DTypeFront microphysics integrator: VODE", text)
+    if match:
+        return "VODE"
+
+    match = re.search(r"DTypeFront microphysics integrator: Rosenbrock \(Rosenbrock tableau \d+: ([^)]+)\)", text)
+    if match:
+        return match.group(1)
+
+    name_map = {
+        "ros2s": "ROS2S",
+        "rodas3p": "Rodas3P",
+        "rodas4p": "Rodas4P",
+        "rodas5p": "Rodas5P",
+        "vode": "VODE",
+    }
+    return name_map.get(path.stem.lower(), path.stem)
+
+
+def parse_profiler_line(text: str, name: str) -> tuple[float, float] | None:
+    # Example:
+    # PhotoChemistry::computePhotoChemistry() 7734 6.474 6.474 6.474 33.65%
+    pattern = re.compile(rf"^\s*{re.escape(name)}\s+\d+\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)%", re.MULTILINE)
+    matches = [(float(match.group(3)), float(match.group(4))) for match in pattern.finditer(text)]
+    if not matches:
+        return None
+    # The same profiler name can appear as exclusive and inclusive entries.  The
+    # inclusive entry is the one with the largest walltime/percentage.
+    return max(matches, key=lambda value: value[1])
+
+
+def parse_log(path: Path) -> Metrics:
+    text = path.read_text(errors="replace")
+    metrics = Metrics(method=method_from_log(text, path))
+
+    match = re.search(r"Performance figure-of-merit:\s+([0-9.eE+-]+)\s+.*?\[([0-9.eE+-]+)\s+Mupdates/s\]", text)
+    if match:
+        metrics.fom_us = float(match.group(1))
+        metrics.mupdates = float(match.group(2))
+
+    match = re.search(r"TinyProfiler total time across processes \[min\.\.\.avg\.\.\.max\]:\s+([0-9.eE+-]+)\s+\.\.\.\s+([0-9.eE+-]+)\s+\.\.\.\s+([0-9.eE+-]+)", text)
+    if match:
+        metrics.tp_wall_s = float(match.group(3))
+
+    match = re.search(r"avg\. num\. of radiation subcycles\s*=\s*([0-9.eE+-]+)", text)
+    if match:
+        metrics.subcycles = float(match.group(1))
+
+    chem = parse_profiler_line(text, "PhotoChemistry::computePhotoChemistry()")
+    if chem is not None:
+        metrics.chem_s, metrics.chem_pct = chem
+
+    advance = parse_profiler_line(text, "QuokkaSimulation::advanceSingleTimestepAtLevel()")
+    if advance is not None:
+        metrics.adv_s, metrics.adv_pct = advance
+
+    hydro = parse_profiler_line(text, "REG::HydroSolver")
+    if hydro is not None:
+        metrics.hydro_s, metrics.hydro_pct = hydro
+
+    match = re.search(r"Photochemistry burn failed.*", text)
+    if match:
+        metrics.failed = match.group(0)
+
+    return metrics
+
+
+def fmt(value: float | None, precision: int = 2) -> str:
+    if value is None:
+        return "n/a"
+    return f"{value:.{precision}f}"
+
+
+def fmt_fom(value: float | None) -> str:
+    if value is None:
+        return "n/a"
+    return f"{value:.4f}"
+
+
+def table_cells(metrics: list[Metrics]) -> list[list[str]]:
+    rows = sorted(metrics, key=lambda row: (METHOD_ORDER.get(row.method, 99), row.method))
+    return [
+        [
+            row.method,
+            fmt_fom(row.fom_us),
+            fmt(row.mupdates, 2),
+            fmt(row.tp_wall_s, 2),
+            fmt(row.chem_s, 2),
+            fmt(row.chem_pct, 2),
+            fmt(row.rad_no_ode_pct, 2),
+            fmt(row.hydro_pct, 2),
+            fmt(row.hydro_only_speedup, 2),
+            fmt(row.rad_substep_cost, 2),
+            fmt(row.subcycles, 2),
+        ]
+        for row in rows
+    ]
+
+
+def make_markdown_table(metrics: list[Metrics]) -> str:
+    lines = [
+        "| Method | FoM us/zone | Mupdate/s | TP wall s | Chem s | Chem % | Rad-noODE % | Hydro % | Hydro-only x | RadSub/HydroStep | Subcyc |",
+        "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
+    ]
+    lines.extend("| " + " | ".join(row) + " |" for row in table_cells(metrics))
+    return "\n".join(lines)
+
+
+def make_pretty_table(metrics: list[Metrics]) -> str:
+    header = ["Method", "FoM us/zone", "Mupdate/s", "TP wall s", "Chem s", "Chem %", "Rad-noODE %", "Hydro %", "Hydro-only x", "RadSub/HydroStep", "Subcyc"]
+    rows = table_cells(metrics)
+    widths = [max(len(row[col]) for row in [header, *rows]) for col in range(len(header))]
+
+    def left(value: str, col: int) -> str:
+        return value.ljust(widths[col])
+
+    def right(value: str, col: int) -> str:
+        return value.rjust(widths[col])
+
+    lines = [
+        "| " + " | ".join(left(value, col) if col == 0 else right(value, col) for col, value in enumerate(header)) + " |",
+        "|" + "|".join("-" * (widths[col] + 2) if col == 0 else "-" * (widths[col] + 1) + ":" for col in range(len(header))) + "|",
+    ]
+    lines.extend("| " + " | ".join(left(value, col) if col == 0 else right(value, col) for col, value in enumerate(row)) + " |" for row in rows)
+    return "\n".join(lines)
+
+
+def make_table(metrics: list[Metrics], *, pretty: bool = False) -> str:
+    if pretty:
+        return make_pretty_table(metrics)
+    return make_markdown_table(metrics)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("logs", nargs="+", type=Path, help="DTypeFront log files or directories containing *.log files")
+    parser.add_argument("--pretty", action="store_true", help="print a Markdown table with fixed-width columns")
+    parser.add_argument("--show-failures", action="store_true", help="print failed-run diagnostics after the table")
+    args = parser.parse_args()
+
+    logs = discover_logs(args.logs)
+    if not logs:
+        raise SystemExit("no log files found")
+
+    metrics = [parse_log(path) for path in logs]
+    print(make_table(metrics, pretty=args.pretty))
+
+    if args.show_failures:
+        failures = [(path, row.failed) for path, row in zip(logs, metrics) if row.failed is not None]
+        if failures:
+            print("\nFailures:")
+            for path, failure in failures:
+                print(f"- {path}: {failure}")
+
+
+if __name__ == "__main__":
+    main()

scripts/slurm/dtypefront_frontier_gpu_sweep.submit

@@ -0,0 +1,69 @@
+#!/bin/bash --login
+#SBATCH -A ast236
+#SBATCH -J dtypefront_frontier
+#SBATCH -o %x-%j.out
+#SBATCH -e %x-%j.err
+#SBATCH -t 00:30:00
+#SBATCH -p batch
+#SBATCH -N 1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=7
+#SBATCH --gpus-per-task=1
+#SBATCH --gpu-bind=closest
+
+set -euo pipefail
+
+ROOT_DIR="${SLURM_SUBMIT_DIR:-$(pwd)}"
+INPUT_FILE="${ROOT_DIR}/inputs/DTypeFront.toml"
+LOG_DIR="${ROOT_DIR}/logs/dtypefront-frontier-${SLURM_JOB_ID:-manual}"
+ROS_BUILD_DIR="${ROOT_DIR}/build/slurm-dtypefront-frontier-rosenbrock"
+VODE_BUILD_DIR="${ROOT_DIR}/build/slurm-dtypefront-frontier-vode"
+BUILD_JOBS="${SLURM_CPUS_PER_TASK:-7}"
+
+. "${ROOT_DIR}/scripts/hpc_profiles/frontier.profile"
+
+GPU_ARCH="${AMREX_AMD_ARCH:-gfx90a}"
+
+mkdir -p "${LOG_DIR}"
+
+run_case() {
+	local exe="$1"
+	local tag="$2"
+	shift 2
+
+	echo "=== $(date -Is) ${tag} ==="
+	echo "ROCR_VISIBLE_DEVICES=${ROCR_VISIBLE_DEVICES:-unset}"
+	echo "HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-unset}"
+	/usr/bin/time -p srun -N 1 -n 1 --gpus-per-task=1 --gpu-bind=closest "${exe}" "${INPUT_FILE}" "$@" \
+		> "${LOG_DIR}/${tag}.log" 2>&1
+}
+
+configure_build() {
+	local build_dir="$1"
+	local integrator="$2"
+
+	cmake -S "${ROOT_DIR}" -B "${build_dir}" \
+		-DCMAKE_BUILD_TYPE=Release \
+		-DDTypeFront_INTEGRATOR="${integrator}" \
+		-DAMReX_GPU_BACKEND=HIP \
+		-DAMReX_AMD_ARCH="${GPU_ARCH}" \
+		-DAMReX_SPACEDIM=3
+	cmake --build "${build_dir}" --target DTypeFront -j "${BUILD_JOBS}"
+}
+
+echo "Running on ${HOSTNAME}"
+echo "Submit dir: ${ROOT_DIR}"
+echo "Log dir: ${LOG_DIR}"
+echo "Build jobs: ${BUILD_JOBS}"
+echo "AMReX AMD arch: ${GPU_ARCH}"
+
+configure_build "${ROS_BUILD_DIR}" Rosenbrock
+run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" ros2s integrator.rosenbrock_tableau=3
+run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas3p integrator.rosenbrock_tableau=2
+run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas4p integrator.rosenbrock_tableau=1
+run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas5p integrator.rosenbrock_tableau=0
+
+configure_build "${VODE_BUILD_DIR}" VODE
+run_case "${VODE_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" vode
+
+echo "Done."

scripts/slurm/dtypefront_gpu_sweep.submit

@@ -0,0 +1,57 @@
+#!/bin/bash --login
+#SBATCH --job-name=dtypefront_gpu
+#SBATCH --output=%x-%j.out
+#SBATCH --error=%x-%j.err
+#SBATCH --time=00:20:00
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=16
+#SBATCH --gpus=h200:1
+#SBATCH --gpu-bind=closest
+#SBATCH --mem=64G
+
+set -euo pipefail
+
+ROOT_DIR="${SLURM_SUBMIT_DIR:-$(pwd)}"
+INPUT_FILE="${ROOT_DIR}/inputs/DTypeFront.toml"
+LOG_DIR="${ROOT_DIR}/logs/dtypefront-${SLURM_JOB_ID:-manual}"
+ROS_BUILD_DIR="${ROOT_DIR}/build/slurm-dtypefront-rosenbrock"
+VODE_BUILD_DIR="${ROOT_DIR}/build/slurm-dtypefront-vode"
+BUILD_JOBS="${SLURM_CPUS_PER_TASK:-16}"
+
+mkdir -p "${LOG_DIR}"
+
+run_case() {
+	local exe="$1"
+	local tag="$2"
+	shift 2
+
+	echo "=== $(date -Is) ${tag} ==="
+	echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-unset}"
+	/usr/bin/time -p "${exe}" "${INPUT_FILE}" "$@" \
+		> "${LOG_DIR}/${tag}.log" 2>&1
+}
+
+configure_build() {
+	local build_dir="$1"
+	local integrator="$2"
+
+	cmake -S "${ROOT_DIR}" -B "${build_dir}" -DDTypeFront_INTEGRATOR="${integrator}" -DAMReX_GPU_BACKEND=CUDA -DAMReX_GPU_ARCH=9.0 -DAMReX_SPACEDIM=3
+	cmake --build "${build_dir}" --target DTypeFront -j "${BUILD_JOBS}"
+}
+
+echo "Running on ${HOSTNAME}"
+echo "Submit dir: ${ROOT_DIR}"
+echo "Log dir: ${LOG_DIR}"
+echo "Build jobs: ${BUILD_JOBS}"
+
+configure_build "${ROS_BUILD_DIR}" Rosenbrock
+run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" ros2s integrator.rosenbrock_tableau=3
+run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas3p integrator.rosenbrock_tableau=2
+run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas4p integrator.rosenbrock_tableau=1
+run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas5p integrator.rosenbrock_tableau=0
+
+configure_build "${VODE_BUILD_DIR}" VODE
+run_case "${VODE_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" vode
+
+echo "Done."

src/problems/DTypeFront/CMakeLists.txt

@@ -3,9 +3,11 @@ if(AMReX_SPACEDIM GREATER_EQUAL 2)
                                                   # network_name to
                                                   # photoionization for this
                                                   # directory only
+  set(DTypeFront_INTEGRATOR "Rosenbrock" CACHE STRING "Microphysics integrator for DTypeFront")
+  set_property(CACHE DTypeFront_INTEGRATOR PROPERTY STRINGS VODE Rosenbrock)
   setup_target_for_microphysics_compilation(${microphysics_network_name}
                                             "${CMAKE_CURRENT_BINARY_DIR}/"
-                                            "Rosenbrock")
+                                            "${DTypeFront_INTEGRATOR}")
 
   # use the BEFORE keyword so that these files get priority in compilation for
   # targets in this directory this is critical to ensure the correct Microphysics
@@ -93,6 +95,11 @@ if(AMReX_SPACEDIM GREATER_EQUAL 2)
 
   # this will add #define CHEMISTRY
   target_compile_definitions(${JOB_NAME} PUBLIC PHOTOCHEMISTRY)
+  if(DTypeFront_INTEGRATOR STREQUAL "Rosenbrock")
+    target_compile_definitions(${JOB_NAME} PUBLIC DTYPEFRONT_USE_ROSENBROCK)
+  else()
+    target_compile_definitions(${JOB_NAME} PUBLIC DTYPEFRONT_USE_VODE)
+  endif()
 
   if(AMReX_GPU_BACKEND MATCHES "CUDA")
     setup_target_for_cuda_compilation(${JOB_NAME})