diff --git a/scripts/python/dtypefront_perf_table.py b/scripts/python/dtypefront_perf_table.py new file mode 100755 index 0000000000..26e9522d60 --- /dev/null +++ b/scripts/python/dtypefront_perf_table.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Generate a DTypeFront performance table from benchmark logs.""" + +from __future__ import annotations + +import argparse +import re +from dataclasses import dataclass +from pathlib import Path + + +METHOD_ORDER = { + "ROS2S": 0, + "Rodas3P": 1, + "Rodas4P": 2, + "VODE": 3, + "Rodas5P": 4, +} + + +@dataclass +class Metrics: + method: str + fom_us: float | None = None + mupdates: float | None = None + tp_wall_s: float | None = None + chem_s: float | None = None + chem_pct: float | None = None + adv_s: float | None = None + adv_pct: float | None = None + hydro_s: float | None = None + hydro_pct: float | None = None + subcycles: float | None = None + failed: str | None = None + + @property + def rad_no_ode_pct(self) -> float | None: + if self.adv_pct is None or self.hydro_pct is None or self.chem_pct is None: + return None + return self.adv_pct - self.hydro_pct - self.chem_pct + + @property + def hydro_only_speedup(self) -> float | None: + if self.hydro_pct is None or self.hydro_pct == 0.0: + return None + return 100.0 / self.hydro_pct + + @property + def rad_substep_cost(self) -> float | None: + if self.adv_s is None or self.hydro_s is None or self.subcycles is None or self.hydro_s == 0.0: + return None + return (self.adv_s - self.hydro_s) / (self.hydro_s * self.subcycles) + + +def discover_logs(paths: list[Path]) -> list[Path]: + logs: list[Path] = [] + for path in paths: + if path.is_dir(): + logs.extend(sorted(path.glob("*.log"))) + else: + logs.append(path) + return logs + + +def method_from_log(text: str, path: Path) -> str: + match = re.search(r"DTypeFront microphysics integrator: VODE", text) + if match: + return "VODE" + + match = re.search(r"DTypeFront microphysics integrator: Rosenbrock \(Rosenbrock tableau \d+: ([^)]+)\)", text) + if match: + return match.group(1) + + name_map = { + "ros2s": "ROS2S", + "rodas3p": "Rodas3P", + "rodas4p": "Rodas4P", + "rodas5p": "Rodas5P", + "vode": "VODE", + } + return name_map.get(path.stem.lower(), path.stem) + + +def parse_profiler_line(text: str, name: str) -> tuple[float, float] | None: + # Example: + # PhotoChemistry::computePhotoChemistry() 7734 6.474 6.474 6.474 33.65% + pattern = re.compile(rf"^\s*{re.escape(name)}\s+\d+\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)%", re.MULTILINE) + matches = [(float(match.group(3)), float(match.group(4))) for match in pattern.finditer(text)] + if not matches: + return None + # The same profiler name can appear as exclusive and inclusive entries. The + # inclusive entry is the one with the largest walltime/percentage. + return max(matches, key=lambda value: value[1]) + + +def parse_log(path: Path) -> Metrics: + text = path.read_text(errors="replace") + metrics = Metrics(method=method_from_log(text, path)) + + match = re.search(r"Performance figure-of-merit:\s+([0-9.eE+-]+)\s+.*?\[([0-9.eE+-]+)\s+Mupdates/s\]", text) + if match: + metrics.fom_us = float(match.group(1)) + metrics.mupdates = float(match.group(2)) + + match = re.search(r"TinyProfiler total time across processes \[min\.\.\.avg\.\.\.max\]:\s+([0-9.eE+-]+)\s+\.\.\.\s+([0-9.eE+-]+)\s+\.\.\.\s+([0-9.eE+-]+)", text) + if match: + metrics.tp_wall_s = float(match.group(3)) + + match = re.search(r"avg\. num\. of radiation subcycles\s*=\s*([0-9.eE+-]+)", text) + if match: + metrics.subcycles = float(match.group(1)) + + chem = parse_profiler_line(text, "PhotoChemistry::computePhotoChemistry()") + if chem is not None: + metrics.chem_s, metrics.chem_pct = chem + + advance = parse_profiler_line(text, "QuokkaSimulation::advanceSingleTimestepAtLevel()") + if advance is not None: + metrics.adv_s, metrics.adv_pct = advance + + hydro = parse_profiler_line(text, "REG::HydroSolver") + if hydro is not None: + metrics.hydro_s, metrics.hydro_pct = hydro + + match = re.search(r"Photochemistry burn failed.*", text) + if match: + metrics.failed = match.group(0) + + return metrics + + +def fmt(value: float | None, precision: int = 2) -> str: + if value is None: + return "n/a" + return f"{value:.{precision}f}" + + +def fmt_fom(value: float | None) -> str: + if value is None: + return "n/a" + return f"{value:.4f}" + + +def table_cells(metrics: list[Metrics]) -> list[list[str]]: + rows = sorted(metrics, key=lambda row: (METHOD_ORDER.get(row.method, 99), row.method)) + return [ + [ + row.method, + fmt_fom(row.fom_us), + fmt(row.mupdates, 2), + fmt(row.tp_wall_s, 2), + fmt(row.chem_s, 2), + fmt(row.chem_pct, 2), + fmt(row.rad_no_ode_pct, 2), + fmt(row.hydro_pct, 2), + fmt(row.hydro_only_speedup, 2), + fmt(row.rad_substep_cost, 2), + fmt(row.subcycles, 2), + ] + for row in rows + ] + + +def make_markdown_table(metrics: list[Metrics]) -> str: + lines = [ + "| Method | FoM us/zone | Mupdate/s | TP wall s | Chem s | Chem % | Rad-noODE % | Hydro % | Hydro-only x | RadSub/HydroStep | Subcyc |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + lines.extend("| " + " | ".join(row) + " |" for row in table_cells(metrics)) + return "\n".join(lines) + + +def make_pretty_table(metrics: list[Metrics]) -> str: + header = ["Method", "FoM us/zone", "Mupdate/s", "TP wall s", "Chem s", "Chem %", "Rad-noODE %", "Hydro %", "Hydro-only x", "RadSub/HydroStep", "Subcyc"] + rows = table_cells(metrics) + widths = [max(len(row[col]) for row in [header, *rows]) for col in range(len(header))] + + def left(value: str, col: int) -> str: + return value.ljust(widths[col]) + + def right(value: str, col: int) -> str: + return value.rjust(widths[col]) + + lines = [ + "| " + " | ".join(left(value, col) if col == 0 else right(value, col) for col, value in enumerate(header)) + " |", + "|" + "|".join("-" * (widths[col] + 2) if col == 0 else "-" * (widths[col] + 1) + ":" for col in range(len(header))) + "|", + ] + lines.extend("| " + " | ".join(left(value, col) if col == 0 else right(value, col) for col, value in enumerate(row)) + " |" for row in rows) + return "\n".join(lines) + + +def make_table(metrics: list[Metrics], *, pretty: bool = False) -> str: + if pretty: + return make_pretty_table(metrics) + return make_markdown_table(metrics) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("logs", nargs="+", type=Path, help="DTypeFront log files or directories containing *.log files") + parser.add_argument("--pretty", action="store_true", help="print a Markdown table with fixed-width columns") + parser.add_argument("--show-failures", action="store_true", help="print failed-run diagnostics after the table") + args = parser.parse_args() + + logs = discover_logs(args.logs) + if not logs: + raise SystemExit("no log files found") + + metrics = [parse_log(path) for path in logs] + print(make_table(metrics, pretty=args.pretty)) + + if args.show_failures: + failures = [(path, row.failed) for path, row in zip(logs, metrics) if row.failed is not None] + if failures: + print("\nFailures:") + for path, failure in failures: + print(f"- {path}: {failure}") + + +if __name__ == "__main__": + main() diff --git a/scripts/slurm/dtypefront_frontier_gpu_sweep.submit b/scripts/slurm/dtypefront_frontier_gpu_sweep.submit new file mode 100644 index 0000000000..112e2a3e4f --- /dev/null +++ b/scripts/slurm/dtypefront_frontier_gpu_sweep.submit @@ -0,0 +1,69 @@ +#!/bin/bash --login +#SBATCH -A ast236 +#SBATCH -J dtypefront_frontier +#SBATCH -o %x-%j.out +#SBATCH -e %x-%j.err +#SBATCH -t 00:30:00 +#SBATCH -p batch +#SBATCH -N 1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=7 +#SBATCH --gpus-per-task=1 +#SBATCH --gpu-bind=closest + +set -euo pipefail + +ROOT_DIR="${SLURM_SUBMIT_DIR:-$(pwd)}" +INPUT_FILE="${ROOT_DIR}/inputs/DTypeFront.toml" +LOG_DIR="${ROOT_DIR}/logs/dtypefront-frontier-${SLURM_JOB_ID:-manual}" +ROS_BUILD_DIR="${ROOT_DIR}/build/slurm-dtypefront-frontier-rosenbrock" +VODE_BUILD_DIR="${ROOT_DIR}/build/slurm-dtypefront-frontier-vode" +BUILD_JOBS="${SLURM_CPUS_PER_TASK:-7}" + +. "${ROOT_DIR}/scripts/hpc_profiles/frontier.profile" + +GPU_ARCH="${AMREX_AMD_ARCH:-gfx90a}" + +mkdir -p "${LOG_DIR}" + +run_case() { + local exe="$1" + local tag="$2" + shift 2 + + echo "=== $(date -Is) ${tag} ===" + echo "ROCR_VISIBLE_DEVICES=${ROCR_VISIBLE_DEVICES:-unset}" + echo "HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-unset}" + /usr/bin/time -p srun -N 1 -n 1 --gpus-per-task=1 --gpu-bind=closest "${exe}" "${INPUT_FILE}" "$@" \ + > "${LOG_DIR}/${tag}.log" 2>&1 +} + +configure_build() { + local build_dir="$1" + local integrator="$2" + + cmake -S "${ROOT_DIR}" -B "${build_dir}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DDTypeFront_INTEGRATOR="${integrator}" \ + -DAMReX_GPU_BACKEND=HIP \ + -DAMReX_AMD_ARCH="${GPU_ARCH}" \ + -DAMReX_SPACEDIM=3 + cmake --build "${build_dir}" --target DTypeFront -j "${BUILD_JOBS}" +} + +echo "Running on ${HOSTNAME}" +echo "Submit dir: ${ROOT_DIR}" +echo "Log dir: ${LOG_DIR}" +echo "Build jobs: ${BUILD_JOBS}" +echo "AMReX AMD arch: ${GPU_ARCH}" + +configure_build "${ROS_BUILD_DIR}" Rosenbrock +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" ros2s integrator.rosenbrock_tableau=3 +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas3p integrator.rosenbrock_tableau=2 +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas4p integrator.rosenbrock_tableau=1 +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas5p integrator.rosenbrock_tableau=0 + +configure_build "${VODE_BUILD_DIR}" VODE +run_case "${VODE_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" vode + +echo "Done." diff --git a/scripts/slurm/dtypefront_gpu_sweep.submit b/scripts/slurm/dtypefront_gpu_sweep.submit new file mode 100644 index 0000000000..891c5c09b3 --- /dev/null +++ b/scripts/slurm/dtypefront_gpu_sweep.submit @@ -0,0 +1,57 @@ +#!/bin/bash --login +#SBATCH --job-name=dtypefront_gpu +#SBATCH --output=%x-%j.out +#SBATCH --error=%x-%j.err +#SBATCH --time=00:20:00 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gpus=h200:1 +#SBATCH --gpu-bind=closest +#SBATCH --mem=64G + +set -euo pipefail + +ROOT_DIR="${SLURM_SUBMIT_DIR:-$(pwd)}" +INPUT_FILE="${ROOT_DIR}/inputs/DTypeFront.toml" +LOG_DIR="${ROOT_DIR}/logs/dtypefront-${SLURM_JOB_ID:-manual}" +ROS_BUILD_DIR="${ROOT_DIR}/build/slurm-dtypefront-rosenbrock" +VODE_BUILD_DIR="${ROOT_DIR}/build/slurm-dtypefront-vode" +BUILD_JOBS="${SLURM_CPUS_PER_TASK:-16}" + +mkdir -p "${LOG_DIR}" + +run_case() { + local exe="$1" + local tag="$2" + shift 2 + + echo "=== $(date -Is) ${tag} ===" + echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-unset}" + /usr/bin/time -p "${exe}" "${INPUT_FILE}" "$@" \ + > "${LOG_DIR}/${tag}.log" 2>&1 +} + +configure_build() { + local build_dir="$1" + local integrator="$2" + + cmake -S "${ROOT_DIR}" -B "${build_dir}" -DDTypeFront_INTEGRATOR="${integrator}" -DAMReX_GPU_BACKEND=CUDA -DAMReX_GPU_ARCH=9.0 -DAMReX_SPACEDIM=3 + cmake --build "${build_dir}" --target DTypeFront -j "${BUILD_JOBS}" +} + +echo "Running on ${HOSTNAME}" +echo "Submit dir: ${ROOT_DIR}" +echo "Log dir: ${LOG_DIR}" +echo "Build jobs: ${BUILD_JOBS}" + +configure_build "${ROS_BUILD_DIR}" Rosenbrock +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" ros2s integrator.rosenbrock_tableau=3 +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas3p integrator.rosenbrock_tableau=2 +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas4p integrator.rosenbrock_tableau=1 +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas5p integrator.rosenbrock_tableau=0 + +configure_build "${VODE_BUILD_DIR}" VODE +run_case "${VODE_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" vode + +echo "Done." diff --git a/src/problems/DTypeFront/CMakeLists.txt b/src/problems/DTypeFront/CMakeLists.txt index f8329fc7c8..85d7e945d2 100644 --- a/src/problems/DTypeFront/CMakeLists.txt +++ b/src/problems/DTypeFront/CMakeLists.txt @@ -3,9 +3,11 @@ if(AMReX_SPACEDIM GREATER_EQUAL 2) # network_name to # photoionization for this # directory only + set(DTypeFront_INTEGRATOR "Rosenbrock" CACHE STRING "Microphysics integrator for DTypeFront") + set_property(CACHE DTypeFront_INTEGRATOR PROPERTY STRINGS VODE Rosenbrock) setup_target_for_microphysics_compilation(${microphysics_network_name} "${CMAKE_CURRENT_BINARY_DIR}/" - "Rosenbrock") + "${DTypeFront_INTEGRATOR}") # use the BEFORE keyword so that these files get priority in compilation for # targets in this directory this is critical to ensure the correct Microphysics @@ -93,6 +95,11 @@ if(AMReX_SPACEDIM GREATER_EQUAL 2) # this will add #define CHEMISTRY target_compile_definitions(${JOB_NAME} PUBLIC PHOTOCHEMISTRY) + if(DTypeFront_INTEGRATOR STREQUAL "Rosenbrock") + target_compile_definitions(${JOB_NAME} PUBLIC DTYPEFRONT_USE_ROSENBROCK) + else() + target_compile_definitions(${JOB_NAME} PUBLIC DTYPEFRONT_USE_VODE) + endif() if(AMReX_GPU_BACKEND MATCHES "CUDA") setup_target_for_cuda_compilation(${JOB_NAME}) diff --git a/src/problems/DTypeFront/testDTypeFront.cpp b/src/problems/DTypeFront/testDTypeFront.cpp index 6ec228a7b9..7ed8ee4156 100644 --- a/src/problems/DTypeFront/testDTypeFront.cpp +++ b/src/problems/DTypeFront/testDTypeFront.cpp @@ -191,6 +191,34 @@ auto compute_equilibrium_temperature_ionized(double n_e) -> double return 0.5 * (T_lo + T_hi); } +#ifdef DTYPEFRONT_USE_ROSENBROCK +auto rosenbrock_tableau_name(int tableau) -> char const * +{ + switch (tableau) { + case 0: + return "Rodas5P"; + case 1: + return "Rodas4P"; + case 2: + return "Rodas3P"; + case 3: + return "ROS2S"; + default: + return "unknown"; + } +} +#endif + +void print_microphysics_integrator() +{ +#ifdef DTYPEFRONT_USE_ROSENBROCK + amrex::Print() << "DTypeFront microphysics integrator: Rosenbrock (Rosenbrock tableau " << integrator_rp::rosenbrock_tableau << ": " + << rosenbrock_tableau_name(integrator_rp::rosenbrock_tableau) << ")\n"; +#else + amrex::Print() << "DTypeFront microphysics integrator: VODE\n"; +#endif +} + } // namespace AMREX_GPU_HOST_DEVICE auto wendland_c2(amrex::Real r) -> amrex::Real @@ -398,6 +426,7 @@ auto problem_main() -> int // Problem initialization QuokkaSimulation sim; + print_microphysics_integrator(); // initialize sim.setInitialConditions(); @@ -589,4 +618,4 @@ auto problem_main() -> int // Cleanup and exit amrex::Print() << "Finished." << '\n'; return status; -} \ No newline at end of file +}