From c06a7e3beb674beaaf196bebd4b445ffc6255b86 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Fri, 26 Jun 2026 11:59:20 -0400 Subject: [PATCH 1/6] add performance scripts for DTypeFront --- scripts/python/dtypefront_perf_table.py | 195 ++++++++++++++++++++++ scripts/slurm/dtypefront_gpu_sweep.submit | 57 +++++++ 2 files changed, 252 insertions(+) create mode 100755 scripts/python/dtypefront_perf_table.py create mode 100644 scripts/slurm/dtypefront_gpu_sweep.submit diff --git a/scripts/python/dtypefront_perf_table.py b/scripts/python/dtypefront_perf_table.py new file mode 100755 index 0000000000..1016d2b1f4 --- /dev/null +++ b/scripts/python/dtypefront_perf_table.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Generate a DTypeFront performance table from benchmark logs.""" + +from __future__ import annotations + +import argparse +import re +from dataclasses import dataclass +from pathlib import Path + + +METHOD_ORDER = { + "ROS2S": 0, + "Rodas3P": 1, + "Rodas4P": 2, + "VODE": 3, + "Rodas5P": 4, +} + + +@dataclass +class Metrics: + method: str + fom_us: float | None = None + mupdates: float | None = None + tp_wall_s: float | None = None + chem_s: float | None = None + chem_pct: float | None = None + adv_s: float | None = None + adv_pct: float | None = None + hydro_s: float | None = None + hydro_pct: float | None = None + subcycles: float | None = None + failed: str | None = None + + @property + def rad_no_ode_pct(self) -> float | None: + if self.adv_pct is None or self.hydro_pct is None or self.chem_pct is None: + return None + return self.adv_pct - self.hydro_pct - self.chem_pct + + @property + def hydro_only_speedup(self) -> float | None: + if self.hydro_pct is None or self.hydro_pct == 0.0: + return None + return 100.0 / self.hydro_pct + + @property + def rad_substep_cost(self) -> float | None: + if self.adv_s is None or self.hydro_s is None or self.subcycles is None or self.hydro_s == 0.0: + return None + return (self.adv_s - self.hydro_s) / (self.hydro_s * self.subcycles) + + +def discover_logs(paths: list[Path]) -> list[Path]: + logs: list[Path] = [] + for path in paths: + if path.is_dir(): + logs.extend(sorted(path.glob("*.log"))) + else: + logs.append(path) + return logs + + +def method_from_log(text: str, path: Path) -> str: + match = re.search(r"DTypeFront microphysics integrator: VODE", text) + if match: + return "VODE" + + match = re.search(r"DTypeFront microphysics integrator: Rosenbrock \(Rosenbrock tableau \d+: ([^)]+)\)", text) + if match: + return match.group(1) + + name_map = { + "ros2s": "ROS2S", + "rodas3p": "Rodas3P", + "rodas4p": "Rodas4P", + "rodas5p": "Rodas5P", + "vode": "VODE", + } + return name_map.get(path.stem.lower(), path.stem) + + +def parse_profiler_line(text: str, name: str) -> tuple[float, float] | None: + # Example: + # PhotoChemistry::computePhotoChemistry() 7734 6.474 6.474 6.474 33.65% + pattern = re.compile(rf"^{re.escape(name)}\s+\d+\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)%", re.MULTILINE) + matches = [(float(match.group(3)), float(match.group(4))) for match in pattern.finditer(text)] + if not matches: + return None + # The same profiler name can appear as exclusive and inclusive entries. The + # inclusive entry is the one with the largest walltime/percentage. + return max(matches, key=lambda value: value[1]) + + +def parse_log(path: Path) -> Metrics: + text = path.read_text(errors="replace") + metrics = Metrics(method=method_from_log(text, path)) + + match = re.search(r"Performance figure-of-merit:\s+([0-9.eE+-]+)\s+.*?\[([0-9.eE+-]+)\s+Mupdates/s\]", text) + if match: + metrics.fom_us = float(match.group(1)) + metrics.mupdates = float(match.group(2)) + + match = re.search(r"TinyProfiler total time across processes \[min\.\.\.avg\.\.\.max\]:\s+([0-9.eE+-]+)\s+\.\.\.\s+([0-9.eE+-]+)\s+\.\.\.\s+([0-9.eE+-]+)", text) + if match: + metrics.tp_wall_s = float(match.group(3)) + + match = re.search(r"avg\. num\. of radiation subcycles\s*=\s*([0-9.eE+-]+)", text) + if match: + metrics.subcycles = float(match.group(1)) + + chem = parse_profiler_line(text, "PhotoChemistry::computePhotoChemistry()") + if chem is not None: + metrics.chem_s, metrics.chem_pct = chem + + advance = parse_profiler_line(text, "QuokkaSimulation::advanceSingleTimestepAtLevel()") + if advance is not None: + metrics.adv_s, metrics.adv_pct = advance + + hydro = parse_profiler_line(text, "REG::HydroSolver") + if hydro is not None: + metrics.hydro_s, metrics.hydro_pct = hydro + + match = re.search(r"Photochemistry burn failed.*", text) + if match: + metrics.failed = match.group(0) + + return metrics + + +def fmt(value: float | None, precision: int = 2) -> str: + if value is None: + return "n/a" + return f"{value:.{precision}f}" + + +def fmt_fom(value: float | None) -> str: + if value is None: + return "n/a" + return f"{value:.4f}" + + +def make_table(metrics: list[Metrics]) -> str: + rows = sorted(metrics, key=lambda row: (METHOD_ORDER.get(row.method, 99), row.method)) + lines = [ + "| Method | FoM us/zone | Mupdate/s | TP wall s | Chem s | Chem % | Rad-noODE % | Hydro % | Hydro-only x | RadSub/HydroStep | Subcyc |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + for row in rows: + lines.append( + "| " + + " | ".join( + [ + row.method, + fmt_fom(row.fom_us), + fmt(row.mupdates, 2), + fmt(row.tp_wall_s, 2), + fmt(row.chem_s, 2), + fmt(row.chem_pct, 2), + fmt(row.rad_no_ode_pct, 2), + fmt(row.hydro_pct, 2), + fmt(row.hydro_only_speedup, 2), + fmt(row.rad_substep_cost, 2), + fmt(row.subcycles, 2), + ] + ) + + " |" + ) + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("logs", nargs="+", type=Path, help="DTypeFront log files or directories containing *.log files") + parser.add_argument("--show-failures", action="store_true", help="print failed-run diagnostics after the table") + args = parser.parse_args() + + logs = discover_logs(args.logs) + if not logs: + raise SystemExit("no log files found") + + metrics = [parse_log(path) for path in logs] + print(make_table(metrics)) + + if args.show_failures: + failures = [(path, row.failed) for path, row in zip(logs, metrics, strict=True) if row.failed is not None] + if failures: + print("\nFailures:") + for path, failure in failures: + print(f"- {path}: {failure}") + + +if __name__ == "__main__": + main() diff --git a/scripts/slurm/dtypefront_gpu_sweep.submit b/scripts/slurm/dtypefront_gpu_sweep.submit new file mode 100644 index 0000000000..26c6cc0a49 --- /dev/null +++ b/scripts/slurm/dtypefront_gpu_sweep.submit @@ -0,0 +1,57 @@ +#!/bin/bash --login +#SBATCH --job-name=dtypefront_gpu +#SBATCH --output=%x-%j.out +#SBATCH --error=%x-%j.err +#SBATCH --time=00:20:00 +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=16 +#SBATCH --gpus=h200:1 +#SBATCH --gpu-bind=closest +#SBATCH --mem=64G + +set -euo pipefail + +ROOT_DIR="${SLURM_SUBMIT_DIR:-$(pwd)}" +INPUT_FILE="${ROOT_DIR}/inputs/DTypeFront.toml" +LOG_DIR="${ROOT_DIR}/logs/dtypefront-${SLURM_JOB_ID:-manual}" +ROS_BUILD_DIR="${ROOT_DIR}/build/slurm-dtypefront-rosenbrock" +VODE_BUILD_DIR="${ROOT_DIR}/build/slurm-dtypefront-vode" +BUILD_JOBS="${SLURM_CPUS_PER_TASK:-16}" + +mkdir -p "${LOG_DIR}" + +run_case() { + local exe="$1" + local tag="$2" + shift 2 + + echo "=== $(date -Is) ${tag} ===" + echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-unset}" + /usr/bin/time -p "${exe}" "${INPUT_FILE}" dtype_front.plot_radii=0 "$@" \ + > "${LOG_DIR}/${tag}.log" 2>&1 +} + +configure_build() { + local build_dir="$1" + local integrator="$2" + + cmake -S "${ROOT_DIR}" -B "${build_dir}" -DDTypeFront_INTEGRATOR="${integrator}" -DAMReX_GPU_BACKEND=CUDA -DAMReX_GPU_ARCH=9.0 -DAMReX_SPACEDIM=3 + cmake --build "${build_dir}" --target DTypeFront -j "${BUILD_JOBS}" +} + +echo "Running on ${HOSTNAME}" +echo "Submit dir: ${ROOT_DIR}" +echo "Log dir: ${LOG_DIR}" +echo "Build jobs: ${BUILD_JOBS}" + +configure_build "${ROS_BUILD_DIR}" Rosenbrock +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" ros2s integrator.rosenbrock_tableau=3 +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas3p integrator.rosenbrock_tableau=2 +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas4p integrator.rosenbrock_tableau=1 +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas5p integrator.rosenbrock_tableau=0 + +configure_build "${VODE_BUILD_DIR}" VODE +run_case "${VODE_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" vode + +echo "Done." From d411805e97e281119b4fc64f0871ab83c3236d4a Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Fri, 26 Jun 2026 12:07:14 -0400 Subject: [PATCH 2/6] Apply suggestions from code review Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- scripts/python/dtypefront_perf_table.py | 4 ++-- scripts/slurm/dtypefront_gpu_sweep.submit | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/python/dtypefront_perf_table.py b/scripts/python/dtypefront_perf_table.py index 1016d2b1f4..67729db88a 100755 --- a/scripts/python/dtypefront_perf_table.py +++ b/scripts/python/dtypefront_perf_table.py @@ -84,7 +84,7 @@ def method_from_log(text: str, path: Path) -> str: def parse_profiler_line(text: str, name: str) -> tuple[float, float] | None: # Example: # PhotoChemistry::computePhotoChemistry() 7734 6.474 6.474 6.474 33.65% - pattern = re.compile(rf"^{re.escape(name)}\s+\d+\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)%", re.MULTILINE) + pattern = re.compile(rf"^\s*{re.escape(name)}\s+\d+\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)\s+([0-9.eE+-]+)%", re.MULTILINE) matches = [(float(match.group(3)), float(match.group(4))) for match in pattern.finditer(text)] if not matches: return None @@ -184,7 +184,7 @@ def main() -> None: print(make_table(metrics)) if args.show_failures: - failures = [(path, row.failed) for path, row in zip(logs, metrics, strict=True) if row.failed is not None] + failures = [(path, row.failed) for path, row in zip(logs, metrics) if row.failed is not None] if failures: print("\nFailures:") for path, failure in failures: diff --git a/scripts/slurm/dtypefront_gpu_sweep.submit b/scripts/slurm/dtypefront_gpu_sweep.submit index 26c6cc0a49..891c5c09b3 100644 --- a/scripts/slurm/dtypefront_gpu_sweep.submit +++ b/scripts/slurm/dtypefront_gpu_sweep.submit @@ -28,7 +28,7 @@ run_case() { echo "=== $(date -Is) ${tag} ===" echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-unset}" - /usr/bin/time -p "${exe}" "${INPUT_FILE}" dtype_front.plot_radii=0 "$@" \ + /usr/bin/time -p "${exe}" "${INPUT_FILE}" "$@" \ > "${LOG_DIR}/${tag}.log" 2>&1 } From 91262beada9eddb09640f31c8eb330c931287844 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Fri, 26 Jun 2026 15:47:08 -0400 Subject: [PATCH 3/6] allow switching between VODE and Rosenbrock --- src/problems/DTypeFront/CMakeLists.txt | 9 ++++++- src/problems/DTypeFront/testDTypeFront.cpp | 31 +++++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/problems/DTypeFront/CMakeLists.txt b/src/problems/DTypeFront/CMakeLists.txt index f8329fc7c8..85d7e945d2 100644 --- a/src/problems/DTypeFront/CMakeLists.txt +++ b/src/problems/DTypeFront/CMakeLists.txt @@ -3,9 +3,11 @@ if(AMReX_SPACEDIM GREATER_EQUAL 2) # network_name to # photoionization for this # directory only + set(DTypeFront_INTEGRATOR "Rosenbrock" CACHE STRING "Microphysics integrator for DTypeFront") + set_property(CACHE DTypeFront_INTEGRATOR PROPERTY STRINGS VODE Rosenbrock) setup_target_for_microphysics_compilation(${microphysics_network_name} "${CMAKE_CURRENT_BINARY_DIR}/" - "Rosenbrock") + "${DTypeFront_INTEGRATOR}") # use the BEFORE keyword so that these files get priority in compilation for # targets in this directory this is critical to ensure the correct Microphysics @@ -93,6 +95,11 @@ if(AMReX_SPACEDIM GREATER_EQUAL 2) # this will add #define CHEMISTRY target_compile_definitions(${JOB_NAME} PUBLIC PHOTOCHEMISTRY) + if(DTypeFront_INTEGRATOR STREQUAL "Rosenbrock") + target_compile_definitions(${JOB_NAME} PUBLIC DTYPEFRONT_USE_ROSENBROCK) + else() + target_compile_definitions(${JOB_NAME} PUBLIC DTYPEFRONT_USE_VODE) + endif() if(AMReX_GPU_BACKEND MATCHES "CUDA") setup_target_for_cuda_compilation(${JOB_NAME}) diff --git a/src/problems/DTypeFront/testDTypeFront.cpp b/src/problems/DTypeFront/testDTypeFront.cpp index 6ec228a7b9..fbb942a7a5 100644 --- a/src/problems/DTypeFront/testDTypeFront.cpp +++ b/src/problems/DTypeFront/testDTypeFront.cpp @@ -191,6 +191,34 @@ auto compute_equilibrium_temperature_ionized(double n_e) -> double return 0.5 * (T_lo + T_hi); } +#ifdef DTYPEFRONT_USE_ROSENBROCK +auto rosenbrock_tableau_name(int tableau) -> char const * +{ + switch (tableau) { + case 0: + return "Rodas5P"; + case 1: + return "Rodas4P"; + case 2: + return "Rodas3P"; + case 3: + return "ROS2S"; + default: + return "unknown"; + } +} +#endif + +void print_microphysics_integrator() +{ +#ifdef DTYPEFRONT_USE_ROSENBROCK + amrex::Print() << "DTypeFront microphysics integrator: Rosenbrock (Rosenbrock tableau " << integrator_rp::rosenbrock_tableau << ": " + << rosenbrock_tableau_name(integrator_rp::rosenbrock_tableau) << ")\n"; +#else + amrex::Print() << "DTypeFront microphysics integrator: VODE\n"; +#endif +} + } // namespace AMREX_GPU_HOST_DEVICE auto wendland_c2(amrex::Real r) -> amrex::Real @@ -398,6 +426,7 @@ auto problem_main() -> int // Problem initialization QuokkaSimulation sim; + print_microphysics_integrator(); // initialize sim.setInitialConditions(); @@ -589,4 +618,4 @@ auto problem_main() -> int // Cleanup and exit amrex::Print() << "Finished." << '\n'; return status; -} \ No newline at end of file +} From 75acaf0491a0b37c4570f720febe55fffe649d84 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:49:23 +0000 Subject: [PATCH 4/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/problems/DTypeFront/testDTypeFront.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/problems/DTypeFront/testDTypeFront.cpp b/src/problems/DTypeFront/testDTypeFront.cpp index fbb942a7a5..7ed8ee4156 100644 --- a/src/problems/DTypeFront/testDTypeFront.cpp +++ b/src/problems/DTypeFront/testDTypeFront.cpp @@ -195,16 +195,16 @@ auto compute_equilibrium_temperature_ionized(double n_e) -> double auto rosenbrock_tableau_name(int tableau) -> char const * { switch (tableau) { - case 0: - return "Rodas5P"; - case 1: - return "Rodas4P"; - case 2: - return "Rodas3P"; - case 3: - return "ROS2S"; - default: - return "unknown"; + case 0: + return "Rodas5P"; + case 1: + return "Rodas4P"; + case 2: + return "Rodas3P"; + case 3: + return "ROS2S"; + default: + return "unknown"; } } #endif From 881b0104174c303287c41d165b656fa7fa4702c6 Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Fri, 26 Jun 2026 16:10:06 -0400 Subject: [PATCH 5/6] add pretty print mode --- scripts/python/dtypefront_perf_table.py | 70 +++++++++++++++++-------- 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/scripts/python/dtypefront_perf_table.py b/scripts/python/dtypefront_perf_table.py index 67729db88a..26e9522d60 100755 --- a/scripts/python/dtypefront_perf_table.py +++ b/scripts/python/dtypefront_perf_table.py @@ -141,38 +141,64 @@ def fmt_fom(value: float | None) -> str: return f"{value:.4f}" -def make_table(metrics: list[Metrics]) -> str: +def table_cells(metrics: list[Metrics]) -> list[list[str]]: rows = sorted(metrics, key=lambda row: (METHOD_ORDER.get(row.method, 99), row.method)) + return [ + [ + row.method, + fmt_fom(row.fom_us), + fmt(row.mupdates, 2), + fmt(row.tp_wall_s, 2), + fmt(row.chem_s, 2), + fmt(row.chem_pct, 2), + fmt(row.rad_no_ode_pct, 2), + fmt(row.hydro_pct, 2), + fmt(row.hydro_only_speedup, 2), + fmt(row.rad_substep_cost, 2), + fmt(row.subcycles, 2), + ] + for row in rows + ] + + +def make_markdown_table(metrics: list[Metrics]) -> str: lines = [ "| Method | FoM us/zone | Mupdate/s | TP wall s | Chem s | Chem % | Rad-noODE % | Hydro % | Hydro-only x | RadSub/HydroStep | Subcyc |", "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", ] - for row in rows: - lines.append( - "| " - + " | ".join( - [ - row.method, - fmt_fom(row.fom_us), - fmt(row.mupdates, 2), - fmt(row.tp_wall_s, 2), - fmt(row.chem_s, 2), - fmt(row.chem_pct, 2), - fmt(row.rad_no_ode_pct, 2), - fmt(row.hydro_pct, 2), - fmt(row.hydro_only_speedup, 2), - fmt(row.rad_substep_cost, 2), - fmt(row.subcycles, 2), - ] - ) - + " |" - ) + lines.extend("| " + " | ".join(row) + " |" for row in table_cells(metrics)) return "\n".join(lines) +def make_pretty_table(metrics: list[Metrics]) -> str: + header = ["Method", "FoM us/zone", "Mupdate/s", "TP wall s", "Chem s", "Chem %", "Rad-noODE %", "Hydro %", "Hydro-only x", "RadSub/HydroStep", "Subcyc"] + rows = table_cells(metrics) + widths = [max(len(row[col]) for row in [header, *rows]) for col in range(len(header))] + + def left(value: str, col: int) -> str: + return value.ljust(widths[col]) + + def right(value: str, col: int) -> str: + return value.rjust(widths[col]) + + lines = [ + "| " + " | ".join(left(value, col) if col == 0 else right(value, col) for col, value in enumerate(header)) + " |", + "|" + "|".join("-" * (widths[col] + 2) if col == 0 else "-" * (widths[col] + 1) + ":" for col in range(len(header))) + "|", + ] + lines.extend("| " + " | ".join(left(value, col) if col == 0 else right(value, col) for col, value in enumerate(row)) + " |" for row in rows) + return "\n".join(lines) + + +def make_table(metrics: list[Metrics], *, pretty: bool = False) -> str: + if pretty: + return make_pretty_table(metrics) + return make_markdown_table(metrics) + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("logs", nargs="+", type=Path, help="DTypeFront log files or directories containing *.log files") + parser.add_argument("--pretty", action="store_true", help="print a Markdown table with fixed-width columns") parser.add_argument("--show-failures", action="store_true", help="print failed-run diagnostics after the table") args = parser.parse_args() @@ -181,7 +207,7 @@ def main() -> None: raise SystemExit("no log files found") metrics = [parse_log(path) for path in logs] - print(make_table(metrics)) + print(make_table(metrics, pretty=args.pretty)) if args.show_failures: failures = [(path, row.failed) for path, row in zip(logs, metrics) if row.failed is not None] From 7acf74359c2c4552cb45bc97cc42043caedb816a Mon Sep 17 00:00:00 2001 From: Ben Wibking Date: Fri, 26 Jun 2026 16:24:45 -0400 Subject: [PATCH 6/6] add frontier batch script --- .../dtypefront_frontier_gpu_sweep.submit | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 scripts/slurm/dtypefront_frontier_gpu_sweep.submit diff --git a/scripts/slurm/dtypefront_frontier_gpu_sweep.submit b/scripts/slurm/dtypefront_frontier_gpu_sweep.submit new file mode 100644 index 0000000000..112e2a3e4f --- /dev/null +++ b/scripts/slurm/dtypefront_frontier_gpu_sweep.submit @@ -0,0 +1,69 @@ +#!/bin/bash --login +#SBATCH -A ast236 +#SBATCH -J dtypefront_frontier +#SBATCH -o %x-%j.out +#SBATCH -e %x-%j.err +#SBATCH -t 00:30:00 +#SBATCH -p batch +#SBATCH -N 1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=7 +#SBATCH --gpus-per-task=1 +#SBATCH --gpu-bind=closest + +set -euo pipefail + +ROOT_DIR="${SLURM_SUBMIT_DIR:-$(pwd)}" +INPUT_FILE="${ROOT_DIR}/inputs/DTypeFront.toml" +LOG_DIR="${ROOT_DIR}/logs/dtypefront-frontier-${SLURM_JOB_ID:-manual}" +ROS_BUILD_DIR="${ROOT_DIR}/build/slurm-dtypefront-frontier-rosenbrock" +VODE_BUILD_DIR="${ROOT_DIR}/build/slurm-dtypefront-frontier-vode" +BUILD_JOBS="${SLURM_CPUS_PER_TASK:-7}" + +. "${ROOT_DIR}/scripts/hpc_profiles/frontier.profile" + +GPU_ARCH="${AMREX_AMD_ARCH:-gfx90a}" + +mkdir -p "${LOG_DIR}" + +run_case() { + local exe="$1" + local tag="$2" + shift 2 + + echo "=== $(date -Is) ${tag} ===" + echo "ROCR_VISIBLE_DEVICES=${ROCR_VISIBLE_DEVICES:-unset}" + echo "HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-unset}" + /usr/bin/time -p srun -N 1 -n 1 --gpus-per-task=1 --gpu-bind=closest "${exe}" "${INPUT_FILE}" "$@" \ + > "${LOG_DIR}/${tag}.log" 2>&1 +} + +configure_build() { + local build_dir="$1" + local integrator="$2" + + cmake -S "${ROOT_DIR}" -B "${build_dir}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DDTypeFront_INTEGRATOR="${integrator}" \ + -DAMReX_GPU_BACKEND=HIP \ + -DAMReX_AMD_ARCH="${GPU_ARCH}" \ + -DAMReX_SPACEDIM=3 + cmake --build "${build_dir}" --target DTypeFront -j "${BUILD_JOBS}" +} + +echo "Running on ${HOSTNAME}" +echo "Submit dir: ${ROOT_DIR}" +echo "Log dir: ${LOG_DIR}" +echo "Build jobs: ${BUILD_JOBS}" +echo "AMReX AMD arch: ${GPU_ARCH}" + +configure_build "${ROS_BUILD_DIR}" Rosenbrock +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" ros2s integrator.rosenbrock_tableau=3 +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas3p integrator.rosenbrock_tableau=2 +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas4p integrator.rosenbrock_tableau=1 +run_case "${ROS_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" rodas5p integrator.rosenbrock_tableau=0 + +configure_build "${VODE_BUILD_DIR}" VODE +run_case "${VODE_BUILD_DIR}/src/problems/DTypeFront/DTypeFront" vode + +echo "Done."