[WIP] Mpi-Pingpong Benchmark

.gitlab/tests/shared_slurm_clusters.yml

@@ -94,7 +94,7 @@ run_tests_slurm_dane:
         VARIANT: [+openmp, '']
       - HOST: dane
         ARCHCONFIG: llnl-cluster
-        BENCHMARK: [laghos, lammps, raja-perf, phloem]
+        BENCHMARK: [laghos, lammps, raja-perf, phloem, mpi-pingpong]
       # Strong scaling runs for "benchpark analyze" test
       - HOST: dane
         ARCHCONFIG: llnl-cluster

experiments/mpi-pingpong/experiment.py

@@ -0,0 +1,45 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from benchpark.directives import variant, maintainers
+from benchpark.experiment import Experiment
+from benchpark.mpi import MpiOnlyExperiment
+from benchpark.caliper import Caliper
+from benchpark.rocm import ROCmExperiment
+from benchpark.cuda import CudaExperiment
+
+
+class MpiPingpong(Experiment, MpiOnlyExperiment, ROCmExperiment, CudaExperiment, Caliper):
+
+    variant(
+        "workload",
+        default="run",
+    )
+
+    maintainers("stephanielam3211")
+
+    def compute_applications_section(self):
+        n_nodes = 64  # max number of nodes
+        expr_vars = {
+            "n_nodes": n_nodes,
+            "iterations": 10000,
+            "msg_size": 16,
+        }
+
+        for pk, pv in expr_vars.items():
+            self.add_experiment_variable(pk, pv, True)
+
+        self.add_experiment_variable("n_ranks", "{n_nodes}*{sys_cores_per_node}", True)
+
+        self.add_experiment_variable("partner_rank", "{n_ranks}-1", True)
+
+        self.set_required_variables(
+            n_resources="{n_ranks}",
+            process_problem_size="{msg_size}",
+            total_problem_size="{msg_size}",
+        )
+
+    def compute_package_section(self):
+        self.add_package_spec(self.name, ["mpi-pingpong@develop"])

repo/mpi-pingpong/application.py

@@ -0,0 +1,19 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+
+from ramble.appkit import *
+
+
+class MpiPingpong(ExecutableApplication):
+    name = "mpi-pingpong"
+
+    tags = ['single-node', 'builtin-caliper']
+
+
+    executable("pingpong", "pingpong -i {iterations} -m {msg_size} -p 0,{partner_rank} -n {n_nodes} -s {sys_cores_per_node}//{sys_sockets_per_node} -c {sys_cores_per_node} -b {experiment_run_dir}/{experiment_name}_metadata.json", use_mpi=True)
+
+    workload("run", executables=["pingpong"])

repo/mpi-pingpong/package.py

@@ -0,0 +1,67 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from spack.package import *
+
+
+class MpiPingpong(CMakePackage, ROCmPackage, CudaPackage):
+
+    git = "https://github.com/LLNL/microbenchmarks.git"
+
+    version("develop", branch="develop")
+
+    variant("caliper", default=False, description="Enable Caliper/Adiak support")
+    variant("rocm", default=False, description="Enable Rocm support")
+    variant("cuda", default=False, description="Enable CUDA support")
+    variant("mpi", default=True, description="Enable MPI support")
+
+    depends_on("mpi", when="+mpi")
+    depends_on("caliper", when="+caliper")
+    depends_on("adiak", when="+caliper")
+    depends_on("hip", when="+rocm")
+    depends_on("cuda", when="+cuda")
+
+    conflicts("+cuda +rocm", msg="Enable only one of +cuda or +rocm")
+
+    root_cmakelists_dir = "repo/pingpong"
+
+    def cmake_args(self):
+        if self.spec.satisfies("+caliper"):
+            args = [
+                "-DUSE_CALIPER=ON"
+            ]
+        else:
+            args = [
+                "-DUSE_CALIPER=OFF"
+            ]
+
+        if self.spec.satisfies("+rocm"):
+            args.append(self.define("CMAKE_EXE_LINKER_FLAGS", self.spec['mpi'].libs.ld_flags))
+            args.append('-DUSE_HIP=ON')
+            args.append(self.define("ROCM_PATH", self.spec["hip"].prefix))
+            hip_vars = self.spec["hip"].variants
+            if "amdgpu_targets" in hip_vars:
+                vals = [t for t in hip_vars["amdgpu_targets"].value if t != "none"]
+                if vals:
+                    archs = ",".join(vals)
+                    args.append(self.define("CMAKE_HIP_ARCHITECTURES", archs))
+        else:
+            args.append('-DUSE_HIP=OFF')
+
+        if self.spec.satisfies("+cuda"):
+            args.append(self.define("USE_CUDA", "ON"))
+            v = self.spec.variants.get("cuda_arch", None)
+            if v:
+                vals = [a for a in v.value if a != "none"]
+                if vals:
+                    args.append(self.define("CMAKE_CUDA_ARCHITECTURES", ";".join(vals)))
+        else:
+            args.append(self.define("USE_CUDA", "OFF"))
+
+        return args
+
+    def install(self, spec, prefix):
+        mkdirp(prefix.bin)
+        install(join_path(self.build_directory, "pingpong"), prefix.bin)

systems/llnl-cluster/system.py

@@ -46,6 +46,7 @@ class LlnlCluster(System):
         "dane": {
             "sys_cores_per_node": 112,
             "sys_cores_os_reserved_per_node": 0,  # No explicit core reservation, first thread on each core reserved (2 threads per core)
+            "sys_sockets_per_node": 2,
             "sys_cores_os_reserved_per_node_list": None,
             "sys_mem_per_node_GB": 256,
             "sys_cpu_mem_per_node_MB": 210,

systems/llnl-elcapitan/system.py

@@ -28,6 +28,7 @@ class LlnlElcapitan(System):
             "sys_cores_per_node": 56,
             "sys_cores_os_reserved_per_node": 8,
             "sys_cores_os_reserved_per_node_list": [0, 8, 16, 24, 32, 40, 48, 56],
+            "sys_sockets_per_node": 2,
             "sys_gpus_per_node": 8,
             "sys_mem_per_node_GB": 512,
             "sys_cpu_mem_per_node_MB": 2048,

systems/llnl-sierra/system.py

@@ -33,6 +33,7 @@ class LlnlSierra(System):
                 22,
                 23,
             ],  # First two cores on each socket reserved.
+            "sys_sockets_per_node": 2,
             "sys_gpus_per_node": 4,
             "sys_mem_per_node_GB": 256,
             "sys_cpu_mem_per_node_MB": 40,