remhos changes

experiments/remhos/experiment.py

@@ -35,6 +35,13 @@ class Remhos(
         description="app version",
     )
 
+    variant(
+        "gpu-aware-mpi",
+        default=False,
+        values=(True, False),
+        description="Use GPU-aware MPI",
+    )
+
     maintainers("rfhaque")
 
     def compute_applications_section(self):
@@ -77,8 +84,18 @@ def compute_applications_section(self):
 
         if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"):
             self.add_experiment_variable("n_gpus", "{n_resources}", True)
+            if self.spec.satisfies("+gpu-aware-mpi"):
+                self.add_experiment_variable("gam", "--gpu-aware-mpi")
+            else:
+                self.add_experiment_variable("gam", "--no-gpu-aware-mpi")
         else:
             self.add_experiment_variable("n_ranks", "{n_resources}", True)
 
     def compute_package_section(self):
-        self.add_package_spec(self.name, [f"remhos{self.determine_version()} +metis"])
+        gam = "~gpu-aware-mpi"
+        if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"):
+            if self.spec.satisfies("+gpu-aware-mpi"):
+                gam = "+gpu-aware-mpi"
+        self.add_package_spec(
+            self.name, [f"remhos{self.determine_version()} +metis {gam}"]
+        )

repo/mfem/package.py

@@ -38,7 +38,19 @@ def get_make_config_options(self, spec, prefix):
         def yes_no(varstr):
             return "YES" if varstr in self.spec else "NO"
 
-        options = super().get_make_config_options(spec, prefix)
+        options = [x for x in super().get_make_config_options(spec, prefix) if not x.startswith("UMPIRE_LIB=")]
+
+        if "+umpire" in spec:
+            umpire = spec["umpire"]
+            umpire_libs = umpire.libs
+            if "^camp" in umpire:
+                umpire_libs += umpire["camp"].libs
+            if "^fmt" in umpire:
+                umpire_libs += umpire["fmt"].libs
+            options += [
+                "UMPIRE_LIB=%s" % self.ld_flags_from_library_list(umpire_libs),
+            ]
+
         options.append("MFEM_USE_CALIPER=%s" % yes_no("+caliper"))
         if "+caliper" in self.spec: 
             options.append("CALIPER_DIR=%s" % self.spec["caliper"].prefix)

repo/remhos/application.py

@@ -20,8 +20,30 @@ class Remhos(ExecutableApplication):
             'network-latency-bound','network-collectives','unstructured-grid']
 
     #TODO: add -ms flag once it's implemented
-    executable('2d', 'remhos'+' -dim 2 -epm {epm}'+' -p 14'+' -rs {rs2d}'+' -o 3 -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -d {device}'+' -pa -no-vis', use_mpi=True)
-    executable('3d', 'remhos'+' -dim 3 -epm {epm}'+' -p 10'+' -rs {rs3d}'+' -o 2'+' -dt {dt}'+' -tf {tf}'+' -ho {ho}' ' -lo {lo}'+' -fct {fct}'+' -vs {vs}'+' -ms {ms}'+' -d {device}'+' -pa -no-vis', use_mpi=True)
+    executable('2d', 'remhos' +
+                     ' -dim 2 -epm {epm}' +
+                     ' -p 14' +
+                     ' -rs {rs2d}' +
+                     ' -o 3 -dt {dt} -tf {tf}' +
+                     ' -ho {ho} -lo {lo}' +
+                     ' -fct {fct}' +
+                     ' -vs {vs}' +
+                     ' -ms {ms}' +
+                     ' {gam}' +
+                     ' -d {device}' +
+                     ' -pa -no-vis', use_mpi=True)
+    executable('3d', 'remhos' +
+                     ' -dim 3 -epm {epm}' +
+                     ' -p 10' +
+                     ' -rs {rs3d}' +
+                     ' -o 2 -dt {dt} -tf {tf}' +
+                     ' -ho {ho} -lo {lo}' +
+                     ' -fct {fct}' +
+                     ' -vs {vs}' +
+                     ' -ms {ms}' +
+                     ' {gam}' +
+                     ' -d {device}' +
+                     ' -pa -no-vis', use_mpi=True)
     workload('2d', executables=['2d'])
     workload('3d', executables=['3d'])
 
@@ -76,8 +98,13 @@ class Remhos(ExecutableApplication):
     workload_variable('ms', default='5',
         description='ms',
         workloads=['2d','3d'])
+
     workload_variable('device', default='cpu',
-        description='cpu or cuda',
+        description='cpu, cuda or hip',
+        workloads=['2d','3d'])
+
+    workload_variable('gam', default='--no-gpu-aware-mpi',
+        description='--gpu-aware-mpi or --no-gpu-aware-mpi',
         workloads=['2d','3d'])
 
     figure_of_merit("FOM", log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'FOM:\s+(?P<fom>[0-9]*\.[0-9]*)', group_name='fom', units='megadofs x time steps / second')

repo/remhos/package.py

@@ -30,6 +30,7 @@ class Remhos(MakefilePackage, CudaPackage, ROCmPackage):
 
     variant("metis", default=True, description="Enable/disable METIS support")
     variant("caliper", default=False, description="Enable/disable Caliper support")
+    variant("gpu-aware-mpi", default=False, description="Enable GPU aware MPI")
 
     depends_on("c", type="build")
     depends_on("cxx", type="build")
@@ -62,14 +63,22 @@ class Remhos(MakefilePackage, CudaPackage, ROCmPackage):
         depends_on(f"mfem cuda_arch={arch}", when=f"cuda_arch={arch}")
     depends_on("mfem +cuda+mpi", when="+cuda")
     depends_on("mfem +rocm+mpi", when="+rocm")
+    depends_on("mfem +umpire", when="+cuda")
+    depends_on("mfem +umpire", when="+rocm")
 
-    depends_on("hypre +rocm +mpi", when="+rocm")
+    depends_on("hypre +rocm+mpi", when="+rocm")
     requires("+rocm", when="^hypre+rocm")
     for target in ("none", "gfx803", "gfx900", "gfx906", "gfx908", "gfx90a", "gfx942"):
         depends_on(f"hypre amdgpu_target={target}", when=f"amdgpu_target={target}")
         depends_on(f"mfem amdgpu_target={target}", when=f"amdgpu_target={target}")
 
-    depends_on("hypre+gpu-aware-mpi", when="^cray-mpich+gtl")
+    depends_on("hypre+umpire", when="+cuda")
+    depends_on("hypre+umpire", when="+rocm")
+    depends_on("hypre+gpu-aware-mpi", when="+gpu-aware-mpi")
+
+    def setup_run_environment(self, env):
+        if "+gpu-aware-mpi" in self.spec:
+            env.set("MFEM_GPU_AWARE_MPI", "1")
 
     def setup_build_environment(self, env):
         if "+cuda" in self.spec: