Crusher: Use GPU buffers in MPI.

Now that Lustre is active on Crusher, try GPU buffers again.

Temporarily disable ADIOS because there appears to be a conflict between the
current ADIOS libs and module craype-accel-amd-gfx90a that manifests as a link
error in the SCORPIO build phase.

Author: ambrad

cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake

@@ -7,7 +7,7 @@ endif()
 
 string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf")
 if (NOT MPILIB STREQUAL mpi-serial)
-  string(APPEND SLIBS " -L$ENV{ADIOS2_DIR}/lib64 -ladios2_c_mpi -ladios2_c -ladios2_core_mpi -ladios2_core -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_atl -ladios2_enet")
+  #string(APPEND SLIBS " -L$ENV{ADIOS2_DIR}/lib64 -ladios2_c_mpi -ladios2_c -ladios2_core_mpi -ladios2_core -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_atl -ladios2_enet")
 endif()
 set(NETCDF_PATH "$ENV{NETCDF_DIR}")
 set(PNETCDF_PATH "$ENV{PNETCDF_DIR}")
@@ -32,4 +32,4 @@ endif()
 
 string(APPEND CPPDEFS " -DCPRCRAY")
 
-set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "See SCREAM issue #2080.")
+#set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "See SCREAM issue #2080.")

cime_config/machines/config_machines.xml

@@ -855,7 +855,7 @@
         <command name="reset"></command>
         <command name="load">PrgEnv-cray</command>
         <!-- See SCREAM issue #2080. -->
-        <!--command name="load">craype-accel-amd-gfx90a</command-->
+        <command name="load">craype-accel-amd-gfx90a</command>
         <command name="load">rocm/5.1.0</command>
       </modules>
       <modules>
@@ -879,7 +879,7 @@
       <env name="PNETCDF_PATH">$ENV{PNETCDF_DIR}</env>
       <env name="MPIR_CVAR_GPU_EAGER_DEVICE_MEM">0</env>
       <!-- See SCREAM issue #2080. -->
-      <env name="MPICH_GPU_SUPPORT_ENABLED">0</env>
+      <env name="MPICH_GPU_SUPPORT_ENABLED">1</env>
       <env name="PNETCDF_HINTS">romio_cb_read=disable</env>
     </environment_variables>
 
@@ -888,9 +888,9 @@
       <env name="OMP_PROC_BIND">spread</env>
       <env name="OMP_PLACES">threads</env>
     </environment_variables>
-    <environment_variables compiler="crayclang-scream" mpilib="mpich">
+    <!--environment_variables compiler="crayclang-scream" mpilib="mpich">
       <env name="ADIOS2_DIR">/lustre/orion/cli133/world-shared/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.17/crayclang-14.0.2</env>
-    </environment_variables>
+    </environment_variables-->
   </machine>