Switch nvidia nightly testing to use CUDA 12.8 (#27822)

Switches nightly testing to use CUDA 12.8

[Reviewed by @e-kayrakli]

Author: jabraham17

util/cron/common-gpu-nvidia-hpe-cray-ex-cuda-12.bash

@@ -0,0 +1,6 @@
+# common settings for running GPU nightly testing on the HPE Cray EX system with CUDA 12
+
+# We need 12.8 for the stream test because the CUDA driver on pinoak
+# only supports PTX for 12.8, until the driver is updated, we need to
+# stick with 12.8 instead of 12.9
+module load cuda/12.8  # default is CUDA 12.9

util/cron/test-gpu-ex-cuda-12.bash

@@ -6,8 +6,7 @@ UTIL_CRON_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
 source $UTIL_CRON_DIR/common-native-gpu.bash
 source $UTIL_CRON_DIR/common-hpe-cray-ex.bash
 source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex.bash
-
-module load cuda/12.4
+source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex-cuda-12.bash
 
 export CHPL_COMM=none
 export CHPL_GPU=nvidia  # amd is also detected automatically

util/cron/test-gpu-ex-cuda-12.colocales.bash

@@ -7,12 +7,11 @@ source $UTIL_CRON_DIR/common-native-gpu.bash
 source $UTIL_CRON_DIR/common-hpe-cray-ex.bash
 source $UTIL_CRON_DIR/common-ofi.bash
 source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex.bash
+source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex-cuda-12.bash
 
 export SLURM_NETWORK=single_node_vni
 export CHPL_RT_LOCALES_PER_NODE=2
-module load cuda/12.4
 
-export CHPL_GPU=nvidia
 export CHPL_NIGHTLY_TEST_DIRS="gpu/native/multiLocale"
 
 export CHPL_NIGHTLY_TEST_CONFIG_NAME="gpu-ex-cuda-12.colocales"

util/cron/test-gpu-ex-cuda-12.interop.bash

@@ -6,12 +6,7 @@ UTIL_CRON_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
 source $UTIL_CRON_DIR/common.bash
 source $UTIL_CRON_DIR/common-hpe-cray-ex.bash
 source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex.bash
-
-
-# We need 12.4 for the stream test because the CUDA driver on pinoak
-# only supports PTX for 12.4, until the driver is updated, we need to
-# stick with 12.4 instead of 12.5
-module load cuda/12.4  # default is CUDA 12.5
+source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex-cuda-12.bash
 
 # We need cublas for the cublas interop test, but since we are using 12.4 above
 # pinoak doesn't have the cublas library for 12.4, so we need to use the cublas

util/cron/test-gpu-ex-cuda-12.ofi.bash

@@ -6,8 +6,7 @@ UTIL_CRON_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
 source $UTIL_CRON_DIR/common-native-gpu.bash
 source $UTIL_CRON_DIR/common-hpe-cray-ex.bash
 source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex.bash
-
-module load cuda/12.4
+source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex-cuda-12.bash
 
 export CHPL_COMM=ofi
 export CHPL_GPU=nvidia  # amd is also detected automatically

util/cron/test-gpu-ex-cuda-12.specialization.bash

@@ -7,8 +7,7 @@ source $UTIL_CRON_DIR/common-native-gpu.bash
 source $UTIL_CRON_DIR/common-hpe-cray-ex.bash
 source $UTIL_CRON_DIR/common-gpu-hpe-cray-ex.bash
 source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex.bash
-
-module load cuda/12.4
+source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex-cuda-12.bash
 
 export CHPL_COMM=none
 export CHPL_GPU=nvidia  # amd is also detected automatically

util/cron/test-perf.gpu-ex-cuda-12.bash

@@ -6,8 +6,7 @@ UTIL_CRON_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
 source $UTIL_CRON_DIR/common-native-gpu.bash
 source $UTIL_CRON_DIR/common-hpe-cray-ex.bash
 source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex.bash
-
-module load cuda/12.4
+source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex-cuda-12.bash
 
 export CHPL_COMM=none
 export CHPL_GPU=nvidia  # amd is detected automatically

util/cron/test-perf.gpu-ex-cuda-12.um.bash

@@ -6,8 +6,7 @@ UTIL_CRON_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) ; pwd)
 source $UTIL_CRON_DIR/common-native-gpu.bash
 source $UTIL_CRON_DIR/common-hpe-cray-ex.bash
 source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex.bash
-
-module load cuda/12.4
+source $UTIL_CRON_DIR/common-gpu-nvidia-hpe-cray-ex-cuda-12.bash
 
 export CHPL_COMM=none
 export CHPL_GPU=nvidia  # amd is detected automatically