add spack job

side note:
- add cuda 12.2 and do not compile bfloat16 below cuda 12.2
- cuda 11.8.0 starts to support 8.9 and 9.0
- current docker image does not support A770. set up another without docker for A770
- tum does not contain HWLOC yet

Author: yhmtsai

.gitlab-ci.yml

@@ -110,7 +110,6 @@ build/cuda110/nompi/gcc/cuda/release/shared:
     # this is seemingly broken with CUDA 11
     # EXTRA_CMAKE_FLAGS: "-DCMAKE_CUDA_FLAGS=-diag-suppress=177"
 
-
 # nvhpc and friends
 build/nvhpc233/cuda120/nompi/nvcpp/release/static:
   extends:
@@ -149,6 +148,63 @@ build/nvhpc227/cuda117/nompi/nvcpp/debug/shared:
     # disable spurious unused argument warning
     EXTRA_CMAKE_FLAGS: "-DCMAKE_CUDA_FLAGS=-diag-suppress=177"
 
+build/cuda118/nompi/gcc/cuda/debug/shared:
+  extends:
+    - .build_and_test_tum_template
+    - .default_variables
+    - .quick_test_condition
+    - .use_tum-nvidia
+  variables:
+    BUILD_CUDA: "ON"
+    BUILD_HWLOC: "OFF"
+    ENABLE_HALF: "ON"
+    BUILD_TYPE: "Debug"
+    MODULE_LOAD: "cmake/3.18.6 cuda/11.8.0 gcc/7.5.0"
+
+build/cuda120/openmpi/gcc/cuda/release/static:
+  extends:
+    - .build_and_test_tum_template
+    - .default_variables
+    - .quick_test_condition
+    - .use_tum-nvidia
+  variables:
+    BUILD_CUDA: "ON"
+    BUILD_HWLOC: "OFF"
+    ENABLE_HALF: "ON"
+    BUILD_MPI: "ON"
+    BUILD_SHARED_LIBS: "OFF"
+    BUILD_TYPE: "Release"
+    MODULE_LOAD: "cmake/3.18.6 cuda/12.0.1 gcc/12.4.0 openmpi/4.1.6"
+
+build/cuda122/openmpi/gcc/cuda/release/static:
+  extends:
+    - .build_and_test_tum_template
+    - .default_variables
+    - .quick_test_condition
+    - .use_tum-nvidia
+  variables:
+    BUILD_CUDA: "ON"
+    BUILD_HWLOC: "OFF"
+    ENABLE_HALF: "ON"
+    ENABLE_BFLOAT16: "ON"
+    BUILD_SHARED_LIBS: "OFF"
+    BUILD_TYPE: "Release"
+    MODULE_LOAD: "cmake/3.20.6 cuda/12.2.2 gcc/12.4.0 openmpi/4.1.6"
+
+build/cuda126/nompi/gcc/cuda/release/shared:
+  extends:
+    - .build_and_test_tum_template
+    - .default_variables
+    - .quick_test_condition
+    - .use_tum-nvidia
+  variables:
+    BUILD_CUDA: "ON"
+    BUILD_HWLOC: "OFF"
+    ENABLE_HALF: "ON"
+    ENABLE_BFLOAT16: "ON"
+    BUILD_TYPE: "Release"
+    MODULE_LOAD: "cmake/3.30.5 cuda/12.6.2 gcc/13.3.0"
+
 # ROCm 4.5 and friends
 build/amd/nompi/gcc/rocm45/release/shared:
   extends:
@@ -230,6 +286,48 @@ build/amd/nompi/gcc/rocm514_wo_omp/release/shared:
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
 
+build/amd/openmpi/gcc/rocm600/release/static:
+  extends:
+    - .build_and_test_tum_template
+    - .default_variables
+    - .quick_test_condition
+    - .use_tum-amd
+  variables:
+    BUILD_HIP: "ON"
+    BUILD_HWLOC: "OFF"
+    BUILD_MPI: "ON"
+    BUILD_TYPE: "Release"
+    BUILD_SHARED_LIBS: "OFF"
+    MODULE_LOAD: "cmake/3.24.4 rocm/6.0.0 gcc/13.3.0 openmpi/5.0.5"
+
+build/amd/nompi/gcc/rocm573/debug/shared:
+  extends:
+    - .build_and_test_tum_template
+    - .default_variables
+    - .quick_test_condition
+    - .use_tum-amd
+  variables:
+    BUILD_HIP: "ON"
+    BUILD_OMP: "OFF"
+    BUILD_HWLOC: "OFF"
+    BUILD_TYPE: "Debug"
+    ENABLE_BFLOAT16: "ON"
+    ENABLE_HALF: "OFF"
+    MODULE_LOAD: "cmake/3.25.3 rocm/5.7.3 gcc/12.4.0"
+
+build/amd/openmpi/gcc/rocm634_wo_omp/release/shared:
+  extends:
+    - .build_and_test_tum_template
+    - .default_variables
+    - .quick_test_condition
+    - .use_tum-amd
+  variables:
+    BUILD_HIP: "ON"
+    BUILD_HWLOC: "OFF"
+    BUILD_MPI: "ON"
+    BUILD_TYPE: "Release"
+    MODULE_LOAD: "cmake/3.29.6 rocm/6.3.4 gcc/13.3.0 openmpi/5.0.5"
+
 # no cuda but latest gcc and clang
 build/nocuda/nompi/gcc/core/debug/static:
   extends:
@@ -350,13 +448,42 @@ build/nocuda-nomixed/nompi/clang/omp/debug/static:
     BUILD_SHARED_LIBS: "OFF"
     MIXED_PRECISION: "OFF"
 
+build/nogpu/nompi/gcc/omp/debug/static:
+  extends:
+    - .build_and_test_tum_template
+    - .default_variables
+    - .quick_test_condition
+    - .use_tum-cpu
+  variables:
+    BUILD_OMP: "ON"
+    BUILD_TYPE: "Debug"
+    BUILD_SHARED_LIBS: "OFF"
+    MIXED_PRECISION: "OFF"
+    BUILD_HWLOC: "OFF"
+    MODULE_LOAD: "cmake/3.16.9 gcc/7.5.0"
+
+build/nogpu/nompi/gcc/omp/release/static:
+  extends:
+    - .build_and_test_tum_template
+    - .default_variables
+    - .quick_test_condition
+    - .use_tum-cpu
+  variables:
+    BUILD_OMP: "ON"
+    BUILD_TYPE: "Release"
+    BUILD_SHARED_LIBS: "OFF"
+    ENABLE_HALF: "ON"
+    ENABLE_BFLOAT16: "ON"
+    BUILD_HWLOC: "OFF"
+    MODULE_LOAD: "cmake/3.16.9 gcc/9.5.0"
+
 # spack oneapi 2023.1
 build/icpx20231/gpu/release/shared:
   extends:
     - .build_and_test_template
     - .default_variables
     - .quick_test_condition
-    - .use_gko-oneapi20231-igpu
+    - .use_gko-oneapi20231-gpu
   variables:
     CXX_COMPILER: "icpx"
     CXX_FLAGS: "-Wpedantic -ffp-model=precise"
@@ -367,13 +494,65 @@ build/icpx20231/gpu/release/shared:
     ONEAPI_DEVICE_SELECTOR: "opencl:gpu"
     BUILD_HWLOC: "OFF"
 
+build/dpcpp20231/gpu/release/shared:
+  extends:
+    - .build_and_test_tum_template
+    - .default_variables
+    - .quick_test_condition
+    - .use_tum-intel-fp32
+  variables:
+    CXX_COMPILER: "dpcpp"
+    CXX_FLAGS: "-Wpedantic -ffp-model=precise"
+    BUILD_SYCL: "ON"
+    BUILD_TYPE: "Release"
+    BUILD_SHARED_LIBS: "ON"
+    DPCPP_SINGLE_MODE: "ON"
+    MODULE_LOAD: "cmake/3.21.7 intel-oneapi-compilers/2023.1.0 intel-oneapi-dpl/2022.1.0 intel-oneapi-tbb/2021.9.0 intel-oneapi-mkl/2023.1.0"
+    ONEAPI_DEVICE_SELECTOR: "opencl:gpu"
+    BUILD_HWLOC: "OFF"
+
+build/icpx202402/gpu/release/shared:
+  extends:
+    - .build_and_test_tum_template
+    - .default_variables
+    - .quick_test_condition
+    - .use_tum-intel
+  variables:
+    CXX_COMPILER: "icpx"
+    CXX_FLAGS: "-Wpedantic -ffp-model=precise -Wno-deprecated-declarations"
+    BUILD_SYCL: "ON"
+    BUILD_HWLOC: "OFF"
+    BUILD_TYPE: "Release"
+    ENABLE_HALF: "OFF"
+    ENABLE_BFLOAT16: "OFF"
+    ONEAPI_DEVICE_SELECTOR: "opencl:gpu"
+    MODULE_LOAD: "cmake/3.21.7 intel-oneapi-compilers/2024.0.2 intel-oneapi-dpl/2022.3.0 intel-oneapi-tbb/2021.11.0 intel-oneapi-mkl/2024.0.0"
+
+build/icpx202421/gpu/release/shared:
+  extends:
+    - .build_and_test_tum_template
+    - .default_variables
+    - .quick_test_condition
+    - .use_tum-intel
+  variables:
+    CXX_COMPILER: "icpx"
+    CXX_FLAGS: "-Wpedantic -ffp-model=precise -Wno-deprecated-declarations"
+    BUILD_SYCL: "ON"
+    BUILD_OMP: "ON"
+    BUILD_HWLOC: "OFF"
+    BUILD_TYPE: "Release"
+    ENABLE_HALF: "ON"
+    ENABLE_BFLOAT16: "ON"
+    ONEAPI_DEVICE_SELECTOR: "level_zero:gpu"
+    MODULE_LOAD: "cmake/3.28.6 intel-oneapi-compilers/2024.2.1 intel-oneapi-dpl/2022.6.1 intel-oneapi-tbb/2021.13.1 intel-oneapi-mkl/2024.2.1"
+
 # TODO: Enable when debug shared library size issues are fixed
 # build/dpcpp/level_zero_igpu/debug/shared:
 #   extends:
 #     - .build_and_test_template
 #     - .default_variables
 #     - .full_test_condition
-#     - .use_gko-oneapi-igpu
+#     - .use_gko-oneapi-gpu
 #   variables:
 #     CXX_COMPILER: "dpcpp"
 #     CXX_FLAGS: "-Wpedantic -ffp-model=precise"
@@ -389,7 +568,7 @@ build/dpcpp/gpu/release/shared:
     - .build_and_test_template
     - .default_variables
     - .full_test_condition
-    - .use_gko-oneapi-igpu
+    - .use_gko-oneapi-gpu
   variables:
     CXX_COMPILER: "dpcpp"
     CXX_FLAGS: "-Wpedantic -ffp-model=precise"

.gitlab/image.yml

@@ -18,26 +18,26 @@
 .use_gko_cuda110-mvapich-gnu9-llvm9:
   image: ginkgohub/cuda:110-mvapich2-gnu9-llvm9-intel2020
   tags:
-    - tum
     - nvidia-gpus-p100
+    - tum
 
 .use_gko_cuda114-openmpi-gnu10-llvm12:
   image: ginkgohub/cuda:114-openmpi-gnu10-llvm12
   tags:
-    - tum
     - nvidia-gpus-p100
+    - tum
 
 .use_gko_nvhpc233-cuda120-openmpi-gnu12-llvm16:
   image: ginkgohub/nvhpc:233-cuda120-openmpi-gnu12-llvm16
   tags:
-    - tum
     - nvidia-gpus
+    - tum
 
 .use_gko_nvhpc227-cuda117-openmpi-gnu11-llvm14:
   image: ginkgohub/nvhpc:227-cuda117-openmpi-gnu11-llvm14
   tags:
-    - tum
     - nvidia-gpus-p100
+    - tum
 
 .use_gko-rocm45-nompi-gnu8-llvm8:
   image: ginkgohub/rocm:45-mvapich2-gnu8-llvm8
@@ -51,17 +51,49 @@
     - private_ci
     - amd-gpu
 
-.use_gko-oneapi-igpu:
+# the image is focal which seems not to support A770
+.use_gko-oneapi-gpu:
   image: ginkgohub/oneapi:latest
   tags:
     - intel-gpus
+    - tum
 
-.use_gko-oneapi20231-igpu:
+# the image is focal which seems not to support A770
+.use_gko-oneapi20231-gpu:
   image: ginkgohub/spack-oneapi:20231-openmpi
   tags:
     - intel-gpus
+    - tum
 
-.use_gko-oneapi-dgpu:
-  image: ginkgohub/oneapi:latest
+
+# contains "s" so it uses tum now
+.use_tum-nvidia:
+  image: rocky_tum
+  tags:
+    - nvidia-gpus
+    - tum
+
+.use_tum-amd:
+  image: rocky_tum
+  tags:
+    - amd-gpus
+    - tum
+
+.use_tum-intel:
+  image: rocky_tum_intel
   tags:
     - intel-gpus
+    - tum
+
+# This gets the gpu without fp64 native support
+.use_tum-intel-fp32:
+  image: rocky_tum_intel
+  tags:
+    - intel-fp32-gpus
+    - tum
+
+.use_tum-cpu:
+  image: rocky_tum
+  tags:
+    - cpu
+    - tum