Enables building mam4xx on Intel GPUs with SYCL

.github/workflows/at2_gcc-cuda.yml

@@ -68,6 +68,9 @@ jobs:
         build-type: ${{ fromJSON(needs.define_matrix.outputs.build_type) }}
         fp-precision: ${{ fromJSON(needs.define_matrix.outputs.precision) }}
     name: gcc-cuda / ${{ matrix.build-type }} - ${{ matrix.fp-precision }}
+    env:
+      # relative path to nvcc_wrapper from mam4xx root
+      haero_nvcw: "haero_src/ext/ekat/extern/kokkos/bin/nvcc_wrapper"
     steps:
       - name: Check out the repository
         uses: actions/checkout@v4
@@ -79,7 +82,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: eagles-project/haero
-          ref: 017fea932381777f48b2585f86d6ab48fe4b8d09
           submodules: recursive
           path: haero_src
       - name: Show action trigger
@@ -100,49 +102,56 @@ jobs:
                   echo "H100 detected--setting Hopper90 architecture"
                   echo "Hopper=ON" >> $GITHUB_ENV
                   echo "CUDA_ARCH=90" >> $GITHUB_ENV
-                  ARCH=90
+                  echo "CUDA_DEVICE=HOPPER90" >> $GITHUB_ENV
                   ;;
               *"A100"*)
                   echo "A100 detected--setting Ampere80 architecture"
                   echo "Ampere=ON" >> $GITHUB_ENV
                   echo "CUDA_ARCH=80" >> $GITHUB_ENV
+                  echo "CUDA_DEVICE=AMPERE80" >> $GITHUB_ENV
                   ;;
               *"V100"*)
                   echo "V100 detected--setting Volta70 architecture"
                   echo "Volta=ON" >> $GITHUB_ENV
                   echo "CUDA_ARCH=70" >> $GITHUB_ENV
+                  echo "CUDA_DEVICE=VOLTA70" >> $GITHUB_ENV
                   ;;
               *)
                   echo "Unsupported GPU model: $gpu_model"
                   exit 1
                   ;;
           esac
+      - name: Set nvcc_wrapper Arch
+        run: |
+          nvcw="$(pwd)/${haero_nvcw}"
+          sed -i s/default_arch=\"sm_70\"/default_arch=\"sm_"$CUDA_ARCH"\"/g "${nvcw}"
+          echo "===================================="
+          grep -i "default_arch=" "${nvcw}"
       - name: Building Haero (${{ matrix.build-type }}, ${{ matrix.fp-precision }} precision)
         run: |
+          nvcw="$(pwd)/${haero_nvcw}"
           cmake -S haero_src -B haero_build \
             -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
             -DCMAKE_INSTALL_PREFIX="haero_install" \
             -DCMAKE_C_COMPILER=gcc \
-            -DCMAKE_CXX_COMPILER=g++ \
+            -DCMAKE_CXX_COMPILER="${nvcw}" \
             -DHAERO_ENABLE_MPI=OFF \
             -DHAERO_ENABLE_GPU=ON \
-            -DHAERO_PRECISION=${{ matrix.fp-precision }}
+            -DHAERO_PRECISION=${{ matrix.fp-precision }} \
+            -DKokkos_ARCH_$CUDA_DEVICE=ON \
+            -DHAERO_DEVICE_ARCH=$CUDA_DEVICE
           cd haero_build
           make -j
           make install
-      - name: Set nvcc_wrapper Arch
-        run: |
-          sed -i s/default_arch=\"sm_70\"/default_arch=\"sm_"$CUDA_ARCH"\"/g `pwd`/haero_install/bin/nvcc_wrapper
-          echo "===================================="
-          grep -i "default_arch=" `pwd`/haero_install/bin/nvcc_wrapper
       - name: Configuring MAM4xx (${{ matrix.build-type }}, ${{ matrix.fp-precision }} precision)
         run: |
+          nvcw="$(pwd)/${haero_nvcw}"
           cmake -S . -B build \
-            -DCMAKE_CXX_COMPILER=`pwd`/haero_install/bin/nvcc_wrapper \
+            -DCMAKE_CXX_COMPILER="${nvcw}" \
             -DCMAKE_C_COMPILER=gcc \
-            -DCMAKE_INSTALL_PREFIX=`pwd`/install \
+            -DCMAKE_INSTALL_PREFIX=$(pwd)/install \
             -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
-            -DMAM4XX_HAERO_DIR=`pwd`/haero_install \
+            -DMAM4XX_HAERO_DIR=$(pwd)/haero_install \
             -DNUM_VERTICAL_LEVELS=72 \
             -DENABLE_COVERAGE=OFF \
             -DENABLE_SKYWALKER=ON \
@@ -151,7 +160,7 @@ jobs:
       - name: Building MAM4xx (${{ matrix.build-type }}, ${{ matrix.fp-precision }} precision)
         run: |
           cd build
-          make
+          make -j
       - name: Running tests (${{ matrix.build-type }}, ${{ matrix.fp-precision }} precision)
         run: |
           cd build

.github/workflows/gh_gcc-cpu.yml

@@ -54,9 +54,9 @@ jobs:
           prec: ${{ inputs.precision || 'ALL' }}
         run: |
           case ${{ env.prec }} in
-            "Debug")
+            "single")
               echo 'precision=["single"]' >> "$GITHUB_OUTPUT" ;;
-            "Release")
+            "double")
               echo 'precision=["double"]' >> "$GITHUB_OUTPUT" ;;
             "ALL")
               echo 'precision=["single", "double"]' >> "$GITHUB_OUTPUT" ;;
@@ -108,7 +108,6 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: eagles-project/haero
-          ref: 017fea932381777f48b2585f86d6ab48fe4b8d09
           submodules: recursive
           path: haero_src
 
@@ -130,9 +129,9 @@ jobs:
       - name: Configuring MAM4xx (${{ matrix.build-type }}, ${{ matrix.fp-precision }} precision)
         run: |
           cmake -S . -B build \
-            -DCMAKE_INSTALL_PREFIX=`pwd`/install \
+            -DCMAKE_INSTALL_PREFIX=$(pwd)/install \
             -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
-            -DMAM4XX_HAERO_DIR=`pwd`/haero_install \
+            -DMAM4XX_HAERO_DIR=$(pwd)/haero_install \
             -DNUM_VERTICAL_LEVELS=72 \
             -DENABLE_COVERAGE=ON \
             -G "Unix Makefiles"

CMakeLists.txt

@@ -42,7 +42,7 @@ set(CLANG_FORMAT_VERSION 14)
 # Set compilers, linkers, and flags from Haero.
 set(CMAKE_CXX_STANDARD ${HAERO_CXX_STANDARD})
 set(CMAKE_CXX_COMPILER ${HAERO_CXX_COMPILER})
-set(CMAKE_CXX_FLAGS    ${HAERO_CXX_FLAGS})
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${HAERO_CXX_FLAGS}")
 
 set(CMAKE_C_STANDARD ${HAERO_C_STANDARD})
 set(CMAKE_C_COMPILER ${HAERO_C_COMPILER})

README.md

@@ -43,19 +43,38 @@ of parameters. Check out the comments at the top of `build-haero.sh`.
 
 You can build a CPU-capable version of HAERO with some defaults set by typing
 
+```shell
+./build-haero.sh <install-path>
 ```
-./build-haero.sh <path>
+
+The extended syntax to fully configure other types of build is
+
+```shell
+./build-haero.sh <install-path> <device> <precision> <build-type> [device-arch]
+```
+
+in which the options are:
+
+```shell
+device: {cpu, gpu}
+precision: {single, double}
+build-type: {Debug, Release}
 ```
 
-where `<path>` is a directory to which HAERO will be installed. If you'd rather
-install HAERO yourself, you can follow the instructions in the
+and the semi-optional `device-arch` argument must correspond to
+[those accepted by Kokkos](https://kokkos.org/kokkos-core-wiki/get-started/configuration-guide.html#architectures)
+and is likely required for a properly-configured GPU build.
+That is to say, the `device-arch` option **should** be set for GPU builds, and
+building for GPU without that argument is unsupported.
+
+If you'd rather install HAERO yourself, you can follow the instructions in the
 [HAERO repository](https://github.com/eagles-project/haero). Make sure you run
 all the steps, including `make install`.
 
 If you're on a machine that requires modules to get access to compilers, etc,
 use
 ```
-source build-haero.sh <path>
+source build-haero.sh <path> [...]
 ```
 to make sure your environment is updated.
 

build-haero.sh

@@ -3,7 +3,7 @@
 # This script builds the Haero high-performance aerosool toolkit with specific
 # settings so mam4xx can be linked against it. Run it like this:
 #
-# `./build-haero.sh <prefix> <device> <precision> <build_type>
+# `./build-haero.sh <prefix> <device> <precision> <build_type> [gpu_type] [gpu_arch]
 #
 # where
 # * <prefix> is the installation prefix (e.g. /usr/local) in which you
@@ -14,6 +14,13 @@
 #   floating point numbers used in Haero. Default: double
 # * <build_type> (either `Debug` or `Release`) determines whether Haero is built
 #   optimized or for debugging. Default: Debug
+# OPTIONAL Args (REQUIRED for GPU builds)
+# * [gpu_type] (`nvidia`, `amd`, `intel`) this is the "brand" of GPU
+# * [gpu_arch] (e.g., HOPPER90, AMD_GFX90A, INTEL_PVC) this is the "model" of
+#              GPU, indicating the generation, compute capability, etc.
+#   * A list of many options is found in $HAERO_ROOT/setup
+#   * The full list of options may be found on the kokkos docs site:
+# https://kokkos.org/kokkos-core-wiki/get-started/configuration-guide.html#architectures
 #
 # NOTE: This script disables MPI, since the mam4xx team is focused on single-
 # NOTE: node parallelism. If you need an MPI-enabled build of Haero, please
@@ -23,20 +30,14 @@ PREFIX=$1
 DEVICE=$2
 PRECISION=$3
 BUILD_TYPE=$4
-# Turn off search for yaml libraries. EKAT will build yaml-cpp from submodules. 
+GPU_TYPE=$5
+DEVICE_ARCH=$6
+# Turn off search for yaml libraries. EKAT will build yaml-cpp from submodules.
 SKIP_FIND_YAML_CPP=ON
 
-# Default compilers (can be overridden by environment variables)
-if [[ -z $CC ]]; then
-  CC=cc
-fi
-if [[ -z $CXX ]]; then
-  CXX=c++
-fi
-
 if [[ "$PREFIX" == "" ]]; then
   echo "Haero installation prefix was not specified!"
-  echo "Usage: $0 <prefix> <device> <precision> <build_type>"
+  echo "Usage: $0 <prefix> <device> <precision> <build_type> [gpu_type] [gpu_arch]"
   exit
 fi
 
@@ -64,7 +65,8 @@ if [[ "$PRECISION" != "single" && "$PRECISION" != "double" ]]; then
   exit
 fi
 if [[ "$BUILD_TYPE" != "Debug" && "$BUILD_TYPE" != "Release" ]]; then
-  echo "Invalid optimization specified: $BUILD_TYPE (must be Debug or Release)"
+  echo "Invalid optimization specified: $BUILD_TYPE"
+  echo "Must be Debug or Release (case-sensitive)"
   exit
 fi
 
@@ -77,32 +79,103 @@ git clone [email protected]:eagles-project/haero.git .haero || exit
 cd .haero || exit
 git submodule update --init --recursive || exit
 
-# Are we on a special machine?
-cd machines
-echo $(pwd)
-for MACHINE_FILE in $(ls)
-do
-  MACHINE=${MACHINE_FILE/\.sh/}
-  echo $MACHINE
-  echo `hostname` | grep -q "$MACHINE" 
-  host_match=$?
-  echo $SYSTEM_NAME | grep -q "$MACHINE"
-  sys_match=$?
-  if  [ $host_match -eq 0 ] || [ $sys_match -eq 0 ]; then
-    echo "Found machine file $MACHINE_FILE. Setting up environment for $MACHINE..."
-    source ./$MACHINE.sh
-  fi
-done
+cd ..
 
-cd ../..
-
-# Configure Haero with the given selections.
+# ==============================================================================
+# below are the default compiler choices for gpu builds and are set based on
+# the command line args.
+# ==============================================================================
 if [[ "$DEVICE" == "gpu" ]]; then
   ENABLE_GPU=ON
+  if [[ "$GPU_TYPE" == "" ]]; then
+    echo "ERROR: GPU_TYPE not provided for GPU build."
+    echo "Must be nvidia, amd, or intel (case-sensitive)"
+    exit
+  elif [[ "$GPU_TYPE" != "nvidia" && "$GPU_TYPE" != "amd" && "$GPU_TYPE" != "intel" ]]; then
+    echo "Device provided as 'gpu', but invalid GPU_TYPE specified: ${GPU_TYPE}."
+    echo "Must be nvidia, amd, or intel (case-sensitive)"
+    exit
+  fi
+  if [[ "$DEVICE_ARCH" == "" ]]; then
+    echo "ERROR: DEVICE_ARCH not provided for GPU build."
+    exit
+  fi
+  # ==============================================================================
+  # NVIDIA GPU + gcc
+  # NOTE: if CXX is set to nvcc_wrapper, then this must be the same path used
+  # in the `sed` command below
+  # This happens by default via the $nvcw variable and should typically work
+  # unless your environment differs from the default
+  # ==============================================================================
+  if [[ "$GPU_TYPE" == "nvidia" ]]; then
+    echo "GPU_TYPE given as nvidia (${DEVICE_ARCH})--setting default compilers."
+    echo "and modifying nvcc_wrapper with correct architecture flag."
+    CXX="$(pwd)/.haero/ext/ekat/extern/kokkos/bin/nvcc_wrapper"
+    nvcw=$CXX
+    CC=gcc
+    echo "C++ compiler: ${CXX}"
+    echo "C compiler: ${CC}"
+    # ==========================================================================
+    if [[ -x "$nvcw" ]]; then
+      # FIXME: this will not work for compute capbility 10.0
+      CUDA_GEN=${DEVICE_ARCH:(-2)}
+      # FIXME: this assumes a default value in nvcc_wrapper
+      sed -i s/default_arch=\"sm_70\"/default_arch=\"sm_"$CUDA_GEN"\"/g "${nvcw}"
+      echo "===================================================================="
+      echo "nvcc_wrapper modified--verify that default_arch=sm_${CUDA_GEN}"
+      echo "===================================================================="
+      grep -i "default_arch=" "${nvcw}"
+      echo "===================================================================="
+    else
+      echo "ERROR: nvcc_wrapper not found at expected location for nvidia gpu build."
+      exit
+    fi
+# ==============================================================================
+# AMD GPU + HIP and clang
+# ==============================================================================
+  elif [[ "$GPU_TYPE" == "amd" ]]; then
+    echo "GPU_TYPE given as amd--setting default compilers."
+    CXX=hipcc
+    CC=amdclang
+    echo "C++ compiler: ${CXX}"
+    echo "C compiler: ${CC}"
+# ==============================================================================
+# Intel GPU + intel compilers
+# ==============================================================================
+  elif [[ "$GPU_TYPE" == "intel" ]]; then
+    echo "GPU_TYPE given as intel--setting default compilers."
+    CXX=icpx
+    CC=icx
+    echo "C++ compiler: ${CXX}"
+    echo "C compiler: ${CC}"
+# ==============================================================================
+  else
+    echo "Device provided as 'gpu', but invalid GPU_TYPE specified: ${GPU_TYPE}."
+    echo "Must be nvidia, amd, or intel (case-sensitive)"
+    echo "More importantly... how did we get here?"
+    exit
+  fi
 else
   ENABLE_GPU=OFF
 fi
 
+if [[ "$DEVICE" == "cpu" ]]; then
+  # Default cpu compilers (can be overridden by environment variables)
+  echo "Setting compilers for CPU device."
+  if [[ -z $CC ]]; then
+    echo "'CC' environment variable not found--setting to default (gcc)"
+    CC=gcc
+  fi
+  if [[ -z $CXX ]]; then
+    echo "'CXX' environment variable not found--setting to default (g++)"
+    CXX=g++
+  fi
+fi
+
+# ==============================================================================
+#     DON'T CHANGE ANYTHING BELOW HERE UNLESS YOU KNOW WHAT YOU'RE DOING
+# ==============================================================================
+
 echo "Configuring Haero with the given selections (WITHOUT MPI)..."
 cmake -S ./.haero -B ./.haero/build \
   -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
@@ -113,11 +186,13 @@ cmake -S ./.haero -B ./.haero/build \
   -DHAERO_ENABLE_MPI=OFF \
   -DHAERO_ENABLE_GPU=$ENABLE_GPU \
   -DHAERO_PRECISION=$PRECISION \
+  -DKokkos_ARCH_$DEVICE_ARCH:BOOL=ON \
+  -DHAERO_DEVICE_ARCH=$DEVICE_ARCH \
   || exit
 
 echo "Building and installing Haero in $PREFIX..."
 cd .haero/build || exit
-make -j8 install
+make -j8 install || exit
 
 cd ../../
 echo "Haero has been installed in $PREFIX. Set HAERO_DIR to this directory in"

setup

@@ -99,7 +99,6 @@ cmake \
  -DMAM4XX_HAERO_DIR=\$HAERO_DIR \
  -DNUM_VERTICAL_LEVELS=\$NUM_VERTICAL_LEVELS \
  -DENABLE_SKYWALKER=ON \
- -DCMAKE_CUDA_ARCHITECTURES=80 \
  \$OPTIONS \
  -G "\$GENERATOR" \
  \$SOURCE_DIR
@@ -120,4 +119,3 @@ echo "  1. cd $1"
 echo "  2. Edit config.sh"
 echo "  3. ./config.sh"
 echo "  4. Build using 'make -j'."
-