Enables building mam4xx on Intel GPUs with SYCL

.github/workflows/at2_gcc-cuda.yml

@@ -68,6 +68,9 @@ jobs:
         build-type: ${{ fromJSON(needs.define_matrix.outputs.build_type) }}
         fp-precision: ${{ fromJSON(needs.define_matrix.outputs.precision) }}
     name: gcc-cuda / ${{ matrix.build-type }} - ${{ matrix.fp-precision }}
+    env:
+      # relative path to nvcc_wrapper from mam4xx root
+      haero_nvcw: "haero_src/ext/ekat/extern/kokkos/bin/nvcc_wrapper"
     steps:
       - name: Check out the repository
         uses: actions/checkout@v4
@@ -79,7 +82,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: eagles-project/haero
-          ref: 017fea932381777f48b2585f86d6ab48fe4b8d09
           submodules: recursive
           path: haero_src
       - name: Show action trigger
@@ -100,49 +102,56 @@ jobs:
                   echo "H100 detected--setting Hopper90 architecture"
                   echo "Hopper=ON" >> $GITHUB_ENV
                   echo "CUDA_ARCH=90" >> $GITHUB_ENV
-                  ARCH=90
+                  echo "CUDA_DEVICE=HOPPER90" >> $GITHUB_ENV
                   ;;
               *"A100"*)
                   echo "A100 detected--setting Ampere80 architecture"
                   echo "Ampere=ON" >> $GITHUB_ENV
                   echo "CUDA_ARCH=80" >> $GITHUB_ENV
+                  echo "CUDA_DEVICE=AMPERE80" >> $GITHUB_ENV
                   ;;
               *"V100"*)
                   echo "V100 detected--setting Volta70 architecture"
                   echo "Volta=ON" >> $GITHUB_ENV
                   echo "CUDA_ARCH=70" >> $GITHUB_ENV
+                  echo "CUDA_DEVICE=VOLTA70" >> $GITHUB_ENV
                   ;;
               *)
                   echo "Unsupported GPU model: $gpu_model"
                   exit 1
                   ;;
           esac
+      - name: Set nvcc_wrapper Arch
+        run: |
+          nvcw="$(pwd)/${haero_nvcw}"
+          sed -i s/default_arch=\"sm_70\"/default_arch=\"sm_"$CUDA_ARCH"\"/g "${nvcw}"
+          echo "===================================="
+          grep -i "default_arch=" "${nvcw}"
       - name: Building Haero (${{ matrix.build-type }}, ${{ matrix.fp-precision }} precision)
         run: |
+          nvcw="$(pwd)/${haero_nvcw}"
           cmake -S haero_src -B haero_build \
             -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
             -DCMAKE_INSTALL_PREFIX="haero_install" \
             -DCMAKE_C_COMPILER=gcc \
-            -DCMAKE_CXX_COMPILER=g++ \
+            -DCMAKE_CXX_COMPILER="${nvcw}" \
             -DHAERO_ENABLE_MPI=OFF \
             -DHAERO_ENABLE_GPU=ON \
-            -DHAERO_PRECISION=${{ matrix.fp-precision }}
+            -DHAERO_PRECISION=${{ matrix.fp-precision }} \
+            -DKokkos_ARCH_$CUDA_DEVICE=ON \
+            -DHAERO_DEVICE_ARCH=$CUDA_DEVICE
           cd haero_build
           make -j
           make install
-      - name: Set nvcc_wrapper Arch
-        run: |
-          sed -i s/default_arch=\"sm_70\"/default_arch=\"sm_"$CUDA_ARCH"\"/g `pwd`/haero_install/bin/nvcc_wrapper
-          echo "===================================="
-          grep -i "default_arch=" `pwd`/haero_install/bin/nvcc_wrapper
       - name: Configuring MAM4xx (${{ matrix.build-type }}, ${{ matrix.fp-precision }} precision)
         run: |
+          nvcw="$(pwd)/${haero_nvcw}"
           cmake -S . -B build \
-            -DCMAKE_CXX_COMPILER=`pwd`/haero_install/bin/nvcc_wrapper \
+            -DCMAKE_CXX_COMPILER="${nvcw}" \
             -DCMAKE_C_COMPILER=gcc \
-            -DCMAKE_INSTALL_PREFIX=`pwd`/install \
+            -DCMAKE_INSTALL_PREFIX=$(pwd)/install \
             -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
-            -DMAM4XX_HAERO_DIR=`pwd`/haero_install \
+            -DMAM4XX_HAERO_DIR=$(pwd)/haero_install \
             -DNUM_VERTICAL_LEVELS=72 \
             -DENABLE_COVERAGE=OFF \
             -DENABLE_SKYWALKER=ON \
@@ -151,7 +160,7 @@ jobs:
       - name: Building MAM4xx (${{ matrix.build-type }}, ${{ matrix.fp-precision }} precision)
         run: |
           cd build
-          make
+          make -j
       - name: Running tests (${{ matrix.build-type }}, ${{ matrix.fp-precision }} precision)
         run: |
           cd build

.github/workflows/gh_gcc-cpu.yml

@@ -54,9 +54,9 @@ jobs:
           prec: ${{ inputs.precision || 'ALL' }}
         run: |
           case ${{ env.prec }} in
-            "Debug")
+            "single")
               echo 'precision=["single"]' >> "$GITHUB_OUTPUT" ;;
-            "Release")
+            "double")
               echo 'precision=["double"]' >> "$GITHUB_OUTPUT" ;;
             "ALL")
               echo 'precision=["single", "double"]' >> "$GITHUB_OUTPUT" ;;
@@ -108,7 +108,6 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: eagles-project/haero
-          ref: 017fea932381777f48b2585f86d6ab48fe4b8d09
           submodules: recursive
           path: haero_src
 
@@ -130,9 +129,9 @@ jobs:
       - name: Configuring MAM4xx (${{ matrix.build-type }}, ${{ matrix.fp-precision }} precision)
         run: |
           cmake -S . -B build \
-            -DCMAKE_INSTALL_PREFIX=`pwd`/install \
+            -DCMAKE_INSTALL_PREFIX=$(pwd)/install \
             -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
-            -DMAM4XX_HAERO_DIR=`pwd`/haero_install \
+            -DMAM4XX_HAERO_DIR=$(pwd)/haero_install \
             -DNUM_VERTICAL_LEVELS=72 \
             -DENABLE_COVERAGE=ON \
             -G "Unix Makefiles"

CMakeLists.txt

@@ -42,7 +42,7 @@ set(CLANG_FORMAT_VERSION 14)
 # Set compilers, linkers, and flags from Haero.
 set(CMAKE_CXX_STANDARD ${HAERO_CXX_STANDARD})
 set(CMAKE_CXX_COMPILER ${HAERO_CXX_COMPILER})
-set(CMAKE_CXX_FLAGS    ${HAERO_CXX_FLAGS})
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${HAERO_CXX_FLAGS}")
 
 set(CMAKE_C_STANDARD ${HAERO_C_STANDARD})
 set(CMAKE_C_COMPILER ${HAERO_C_COMPILER})

README.md

@@ -43,19 +43,38 @@ of parameters. Check out the comments at the top of `build-haero.sh`.
 
 You can build a CPU-capable version of HAERO with some defaults set by typing
 
+```shell
+./build-haero.sh <install-path>
 ```
-./build-haero.sh <path>
+
+The extended syntax to fully configure other types of build is
+
+```shell
+./build-haero.sh <install-path> <device> <precision> <build-type> [device-arch]
+```
+
+in which the options are:
+
+```shell
+device: {cpu, gpu}
+precision: {single, double}
+build-type: {Debug, Release}
 ```
 
-where `<path>` is a directory to which HAERO will be installed. If you'd rather
-install HAERO yourself, you can follow the instructions in the
+and the semi-optional `device-arch` argument must correspond to
+[those accepted by Kokkos](https://kokkos.org/kokkos-core-wiki/get-started/configuration-guide.html#architectures)
+and is likely required for a properly-configured GPU build.
+That is to say, the `device-arch` option **should** be set for GPU builds, and
+building for GPU without that argument is unsupported.
+
+If you'd rather install HAERO yourself, you can follow the instructions in the
 [HAERO repository](https://github.com/eagles-project/haero). Make sure you run
 all the steps, including `make install`.
 
 If you're on a machine that requires modules to get access to compilers, etc,
 use
 ```
-source build-haero.sh <path>
+source build-haero.sh <path> [...]
 ```
 to make sure your environment is updated.
 

build-haero.sh

@@ -23,17 +23,10 @@ PREFIX=$1
 DEVICE=$2
 PRECISION=$3
 BUILD_TYPE=$4
-# Turn off search for yaml libraries. EKAT will build yaml-cpp from submodules. 
+DEVICE_ARCH=$5
+# Turn off search for yaml libraries. EKAT will build yaml-cpp from submodules.
 SKIP_FIND_YAML_CPP=ON
 
-# Default compilers (can be overridden by environment variables)
-if [[ -z $CC ]]; then
-  CC=cc
-fi
-if [[ -z $CXX ]]; then
-  CXX=c++
-fi
-
 if [[ "$PREFIX" == "" ]]; then
   echo "Haero installation prefix was not specified!"
   echo "Usage: $0 <prefix> <device> <precision> <build_type>"
@@ -75,34 +68,51 @@ fi
 echo "Cloning Haero repository into $(pwd)/.haero..."
 git clone [email protected]:eagles-project/haero.git .haero || exit
 cd .haero || exit
+git checkout main
 git submodule update --init --recursive || exit
 
-# Are we on a special machine?
-cd machines
-echo $(pwd)
-for MACHINE_FILE in $(ls)
-do
-  MACHINE=${MACHINE_FILE/\.sh/}
-  echo $MACHINE
-  echo `hostname` | grep -q "$MACHINE" 
-  host_match=$?
-  echo $SYSTEM_NAME | grep -q "$MACHINE"
-  sys_match=$?
-  if  [ $host_match -eq 0 ] || [ $sys_match -eq 0 ]; then
-    echo "Found machine file $MACHINE_FILE. Setting up environment for $MACHINE..."
-    source ./$MACHINE.sh
-  fi
-done
-
-cd ../..
-
-# Configure Haero with the given selections.
-if [[ "$DEVICE" == "gpu" ]]; then
+cd ..
+
+# these are (at least close to) the standard compiler choices for gpu builds
+# to use one, uncomment or replace in the if [] blocks below
+
+# NVIDIA GPU + gcc
+# NOTE: if CXX is set to nvcc_wrapper, then this must be the same path used
+# in the `sed` command below
+# This happens by default via the $nvcw variable
+CXX="$(pwd)/.haero/ext/ekat/extern/kokkos/bin/nvcc_wrapper"
+nvcw=$CXX
+CC=gcc
+
+if [[ "$DEVICE" == "gpu" && ! -z "$nvcw" ]]; then
   ENABLE_GPU=ON
+  # nvcw="$(pwd)/.haero/ext/ekat/extern/kokkos/bin/nvcc_wrapper"
+  CUDA_GEN=${DEVICE_ARCH:(-2)}
+  # FIXME: this assumes a default value in nvcc_wrapper
+  sed -i s/default_arch=\"sm_70\"/default_arch=\"sm_"$CUDA_GEN"\"/g "${nvcw}"
+  echo "===================================="
+  grep -i "default_arch=" "${nvcw}"
+  echo "===================================="
 else
   ENABLE_GPU=OFF
 fi
 
+# AMD GPU
+# CXX=hipcc
+# CC=amdclang
+
+# Intel GPU
+# CXX=icpx
+# CC=icx
+
+# Default compilers (can be overridden by environment variables)
+if [[ -z $CC ]]; then
+  CC=cc
+fi
+if [[ -z $CXX ]]; then
+  CXX=c++
+fi
+
 echo "Configuring Haero with the given selections (WITHOUT MPI)..."
 cmake -S ./.haero -B ./.haero/build \
   -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
@@ -113,11 +123,13 @@ cmake -S ./.haero -B ./.haero/build \
   -DHAERO_ENABLE_MPI=OFF \
   -DHAERO_ENABLE_GPU=$ENABLE_GPU \
   -DHAERO_PRECISION=$PRECISION \
+  -DKokkos_ARCH_$DEVICE_ARCH:BOOL=ON \
+  -DHAERO_DEVICE_ARCH=$DEVICE_ARCH \
   || exit
 
 echo "Building and installing Haero in $PREFIX..."
 cd .haero/build || exit
-make -j8 install
+make -j8 install || exit
 
 cd ../../
 echo "Haero has been installed in $PREFIX. Set HAERO_DIR to this directory in"

setup

@@ -99,7 +99,6 @@ cmake \
  -DMAM4XX_HAERO_DIR=\$HAERO_DIR \
  -DNUM_VERTICAL_LEVELS=\$NUM_VERTICAL_LEVELS \
  -DENABLE_SKYWALKER=ON \
- -DCMAKE_CUDA_ARCHITECTURES=80 \
  \$OPTIONS \
  -G "\$GENERATOR" \
  \$SOURCE_DIR
@@ -120,4 +119,3 @@ echo "  1. cd $1"
 echo "  2. Edit config.sh"
 echo "  3. ./config.sh"
 echo "  4. Build using 'make -j'."
-

src/mam4xx/conversions.hpp

@@ -251,10 +251,28 @@ KOKKOS_INLINE_FUNCTION Real saturation_mixing_ratio_hardy(Real T, Real P) {
 ///                 the temperature. If not supplied,
 ///                 @ref saturation_mixing_ratio_hardy is used.
 /// @return relative humidity [1]
-KOKKOS_INLINE_FUNCTION Real relative_humidity_from_vapor_mixing_ratio(
-    Real w, Real T, Real p,
-    Real (*wsat)(Real, Real) = saturation_mixing_ratio_hardy) {
-  const auto ws = wsat(T, p);
+// KOKKOS_INLINE_FUNCTION Real relative_humidity_from_vapor_mixing_ratio(
+//     Real w, Real T, Real p,
+//     Real (*wsat)(Real, Real) = saturation_mixing_ratio_hardy) {
+//   const auto ws = wsat(T, p);
+//   return w / ws;
+// }
+
+/// Computes the relative humidity from the water vapor mixing ratio and the
+/// pressure and temperature, given the relationship between temperature and
+/// the water vapor saturation pressure.
+///
+/// Use this formula with parameterizations that are defined with respect to
+/// air air (and note that mixing ratio is defined with respect to dry air).
+///
+/// @param [in] w water vapor mixing ratio [kg vapor / kg dry air]
+/// @param [in] p total pressure [Pa]
+/// @param [in] T temperature [K]
+/// @return relative humidity [1]
+KOKKOS_INLINE_FUNCTION Real relative_humidity_from_vapor_mixing_ratio(Real w,
+                                                                      Real T,
+                                                                      Real p) {
+  const auto ws = saturation_mixing_ratio_hardy(T, p);
   return w / ws;
 }