CI: oneAPI SYCL for AMD GPUs (#3341)

Author: nmnobre

.github/workflows/dependencies/dependencies_codeplay.sh

@@ -9,3 +9,7 @@ set -eu -o pipefail
 curl -o oneapi_nvidia.sh -L "https://developer.codeplay.com/api/v1/products/download?product=oneapi&variant=nvidia&filters[]=linux&aat=$1"
 chmod +x oneapi_nvidia.sh
 sudo ./oneapi_nvidia.sh --yes
+
+curl -o oneapi_amd.sh -L "https://developer.codeplay.com/api/v1/products/download?product=oneapi&variant=amd&filters[]=linux&aat=$1"
+chmod +x oneapi_amd.sh
+sudo ./oneapi_amd.sh --yes

.github/workflows/dependencies/dependencies_hip.sh

@@ -20,7 +20,7 @@ echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
 # Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu
 curl -O https://repo.radeon.com/rocm/rocm.gpg.key
 sudo apt-key add rocm.gpg.key
-echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' \
+echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${1-debian}/ ubuntu main" \
   | sudo tee /etc/apt/sources.list.d/rocm.list
 echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \
   | sudo tee -a /etc/profile.d/rocm.sh

.github/workflows/intel.yml

@@ -148,6 +148,57 @@ jobs:
         ccache -s
         du -hs ~/.cache/ccache
 
+  no-tests-oneapi-sycl-amd:
+    name: oneAPI SYCL for AMD GPUs
+    runs-on: ubuntu-20.04
+    env:
+      CODEPLAYTOKEN: ${{ secrets.CODEPLAYTOKEN }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Dependencies
+      if: ${{ env.CODEPLAYTOKEN != '' }}
+      run: |
+        .github/workflows/dependencies/dependencies_hip.sh 5.4.6
+        .github/workflows/dependencies/dependencies_dpcpp.sh
+        .github/workflows/dependencies/dependencies_codeplay.sh ${{ env.CODEPLAYTOKEN }}
+        .github/workflows/dependencies/dependencies_ccache.sh
+    - name: Set Up Cache
+      if: ${{ env.CODEPLAYTOKEN != '' }}
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
+    - name: Build & Install
+      if: ${{ env.CODEPLAYTOKEN != '' }}
+      # mkl/rng/device/detail/mrg32k3a_impl.hpp has a number of sign-compare error
+      # mkl/rng/device/detail/mrg32k3a_impl.hpp has missing braces in array-array initalization
+      env: {CXXFLAGS: "-fsycl -fsycl-targets=amdgcn-amd-amdhsa -Xsycl-target-backend --offload-arch=gfx90a -fno-operator-names -Werror -Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wnon-virtual-dtor -Wno-sign-compare -Wno-missing-braces"}
+      run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=55M
+        export CCACHE_DEPEND=1
+        ccache -z
+
+        set +e
+        source /opt/intel/oneapi/setvars.sh --include-intel-llvm
+        set -e
+        cmake -S . -B build                                \
+            -DCMAKE_VERBOSE_MAKEFILE=ON                    \
+            -DAMReX_EB=OFF                                 \
+            -DAMReX_ENABLE_TESTS=OFF                       \
+            -DAMReX_GPU_BACKEND=SYCL                       \
+            -DAMReX_SYCL_SUB_GROUP_SIZE=64                 \
+            -DCMAKE_C_COMPILER=$(which icx)                \
+            -DCMAKE_CXX_COMPILER=$(which clang++)          \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+        cmake --build build --parallel 2
+
+        ccache -s
+        du -hs ~/.cache/ccache
+
 # "Classic" EDG Intel Compiler
   tests-icc:
     name: ICC [tests]

Docs/sphinx_documentation/source/GPU.rst

@@ -404,7 +404,7 @@ Below is an example configuration for SYCL:
    +------------------------------+-------------------------------------------------+-------------+-----------------+
    | AMReX_SYCL_ONEDPL            | Enable SYCL's oneDPL algorithms                 | NO          | YES, NO         |
    +------------------------------+-------------------------------------------------+-------------+-----------------+
-   | AMReX_SYCL_SUB_GROUP_SIZE    | Specify subgroup size                           | 32          | 32, 16          |
+   | AMReX_SYCL_SUB_GROUP_SIZE    | Specify subgroup size                           | 32          | 64, 32, 16      |
    +------------------------------+-------------------------------------------------+-------------+-----------------+
 .. raw:: latex
 

Tools/CMake/AMReXOptions.cmake

@@ -169,7 +169,7 @@ if (AMReX_SYCL)
    unset(_valid_sycl_compiler_ids)
 
    set(AMReX_SYCL_SUB_GROUP_SIZE_DEFAULT 32)
-   set(AMReX_SYCL_SUB_GROUP_SIZE_VALUES 16 32)
+   set(AMReX_SYCL_SUB_GROUP_SIZE_VALUES 16 32 64)
    set(AMReX_SYCL_SUB_GROUP_SIZE ${AMReX_SYCL_SUB_GROUP_SIZE_DEFAULT} CACHE STRING
        "SYCL sub-group size")
    if (NOT AMReX_SYCL_SUB_GROUP_SIZE IN_LIST AMReX_SYCL_SUB_GROUP_SIZE_VALUES)