Skip to content

Commit d8a408c

Browse files
oksanagubaoksanaguba
committed
Merge 'ABHI/abagusetty/homme/aurora' (PR #6690)
Update Cmake machine config for Aurora Cleanup certain warnings related to headers and Intel compilers Linking Fortran/C++/SYCL with CXX linker (-fortlib) instead of Fortran language for SYCL builds. Since Fortran linking with SYCL is no longer supported (link is in the PR).
2 parents d4a5531 + aab68c6 commit d8a408c

File tree

6 files changed

+28
-34
lines changed

6 files changed

+28
-34
lines changed

‎components/homme/cmake/HommeMacros.cmake‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,13 @@ macro(createTestExec execName execType macroNP macroNC
112112
ADD_DEFINITIONS(-DHAVE_CONFIG_H)
113113

114114
ADD_EXECUTABLE(${execName} ${EXEC_SOURCES})
115-
SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran)
115+
# For SYCL builds it is suggested to use CXX linker with `-fortlib`
116+
# for mixed-language setups
117+
IF(SYCL_BUILD)
118+
SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE CXX)
119+
ELSE()
120+
SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran)
121+
ENDIF()
116122
IF(BUILD_HOMME_WITHOUT_PIOLIBRARY)
117123
TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY)
118124
ENDIF()

‎components/homme/cmake/machineFiles/aurora-aot.cmake‎

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
#module restore
2-
#module load oneapi/eng-compiler/2022.12.30.005
3-
#module load intel_compute_runtime/release/agama-devel-627
4-
#module load spack cmake
2+
#module load spack-pe-base cmake
53
#module list
64

75

8-
SET (SUNSPOT_MACHINE TRUE CACHE BOOL "")
6+
SET (AURORA_MACHINE TRUE CACHE BOOL "")
97

108
SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")
119
SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "")
@@ -18,12 +16,12 @@ SET(USE_QUEUING FALSE CACHE BOOL "")
1816

1917
#temp hack
2018
SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "")
19+
SET(HOMME_USE_MKL TRUE CACHE BOOL "")
2120

2221
SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
2322
SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
2423

25-
#set(KOKKOS_HOME "/home/onguba/kokkos-build/mar05-aot/install" CACHE STRING "")
26-
#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "")
24+
set(Kokkos_ROOT $ENV{KOKKOS_HOME} CACHE STRING "")
2725

2826
SET(USE_TRILINOS OFF CACHE BOOL "")
2927

@@ -36,19 +34,14 @@ SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "")
3634
SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "")
3735
SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "")
3836

39-
# -fsycl-link-huge-device-code for theta to get build
40-
#JIT flags
41-
#SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
42-
#SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64")
43-
4437
#AOT flags
4538
SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
46-
SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"")
39+
SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=intel_gpu_pvc")
4740

4841
SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "")
4942
SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "")
5043

51-
SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "")
44+
SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "")
5245
SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "")
5346

5447
set (ENABLE_OPENMP OFF CACHE BOOL "")
@@ -57,8 +50,6 @@ set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")
5750

5851
set (HOMME_TESTING_PROFILE "dev" CACHE STRING "")
5952

60-
set (USE_NUM_PROCS 4 CACHE STRING "")
61-
62-
SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "")
63-
53+
set (USE_NUM_PROCS 12 CACHE STRING "")
6454

55+
SET (USE_MPI_OPTIONS "--pmi=pmix --cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 gpu_tile_compact.sh" CACHE FILEPATH "")

‎components/homme/cmake/machineFiles/aurora-jit.cmake‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
#module restore
2-
#module load oneapi/eng-compiler/2022.12.30.005
3-
#module load intel_compute_runtime/release/agama-devel-627
4-
#module load spack cmake
2+
#module load spack-pe-base cmake
53
#module list
64

75

6+
SET (AURORA_MACHINE TRUE CACHE BOOL "")
87

98
SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")
109
SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "")
@@ -17,12 +16,12 @@ SET(USE_QUEUING FALSE CACHE BOOL "")
1716

1817
#temp hack
1918
SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "")
19+
SET(HOMME_USE_MKL TRUE CACHE BOOL "")
2020

2121
SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
2222
SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")
2323

24-
#set(KOKKOS_HOME "/home/onguba/kokkos-build/jan03-2024/install" CACHE STRING "")
25-
#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "")
24+
set(Kokkos_ROOT $ENV{KOKKOS_HOME} CACHE STRING "")
2625

2726
SET(USE_TRILINOS OFF CACHE BOOL "")
2827

@@ -35,14 +34,14 @@ SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "")
3534
SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "")
3635
SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "")
3736

38-
# -fsycl-link-huge-device-code for theta to get build
37+
#AOT flags
3938
SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
40-
SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64")
39+
SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl")
4140

42-
SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "")
41+
SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "")
4342
SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "")
4443

45-
SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "")
44+
SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "")
4645
SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "")
4746

4847
set (ENABLE_OPENMP OFF CACHE BOOL "")
@@ -51,8 +50,6 @@ set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")
5150

5251
set (HOMME_TESTING_PROFILE "dev" CACHE STRING "")
5352

54-
set (USE_NUM_PROCS 4 CACHE STRING "")
55-
56-
SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "")
57-
53+
set (USE_NUM_PROCS 12 CACHE STRING "")
5854

55+
SET (USE_MPI_OPTIONS "--pmi=pmix --cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 gpu_tile_compact.sh" CACHE FILEPATH "")

‎components/homme/src/share/cxx/ExecSpaceDefs.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
#endif
2323

2424
#ifdef KOKKOS_ENABLE_SYCL
25-
#include <CL/sycl.hpp>
25+
#include <sycl/sycl.hpp>
2626
#endif
2727

2828
namespace Homme {

‎components/homme/src/share/cxx/utilities/scream_tridiag.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ int get_team_nthr (const TeamMember& team) {
120120
return team.team_size();
121121
}
122122

123-
// Impl details for Nvidia and AMD GPUs.
123+
// Impl details for Nvidia, AMD and Intel GPUs.
124124

125125
template <typename TeamMember> KOKKOS_FORCEINLINE_FUNCTION
126126
int get_thread_id_within_team_gpu (const TeamMember& team) {

‎components/homme/src/share/cxx/vector/vector_pragmas.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
#ifndef HOMMEXX_VECTOR_PRAGMAS_HPP
88
#define HOMMEXX_VECTOR_PRAGMAS_HPP
99

10-
#if defined(__INTEL_COMPILER)
10+
#if defined(__INTEL_COMPILER) || defined(__INTEL_CLANG_COMPILER) || defined(__INTEL_LLVM_COMPILER)
1111

1212
#define VECTOR_IVDEP_LOOP _Pragma("ivdep")
1313
#define ALWAYS_VECTORIZE_LOOP _Pragma("vector always")

0 commit comments

Comments
 (0)