Skip to content

Update HOMME Aurora machine cmake config #6690

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 29, 2024
Merged

Update HOMME Aurora machine cmake config #6690

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9bb456c
Update to HOMME Auroro machine cmake config
abagusetty Oct 16, 2024
aab68c6
[SYCL] revert -qmkl to -mkl to have backward compatibility with tradi…
abagusetty Oct 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion components/homme/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ endif()

OPTION(HOMME_USE_MKL "Whether to use Intel's MKL instead of blas/lapack" FALSE)
IF(HOMME_USE_MKL)
MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -mkl will be added to each executable/library.")
MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -qmkl will be added to each executable/library.")
ELSE()
OPTION(HOMME_FIND_BLASLAPACK "Whether to use system blas/lapack" FALSE)
MESSAGE(STATUS "HOMME_FIND_BLASLAPACK=${HOMME_FIND_BLASLAPACK}")
Expand Down
16 changes: 11 additions & 5 deletions components/homme/cmake/HommeMacros.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,13 @@ macro(createTestExec execName execType macroNP macroNC
ADD_DEFINITIONS(-DHAVE_CONFIG_H)

ADD_EXECUTABLE(${execName} ${EXEC_SOURCES})
SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran)
# For SYCL builds it is suggested to use CXX linker with `-fortlib`
# for mixed-language setups
IF(SYCL_BUILD)
SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE CXX)
ELSE()
SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran)
ENDIF()
IF(BUILD_HOMME_WITHOUT_PIOLIBRARY)
TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY)
ENDIF()
Expand Down Expand Up @@ -165,8 +171,8 @@ macro(createTestExec execName execType macroNP macroNC
PROPERTIES Fortran_MODULE_DIRECTORY ${EXEC_MODULE_DIR})

IF (HOMME_USE_MKL)
TARGET_COMPILE_OPTIONS(${execName} PUBLIC -mkl)
TARGET_LINK_LIBRARIES(${execName} -mkl)
TARGET_COMPILE_OPTIONS(${execName} PUBLIC -qmkl)
TARGET_LINK_LIBRARIES(${execName} -qmkl)
ELSE()
IF (NOT HOMME_FIND_BLASLAPACK)
TARGET_LINK_LIBRARIES(${execName} lapack blas)
Expand Down Expand Up @@ -264,8 +270,8 @@ macro(createExecLib libName execType libSrcs inclDirs macroNP
ENDIF ()

IF (HOMME_USE_MKL)
TARGET_COMPILE_OPTIONS(${libName} PUBLIC -mkl)
TARGET_LINK_LIBRARIES(${libName} -mkl)
TARGET_COMPILE_OPTIONS(${libName} PUBLIC -qmkl)
TARGET_LINK_LIBRARIES(${libName} -qmkl)
ELSE()
IF (NOT HOMME_FIND_BLASLAPACK)
TARGET_LINK_LIBRARIES(${libName} lapack blas)
Expand Down
25 changes: 8 additions & 17 deletions components/homme/cmake/machineFiles/aurora-aot.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
#module restore
#module load oneapi/eng-compiler/2022.12.30.005
#module load intel_compute_runtime/release/agama-devel-627
#module load spack cmake
#module load spack-pe-base cmake
#module list


SET (SUNSPOT_MACHINE TRUE CACHE BOOL "")
SET (AURORA_MACHINE TRUE CACHE BOOL "")

SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")
SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "")
Expand All @@ -18,12 +16,12 @@ SET(USE_QUEUING FALSE CACHE BOOL "")

#temp hack
SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "")
SET(HOMME_USE_MKL TRUE CACHE BOOL "")

SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")

#set(KOKKOS_HOME "/home/onguba/kokkos-build/mar05-aot/install" CACHE STRING "")
#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "")
set(Kokkos_ROOT $ENV{KOKKOS_HOME} CACHE STRING "")

SET(USE_TRILINOS OFF CACHE BOOL "")

Expand All @@ -36,19 +34,14 @@ SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "")
SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "")
SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "")

# -fsycl-link-huge-device-code for theta to get build
#JIT flags
#SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
#SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64")

#AOT flags
SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"")
SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=intel_gpu_pvc")

SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "")
SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "")

SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "")
SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "")
SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "")

set (ENABLE_OPENMP OFF CACHE BOOL "")
Expand All @@ -57,8 +50,6 @@ set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")

set (HOMME_TESTING_PROFILE "dev" CACHE STRING "")

set (USE_NUM_PROCS 4 CACHE STRING "")

SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "")

set (USE_NUM_PROCS 12 CACHE STRING "")

SET (USE_MPI_OPTIONS "--pmi=pmix --cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 gpu_tile_compact.sh" CACHE FILEPATH "")
Copy link
Contributor

@oksanaguba oksanaguba Oct 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is tile script always in a user's path?

Copy link
Contributor Author

@abagusetty abagusetty Oct 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, the tile binding script is always in the SYSTEM path for the users

23 changes: 10 additions & 13 deletions components/homme/cmake/machineFiles/aurora-jit.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
#module restore
#module load oneapi/eng-compiler/2022.12.30.005
#module load intel_compute_runtime/release/agama-devel-627
#module load spack cmake
#module load spack-pe-base cmake
#module list


SET (AURORA_MACHINE TRUE CACHE BOOL "")

SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "")
SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "")
Expand All @@ -17,12 +16,12 @@ SET(USE_QUEUING FALSE CACHE BOOL "")

#temp hack
SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "")
SET(HOMME_USE_MKL TRUE CACHE BOOL "")

SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "")
SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "")

#set(KOKKOS_HOME "/home/onguba/kokkos-build/jan03-2024/install" CACHE STRING "")
#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "")
set(Kokkos_ROOT $ENV{KOKKOS_HOME} CACHE STRING "")

SET(USE_TRILINOS OFF CACHE BOOL "")

Expand All @@ -35,14 +34,14 @@ SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "")
SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "")
SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "")

# -fsycl-link-huge-device-code for theta to get build
#AOT flags
SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda")
SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64")
SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl")

SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "")
SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "")
SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "")

SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "")
SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "")
SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "")

set (ENABLE_OPENMP OFF CACHE BOOL "")
Expand All @@ -51,8 +50,6 @@ set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "")

set (HOMME_TESTING_PROFILE "dev" CACHE STRING "")

set (USE_NUM_PROCS 4 CACHE STRING "")

SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "")

set (USE_NUM_PROCS 12 CACHE STRING "")

SET (USE_MPI_OPTIONS "--pmi=pmix --cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 gpu_tile_compact.sh" CACHE FILEPATH "")
2 changes: 1 addition & 1 deletion components/homme/src/share/cxx/ExecSpaceDefs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
#endif

#ifdef KOKKOS_ENABLE_SYCL
#include <CL/sycl.hpp>
#include <sycl/sycl.hpp>
Copy link
Contributor

@oksanaguba oksanaguba Oct 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could you explain this change please?

Copy link
Contributor Author

@abagusetty abagusetty Oct 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SYCL specification has long deprecated these headers. <CL/sycl.hpp> header was valid only in SYCL 1.2.1 but it has been deprecated for <sycl/sycl.hpp> in SYCL 2020 Standards.

From the SYCL spec

For consistency, the programming API will only refer to the <sycl/sycl.hpp> header and the ::sycl namespace, but this should be considered synonymous with the SYCL 1.2.1 header and namespace.```

#endif

namespace Homme {
Expand Down
2 changes: 1 addition & 1 deletion components/homme/src/share/cxx/utilities/scream_tridiag.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ int get_team_nthr (const TeamMember& team) {
return team.team_size();
}

// Impl details for Nvidia and AMD GPUs.
// Impl details for Nvidia, AMD and Intel GPUs.

template <typename TeamMember> KOKKOS_FORCEINLINE_FUNCTION
int get_thread_id_within_team_gpu (const TeamMember& team) {
Expand Down
2 changes: 1 addition & 1 deletion components/homme/src/share/cxx/vector/vector_pragmas.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
#ifndef HOMMEXX_VECTOR_PRAGMAS_HPP
#define HOMMEXX_VECTOR_PRAGMAS_HPP

#if defined(__INTEL_COMPILER)
#if defined(__INTEL_COMPILER) || defined(__INTEL_CLANG_COMPILER) || defined(__INTEL_LLVM_COMPILER)
Copy link
Contributor

@oksanaguba oksanaguba Oct 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

which one will be INTEL_CLANG compiler and which one is LLVM? thanks

Copy link
Contributor Author

@abagusetty abagusetty Oct 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Intel compilers are identified as both clang and llvm based. Both the macros are defined by oneAPI compilers.

The difference being the naming convention: If it is a compiler released by Intel as a product (both the macros are defined) or if a user builds the compiler themselves from upstream repo (then either or both could be defined)


#define VECTOR_IVDEP_LOOP _Pragma("ivdep")
#define ALWAYS_VECTORIZE_LOOP _Pragma("vector always")
Expand Down