Skip to content

Commit 84a24a8

Browse files
authored
Set up A100 environment on ExCL (#2147)
* Add milan0 scripts * Add strict linking to ORANGE builds * Add hidden flags * Fix orange builds * Fix clangd setup * Change cache dir * Auto-install environment setup * Use ccache for cuda and hip * Use git hook path
1 parent d66098c commit 84a24a8

File tree

9 files changed

+304
-89
lines changed

9 files changed

+304
-89
lines changed

CMakePresets.json

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -50,37 +50,55 @@
5050
"CELERITAS_USE_covfie": {"type": "BOOL", "value": "OFF"}
5151
}
5252
},
53+
{
54+
"name": ".cuda",
55+
"hidden": true,
56+
"description": "Enable CUDA",
57+
"cacheVariables": {
58+
"CELERITAS_USE_CUDA": {"type": "BOOL", "value": "ON"},
59+
"CELERITAS_USE_HIP": {"type": "BOOL", "value": "OFF"},
60+
"CMAKE_CUDA_FLAGS_RELEASE": "-O3 -DNDEBUG",
61+
"CMAKE_CUDA_COMPILER_LAUNCHER": {"type": "STRING", "value": "$env{CCACHE_PROGRAM}"}
62+
}
63+
},
5364
{
5465
"name": ".cuda-volta",
5566
"hidden": true,
56-
"description": "Options to enable CUDA 11 on Volta architecture",
67+
"description": "Enable CUDA 11 on Volta architecture",
68+
"inherits": [".cuda"],
5769
"cacheVariables": {
58-
"CELERITAS_USE_CUDA": {"type": "BOOL", "value": "ON"},
5970
"CMAKE_CUDA_ARCHITECTURES": {"type": "STRING", "value": "70"}
6071
}
6172
},
6273
{
63-
"name": ".hip-mi100",
74+
"name": ".hip",
6475
"hidden": true,
65-
"description": "Options to enable HIP on AMD MI100",
76+
"description": "Enable HIP",
6677
"cacheVariables": {
67-
"CELERITAS_USE_CUDA": {"type": "BOOL", "value": "OFF"},
68-
"CELERITAS_USE_HIP": {"type": "BOOL", "value": "ON"},
69-
"CMAKE_HIP_ARCHITECTURES": {"type": "STRING", "value": "gfx908"},
78+
"CELERITAS_USE_CUDA": {"type": "BOOL", "value": "OFF"},
79+
"CELERITAS_USE_HIP": {"type": "BOOL", "value": "ON"},
80+
"CMAKE_HIP_COMPILER_LAUNCHER": {"type": "STRING", "value": "$env{CCACHE_PROGRAM}"},
7081
"CMAKE_HIP_FLAGS": "-munsafe-fp-atomics -Wno-#warnings",
82+
"CMAKE_HIP_FLAGS_RELEASE": "-O3 -DNDEBUG",
7183
"CMAKE_HIP_FLAGS_DEBUG": "-g -ggdb -O"
7284
}
7385
},
86+
{
87+
"name": ".hip-mi100",
88+
"hidden": true,
89+
"description": "Enable HIP on AMD MI100",
90+
"inherits": [".hip"],
91+
"cacheVariables": {
92+
"CMAKE_HIP_ARCHITECTURES": {"type": "STRING", "value": "gfx908"}
93+
}
94+
},
7495
{
7596
"name": ".hip-mi250",
7697
"hidden": true,
77-
"description": "Options to enable HIP on AMD MI250",
98+
"description": "Enable HIP on AMD MI250",
99+
"inherits": [".hip"],
78100
"cacheVariables": {
79-
"CELERITAS_USE_CUDA": {"type": "BOOL", "value": "OFF"},
80-
"CELERITAS_USE_HIP": {"type": "BOOL", "value": "ON"},
81-
"CMAKE_HIP_ARCHITECTURES": {"type": "STRING", "value": "gfx90a"},
82-
"CMAKE_HIP_FLAGS": "-munsafe-fp-atomics",
83-
"CMAKE_HIP_FLAGS_DEBUG": "-g -ggdb -O"
101+
"CMAKE_HIP_ARCHITECTURES": {"type": "STRING", "value": "gfx90a"}
84102
}
85103
},
86104
{
@@ -89,9 +107,7 @@
89107
"description": "Build with optimizations and without debug assertions",
90108
"cacheVariables": {
91109
"CELERITAS_DEBUG": {"type": "BOOL", "value": "OFF"},
92-
"CMAKE_BUILD_TYPE": {"type": "STRING", "value": "Release"},
93-
"CMAKE_CUDA_FLAGS_RELEASE": "-O3 -DNDEBUG",
94-
"CMAKE_HIP_FLAGS_RELEASE": "-O3 -DNDEBUG"
110+
"CMAKE_BUILD_TYPE": {"type": "STRING", "value": "Release"}
95111
}
96112
},
97113
{

scripts/build.sh

Lines changed: 129 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -59,19 +59,36 @@ fancy_hostname() {
5959
printf '%s\n' "${sys}"
6060
}
6161

62-
# Load environment (make sure this is not executed as a subshell)
63-
load_system_env() {
62+
# Determine the environment script for the given hostname
63+
get_system_env() {
6464
ENV_SCRIPT="${CELER_SOURCE_DIR}/scripts/env/$1.sh"
65-
if [ -f "${ENV_SCRIPT}" ]; then
66-
log info "Sourcing environment script at ${ENV_SCRIPT}"
67-
set +e
68-
if ! . "$ENV_SCRIPT"; then
69-
log error "Environment setup for $1 failed"
70-
exit 1
71-
fi
72-
set -e
73-
else
65+
if [ ! -f "${ENV_SCRIPT}" ]; then
7466
log debug "No environment script exists at ${ENV_SCRIPT}"
67+
else
68+
log info "Found environment script at ${ENV_SCRIPT}"
69+
printf '%s\n' "${ENV_SCRIPT}"
70+
fi
71+
}
72+
73+
# Source a script safely (make sure this is not executed as a subshell)
74+
source_script() {
75+
set +e
76+
if ! . "$1"; then
77+
log error "Failed to source '$1'"
78+
exit 1
79+
fi
80+
_celer_build=0
81+
set -e
82+
}
83+
84+
# Allow downstream scripts (e.g., excl) to load additional source files
85+
load_system_env() {
86+
env_script_="$(get_system_env "${1}")"
87+
if [ -z "${env_script_}" ]; then
88+
log error "Failed to find environment script for '$1'"
89+
return 1
90+
else
91+
source_script "${env_script_}"
7592
fi
7693
}
7794

@@ -121,31 +138,81 @@ check_ccache_usage() {
121138
fi
122139
}
123140

141+
# Find a command in PATH, suppressing error messages
142+
find_command() {
143+
command -v "$1" 2>/dev/null || printf ""
144+
}
145+
124146
# Auto-detect and configure ccache if available
125147
setup_ccache() {
126-
if CCACHE_PROGRAM="$(command -v ccache 2>/dev/null)"; then
148+
CCACHE_PROGRAM="$(find_command ccache)"
149+
if [ -n "${CCACHE_PROGRAM}" ]; then
127150
log info "Using ccache: ${CCACHE_PROGRAM}"
128151
check_ccache_usage
129-
[ -n "${CCACHE_PROGRAM}" ] && export CCACHE_PROGRAM
152+
export CCACHE_PROGRAM
130153
fi
131154
}
132155

133156
# Check if pre-commit hook is installed and install if missing
134157
install_precommit_if_git() {
135-
if git_dir=$(git rev-parse --git-dir 2>/dev/null); then
136-
:
137-
else
158+
if ! git_hook_path="$(git rev-parse --git-path hooks/pre-commit 2>/dev/null)"; then
138159
log debug "Not in a git repository, skipping pre-commit check"
139160
return 1
140161
fi
162+
log debug "Checking for pre-commit hooks at '${git_hook_path}'"
141163

142-
if [ ! -f "${git_dir}/hooks/pre-commit" ]; then
164+
if [ ! -f "${git_hook_path}" ]; then
143165
log info "Pre-commit hook not found, installing commit hooks"
144166
./scripts/dev/install-commit-hooks.sh
145167
fi
146168
return 0
147169
}
148170

171+
# Determine the RC file for the current shell
172+
get_shell_rc_file() {
173+
shell_name=$(basename "${SHELL}")
174+
case "${shell_name}" in
175+
bash) printf '%s\n' "${HOME}/.bashrc" ;;
176+
zsh) printf '%s\n' "${HOME}/.zshrc" ;;
177+
ksh) printf '%s\n' "${HOME}/.kshrc" ;;
178+
*) log warning "Unknown shell: ${shell_name}"; printf '%s\n' "${HOME}/.bashrc" ;;
179+
esac
180+
}
181+
182+
sentinel_marker_='celeritas-build-env'
183+
184+
# Check if sentinel marker exists in rc file
185+
has_sentinel() {
186+
rc_file=$1
187+
if [ ! -f "${rc_file}" ]; then
188+
return 1
189+
fi
190+
grep -q "${sentinel_marker_}" "${rc_file}" 2>/dev/null
191+
}
192+
193+
# Install environment sourcing to shell rc file
194+
install_shell_env() {
195+
rc_file=$1
196+
197+
if has_sentinel "${rc_file}"; then
198+
log debug "Skipping modification of ${rc_file}: sentinel exists"
199+
return 1
200+
fi
201+
202+
log info "Installing environment sourcing to ${rc_file}"
203+
cat >> "${rc_file}" <<EOF
204+
# >>> ${sentinel_marker_} >>>
205+
load_system_env() {
206+
CELER_SOURCE_DIR="${CELER_SOURCE_DIR}"
207+
if [ -d "\${CELER_SOURCE_DIR}" ]; then
208+
. "\${CELER_SOURCE_DIR}/scripts/env/\$1.sh"
209+
fi
210+
}
211+
load_system_env "${SYSTEM_NAME}"
212+
# <<< ${sentinel_marker_} <<<
213+
EOF
214+
}
215+
149216
#-----------------------------------------------------------------------------#
150217

151218
# Determine the source directory from the build script
@@ -155,22 +222,48 @@ export CELER_SOURCE_DIR="$(cd "$(dirname "$0")"/.. && pwd)"
155222
cd ${CELER_SOURCE_DIR}
156223

157224
# Determine system name, failing on an empty string
158-
SYSTEM_NAME=$(fancy_hostname)
225+
SYSTEM_NAME="$(fancy_hostname)"
159226
if [ -z "${SYSTEM_NAME}" ]; then
160227
log warning "Could not determine SYSTEM_NAME from LMOD_SYSTEM_NAME or HOSTNAME"
161228
log error "Empty SYSTEM_NAME, needed to load environment and presets"
162229
exit 1
163230
fi
164231

165232
# Check whether cmake/pre-commit change from environment
166-
OLD_CMAKE="$(command -v cmake 2>/dev/null || printf '')"
167-
OLD_PRE_COMMIT="$(command -v pre-commit 2>/dev/null || printf '')"
233+
OLD_CMAKE="$(find_command cmake)"
234+
OLD_PRE_COMMIT="$(find_command pre-commit)"
235+
OLD_XDG_CACHE_HOME="${XDG_CACHE_HOME}"
168236

169237
# Load environment paths using the system name
170-
load_system_env "${SYSTEM_NAME}"
238+
ENV_SCRIPT="$(get_system_env "${SYSTEM_NAME}")"
239+
if [ -n "${ENV_SCRIPT}" ]; then
240+
source_script "${ENV_SCRIPT}"
241+
fi
242+
CMAKE="$(find_command cmake)"
171243

172-
NEW_CMAKE="$(command -v cmake 2>/dev/null || printf 'cmake unavailable')"
173-
NEW_PRE_COMMIT="$(command -v pre-commit 2>/dev/null || printf '')"
244+
# Check whether the environment changed
245+
needs_env=false
246+
if [ "$(find_command cmake)" != "${OLD_PRE_COMMIT}" ]; then
247+
log warning "Local environment script uses a different pre-commit than your \$PATH"
248+
needs_env=true
249+
fi
250+
if [ "${CMAKE}" != "${OLD_CMAKE}" ]; then
251+
log warning "Local environment script uses a different CMake than your \$PATH"
252+
needs_env=true
253+
fi
254+
if [ "${OLD_XDG_CACHE_HOME}" != "${XDG_CACHE_HOME}" ]; then
255+
log warning "Cache directory changed from XDG_CACHE_HOME=${OLD_XDG_CACHE_HOME} to ${XDG_CACHE_HOME}"
256+
needs_env=true
257+
fi
258+
if ${needs_env}; then
259+
rc_file="$(get_shell_rc_file)"
260+
if install_shell_env "${rc_file}" "${ENV_SCRIPT}"; then
261+
needs_env=false
262+
else
263+
log warning "Please manually add the following to ${rc_file}:"
264+
printf ' . %s\n' "${ENV_SCRIPT}" >&2
265+
fi
266+
fi
174267

175268
# Link preset file
176269
ln_presets "${SYSTEM_NAME}"
@@ -179,16 +272,16 @@ ln_presets "${SYSTEM_NAME}"
179272
setup_ccache
180273

181274
# Check arguments and give presets if missing
182-
if [ $# -eq 0 ]; then
275+
if [ $# -eq 0 ] ; then
183276
printf '%s\n' "Usage: $0 PRESET [config_args...]" >&2
184-
if command -v cmake >/dev/null 2>&1; then
185-
if cmake --list-presets >&2 ; then
186-
log info "Try using the '${SYSTEM_NAME}' or 'dev' presets ('base'), or 'default' if not"
187-
else
188-
log error "CMake may be too old or JSON file may be broken"
189-
fi
190-
else
277+
if [ -n "${CMAKE}" ]; then
191278
log error "cmake unavailable: cannot call --list-presets"
279+
exit 1
280+
fi
281+
if "${CMAKE}" --list-presets >&2 ; then
282+
log info "Try using the '${SYSTEM_NAME}' or 'dev' presets ('base'), or 'default' if not"
283+
else
284+
log error "CMake may be too old or JSON file may be broken"
192285
fi
193286
exit 2
194287
fi
@@ -209,15 +302,11 @@ if cmake --build --preset="${CMAKE_PRESET}"; then
209302
fi
210303

211304
install_precommit_if_git
212-
if [ "${NEW_PRE_COMMIT}" != "${OLD_PRE_COMMIT}" ]; then
213-
log warning "Local environment script uses a different pre-commit than your \$PATH:"
214-
log info "Recommend adding '. ${CELER_SOURCE_DIR}/${ENV_SCRIPT}' to your shell rc"
215-
fi
216-
217-
if [ "${NEW_CMAKE}" != "${OLD_CMAKE}" ]; then
218-
log warning "Local environment script uses a different CMake than your \$PATH:"
219-
log info "Recommend adding '. ${CELER_SOURCE_DIR}/${ENV_SCRIPT}' to your shell rc"
220-
fi
221305
else
222306
log error "build failed: check configuration and build errors above"
223307
fi
308+
309+
if ${needs_env}; then
310+
log warning "Environment changed: please manually add the following to ${rc_file}:"
311+
printf ' . %s\n' "${ENV_SCRIPT}" >&2
312+
fi

scripts/cmake-presets/faraday.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
{
66
"name": ".base",
77
"hidden": true,
8-
"inherits": ["full", ".spack-base"],
8+
"inherits": [".hip", "full", ".spack-base"],
99
"binaryDir": "$env{SCRATCHDIR}/build/celeritas-${presetName}",
1010
"generator": "Ninja",
1111
"cacheVariables": {
@@ -19,6 +19,8 @@
1919
"CMAKE_CXX_FLAGS": "-Wall -Wextra -pedantic -Werror -fdiagnostics-color=always",
2020
"CMAKE_CXX_STANDARD": "20",
2121
"CMAKE_CXX_EXTENSIONS": {"type": "BOOL", "value": "OFF"},
22+
"CMAKE_EXE_LINKER_FLAGS": "-Wl,-z,defs",
23+
"CMAKE_SHARED_LINKER_FLAGS": "-Wl,-z,defs",
2224
"CMAKE_HIP_FLAGS": "-Wall -Wextra -pedantic -Werror -fdiagnostics-color=always",
2325
"CMAKE_HIP_PLATFORM": "amd",
2426
"CMAKE_HIP_ARCHITECTURES": "gfx942",

scripts/cmake-presets/hudson.json

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,12 @@
55
{
66
"name": ".base",
77
"hidden": true,
8-
"inherits": ["full", ".spack-base"],
8+
"inherits": ["full", ".cuda", ".spack-base"],
99
"binaryDir": "/scratch/$env{USER}/build/celeritas-${presetName}",
1010
"generator": "Ninja",
1111
"cacheVariables": {
1212
"BUILD_SHARED_LIBS": {"type": "BOOL", "value": "ON"},
1313
"CELERITAS_BUILD_DOCS": {"type": "BOOL", "value": "OFF"},
14-
"CELERITAS_USE_CUDA": {"type": "BOOL", "value": "ON"},
15-
"CELERITAS_USE_HIP": {"type": "BOOL", "value": "OFF"},
1614
"CELERITAS_USE_OpenMP": {"type": "BOOL", "value": "OFF"},
1715
"CELERITAS_USE_MPI": {"type": "BOOL", "value": "OFF"},
1816
"CMAKE_CXX_FLAGS": "-Wall -Wextra -pedantic -Werror -Wno-stringop-overread",
@@ -26,6 +24,15 @@
2624
"CMAKE_EXPORT_COMPILE_COMMANDS": {"type": "BOOL", "value": "ON"}
2725
}
2826
},
27+
{
28+
"name": ".orange",
29+
"hidden": true,
30+
"cacheVariables": {
31+
"CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"},
32+
"CMAKE_EXE_LINKER_FLAGS": "-Wl,-z,defs",
33+
"CMAKE_SHARED_LINKER_FLAGS": "-Wl,-z,defs"
34+
}
35+
},
2936
{
3037
"name": "release",
3138
"displayName": "Build with full optimizations (VecGeom)",
@@ -42,26 +49,19 @@
4249
{
4350
"name": "release-orange",
4451
"displayName": "Build with full optimizations (ORANGE)",
45-
"inherits": ["release"],
46-
"cacheVariables": {
47-
"CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"}
48-
}
52+
"inherits": [".orange", "release"]
4953
},
5054
{
5155
"name": "reldeb-orange",
5256
"displayName": "Build with optimizations, but enable host debugging (ORANGE)",
53-
"inherits": ["reldeb"],
54-
"cacheVariables": {
55-
"CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"}
56-
}
57+
"inherits": [".orange", "reldeb"]
5758
},
5859
{
5960
"name": "debug-orange",
6061
"displayName": "Build without optimization, enabling full host debugging (ORANGE)",
61-
"inherits": [".base", ".debug"],
62+
"inherits": [".orange", ".base", ".debug"],
6263
"cacheVariables": {
63-
"CELERITAS_DEVICE_DEBUG":{"type": "BOOL", "value": "OFF"},
64-
"CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"}
64+
"CELERITAS_DEVICE_DEBUG":{"type": "BOOL", "value": "OFF"}
6565
}
6666
},
6767
{

0 commit comments

Comments
 (0)