Set up A100 environment on ExCL

CMakePresets.json

@@ -50,37 +50,55 @@
         "CELERITAS_USE_covfie":  {"type": "BOOL", "value": "OFF"}
       }
     },
+    {
+      "name": ".cuda",
+      "hidden": true,
+      "description": "Enable CUDA",
+      "cacheVariables": {
+        "CELERITAS_USE_CUDA": {"type": "BOOL",   "value": "ON"},
+        "CELERITAS_USE_HIP": {"type": "BOOL",   "value": "OFF"},
+        "CMAKE_CUDA_FLAGS_RELEASE": "-O3 -DNDEBUG",
+        "CMAKE_CUDA_COMPILER_LAUNCHER": {"type": "STRING", "value": "$env{CCACHE_PROGRAM}"}
+      }
+    },
     {
       "name": ".cuda-volta",
       "hidden": true,
-      "description": "Options to enable CUDA 11 on Volta architecture",
+      "description": "Enable CUDA 11 on Volta architecture",
+      "inherits": [".cuda"],
       "cacheVariables": {
-        "CELERITAS_USE_CUDA":       {"type": "BOOL",   "value": "ON"},
         "CMAKE_CUDA_ARCHITECTURES": {"type": "STRING", "value": "70"}
       }
     },
     {
-      "name": ".hip-mi100",
+      "name": ".hip",
       "hidden": true,
-      "description": "Options to enable HIP on AMD MI100",
+      "description": "Enable HIP",
       "cacheVariables": {
-        "CELERITAS_USE_CUDA":       {"type": "BOOL",   "value": "OFF"},
-        "CELERITAS_USE_HIP":        {"type": "BOOL",   "value": "ON"},
-        "CMAKE_HIP_ARCHITECTURES":  {"type": "STRING", "value": "gfx908"},
+        "CELERITAS_USE_CUDA": {"type": "BOOL", "value": "OFF"},
+        "CELERITAS_USE_HIP": {"type": "BOOL",   "value": "ON"},
+        "CMAKE_HIP_COMPILER_LAUNCHER": {"type": "STRING", "value": "$env{CCACHE_PROGRAM}"},
         "CMAKE_HIP_FLAGS": "-munsafe-fp-atomics -Wno-#warnings",
+        "CMAKE_HIP_FLAGS_RELEASE": "-O3 -DNDEBUG",
         "CMAKE_HIP_FLAGS_DEBUG": "-g -ggdb -O"
       }
     },
+    {
+      "name": ".hip-mi100",
+      "hidden": true,
+      "description": "Enable HIP on AMD MI100",
+      "inherits": [".hip"],
+      "cacheVariables": {
+        "CMAKE_HIP_ARCHITECTURES":  {"type": "STRING", "value": "gfx908"}
+      }
+    },
     {
       "name": ".hip-mi250",
       "hidden": true,
-      "description": "Options to enable HIP on AMD MI250",
+      "description": "Enable HIP on AMD MI250",
+      "inherits": [".hip"],
       "cacheVariables": {
-        "CELERITAS_USE_CUDA":       {"type": "BOOL",   "value": "OFF"},
-        "CELERITAS_USE_HIP":        {"type": "BOOL",   "value": "ON"},
-        "CMAKE_HIP_ARCHITECTURES":  {"type": "STRING", "value": "gfx90a"},
-        "CMAKE_HIP_FLAGS": "-munsafe-fp-atomics",
-        "CMAKE_HIP_FLAGS_DEBUG": "-g -ggdb -O"
+        "CMAKE_HIP_ARCHITECTURES":  {"type": "STRING", "value": "gfx90a"}
       }
     },
     {
@@ -89,9 +107,7 @@
       "description": "Build with optimizations and without debug assertions",
       "cacheVariables": {
         "CELERITAS_DEBUG":  {"type": "BOOL",   "value": "OFF"},
-        "CMAKE_BUILD_TYPE": {"type": "STRING", "value": "Release"},
-        "CMAKE_CUDA_FLAGS_RELEASE": "-O3 -DNDEBUG",
-        "CMAKE_HIP_FLAGS_RELEASE": "-O3 -DNDEBUG"
+        "CMAKE_BUILD_TYPE": {"type": "STRING", "value": "Release"}
       }
     },
     {

scripts/build.sh

@@ -59,19 +59,36 @@ fancy_hostname() {
   printf '%s\n' "${sys}"
 }
 
-# Load environment (make sure this is not executed as a subshell)
-load_system_env() {
+# Determine the environment script for the given hostname
+get_system_env() {
   ENV_SCRIPT="${CELER_SOURCE_DIR}/scripts/env/$1.sh"
-  if [ -f "${ENV_SCRIPT}" ]; then
-    log info "Sourcing environment script at ${ENV_SCRIPT}"
-    set +e
-    if ! . "$ENV_SCRIPT"; then
-      log error "Environment setup for $1 failed"
-      exit 1
-    fi
-    set -e
-  else
+  if [ ! -f "${ENV_SCRIPT}" ]; then
     log debug "No environment script exists at ${ENV_SCRIPT}"
+  else
+    log info "Found environment script at ${ENV_SCRIPT}"
+    printf '%s\n' "${ENV_SCRIPT}"
+  fi
+}
+
+# Source a script safely (make sure this is not executed as a subshell)
+source_script() {
+  set +e
+  if ! . "$1"; then
+    log error "Failed to source '$1'"
+    exit 1
+  fi
+  _celer_build=0
+  set -e
+}
+
+# Allow downstream scripts (e.g., excl) to load additional source files
+load_system_env() {
+  env_script_="$(get_system_env "${1}")"
+  if [ -z "${env_script_}" ]; then
+    log error "Failed to find environment script for '$1'"
+    return 1
+  else
+    source_script "${env_script_}"
   fi
 }
 
@@ -121,31 +138,81 @@ check_ccache_usage() {
   fi
 }
 
+# Find a command in PATH, suppressing error messages
+find_command() {
+    command -v "$1" 2>/dev/null || printf ""
+}
+
 # Auto-detect and configure ccache if available
 setup_ccache() {
-  if CCACHE_PROGRAM="$(command -v ccache 2>/dev/null)"; then
+  CCACHE_PROGRAM="$(find_command ccache)"
+  if [ -n "${CCACHE_PROGRAM}" ]; then
     log info "Using ccache: ${CCACHE_PROGRAM}"
     check_ccache_usage
-    [ -n "${CCACHE_PROGRAM}" ] && export CCACHE_PROGRAM
+    export CCACHE_PROGRAM
   fi
 }
 
 # Check if pre-commit hook is installed and install if missing
 install_precommit_if_git() {
-  if git_dir=$(git rev-parse --git-dir 2>/dev/null); then
-    :
-  else
+  if ! git_hook_path="$(git rev-parse --git-path hooks/pre-commit 2>/dev/null)"; then
     log debug "Not in a git repository, skipping pre-commit check"
     return 1
   fi
+  log debug "Checking for pre-commit hooks at '${git_hook_path}'"
 
-  if [ ! -f "${git_dir}/hooks/pre-commit" ]; then
+  if [ ! -f "${git_hook_path}" ]; then
     log info "Pre-commit hook not found, installing commit hooks"
     ./scripts/dev/install-commit-hooks.sh
   fi
   return 0
 }
 
+# Determine the RC file for the current shell
+get_shell_rc_file() {
+  shell_name=$(basename "${SHELL}")
+  case "${shell_name}" in
+    bash) printf '%s\n' "${HOME}/.bashrc" ;;
+    zsh) printf '%s\n' "${HOME}/.zshrc" ;;
+    ksh) printf '%s\n' "${HOME}/.kshrc" ;;
+    *) log warning "Unknown shell: ${shell_name}"; printf '%s\n' "${HOME}/.bashrc" ;;
+  esac
+}
+
+sentinel_marker_='celeritas-build-env'
+
+# Check if sentinel marker exists in rc file
+has_sentinel() {
+  rc_file=$1
+  if [ ! -f "${rc_file}" ]; then
+    return 1
+  fi
+  grep -q "${sentinel_marker_}" "${rc_file}" 2>/dev/null
+}
+
+# Install environment sourcing to shell rc file
+install_shell_env() {
+  rc_file=$1
+
+  if has_sentinel "${rc_file}"; then
+    log debug "Skipping modification of ${rc_file}: sentinel exists"
+    return 1
+  fi
+
+  log info "Installing environment sourcing to ${rc_file}"
+  cat >> "${rc_file}" <<EOF
+# >>> ${sentinel_marker_} >>>
+load_system_env() {
+  CELER_SOURCE_DIR="${CELER_SOURCE_DIR}"
+  if [ -d "\${CELER_SOURCE_DIR}" ]; then
+    . "\${CELER_SOURCE_DIR}/scripts/env/\$1.sh"
+  fi
+}
+load_system_env "${SYSTEM_NAME}"
+# <<< ${sentinel_marker_} <<<
+EOF
+}
+
 #-----------------------------------------------------------------------------#
 
 # Determine the source directory from the build script
@@ -155,22 +222,48 @@ export CELER_SOURCE_DIR="$(cd "$(dirname "$0")"/.. && pwd)"
 cd ${CELER_SOURCE_DIR}
 
 # Determine system name, failing on an empty string
-SYSTEM_NAME=$(fancy_hostname)
+SYSTEM_NAME="$(fancy_hostname)"
 if [ -z "${SYSTEM_NAME}" ]; then
   log warning "Could not determine SYSTEM_NAME from LMOD_SYSTEM_NAME or HOSTNAME"
   log error "Empty SYSTEM_NAME, needed to load environment and presets"
   exit 1
 fi
 
 # Check whether cmake/pre-commit change from environment
-OLD_CMAKE="$(command -v cmake 2>/dev/null || printf '')"
-OLD_PRE_COMMIT="$(command -v pre-commit 2>/dev/null || printf '')"
+OLD_CMAKE="$(find_command cmake)"
+OLD_PRE_COMMIT="$(find_command pre-commit)"
+OLD_XDG_CACHE_HOME="${XDG_CACHE_HOME}"
 
 # Load environment paths using the system name
-load_system_env "${SYSTEM_NAME}"
+ENV_SCRIPT="$(get_system_env "${SYSTEM_NAME}")"
+if [ -n "${ENV_SCRIPT}" ]; then
+  source_script "${ENV_SCRIPT}"
+fi
+CMAKE="$(find_command cmake)"
 
-NEW_CMAKE="$(command -v cmake 2>/dev/null || printf 'cmake unavailable')"
-NEW_PRE_COMMIT="$(command -v pre-commit 2>/dev/null || printf '')"
+# Check whether the environment changed
+needs_env=false
+if [ "$(find_command cmake)" != "${OLD_PRE_COMMIT}" ]; then
+  log warning "Local environment script uses a different pre-commit than your \$PATH"
+  needs_env=true
+fi
+if [ "${CMAKE}" != "${OLD_CMAKE}" ]; then
+  log warning "Local environment script uses a different CMake than your \$PATH"
+  needs_env=true
+fi
+if [ "${OLD_XDG_CACHE_HOME}" != "${XDG_CACHE_HOME}" ]; then
+  log warning "Cache directory changed from XDG_CACHE_HOME=${OLD_XDG_CACHE_HOME} to ${XDG_CACHE_HOME}"
+  needs_env=true
+fi
+if ${needs_env}; then
+  rc_file="$(get_shell_rc_file)"
+  if install_shell_env "${rc_file}" "${ENV_SCRIPT}"; then
+    needs_env=false
+  else
+    log warning "Please manually add the following to ${rc_file}:"
+    printf ' . %s\n' "${ENV_SCRIPT}" >&2
+  fi
+fi
 
 # Link preset file
 ln_presets "${SYSTEM_NAME}"
@@ -179,16 +272,16 @@ ln_presets "${SYSTEM_NAME}"
 setup_ccache
 
 # Check arguments and give presets if missing
-if [ $# -eq 0 ]; then
+if [ $# -eq 0 ] ; then
   printf '%s\n' "Usage: $0 PRESET [config_args...]" >&2
-  if command -v cmake >/dev/null 2>&1; then
-    if cmake --list-presets >&2 ; then
-      log info "Try using the '${SYSTEM_NAME}' or 'dev' presets ('base'), or 'default' if not"
-    else
-      log error "CMake may be too old or JSON file may be broken"
-    fi
-  else
+  if [ -n "${CMAKE}" ]; then
     log error "cmake unavailable: cannot call --list-presets"
+    exit 1
+  fi
+  if "${CMAKE}" --list-presets >&2 ; then
+    log info "Try using the '${SYSTEM_NAME}' or 'dev' presets ('base'), or 'default' if not"
+  else
+    log error "CMake may be too old or JSON file may be broken"
   fi
   exit 2
 fi
@@ -209,15 +302,11 @@ if cmake --build --preset="${CMAKE_PRESET}"; then
   fi
 
   install_precommit_if_git
-  if [ "${NEW_PRE_COMMIT}" != "${OLD_PRE_COMMIT}" ]; then
-    log warning "Local environment script uses a different pre-commit than your \$PATH:"
-    log info "Recommend adding '. ${CELER_SOURCE_DIR}/${ENV_SCRIPT}' to your shell rc"
-  fi
-
-  if [ "${NEW_CMAKE}" != "${OLD_CMAKE}" ]; then
-    log warning "Local environment script uses a different CMake than your \$PATH:"
-    log info "Recommend adding '. ${CELER_SOURCE_DIR}/${ENV_SCRIPT}' to your shell rc"
-  fi
 else
   log error "build failed: check configuration and build errors above"
 fi
+
+if ${needs_env}; then
+  log warning "Environment changed: please manually add the following to ${rc_file}:"
+  printf ' . %s\n' "${ENV_SCRIPT}" >&2
+fi

scripts/cmake-presets/faraday.json

@@ -5,7 +5,7 @@
     {
       "name": ".base",
       "hidden": true,
-      "inherits": ["full", ".spack-base"],
+      "inherits": [".hip", "full", ".spack-base"],
       "binaryDir": "$env{SCRATCHDIR}/build/celeritas-${presetName}",
       "generator": "Ninja",
       "cacheVariables": {
@@ -19,6 +19,8 @@
         "CMAKE_CXX_FLAGS": "-Wall -Wextra -pedantic -Werror -fdiagnostics-color=always",
         "CMAKE_CXX_STANDARD": "20",
         "CMAKE_CXX_EXTENSIONS": {"type": "BOOL", "value": "OFF"},
+        "CMAKE_EXE_LINKER_FLAGS": "-Wl,-z,defs",
+        "CMAKE_SHARED_LINKER_FLAGS": "-Wl,-z,defs",
         "CMAKE_HIP_FLAGS": "-Wall -Wextra -pedantic -Werror -fdiagnostics-color=always",
         "CMAKE_HIP_PLATFORM": "amd",
         "CMAKE_HIP_ARCHITECTURES": "gfx942",

scripts/cmake-presets/hudson.json

@@ -5,14 +5,12 @@
     {
       "name": ".base",
       "hidden": true,
-      "inherits": ["full", ".spack-base"],
+      "inherits": ["full", ".cuda", ".spack-base"],
       "binaryDir": "/scratch/$env{USER}/build/celeritas-${presetName}",
       "generator": "Ninja",
       "cacheVariables": {
         "BUILD_SHARED_LIBS":     {"type": "BOOL", "value": "ON"},
         "CELERITAS_BUILD_DOCS":  {"type": "BOOL", "value": "OFF"},
-        "CELERITAS_USE_CUDA":    {"type": "BOOL", "value": "ON"},
-        "CELERITAS_USE_HIP":     {"type": "BOOL", "value": "OFF"},
         "CELERITAS_USE_OpenMP":  {"type": "BOOL", "value": "OFF"},
         "CELERITAS_USE_MPI":     {"type": "BOOL", "value": "OFF"},
         "CMAKE_CXX_FLAGS": "-Wall -Wextra -pedantic -Werror -Wno-stringop-overread",
@@ -26,6 +24,15 @@
         "CMAKE_EXPORT_COMPILE_COMMANDS": {"type": "BOOL", "value": "ON"}
       }
     },
+    {
+      "name": ".orange",
+      "hidden": true,
+      "cacheVariables": {
+        "CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"},
+        "CMAKE_EXE_LINKER_FLAGS": "-Wl,-z,defs",
+        "CMAKE_SHARED_LINKER_FLAGS": "-Wl,-z,defs"
+      }
+    },
     {
       "name": "release",
       "displayName": "Build with full optimizations (VecGeom)",
@@ -42,26 +49,19 @@
     {
       "name": "release-orange",
       "displayName": "Build with full optimizations (ORANGE)",
-      "inherits": ["release"],
-      "cacheVariables": {
-        "CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"}
-      }
+      "inherits": [".orange", "release"]
     },
     {
       "name": "reldeb-orange",
       "displayName": "Build with optimizations, but enable host debugging (ORANGE)",
-      "inherits": ["reldeb"],
-      "cacheVariables": {
-        "CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"}
-      }
+      "inherits": [".orange", "reldeb"]
     },
     {
       "name": "debug-orange",
       "displayName": "Build without optimization, enabling full host debugging (ORANGE)",
-      "inherits": [".base", ".debug"],
+      "inherits": [".orange", ".base", ".debug"],
       "cacheVariables": {
-        "CELERITAS_DEVICE_DEBUG":{"type": "BOOL", "value": "OFF"},
-        "CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"}
+        "CELERITAS_DEVICE_DEBUG":{"type": "BOOL", "value": "OFF"}
       }
     },
     {