Merge pull request #471 from llnl/bugfix/fix_broken_schedules

Fix broken schedules and redundant lib installs

Author: ldowen

.gitlab-ci.yml

@@ -13,7 +13,7 @@ variables:
 
   SCRIPT_DIR: "scripts"
   ATS_FILE: "tests/integration.ats"
-  PERF_ATS_FILE: "tests/performance/run_perf.py"
+  PERF_ATS_FILE: "tests/run_perf.py"
   SPHERAL_MODULE: "Spheral"
   SPACK_PKG_NAME: spheral
   DEV_PKG_SPEC: "spheral@develop+mpi~caliper~network%gcc"
@@ -26,14 +26,14 @@ variables:
 stages:
   - init
   - update_tpls
+  - generate_buildcache
+  - install_production
+  - update_permissions
   - tpls
   - build_and_install
   - perf_install
   - run_ats
   - run_perf
-  - generate_buildcache
-  - install_production
-  - update_permissions
   - cleanup
   - deploy
 

.gitlab/pages.yml

@@ -15,7 +15,7 @@ create-pages:
     - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "Deploy"'
     - if: '$CI_COMMIT_MESSAGE =~ /test-deploy/'
   script:
-    - source $HOME/docsvenv/bin/activate
+    - source $HOME/docsenv/bin/activate
     - srun -n 8 --cpus-per-task 14 python3 $SCRIPT_DIR/lc/plot_hist_data.py --threads 14 --pkg-name $SPHERAL_MODULE
   artifacts:
     paths:

.gitlab/performance.yml

@@ -8,13 +8,15 @@
   rules:
     - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "Performance tests"'
     - if: '$CI_COMMIT_MESSAGE =~ /test-perf/'
+  dependencies: []
   script:
-    # This script submits a job that runs at 7
-    # Benchmark directory is set in scripts/spheral_ats.py
+    # Directory storing benchmark data is set in scripts/spheral_ats.py
     - INSTALL_DIR=$PERF_INSTALL_DIR/${SPEC#?}
+    - cd $INSTALL_DIR
     - echo "Running performance tests using install at $INSTALL_DIR"
-    - $INSTALL_DIR/bin/spheral-ats --ciRun --numNodes 2 --delay --batch --logs perf_logs $INSTALL_DIR/$PERF_ATS_FILE || exit_code=$?
-    - exit $exit_code
+    - rm -rf perf_logs
+    - find tests/ -name '*.cali' -print -delete
+    - ./bin/spheral-ats --ciRun --numNodes 2 --logs perf_logs $INSTALL_DIR/$PERF_ATS_FILE
 
 toss_gcc_mpi_perf:
   extends: [.toss_perf_shell, .gcc_mpi, .run_perf]

.gitlab/scripts.yml

@@ -62,6 +62,7 @@
       - job-name.txt
 
 # This copies the install so scheduled perf tests can use it
+# Adding test-perf to commit message will trigger the perf install from that branch
 .perf_install:
   extends: [.retry_rule]
   stage: perf_install
@@ -73,11 +74,11 @@
   script:
     - CI_BUILD_DIR=$(cat ci-dir.txt)
     - INSTALL_DIR=$PERF_INSTALL_DIR/${SPEC#?}
-    - echo "Copying install from $CI_BUILD_DIR/build_gitlab/install to $INSTALL_DIR"
     - ml load mpifileutils
     - if [[ -d $INSTALL_DIR ]]; then $RUN_CMD -n $NPROC drm $INSTALL_DIR; else echo "No install exists."; fi
-    - cd $CI_BUILD_DIR
-    - $RUN_CMD -n $NPROC dcp ./build_gitlab/install $INSTALL_DIR
+    - cd $CI_BUILD_DIR/build_gitlab/build
+    - cmake ../../ -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR
+    - make -j $NPROC install
     - $RUN_CMD -n $NPROC dchmod --mode go+rx $INSTALL_DIR
   artifacts:
     paths:
@@ -95,6 +96,8 @@
     - cd $CI_BUILD_DIR && cat job-name.txt
 
     - ./build_gitlab/install/bin/spheral-ats --ciRun ./build_gitlab/install/$ATS_FILE || exit_code=$?
+    # Clean up Caliper files
+    - find ./build_gitlab/install/tests -name '*.cali' -print0 -delete
     - cp -r test-logs $CI_PROJECT_DIR
     - exit $exit_code
   artifacts:
@@ -112,23 +115,24 @@
     GIT_STRATEGY: empty
   script:
     - CI_BUILD_DIR=$(cat ci-dir.txt)
-
     - ml load mpifileutils
     - cd $SPHERAL_BUILDS_DIR
-    - $RUN_CMD -n 20 drm $CI_BUILD_DIR/..
+    - $RUN_CMD -n $NPROC drm $CI_BUILD_DIR/..
   interruptible: true
 
 # ------------------------------------------------------------------------------
 # Shared TPL scripts.
 # ------------------------------------------------------------------------------
 
+# Updates the TPLs in the upstream. Adding update-tpls to commit message will trigger this
 .update_tpls:
   variables:
     GIT_STRATEGY: clone
     GIT_SUBMODULE_STRATEGY: recursive
   stage: update_tpls
   rules:
     - !reference [.merge_pr_rule, rules]
+    - if: '$CI_COMMIT_MESSAGE =~ /update-tpls/'
   script:
     - ./$SCRIPT_DIR/devtools/tpl-manager.py --update-upstream
 
@@ -138,6 +142,7 @@
     GIT_STRATEGY: none
   rules:
     - !reference [.merge_pr_rule, rules]
+    - if: '$CI_COMMIT_MESSAGE =~ /update-tpls/'
   script:
     - ml load mpifileutils
     - $RUN_CMD -n $NPROC dchmod --mode go+rx $UPSTREAM_DIR
@@ -204,7 +209,7 @@
     - env INSTALL_DIR=$INSTALL_DIR DEV_PKG_SPEC=$DEV_PKG_SPEC BUILD_ALLOC="" SCRIPT_DIR=$SCRIPT_DIR
       bash ./$SCRIPT_DIR/lc/install-from-dev-pkg.sh
     - ml load mpifileutils
-    - dchmod --mode go+rx $INSTALL_DIR
+    - $RUN_CMD -n $NPROC dchmod --mode go+rx $INSTALL_DIR
 
   artifacts:
     paths:
@@ -243,12 +248,14 @@
   dependencies: []
   script:
     - cd $SPHERAL_BUILDS_DIR
-    - MAX_DIR=30
-    - DIR_LIST=$(ls -ltd * | sed "1, $MAX_DIR d" | rev | cut -d ' ' -f1 | rev | paste -sd ' ' - )
-    - echo $DIR_LIST
     - ml load mpifileutils
-    - if [[ $DIR_LIST ]]; then $RUN_CMD -n $NPROC drm $DIR_LIST; else echo "No directories to remove at this time."; fi
+    - |
+      find . -mindepth 1 -maxdepth 1 -type d -mtime +15 -print0 \
+        | while IFS= read -r -d '' dir; do
+            $RUN_CMD -n $NPROC drm "$dir"
+          done
 
+# Remove performance benchmarks that are older than 6 months
 .clean_old_perfs:
   stage: cleanup
   variables:
@@ -260,9 +267,14 @@
   dependencies: []
   script:
     - BENCHMARK_LOC=$(python3 $SCRIPT_DIR/spheral_ats.py --get-benchmark)
-    - ml load mpifileutils
     - cd $BENCHMARK_LOC
-    - env TARGET_DIR=$BENCHMARK_LOC bash ./$SCRIPT_DIR/lc/cleanup_files
+    - ml load mpifileutils
+    - |
+      find . -mindepth 2 -type d -mtime +200 -print0 \
+        | while IFS= read -r -d '' dir; do
+            echo "Removing $dir"
+            $RUN_CMD -n $NPROC drm "$dir"
+          done
 
 # ------------------------------------------------------------------------------
 # Script Utilities
@@ -273,6 +285,7 @@
     - if: '$CI_COMMIT_REF_NAME == "develop"'
       when: always
 
+# Adding test-prod to commit message triggers jobs that rely on this rule
 .tag_release_rule:
   rules:
     - if: $CI_COMMIT_TAG

CMakeLists.txt

@@ -6,7 +6,7 @@ project(spheral LANGUAGES C CXX Fortran VERSION ${SPHERAL_VERSION})
 
 set(SPHERAL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH "Path to Spheral source directory")
 set(SPHERAL_BINARY_DIR ${CMAKE_BINARY_DIR})
-set(SPHERAL_TEST_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX}/tests)
+set(SPHERAL_TEST_INSTALL_PREFIX tests)
 
 include(cmake/SetupSpheral.cmake)
 

Dockerfile

@@ -60,6 +60,7 @@ RUN apt-get update -y && \
         python3-dev \
         python3-venv \
         python3-pip \
+        python3-tk \
         unzip \
         libtool \
         wget \

RELEASE_NOTES.md

@@ -7,24 +7,33 @@ Notable changes include:
   * New features / API changes:
     * Added view class for PairwiseField (PairwiseFieldView).
     * Refactored use of pair-wise fields in hydro packages to avoid using pointers and allow empty PairwiseFields.
-    * Bin files in install (bin/spheral and bin/spheral-ats) now use relative paths instead of being configured for one specific path. This allows installs to be relocatable.
+    * Bin files in install (bin/spheral and bin/spheral-ats) now use relative paths instead of being configured for one specific path.
 
-  * Bug fixes
+  * Bug fixes:
+    * Adiak memory leak is fixed by calling adiak::clean() before exit.
+    * Performance tests no longer import from Spheral proper but only rely on SpheralConfigs.py.
 
   * Build changes / improvements:
     * Updated to PYB11Generator 2025.12.1.
     * Converted all Spheral Python modules to be submodules of a single PYB11Generator module (SpheralCompiledModules).
       For users importing from the master Spheral.py file (or it's dimensional specialization) this change is hidden,
       so there is no user interface impact.
+    * Added update-tpls commit message trigger.
+    * Spheral mpi python interface ensures proper allocation calls are used on Flux machines to avoid strange hangs that can occur if running outside of allocations.
     * Performance testing and CI improvements:
+      * Structure for performance tests is different, with the tests themselves set in a perf_tests.py file and run_perf.py runs the perf tests using ATS.
       * The cleanup old directories job is now a scheduled pipelines that runs weekly.
-      * The performance tests and deploy jobs are now scheduled pipelines that run nightly.
+      * The performance tests and deploy jobs are now scheduled pipelines that run on a schedule.
       * Gitlab pages now uses Plotly for interactive visualizations.
       * Gathering of performance data for the deploy stage is now parallelized over ranks and threads.
-      * Merges to develop will copy installs to a shared directory for use by the performance testing scripts.
-      * Updated build system support spack v1
-        * Updated TPL Manager to support spack v1.1.0
-        * Updated CI to support python v3.12 and Ubuntu 24.04
+      * Merges to develop will create installs to a shared directory for use by the performance testing scripts.
+    * Builds and installs are cleaner:
+      * Rpaths are no longer overwritten, allowing things set in the Spack host config file to be used.
+      * Spheral libraries are only installed once now and a Spheral.pth with a relative path to the install lib is used in the virtual python environment.
+    * Updated build system support spack v1.
+      * Updated TPL Manager to support spack v1.1.0.
+      * Updated CI to support python v3.12 and Ubuntu 24.04.
+      * Added actual Spheral release versions and TPL conditions to Spheral package.
 
 Version v2025.12.0 -- Release date 2025-12-19
 ==============================================

cmake/InstallTPLs.cmake

@@ -124,8 +124,13 @@ if(POLYTOPE_FOUND)
   # Install Polytope python library to our site-packages
   if (SPHERAL_ENABLE_PYTHON)
     install(FILES ${POLYTOPE_INSTALL_PREFIX}/${POLYTOPE_SITE_PACKAGES_PATH}/polytope.so
-      DESTINATION ${CMAKE_INSTALL_PREFIX}/.venv/${SPHERAL_SITE_PACKAGES_PATH}/polytope/
+      DESTINATION .venv/${SPHERAL_SITE_PACKAGES_PATH}/polytope/
     )
+    configure_file(
+      ${POLYTOPE_INSTALL_PREFIX}/${SPHERAL_SITE_PACKAGES_PATH}/polytope/polytope.so
+      ${CMAKE_BINARY_DIR}/.venv/${SPHERAL_SITE_PACKAGES_PATH}/polytope/polytope.so
+      FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
+      COPYONLY)
     if (NOT EXISTS ${POLYTOPE_INSTALL_PREFIX}/${POLYTOPE_SITE_PACKAGES_PATH}/polytope.so)
       message(FATAL_ERROR
         "${POLYTOPE_INSTALL_PREFIX}/${POLYTOPE_SITE_PACKAGES_PATH}/polytope.so not found")
@@ -245,15 +250,3 @@ blt_convert_to_system_includes(TARGETS "${SPHERAL_BLT_DEPENDS}")
 if (EXISTS ${EXTERNAL_SPHERAL_TPL_CMAKE})
   include(${EXTERNAL_SPHERAL_TPL_CMAKE})
 endif()
-
-if (SPHERAL_ENABLE_PYTHON)
-  configure_file(
-    ${POLYTOPE_INSTALL_PREFIX}/${SPHERAL_SITE_PACKAGES_PATH}/polytope/polytope.so
-    ${CMAKE_BINARY_DIR}/.venv/${SPHERAL_SITE_PACKAGES_PATH}/polytope/polytope.so
-    FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
-    COPYONLY)
-
-  install(FILES ${POLYTOPE_INSTALL_PREFIX}/${SPHERAL_SITE_PACKAGES_PATH}/polytope/polytope.so
-    DESTINATION ${CMAKE_INSTALL_PREFIX}/.venv/${SPHERAL_SITE_PACKAGES_PATH}/polytope/
-  )
-endif()

cmake/SetupSpheral.cmake

@@ -137,7 +137,7 @@ set(CMAKE_SKIP_BUILD_RPATH FALSE)
 # (but later on when installing)
 set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
 
-set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}")
+list(APPEND CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
 
 # add the automatically determined parts of the RPATH
 # which point to directories outside the build tree to the install RPATH

cmake/spheral/SpheralAddLibs.cmake

@@ -139,9 +139,6 @@ function(spheral_add_cxx_library package_name _cxx_obj_list)
 
   # Export Spheral target
   install(EXPORT ${export_target_name} DESTINATION lib/cmake)
-
-  # Set the r-path of the C++ lib such that it is independent of the build dir when installed
-  set_target_properties(Spheral_${package_name} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
 endfunction()
 
 #----------------------------------------------------------------------------------------
@@ -286,7 +283,4 @@ function(spheral_add_pybind11_library package_name module_list_name)
 
   set_property(GLOBAL APPEND PROPERTY ${module_list_name} ${package_name})
 
-  # Set the r-path of the C++ lib such that it is independent of the build dir when installed
-  set_target_properties(${MODULE_NAME} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
-
 endfunction()