Merge pull request #467 from llnl/466-fix-scheduled-pipeline-bug

Prevent jobs-prod from running in scheduled pipelines

Author: ldowen

.gitlab-ci.yml

@@ -28,6 +28,7 @@ stages:
   - update_tpls
   - tpls
   - build_and_install
+  - perf_install
   - run_ats
   - run_perf
   - generate_buildcache
@@ -57,4 +58,9 @@ include:
       - if: '$CI_PIPELINE_SOURCE == "push"'
         when: always
   - local: .gitlab/jobs-prod.yml
+    rules:
+      - if: '$CI_PIPELINE_SOURCE == "scheduled" || $CI_COMMIT_TAG'
+        when: never
+      - if: '$CI_PIPELINE_SOURCE == "push"'
+        when: always
   - local: .gitlab/pages.yml

.gitlab/jobs-mpi.yml

@@ -6,7 +6,10 @@ toss_gcc_mpi_cxxonly_init:
 
 toss_gcc_mpi_cxxonly_tpls:
   extends: [.toss_resource1, .gcc_mpi_cxxonly, .tpls]
-  needs: [toss_gcc_mpi_cxxonly_init]
+  needs:
+    - job: toss_update_tpls
+      optional: true
+    - job: toss_gcc_mpi_cxxonly_init
 
 toss_gcc_mpi_cxxonly_build:
   extends: [.toss_resource1, .gcc_mpi_cxxonly, .build]
@@ -26,14 +29,19 @@ toss_gcc_mpi_init:
 
 toss_gcc_mpi_tpls:
   extends: [.toss_resource2, .gcc_mpi, .tpls]
-  needs: [toss_gcc_mpi_init]
+  needs:
+    - job: toss_update_tpls
+      optional: true
+    - job: toss_gcc_mpi_init
 
 toss_gcc_mpi_build:
-  variables:
-    DO_PERF_INSTALL: 1
   extends: [.toss_resource2, .gcc_mpi, .build_and_test]
   needs: [toss_gcc_mpi_tpls]
 
+toss_gcc_mpi_perf_install:
+  extends: [.toss_resource2, .gcc_mpi, .perf_install]
+  needs: [toss_gcc_mpi_build]
+
 toss_gcc_mpi_test:
   extends: [.toss_resource2, .gcc_mpi, .run_ats]
   needs: [toss_gcc_mpi_build]
@@ -48,15 +56,20 @@ toss_clang_mpi_dev_build_init:
 
 toss_clang_mpi_dev_build_tpls:
   extends: [.toss_resource3, .clang_mpi_dev_build, .tpls]
-  needs: [toss_clang_mpi_dev_build_init]
+  needs:
+    - job: toss_update_tpls
+      optional: true
+    - job: toss_clang_mpi_dev_build_init
 
 toss_clang_mpi_dev_build_build:
   extends: [.toss_resource3, .clang_mpi_dev_build, .build_and_test]
   needs: [toss_clang_mpi_dev_build_tpls]
 
+toss_clang_mpi_dev_build_perf_install:
+  extends: [.toss_resource3, .clang_mpi_dev_build, .perf_install]
+  needs: [toss_clang_mpi_dev_build_build]
+
 toss_clang_mpi_dev_build_test:
-  variables:
-    DO_PERF_INSTALL: 1
   extends: [.toss_resource3, .clang_mpi_dev_build, .run_ats]
   needs: [toss_clang_mpi_dev_build_build]
 
@@ -74,14 +87,19 @@ cray_rocm_mpi_init:
 
 cray_rocm_mpi_tpls:
   extends: [.cray_resource1, .rocm_mpi, .tpls]
-  needs: [cray_rocm_mpi_init]
+  needs:
+    - job: cray_update_tpls
+      optional: true
+    - job: cray_rocm_mpi_init
 
 cray_rocm_mpi_build:
-  variables:
-    DO_PERF_INSTALL: 1
   extends: [.cray_resource1, .rocm_mpi, .build_and_test]
   needs: [cray_rocm_mpi_tpls]
 
+cray_rocm_mpi_perf_install:
+  extends: [.cray_resource1, .rocm_mpi, .perf_install]
+  needs: [cray_rocm_mpi_build]
+
 cray_rocm_mpi_test:
   extends: [.cray_resource1, .rocm_mpi, .run_ats]
   needs: [cray_rocm_mpi_build]
@@ -96,14 +114,19 @@ cray_hip_rocm_mpi_init:
 
 cray_hip_rocm_mpi_tpls:
   extends: [.cray_resource2, .hip_rocm_mpi, .tpls]
-  needs: [cray_hip_rocm_mpi_init]
+  needs:
+    - job: cray_update_tpls
+      optional: true
+    - job: cray_hip_rocm_mpi_init
 
 cray_hip_rocm_mpi_build:
-  variables:
-    DO_PERF_INSTALL: 1
   extends: [.cray_resource2, .hip_rocm_mpi, .build_and_test]
   needs: [cray_hip_rocm_mpi_tpls]
 
+cray_hip_rocm_mpi_perf_install:
+  extends: [.cray_resource2, .hip_rocm_mpi, .perf_install]
+  needs: [cray_hip_rocm_mpi_build]
+
 cray_hip_rocm_mpi_test:
   extends: [.cray_resource2, .hip_rocm_mpi, .run_ats]
   needs: [cray_hip_rocm_mpi_build]

.gitlab/jobs-seq.yml

@@ -6,7 +6,10 @@ toss_gcc_~mpi_init:
 
 toss_gcc_~mpi_tpls:
   extends: [.gcc_~mpi, .tpls, .toss_resource1]
-  needs: [toss_gcc_~mpi_init]
+  needs:
+    - job: toss_update_tpls
+      optional: true
+    - job: toss_gcc_~mpi_init
 
 toss_gcc_~mpi_build:
   extends: [.gcc_~mpi, .build_and_test, .toss_resource1]
@@ -27,7 +30,10 @@ cray_hip_rocm_~mpi_init:
 
 cray_hip_rocm_~mpi_tpls:
   extends: [.cray_resource2, .hip_rocm_~mpi, .tpls]
-  needs: [cray_hip_rocm_~mpi_init]
+  needs:
+    - job: cray_update_tpls
+      optional: true
+    - job: cray_hip_rocm_~mpi_init
 
 cray_hip_rocm_~mpi_build:
   extends: [.cray_resource2, .hip_rocm_~mpi, .build_and_test]
@@ -48,7 +54,10 @@ cray_rocm_~mpi_Debug_init:
 
 cray_rocm_~mpi_Debug_tpls:
   extends: [.cray_resource1, .rocm_~mpi_Debug, .tpls]
-  needs: [cray_rocm_~mpi_Debug_init]
+  needs:
+    - job: cray_update_tpls
+      optional: true
+    - job: cray_rocm_~mpi_Debug_init
 
 cray_rocm_~mpi_Debug_build:
   extends: [.cray_resource1, .rocm_~mpi_Debug, .build_and_test]

.gitlab/os.yml

@@ -5,20 +5,22 @@
   variables:
     LLNL_SERVICE_USER: sphapp
     UPSTREAM_DIR: /usr/WS2/sduser/Spheral/spack_upstream/0.22
-    # On merge to develop, do the install in this directory
-    PERF_INSTALL_DIR: /usr/workspace/sduser/Spheral/performance_installs
     DISPLAY: ':0.0'
 
 .on_toss_4_x86:
   variables:
     ARCH: 'toss_4_x86_64_ib'
     SPHERAL_BUILDS_DIR: /p/lustre1/sphapp/spheral-ci-builds
+    # On merge to develop, do the install in this directory
+    PERF_INSTALL_DIR: /usr/workspace/sduser/Spheral/performance_installs/$ARCH
     RUN_CMD: 'srun'
   extends: [.sys_config]
 
 .on_toss_4_x86_cray:
   variables:
     ARCH: 'toss_4_x86_64_ib_cray'
     SPHERAL_BUILDS_DIR: /p/lustre1/sphapp/spheral-ci-builds
+    # On merge to develop, do the install in this directory
+    PERF_INSTALL_DIR: /usr/workspace/sduser/Spheral/performance_installs/$ARCH
     RUN_CMD: 'flux run'
   extends: [.sys_config]

.gitlab/performance.yml

@@ -5,15 +5,15 @@
     GIT_STRATEGY: none
     FF_ENABLE_BASH_EXIT_CODE_CHECK: 1
     FF_USE_NEW_BASH_EVAL_STRATEGY: 1
-    INSTALL_LOCATION: "$PERF_INSTALL_DIR/$SYS_TYPE/${SPEC#?}"
   rules:
     - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "Performance tests"'
     - if: '$CI_COMMIT_MESSAGE =~ /test-perf/'
   script:
     # This script submits a job that runs at 7
     # Benchmark directory is set in scripts/spheral_ats.py
-    - echo "Submitting performance tests for install at $INSTALL_LOCATION"
-    - $INSTALL_LOCATION/bin/spheral-ats --ciRun --numNodes 2 --delay --batch --logs perf_logs $INSTALL_LOCATION/$PERF_ATS_FILE || exit_code=$?
+    - INSTALL_DIR=$PERF_INSTALL_DIR/${SPEC#?}
+    - echo "Running performance tests using install at $INSTALL_DIR"
+    - $INSTALL_DIR/bin/spheral-ats --ciRun --numNodes 2 --delay --batch --logs perf_logs $INSTALL_DIR/$PERF_ATS_FILE || exit_code=$?
     - exit $exit_code
 
 toss_gcc_mpi_perf:

.gitlab/scripts.yml

@@ -40,18 +40,10 @@
   stage: build_and_install
   variables:
     GIT_STRATEGY: none
-  rules:
-    # If test-perf is in commit message or this is a merge to develop, do install in $PERF_INSTALL_DIR
-    - if: '($CI_COMMIT_MESSAGE =~ /test-perf/ || $CI_COMMIT_REF_NAME == "develop") && $DO_PERF_INSTALL'
-      variables:
-        EXTRA_INSTALL_ARGS: "-i $PERF_INSTALL_DIR/$SYS_TYPE/${SPEC#?}" # Removes the leading %
-    - when: always
-      variables:
-        EXTRA_INSTALL_ARGS: "--no-clean"
   script:
     - CI_BUILD_DIR=$(cat ci-dir.txt)
     - cd $CI_BUILD_DIR && cat job-name.txt
-    - ./$SCRIPT_DIR/devtools/host-config-build.py $EXTRA_INSTALL_ARGS --build --ctest --nprocs $NPROC --host-config gitlab.cmake -DSPHERAL_ENABLE_TIMERS=ON $EXTRA_CMAKE_ARGS
+    - ./$SCRIPT_DIR/devtools/host-config-build.py --no-clean --build --ctest --nprocs $NPROC --host-config gitlab.cmake -DSPHERAL_ENABLE_TIMERS=ON $EXTRA_CMAKE_ARGS
   artifacts:
     paths:
       - ci-dir.txt
@@ -69,6 +61,28 @@
       - ci-dir.txt
       - job-name.txt
 
+# This copies the install so scheduled perf tests can use it
+.perf_install:
+  extends: [.retry_rule]
+  stage: perf_install
+  variables:
+    GIT_STRATEGY: none
+  rules:
+    - if: '($CI_COMMIT_MESSAGE =~ /test-perf/ || $CI_COMMIT_REF_NAME == "develop")'
+      when: always
+  script:
+    - CI_BUILD_DIR=$(cat ci-dir.txt)
+    - INSTALL_DIR=$PERF_INSTALL_DIR/${SPEC#?}
+    - echo "Copying install from $CI_BUILD_DIR/build_gitlab/install to $INSTALL_DIR"
+    - ml load mpifileutils
+    - if [[ -d $INSTALL_DIR ]]; then $RUN_CMD -n $NPROC drm $INSTALL_DIR; else echo "No install exists."; fi
+    - cd $CI_BUILD_DIR
+    - $RUN_CMD -n $NPROC dcp ./build_gitlab/install $INSTALL_DIR
+    - $RUN_CMD -n $NPROC dchmod --mode go+rx $INSTALL_DIR
+  artifacts:
+    paths:
+      - ci-dir.txt
+
 .run_ats:
   extends: [.retry_rule]
   stage: run_ats

RELEASE_NOTES.md

@@ -7,11 +7,12 @@ Notable changes include:
   * New features / API changes:
     * Added view class for PairwiseField (PairwiseFieldView).
     * Refactored use of pair-wise fields in hydro packages to avoid using pointers and allow empty PairwiseFields.
+    * Bin files in install (bin/spheral and bin/spheral-ats) now use relative paths instead of being configured for one specific path. This allows installs to be relocatable.
 
   * Bug fixes
 
   * Build changes / improvements:
-    * Updated to PYB11Genertor 2025.12.1
+    * Updated to PYB11Generator 2025.12.1.
     * Converted all Spheral Python modules to be submodules of a single PYB11Generator module (SpheralCompiledModules).
       For users importing from the master Spheral.py file (or it's dimensional specialization) this change is hidden,
       so there is no user interface impact.
@@ -20,7 +21,7 @@ Notable changes include:
       * The performance tests and deploy jobs are now scheduled pipelines that run nightly.
       * Gitlab pages now uses Plotly for interactive visualizations.
       * Gathering of performance data for the deploy stage is now parallelized over ranks and threads.
-      * Merges to develop will trigger builds to be done in a shared directory for use by the performance testing scripts.
+      * Merges to develop will copy installs to a shared directory for use by the performance testing scripts.
 
 Version v2025.12.0 -- Release date 2025-12-19
 ==============================================

scripts/CMakeLists.txt

@@ -11,18 +11,9 @@ if (SPHERAL_ENABLE_PYTHON)
   # Build Stage
   # ***********
 
-  # Configure the build time scripts to run a build dir spheral environment.
-  set(SPHERAL_EXEC_DIR ${CMAKE_BINARY_DIR})
-  set(SPHERAL_SCRIPT_DIR ${SPHERAL_ROOT_DIR})
-  configure_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/spheral-env.in"
-    "${CMAKE_BINARY_DIR}/bin/spheral"
-    FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
-  )
-
-  configure_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/atstest.in"
-    "${CMAKE_BINARY_DIR}/bin/spheral-ats"
+  # Copy the spheral-env to run a build time environment.
+  file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/spheral-env.sh"
+    DESTINATION "${CMAKE_BINARY_DIR}/bin/spheral"
     FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
   )
 
@@ -47,28 +38,25 @@ if (SPHERAL_ENABLE_PYTHON)
     DESTINATION "${CMAKE_INSTALL_PREFIX}/scripts"
   )
 
+  install(FILES
+    "${CMAKE_CURRENT_SOURCE_DIR}/spheral-env.sh"
+    RENAME "spheral"
+    DESTINATION "${CMAKE_INSTALL_PREFIX}/bin"
+    PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
+  )
+
+  install(FILES
+    "${CMAKE_CURRENT_SOURCE_DIR}/atstest.sh"
+    RENAME "spheral-ats"
+    DESTINATION "${CMAKE_INSTALL_PREFIX}/bin"
+    PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
+  )
+
   # Configure scripts for the install stage.
   install( CODE "
   message(\"-- Setting up spheral python virtual environment.\")
 
   execute_process(COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target python_runtime_env)
-
-  message(\"-- Setting up spheral executable scripts.\")
-
-  set(SPHERAL_EXEC_DIR ${CMAKE_INSTALL_PREFIX})
-  set(SPHERAL_SCRIPT_DIR ${CMAKE_INSTALL_PREFIX})
-
-  configure_file(
-    \"${CMAKE_CURRENT_SOURCE_DIR}/spheral-env.in\"
-    \"${CMAKE_INSTALL_PREFIX}/bin/spheral\"
-    FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
-  )
-
-  configure_file(
-    \"${CMAKE_CURRENT_SOURCE_DIR}/atstest.in\"
-    \"${CMAKE_INSTALL_PREFIX}/bin/spheral-ats\"
-    FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
-  )
   ")
 
   # byte compile all of the venv site-packages.
@@ -80,8 +68,9 @@ if (SPHERAL_ENABLE_PYTHON)
   message(\"-- Byte compilation complete.\")
   ")
 
-  message("-- Creating local Silo module path file.")
-  file(WRITE "${CMAKE_BINARY_DIR}/.venv/${SPHERAL_SITE_PACKAGES_PATH}/silo_lib.pth"
-    "@CONFIG_SILO_DIR@/lib"
-  )
+  # This is unused so no need to do it at the moment
+  # message("-- Creating local Silo module path file.")
+  # file(WRITE "${CMAKE_BINARY_DIR}/.venv/${SPHERAL_SITE_PACKAGES_PATH}/silo_lib.pth"
+  #   "@CONFIG_SILO_DIR@/lib"
+  # )
 endif()

scripts/atstest.in

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# Determine the path of this script file
+SPHERAL_BIN_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+$SPHERAL_BIN_DIR/spheral $SPHERAL_BIN_DIR/../scripts/spheral_ats.py "$@"