feat: generalize use of sparse matrices in kernels

.github/ci/README.md

@@ -29,6 +29,17 @@ Required fields (all of them, always):
   Path under the bucket where integrated-test artifacts are uploaded. For providers whose URI
   scheme is `<scheme>://<bucket>/<path>`, the first path segment is the bucket name.
 
+- `integrated_tests_baseline_fallback_public_url_prefix`
+  Public URL prefix used by integrated tests when the requested baseline archive is missing from
+  both the runner-local baseline cache and this org's public artifact URL prefix. Use an empty
+  string to disable fallback. The workflow appends the baseline archive filename from
+  `.integrated_tests.yaml`, e.g. `baseline_integratedTests-pr3959-16478-faf1698.tar.gz`.
+
+- `local_baseline_dir`
+  Host-side directory used as a persistent integrated-test baseline cache on runners that mount
+  one. Use an empty string for GitHub-hosted, Azure-hosted, or other runners without this mount;
+  those runners will use the configured artifact URLs instead.
+
 - `artifact_public_url_base`
   Public URL root for uploaded artifacts. The consumer builds each final URL as
   `${artifact_public_url_base}/${integrated_tests_artifact_bucket_path}/${filename}`, so the base
@@ -60,6 +71,22 @@ Required fields (all of them, always):
   `CMAKE_CUDA_ARCHITECTURES` for CUDA jobs on that runner. The same exact-match then prefix
   fallback used by `runner_ca_bundle_host_paths` applies. Example: `{"streak2": "86"}`.
 
+- `runner_resource_overrides`
+  Map keyed first by the actual GitHub runner name (`RUNNER_NAME`), then by runner label, then by
+  the prefix before the first `-`. Each value is an object with optional string fields:
+  `docker_run_args`, `nproc`, and `ctest_parallel_level`.
+  When `nproc` or `ctest_parallel_level` is set here, it takes precedence over the workflow
+  matrix value for jobs that land on that runner. `nproc` and `ctest_parallel_level` must be
+  positive integer strings when present.
+  Use a runner-label key for a common cap that applies to any runner selected for that label, for
+  example `{"thinkpad": {"docker_run_args": "--cpus=8", "nproc": "8"}}`.
+  Use actual `RUNNER_NAME` keys only when runner services have known stable names and need
+  exclusive CPU pinning, for example
+  `{"thinkpad1": {"docker_run_args": "--cpuset-cpus=0-7", "nproc": "8"}}`.
+  `docker_run_args` is appended to the job's Docker arguments and can affect host isolation, so
+  review changes to this field with the same care as workflow changes.
+  Use `{}` for orgs that do not need runner-specific resource overrides.
+
 - `runners`
   Map from runner role to runner label. Required roles:
   - `default`

.github/ci/orgs/EarthFlowAI.json

@@ -2,15 +2,34 @@
   "storage_provider": "cloudflare",
   "sccache_profile": "cloudflare",
   "integrated_tests_artifact_bucket_path": "geosx/integratedTests",
-  "artifact_public_url_base": "https://pub-17b272fb651e459ca81dd0fd457099ed.r2.dev",
+  "integrated_tests_baseline_fallback_public_url_prefix": "https://storage.googleapis.com/geosx/integratedTests",
+  "local_baseline_dir": "",
+  "artifact_public_url_base": "https://pub-a0f706ed868c4d19b0bc6c13e7f0bc29.r2.dev",
   "artifact_public_url_bucket_scoped": true,
   "runner_ca_bundle_host_paths": {},
   "runner_cuda_architectures": { "thinkpad": "120" },
+  "runner_resource_overrides": {
+    "thinkpad0": {
+      "docker_run_args": "--cpuset-cpus=0-7",
+      "nproc": "8",
+      "ctest_parallel_level": "1"
+    },
+    "thinkpad1": {
+      "docker_run_args": "--cpuset-cpus=8-15",
+      "nproc": "8",
+      "ctest_parallel_level": "1"
+    },
+    "thinkpad2": {
+      "docker_run_args": "--cpuset-cpus=16-23",
+      "nproc": "8",
+      "ctest_parallel_level": "1"
+    }
+  },
   "runners": {
-    "default": "ubuntu-22.04",
-    "cpu_heavy": "32core",
-    "integrated_tests": "32core",
-    "code_coverage": "32core",
+    "default": "thinkpad",
+    "cpu_heavy": "thinkpad",
+    "integrated_tests": "32-physical-core",
+    "code_coverage": "thinkpad",
     "cuda": "thinkpad"
   }
 }

.github/ci/orgs/GEOS-DEV.json

@@ -2,10 +2,13 @@
   "storage_provider": "gcp",
   "sccache_profile": "gcp",
   "integrated_tests_artifact_bucket_path": "geosx/integratedTests",
+  "integrated_tests_baseline_fallback_public_url_prefix": "",
+  "local_baseline_dir": "/data/GEOS/baselines",
   "artifact_public_url_base": "https://storage.googleapis.com",
   "artifact_public_url_bucket_scoped": false,
   "runner_ca_bundle_host_paths": { "streak2": "/etc/pki/tls/certs/ca-bundle.crt" },
   "runner_cuda_architectures": { "streak2": "86" },
+  "runner_resource_overrides": {},
   "runners": {
     "default": "ubuntu-22.04",
     "cpu_heavy": "streak2",

.github/workflows/build_and_test.yml

@@ -37,6 +37,10 @@ on:
       DOCKER_RUN_ARGS:
         required: false
         type: string
+      RUNNER_RESOURCE_OVERRIDES_JSON:
+        required: false
+        type: string
+        default: '{}'
       DOCKER_CA_BUNDLE_HOST_PATH:
         required: false
         type: string
@@ -70,6 +74,14 @@ on:
       LOCAL_BASELINE_DIR:
         required: false
         type: string
+      INTEGRATED_TESTS_BASELINE_FALLBACK_PUBLIC_URL_PREFIX:
+        description: >
+          Optional public URL prefix used by integrated tests when the
+          baseline archive is missing from the current org's public artifact
+          location and local runner cache.
+        required: false
+        type: string
+        default: ''
       BUILD_GENERATOR:
         required: false
         type: string
@@ -503,6 +515,7 @@ jobs:
         ARTIFACT_PUBLIC_URL_BUCKET_SCOPED="${{ inputs.ARTIFACT_PUBLIC_URL_BUCKET_SCOPED }}"
         ARTIFACT_UPLOAD_URI_PREFIX="${{ inputs.ARTIFACT_UPLOAD_URI_PREFIX }}"
         ARTIFACT_PUBLIC_URL_PREFIX="${{ inputs.ARTIFACT_PUBLIC_URL_PREFIX }}"
+        INTEGRATED_TESTS_BASELINE_FALLBACK_PUBLIC_URL_PREFIX="${{ inputs.INTEGRATED_TESTS_BASELINE_FALLBACK_PUBLIC_URL_PREFIX }}"
         ARTIFACT_UPLOAD_CREDENTIALS_FILE="${{ steps.artifact_upload_credentials.outputs.credentials_file }}"
         ARTIFACT_UPLOAD_COMMAND="$(cat <<'__ARTIFACT_UPLOAD_COMMAND__'
         ${{ inputs.ARTIFACT_UPLOAD_COMMAND }}
@@ -512,6 +525,27 @@ jobs:
         ${{ inputs.ARTIFACT_UPLOAD_PRE_COMMAND }}
         __ARTIFACT_UPLOAD_PRE_COMMAND__
         )"
+        RUNNER_RESOURCE_OVERRIDES_JSON='${{ inputs.RUNNER_RESOURCE_OVERRIDES_JSON }}'
+        if [[ -z "${RUNNER_RESOURCE_OVERRIDES_JSON}" ]]; then
+          RUNNER_RESOURCE_OVERRIDES_JSON='{}'
+        fi
+        if ! jq -e 'type == "object"' <<< "${RUNNER_RESOURCE_OVERRIDES_JSON}" >/dev/null 2>&1; then
+          echo "::error::RUNNER_RESOURCE_OVERRIDES_JSON must be a JSON object."
+          exit 1
+        fi
+        RUNNER_RESOURCE_OVERRIDE_JSON="$(jq -c \
+          --arg runner_name "${RUNNER_NAME:-}" \
+          --arg runner_label "${{ inputs.RUNS_ON }}" '
+          def prefix($value): ($value | split("-")[0]);
+          .[$runner_name] // .[$runner_label] // .[(prefix($runner_label))] // {}
+        ' <<< "${RUNNER_RESOURCE_OVERRIDES_JSON}")"
+        RUNNER_RESOURCE_DOCKER_RUN_ARGS="$(jq -r '.docker_run_args // ""' <<< "${RUNNER_RESOURCE_OVERRIDE_JSON}")"
+        RUNNER_RESOURCE_NPROC="$(jq -r '.nproc // ""' <<< "${RUNNER_RESOURCE_OVERRIDE_JSON}")"
+        RUNNER_RESOURCE_CTEST_PARALLEL_LEVEL="$(jq -r '.ctest_parallel_level // ""' <<< "${RUNNER_RESOURCE_OVERRIDE_JSON}")"
+
+        if [[ "${RUNNER_RESOURCE_OVERRIDE_JSON}" != "{}" ]]; then
+          echo "Resolved runner resource override for RUNNER_NAME=${RUNNER_NAME:-}, RUNS_ON=${{ inputs.RUNS_ON }}: ${RUNNER_RESOURCE_OVERRIDE_JSON}"
+        fi
 
         if [[ -z "${ARTIFACT_UPLOAD_URI_PREFIX}" && -n "${ARTIFACT_UPLOAD_BUCKET_PATH}" ]]; then
           ARTIFACT_UPLOAD_URI_PREFIX="${ARTIFACT_UPLOAD_URI_ROOT%/}/${ARTIFACT_UPLOAD_BUCKET_PATH}"
@@ -557,16 +591,22 @@ jobs:
           HAS_UPLOAD_TARGET=true
         fi
 
-        if [[ -n "${{ inputs.NPROC }}" ]]; then
-          NPROC=${{ inputs.NPROC }}
-          script_args+=(--nproc ${NPROC})
+        EFFECTIVE_NPROC="${RUNNER_RESOURCE_NPROC}"
+        if [[ -z "${EFFECTIVE_NPROC}" ]]; then
+          EFFECTIVE_NPROC="${{ inputs.NPROC }}"
+        fi
+        if [[ -n "${EFFECTIVE_NPROC}" ]]; then
+          script_args+=(--nproc "${EFFECTIVE_NPROC}")
         fi
 
         if [[ -n "${{ inputs.BUILD_GENERATOR }}" ]]; then
           script_args+=(${{ inputs.BUILD_GENERATOR }})
         fi
 
         docker_args+=(${{ inputs.DOCKER_RUN_ARGS }})
+        if [[ -n "${RUNNER_RESOURCE_DOCKER_RUN_ARGS}" ]]; then
+          docker_args+=(${RUNNER_RESOURCE_DOCKER_RUN_ARGS})
+        fi
 
         if [[ -n "${INPUT_DOCKER_CA_BUNDLE_HOST_PATH}" ]]; then
           if [[ ! -f "${INPUT_DOCKER_CA_BUNDLE_HOST_PATH}" ]]; then
@@ -579,15 +619,26 @@ jobs:
           docker_args+=(-e CURL_CA_BUNDLE=/certs/ca-bundle.crt)
         fi
 
-        COMMIT=${{ github.event.pull_request.head.sha }}
+        if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+          COMMIT="${{ github.event.pull_request.head.sha }}"
+          INTEGRATED_TESTS_ARTIFACT_ID="pr${{ github.event.number }}"
+        else
+          COMMIT="${GITHUB_SHA}"
+          INTEGRATED_TESTS_ARTIFACT_ID="branch"
+        fi
+
+        if [[ -z "${COMMIT}" ]]; then
+          COMMIT="${GITHUB_SHA}"
+        fi
+
         SHORT_COMMIT=${COMMIT:0:7}
         script_args+=(--install-dir-basename GEOS-${SHORT_COMMIT})
 
         if [[ "${{ inputs.BUILD_TYPE }}" == "integrated_tests" || "${HAS_UPLOAD_TARGET}" == "true" ]]; then
           HAS_DATA_EXCHANGE=true
 
           if [[ "${{ inputs.BUILD_TYPE }}" == "integrated_tests" ]]; then
-            DATA_BASENAME=integratedTests-pr${{ github.event.number }}-${{ github.run_number }}-${SHORT_COMMIT}.tar.gz
+            DATA_BASENAME=integratedTests-${INTEGRATED_TESTS_ARTIFACT_ID}-${{ github.run_number }}-${SHORT_COMMIT}.tar.gz
             script_args+=(--run-integrated-tests)
           else
             DATA_BASENAME=GEOS-and-TPL-${SHORT_COMMIT}.tar.gz
@@ -681,15 +732,19 @@ jobs:
         docker_args+=(--name ${CONTAINER_NAME})
 
 
-        if [[ -n "${{ inputs.CTEST_PARALLEL_LEVEL }}" ]]; then
-          script_args+=(--ctest-parallel-level "${{ inputs.CTEST_PARALLEL_LEVEL }}")
+        EFFECTIVE_CTEST_PARALLEL_LEVEL="${RUNNER_RESOURCE_CTEST_PARALLEL_LEVEL}"
+        if [[ -z "${EFFECTIVE_CTEST_PARALLEL_LEVEL}" ]]; then
+          EFFECTIVE_CTEST_PARALLEL_LEVEL="${{ inputs.CTEST_PARALLEL_LEVEL }}"
+        fi
+        if [[ -n "${EFFECTIVE_CTEST_PARALLEL_LEVEL}" ]]; then
+          script_args+=(--ctest-parallel-level "${EFFECTIVE_CTEST_PARALLEL_LEVEL}")
         fi
 
         if ${{ inputs.CODE_COVERAGE }} == 'true'; then
           script_args+=(--code-coverage)
         fi
 
-        if [[ -n "${{ inputs.LOCAL_BASELINE_DIR }}" ]]; then
+        if [[ "${{ inputs.BUILD_TYPE }}" = "integrated_tests" ]]; then
           # Extract the 'baseline' value
 
           # Define the path to the YAML file
@@ -714,18 +769,64 @@ jobs:
           PR_BASELINE_FOLDER_NAME=baselines_${PR_NUMBER}
           echo "Baseline folder name: ${PR_BASELINE_FOLDER_NAME}"
 
-          CURRENT_BASELINE_DIR=${{ inputs.LOCAL_BASELINE_DIR }}/${PR_BASELINE_FOLDER_NAME}
-          echo "Current baseline dir: ${CURRENT_BASELINE_DIR}"
+          LOCAL_BASELINE_DIR_INPUT="${{ inputs.LOCAL_BASELINE_DIR }}"
+          PERSISTENT_BASELINE_DIR=""
+          CURRENT_BASELINE_DIR=""
+
+          if [[ -n "${LOCAL_BASELINE_DIR_INPUT}" ]]; then
+            PERSISTENT_BASELINE_DIR="${LOCAL_BASELINE_DIR_INPUT}/${PR_BASELINE_FOLDER_NAME}"
+            CURRENT_BASELINE_DIR="${PERSISTENT_BASELINE_DIR}"
+            echo "Current baseline dir: ${CURRENT_BASELINE_DIR}"
+          else
+            echo "No persistent local baseline directory configured."
+          fi
+
+          if [[ -z "${CURRENT_BASELINE_DIR}" || ! -d "${CURRENT_BASELINE_DIR}" ]]; then
+            BASELINE_ARCHIVE_NAME="${BASELINE_TAG%.tar.gz}.tar.gz"
+            BASELINE_DOWNLOAD_URLS=()
+
+            if [[ -n "${ARTIFACT_PUBLIC_URL_PREFIX}" ]]; then
+              BASELINE_DOWNLOAD_URLS+=("${ARTIFACT_PUBLIC_URL_PREFIX%/}/${BASELINE_ARCHIVE_NAME}")
+            fi
+
+            if [[ -n "${INTEGRATED_TESTS_BASELINE_FALLBACK_PUBLIC_URL_PREFIX}" ]]; then
+              BASELINE_DOWNLOAD_URLS+=("${INTEGRATED_TESTS_BASELINE_FALLBACK_PUBLIC_URL_PREFIX%/}/${BASELINE_ARCHIVE_NAME}")
+            fi
+
+            if (( ${#BASELINE_DOWNLOAD_URLS[@]} > 0 )); then
+              BASELINE_DOWNLOAD_DIR="${RUNNER_TEMP}/geos-baselines/${PR_BASELINE_FOLDER_NAME}"
+              BASELINE_DOWNLOAD_TMP="${RUNNER_TEMP}/geos-baselines-${BASELINE_ARCHIVE_NAME}"
+              rm -f "${BASELINE_DOWNLOAD_TMP}"
+
+              for BASELINE_DOWNLOAD_URL in "${BASELINE_DOWNLOAD_URLS[@]}"; do
+                if [[ -n "${PERSISTENT_BASELINE_DIR}" ]]; then
+                  echo "Current baselines directory (${PERSISTENT_BASELINE_DIR}) not found; trying ${BASELINE_DOWNLOAD_URL}"
+                else
+                  echo "Trying ${BASELINE_DOWNLOAD_URL}"
+                fi
+                if curl --fail --location --silent --show-error --retry 3 --retry-delay 5 --output "${BASELINE_DOWNLOAD_TMP}" "${BASELINE_DOWNLOAD_URL}"; then
+                  mkdir -p "${BASELINE_DOWNLOAD_DIR}"
+                  mv "${BASELINE_DOWNLOAD_TMP}" "${BASELINE_DOWNLOAD_DIR}/${BASELINE_ARCHIVE_NAME}"
+                  CURRENT_BASELINE_DIR="${BASELINE_DOWNLOAD_DIR}"
+                  echo "Downloaded integrated test baseline archive to transient cache ${CURRENT_BASELINE_DIR}/${BASELINE_ARCHIVE_NAME}"
+                  break
+                fi
+                rm -f "${BASELINE_DOWNLOAD_TMP}"
+              done
+            fi
+          fi
 
-          if [ -d ${CURRENT_BASELINE_DIR} ];then
+          if [ -d "${CURRENT_BASELINE_DIR}" ];then
             echo "Current baseline dir found."
-            ls -l ${CURRENT_BASELINE_DIR}
+            ls -l "${CURRENT_BASELINE_DIR}"
 
             # We defined a mount point and mount it read-only inside the container.
             CURRENT_BASELINE_DIR_MOUNT=/tmp/geos/baselines
             docker_args+=(--volume=${CURRENT_BASELINE_DIR}:${CURRENT_BASELINE_DIR_MOUNT}:ro)
+          elif [[ -n "${PERSISTENT_BASELINE_DIR}" ]]; then
+            echo "Current baselines directory (${PERSISTENT_BASELINE_DIR}) not found"
           else
-            echo "Current baselines directory (${CURRENT_BASELINE_DIR}) not found"
+            echo "No integrated test baselines were found to mount."
           fi
         fi
 
@@ -803,8 +904,7 @@ jobs:
                 # 1. We copy the baselines to a local directory to store them
 
                 # 1.a Create the new target directory to store the new baselines
-                THIS_PR_NUMBER=pr${{ github.event.number }}
-                NEW_PR_BASELINE_FOLDER_NAME=baselines_${THIS_PR_NUMBER}
+                NEW_PR_BASELINE_FOLDER_NAME=baselines_${INTEGRATED_TESTS_ARTIFACT_ID}
                 TARGET_DIR="${{ inputs.LOCAL_BASELINE_DIR }}/${NEW_PR_BASELINE_FOLDER_NAME}"
                 echo "Create folder ${TARGET_DIR}"
                 mkdir -p "${TARGET_DIR}"