tests: run MPI integration tests in parallel, sized to the host

Integration tests launched MPIEXEC_MAX_NUMPROCS ranks, which FindMPI sets to
the host physical core count -> 96 ranks/test on the Jenkins box, massively
oversubscribing and timing out (svo_cthyb_basic_tf).

- Introduce SOLID_DMFT_TEST_NUM_PROC = min(4, cores) for the rank count per
  integration test, use it for all MPI-launched tests, and set the PROCESSORS
  test property so 'ctest -j' budgets concurrency correctly.
- Jenkins: pin SOLID_DMFT_TEST_NUM_PROC=4 (the build pod's cgroup makes nproc
  misreport as 1), disable OpenMPI binding and allow oversubscribe so
  concurrent mpiruns don't stack on cores 0-3, and run 'ctest -j2' (the pod's
  cpuset is only 2 physical cores + HT siblings). macOS m1: 'ctest -j8'.

Author: the-hampel

jenkins/Jenkinsfile

@@ -37,7 +37,14 @@ def runBuild = { String platform, String prefix, List<String> envs ->
   envs += [
     "OPENBLAS_NUM_THREADS=1",
     "MKL_NUM_THREADS=1",
-    "OMP_NUM_THREADS=1"]
+    "OMP_NUM_THREADS=1",
+    /* Tests run concurrently via `ctest -j`, each launching its own mpirun.
+       OpenMPI binds-to-core by default and every mpirun starts binding at
+       core 0, so all concurrent tests pile onto the same few cores while the
+       rest idle (8x slowdowns observed). Disable binding so the scheduler
+       spreads ranks, and allow oversubscription as a safety net. */
+    "OMPI_MCA_hwloc_base_binding_policy=none",
+    "OMPI_MCA_rmaps_base_oversubscribe=yes"]
   /* the triqs venv on macOS is built from the brew python, which may differ from the system python3 */
   def python3 = macos ? "${env.BREW}/bin/python3" : "python3"
   def pythonVer = sh(returnStdout: true, script: """$python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")'""").trim()
@@ -84,13 +91,21 @@ def runBuild = { String platform, String prefix, List<String> envs ->
       args += ' -DASAN=ON -DUBSAN=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo'
     dir(buildDir) {
       sh """#!/bin/sh -ex
-        cmake $WORKSPACE -DCMAKE_INSTALL_PREFIX=$installDir -DTRIQS_ROOT=$triqsDir $args
+        cmake $WORKSPACE -DCMAKE_INSTALL_PREFIX=$installDir -DTRIQS_ROOT=$triqsDir -DSOLID_DMFT_TEST_NUM_PROC=4 $args
         make -j\$PARALLEL || make -j1 VERBOSE=1
       """
       catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
         try {
-          /* run tests in a single rank since they use MPI */
-          sh "ctest --output-on-failure -j1"
+          /* Tests run 4 MPI ranks each (PROCESSORS property) with binding
+             disabled (see envs), so ctest -j is a core budget. The Linux build
+             pod's cpuset (3-4,99-100) is only 2 physical cores + their HT
+             siblings, so -j2 matches the 2 real cores: integration tests
+             (PROCESSORS=4 > 2) run one at a time across all 4 logical CPUs,
+             and unit tests pack 2-up without oversubscribing. Total wall-time
+             is bounded by the 2 physical cores regardless. macOS m1 nodes have
+             ~10 cores, so -j8 = 2 concurrent 4-rank tests. */
+          def ctestJobs = macos ? 8 : 2
+          sh "ctest --output-on-failure -j${ctestJobs}"
         } catch (exc) {
           archiveArtifacts(artifacts: 'Testing/Temporary/LastTest.log')
           throw exc

test/python/CMakeLists.txt

@@ -36,6 +36,21 @@ endforeach()
 
 # ------------------------------#
 
+# MPI ranks per integration test = min(4, cores). Not the raw core count:
+# FindMPI's MPIEXEC_MAX_NUMPROCS = physical cores would spawn 96 ranks/test on
+# the CI box (OOM/timeouts); the cap avoids that, and never exceeding the cores
+# keeps mpirun from requesting more slots than exist. The PROCESSORS property
+# below lets `ctest -j` schedule concurrently. Override with
+# -DSOLID_DMFT_TEST_NUM_PROC=N.
+include(ProcessorCount)
+ProcessorCount(_solid_dmft_ncores)
+if(_solid_dmft_ncores GREATER 0 AND _solid_dmft_ncores LESS 4)
+  set(_solid_dmft_default_nproc ${_solid_dmft_ncores})
+else()
+  set(_solid_dmft_default_nproc 4)
+endif()
+set(SOLID_DMFT_TEST_NUM_PROC ${_solid_dmft_default_nproc} CACHE STRING "MPI ranks launched per integration test")
+
 # integration tests
 set (integration_tests
      svo_hubbardI_basic
@@ -62,12 +77,13 @@ foreach(test ${integration_tests})
 
   add_test(NAME ${test}
            # COMMAND bash ${test}.sh
-           COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ${TRIQS_PYTHON_EXECUTABLE} test.py
+           COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SOLID_DMFT_TEST_NUM_PROC} ${MPIEXEC_PREFLAGS} ${TRIQS_PYTHON_EXECUTABLE} test.py
            # COMMAND ${TRIQS_PYTHON_EXECUTABLE} test.py
            WORKING_DIRECTORY ${test_dir}
           )
 
   set_property(TEST ${test} APPEND PROPERTY ENVIRONMENT PYTHONPATH=${PROJECT_BINARY_DIR}/python:$ENV{PYTHONPATH})
+  set_property(TEST ${test} PROPERTY PROCESSORS ${SOLID_DMFT_TEST_NUM_PROC})
 endforeach()
 
 # ------------------------------#
@@ -80,11 +96,12 @@ foreach(file test_maxent.py helper.py)
 endforeach()
 
 add_test(NAME maxent_${test}
-         COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ${TRIQS_PYTHON_EXECUTABLE} test_maxent.py
+         COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SOLID_DMFT_TEST_NUM_PROC} ${MPIEXEC_PREFLAGS} ${TRIQS_PYTHON_EXECUTABLE} test_maxent.py
          WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
         )
 
 set_property(TEST maxent_${test} APPEND PROPERTY ENVIRONMENT PYTHONPATH=${PROJECT_BINARY_DIR}/python:$ENV{PYTHONPATH})
+set_property(TEST maxent_${test} PROPERTY PROCESSORS ${SOLID_DMFT_TEST_NUM_PROC})
 set_property(TEST maxent_${test} APPEND PROPERTY DEPENDS "svo_hubbardI_basic")
 
 # ------------------------------#
@@ -101,11 +118,12 @@ foreach(test ${restart_integration_test})
   endforeach()
 
   add_test(NAME restart_${test}
-           COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ${TRIQS_PYTHON_EXECUTABLE} test_restart.py
+           COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SOLID_DMFT_TEST_NUM_PROC} ${MPIEXEC_PREFLAGS} ${TRIQS_PYTHON_EXECUTABLE} test_restart.py
            WORKING_DIRECTORY ${test_dir}
           )
 
   set_property(TEST restart_${test} APPEND PROPERTY ENVIRONMENT PYTHONPATH=${PROJECT_BINARY_DIR}/python:$ENV{PYTHONPATH})
+  set_property(TEST restart_${test} PROPERTY PROCESSORS ${SOLID_DMFT_TEST_NUM_PROC})
 endforeach()
 set_property(TEST maxent_${test} APPEND PROPERTY DEPENDS "svo_hubbardI_basic")
 
@@ -142,12 +160,13 @@ foreach(test ${gw_emb_integration_tests})
 
   add_test(NAME ${test}
            # COMMAND bash ${test}.sh
-           COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} ${TRIQS_PYTHON_EXECUTABLE} test.py
+           COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${SOLID_DMFT_TEST_NUM_PROC} ${MPIEXEC_PREFLAGS} ${TRIQS_PYTHON_EXECUTABLE} test.py
            # COMMAND ${TRIQS_PYTHON_EXECUTABLE} test.py
            WORKING_DIRECTORY ${test_dir}
           )
 
   set_property(TEST ${test} APPEND PROPERTY ENVIRONMENT PYTHONPATH=${PROJECT_BINARY_DIR}/python:$ENV{PYTHONPATH})
+  set_property(TEST ${test} PROPERTY PROCESSORS ${SOLID_DMFT_TEST_NUM_PROC})
 endforeach()
 
 endif()