Bugfix flux hangs on Corona and Lassen CI jobs. (#2398)

* Bugfix time limits on Corona CI jobs.

* Removed quotes from the lassen catch2 filesystem tests argument.

* Fix the invocation of the unit and integration tests in the lassen distconv CI pipeline.

* Fixed the LC contrib launcher to properly set the correct version of
the PMI (pmix) field for the Corona system.  This resolves hangs in
flux as jobs are finalizing MPI.

* Reset time limits now that the Corona hang has been resolved.

* Renamed the cosmoflow integration test to have a _distconv.py suffix
so that it is picked up by CI.

Removed deadcode.

Author: bvanessen

.gitlab/common/common.yml

@@ -116,6 +116,7 @@
   rules:
     - exists:
       - spack-ci-env-name.sh
+      - ${LBANN_BUILD_PARENT_DIR}/build
   script:
     - echo "== CLEAN BUILD DIRECTORY =="
     - source spack-ci-env-name.sh

.gitlab/lassen/run_integration_tests.sh

@@ -29,8 +29,7 @@ echo "Task: Intergation Tests"
 cd ci_test/integration_tests
 echo "Running integration tests with file pattern: ${TEST_FLAG}"
 export OMP_NUM_THREADS=10
-lrun -N ${LBANN_NNODES} -T $TEST_TASKS_PER_NODE lbann_pfe.sh -m pytest -s -vv --durations=0 ${WEEKLY_FLAG} --junitxml=results.xml ${TEST_FLAG}
-#lrun -N ${LBANN_NNODES} -T $TEST_TASKS_PER_NODE python3 -m pytest -s -vv --durations=0 ${WEEKLY_FLAG} --junitxml=results.xml ${TEST_FLAG}
+lbann_pfe.sh -m pytest -s -vv --durations=0 ${WEEKLY_FLAG} --junitxml=results.xml ${TEST_FLAG}
 status=$(($status + $?))
 
 echo "Task: Finished"

.gitlab/lassen/run_unit_tests.sh

@@ -28,8 +28,7 @@ echo "Task: Unit Tests"
 cd ci_test/unit_tests
 echo "Running unit tests with file pattern: ${TEST_FLAG}"
 export OMP_NUM_THREADS=10
-lrun -N ${LBANN_NNODES} -T $TEST_TASKS_PER_NODE lbann_pfe.sh -m pytest -s -vv --durations=0 --junitxml=results.xml ${TEST_FLAG}
-#lrun -N ${LBANN_NNODES} -T $TEST_TASKS_PER_NODE python3 -m pytest -s -vv --durations=0 --junitxml=results.xml ${TEST_FLAG}
+lbann_pfe.sh -m pytest -s -vv --durations=0 --junitxml=results.xml ${TEST_FLAG}
 status=$(($status + $?))
 
 echo "Task: Finished"

ci_test/integration_tests/test_integration_cosmoflow.py renamed to ci_test/integration_tests/test_integration_cosmoflow_distconv.py

@@ -93,7 +93,7 @@ def test_run_parallel_filesystem_catch_tests(cluster, dirname):
     mpi_launch = get_system_mpi_launch(cluster)
     mpi_output_file_name = 'mpi_filesystem_catch_tests_output-%s-rank=%%r-size=%%s.xml' % (cluster)
     mpi_output_file = os.path.join(output_dir, mpi_output_file_name)
-    mpi_catch_args = [mpi_catch_exe, '"[filesystem]"', '-r', 'junit', '-o', mpi_output_file]
+    mpi_catch_args = [mpi_catch_exe, '[filesystem]', '-r', 'junit', '-o', mpi_output_file]
     output = sp.run(mpi_launch + mpi_catch_args, cwd=build_dir)
     if output.returncode != 0:
         raise AssertionError('return_code={%d}' % output.returncode)

ci_test/unit_tests/test_catch2_unit_tests.py

@@ -132,6 +132,8 @@ def prepend_environment_path(key, prefix):
         set_environment('MIOPEN_CUSTOM_CACHE_DIR', f'{tmpdir}/MIOpen_custom_cache')
         if os.getenv('ROCM_PATH') is not None:
             prepend_environment_path('LD_LIBRARY_PATH', os.path.join(os.getenv('ROCM_PATH'), 'llvm', 'lib'))
+        # Ensure that the correct PMI is set to avoid hangs, etc.
+        launcher_args.append('-o pmi=pmix')
 
     # Optimizations for Sierra-like systems
     if system in ('sierra', 'lassen', 'rzansel'):