Merge pull request #3514 from jsiirola/conda-timeout-update

Be more careful when/how we timeout conda install

Author: mrmundt

.github/workflows/test_branches.yml

@@ -304,17 +304,21 @@ jobs:
         # Set up environment
         conda config --set always_yes yes
         conda config --set auto_update_conda false
+        conda config --set channel_priority strict
         conda config --remove channels defaults
         conda config --append channels nodefaults
         conda config --append channels conda-forge
         # Try to install mamba
-        # Note: removed '--update-deps' on 2025-02-28 to work around
-        #       broken libffi(?)
-        conda install -q -y -n base conda-libmamba-solver \
-            || MAMBA_FAILED=1
-        if test -z "$MAMBA_FAILED"; then
-            echo "*** Activating the mamba environment solver ***"
-            conda config --set solver libmamba
+        CONDA_VER=$(conda --version | cut -d\  -f2)
+        if test 23.10 = "`echo -e "23.10\n$CONDA_VER" | sort -V | tail -1`"; then
+            # Note: removed '--update-deps' on 2025-02-28 to work around
+            #       broken libffi(?)
+            conda install -q -y -n base conda-libmamba-solver \
+                || MAMBA_FAILED=1
+            if test -z "$MAMBA_FAILED"; then
+                echo "*** Activating the mamba environment solver ***"
+                conda config --set solver libmamba
+            fi
         fi
         # Add the rest of the channels
         conda config --append channels gurobi
@@ -366,17 +370,19 @@ jobs:
         # possibly if it outputs messages to stderr)
         conda install --update-deps -q -y $CONDA_DEPENDENCIES
         if test -z "${{matrix.slim}}"; then
-            # xpress.init() (from conda) hangs indefinitely on GHA/Windows under
-            # Python 3.10 and 3.11.  Exclude that release on that platform.
+            # xpress.init() (xpress 9.5.1 from conda) hangs indefinitely
+            # on GHA/Windows under Python 3.10 and 3.11.  Exclude that
+            # release on that platform.
             if [[ ${{matrix.TARGET}} == win && ${{matrix.python}} =~ 3.1[01] ]]; then
                 # We would like to just use something like:
                 #  - "!=9.5.1" (conda errors)
                 #  - "<9.5.1|>9.5.1" (conda installs 9.1.2, which also hangs)
                 #  - "<=9.5.0|>9.5.1" (conda seg faults)
-                XPRESS='xpress=9.5.0'
+                XPRESS='xpress>=9.5.2'
             else
                 XPRESS='xpress'
             fi
+            TIMEOUT_MSG="TIMEOUT: killing conda install process"
             PYVER=$(echo "py${{matrix.python}}" | sed 's/\.//g')
             echo "Installing for $PYVER"
             for PKG in 'cplex>=12.10' docplex gurobi "$XPRESS" cyipopt pymumps scip; do
@@ -405,13 +411,22 @@ jobs:
                         # indefinitely (lately while installing SCIP),
                         # we will run "conda install" in a subshell and
                         # kill it after 5-minutes.
-                        (conda install -q -y $PKG) &
-                        conda_pid=$!
-                        (sleep 300 && kill "$conda_pid" 2>/dev/null) &
+                        (conda install -q -y $PKG |& tee conda.log) &
+                        conda_subshell_pid=$!
+                        sleep 1
+                        conda_pid=$(ps | grep conda | sed 's/^ *//' | sed 's/ .*//')
+                        # Only timeout the conda process if it's not to
+                        # the installation phase yet.  Signalling conda
+                        # after it has started the installation process
+                        # can irrecoverably corrupt the environment.
+                        (sleep 300 \
+                            && if test `grep 'Package Plan' conda.log | wc -l` -eq 0; \
+                            then echo "$TIMEOUT_MSG $conda_pid (TERM)" \
+                            && kill -TERM $conda_pid 2>/dev/null ; fi) &
                         timeout_pid=$!
-                        wait "$conda_pid" || _BUILDS=""
-                        kill "$timeout_pid" 2>/dev/null \
-                            || echo "TIMEOUT: killed conda install process"
+                        wait $conda_subshell_pid || _BUILDS=""
+                        kill $timeout_pid 2>/dev/null \
+                            || echo "TIMEOUT: time limit expired"
                     fi
                 fi
                 echo ""

.github/workflows/test_pr_and_main.yml

@@ -341,17 +341,21 @@ jobs:
         # Set up environment
         conda config --set always_yes yes
         conda config --set auto_update_conda false
+        conda config --set channel_priority strict
         conda config --remove channels defaults
         conda config --append channels nodefaults
         conda config --append channels conda-forge
         # Try to install mamba
-        # Note: removed '--update-deps' on 2025-02-28 to work around
-        #       broken libffi(?)
-        conda install -q -y -n base conda-libmamba-solver \
-            || MAMBA_FAILED=1
-        if test -z "$MAMBA_FAILED"; then
-            echo "*** Activating the mamba environment solver ***"
-            conda config --set solver libmamba
+        CONDA_VER=$(conda --version | cut -d\  -f2)
+        if test 23.10 = "`echo -e "23.10\n$CONDA_VER" | sort -V | tail -1`"; then
+            # Note: removed '--update-deps' on 2025-02-28 to work around
+            #       broken libffi(?)
+            conda install -q -y -n base conda-libmamba-solver \
+                || MAMBA_FAILED=1
+            if test -z "$MAMBA_FAILED"; then
+                echo "*** Activating the mamba environment solver ***"
+                conda config --set solver libmamba
+            fi
         fi
         # Add the rest of the channels
         conda config --append channels gurobi
@@ -403,17 +407,19 @@ jobs:
         # possibly if it outputs messages to stderr)
         conda install --update-deps -q -y $CONDA_DEPENDENCIES
         if test -z "${{matrix.slim}}"; then
-            # xpress.init() (from conda) hangs indefinitely on GHA/Windows under
-            # Python 3.10 and 3.11.  Exclude that release on that platform.
+            # xpress.init() (xpress 9.5.1 from conda) hangs indefinitely
+            # on GHA/Windows under Python 3.10 and 3.11.  Exclude that
+            # release on that platform.
             if [[ ${{matrix.TARGET}} == win && ${{matrix.python}} =~ 3.1[01] ]]; then
                 # We would like to just use something like:
                 #  - "!=9.5.1" (conda errors)
                 #  - "<9.5.1|>9.5.1" (conda installs 9.1.2, which also hangs)
                 #  - "<=9.5.0|>9.5.1" (conda seg faults)
-                XPRESS='xpress=9.5.0'
+                XPRESS='xpress>=9.5.2'
             else
                 XPRESS='xpress'
             fi
+            TIMEOUT_MSG="TIMEOUT: killing conda install process"
             PYVER=$(echo "py${{matrix.python}}" | sed 's/\.//g')
             echo "Installing for $PYVER"
             for PKG in 'cplex>=12.10' docplex gurobi "$XPRESS" cyipopt pymumps scip; do
@@ -442,13 +448,22 @@ jobs:
                         # indefinitely (lately while installing SCIP),
                         # we will run "conda install" in a subshell and
                         # kill it after 5-minutes.
-                        (conda install -q -y $PKG) &
-                        conda_pid=$!
-                        (sleep 300 && kill "$conda_pid" 2>/dev/null) &
+                        (conda install -q -y $PKG |& tee conda.log) &
+                        conda_subshell_pid=$!
+                        sleep 1
+                        conda_pid=$(ps | grep conda | sed 's/^ *//' | sed 's/ .*//')
+                        # Only timeout the conda process if it's not to
+                        # the installation phase yet.  Signalling conda
+                        # after it has started the installation process
+                        # can irrecoverably corrupt the environment.
+                        (sleep 300 \
+                            && if test `grep 'Package Plan' conda.log | wc -l` -eq 0; \
+                            then echo "$TIMEOUT_MSG $conda_pid (TERM)" \
+                            && kill -TERM $conda_pid 2>/dev/null ; fi) &
                         timeout_pid=$!
-                        wait "$conda_pid" || _BUILDS=""
-                        kill "$timeout_pid" 2>/dev/null \
-                            || echo "TIMEOUT: killed conda install process"
+                        wait $conda_subshell_pid || _BUILDS=""
+                        kill $timeout_pid 2>/dev/null \
+                            || echo "TIMEOUT: time limit expired"
                     fi
                 fi
                 echo ""