host-select: fix compatibility with force-condemned hosts

Author: oliver-sanders

changes.d/6623.fix.md

@@ -0,0 +1,4 @@
+Auto restart: The "force condemn" option (that tells workflows running on a
+server to shutdown as opposed to migrate) hasn't worked with the host-selection
+mechanism since Cylc 8.0.0. This has now been fixed and the "force condemn"
+option has been restored in the documentation.

cylc/flow/cfgspec/globalcfg.py

@@ -826,16 +826,52 @@ def default_for(
                    range.
             ''')
             Conf('condemned', VDR.V_ABSOLUTE_HOST_LIST, desc=f'''
-                These hosts will not be used to run jobs.
+                List run hosts that workflows should *not* run on.
 
-                If workflows are already running on
-                condemned hosts, Cylc will shut them down and
-                restart them on different hosts.
+                These hosts will be subtracted from the
+                `available <global.cylc[scheduler][run hosts]>` hosts
+                preventing new workflows from starting on the "condemned" host.
+
+                Any workflows running on these hosts will either migrate
+                to another host, or shutdown according to
+                :py:mod:`the configuration <cylc.flow.main_loop.auto_restart>`.
+
+                This feature requires ``auto restart`` to be listed
+                in `global.cylc[scheduler][main loop]plugins`.
+
+                For more information, see the
+                :py:mod:`auto restart <cylc.flow.main_loop.auto_restart>`
+                plugin.
+
+                .. rubric:: Example:
+
+                .. code-block:: cylc
+
+                   [scheduler]
+                       [[main loop]]
+                           # activate the "auto restart" plugin
+                           plugins = auto restart
+                       [[run hosts]]
+                           # there are three hosts in the "pool"
+                           available = host1, host2, host3
+
+                           # however two have been taken out:
+                           # * workflows running on "host1" will attempt to
+                           #   restart on "host3"
+                           # * workflows running on "host2" will shutdown
+                           condemned = host1, host2!
 
                 .. seealso::
 
+                   :py:mod:`cylc.flow.main_loop.auto_restart`
                    :ref:`auto-stop-restart`
 
+                .. versionchanged:: 8.4.2
+
+                   The force-condemn ("!") option caused issues at workflow
+                   startup for Cylc versions between 8.0.0 and 8.4.1
+                   inclusive.
+
                 .. versionchanged:: 8.0.0
 
                    {REPLACES}``[suite servers]condemned hosts``.
@@ -1336,7 +1372,7 @@ def default_for(
                 The means by which task progress messages are reported back to
                 the running workflow.
 
-                ..rubric:: Options:
+                .. rubric:: Options:
 
                 zmq
                    Direct client-server TCP communication via network ports

cylc/flow/host_select.py

@@ -128,6 +128,13 @@ def select_workflow_host(cached=True):
     # be returned with the up-to-date configuration.
     global_config = glbl_cfg(cached=cached)
 
+    # condemned hosts may be suffixed with an "!" to activate "force mode"
+    blacklist = []
+    for host in global_config.get(['scheduler', 'run hosts', 'condemned'], []):
+        if host.endswith('!'):
+            host = host[:-1]
+        blacklist.append(host)
+
     return select_host(
         # list of workflow hosts
         global_config.get([
@@ -138,9 +145,7 @@ def select_workflow_host(cached=True):
             'scheduler', 'run hosts', 'ranking'
         ]),
         # list of condemned hosts
-        blacklist=global_config.get(
-            ['scheduler', 'run hosts', 'condemned']
-        ),
+        blacklist=blacklist,
         blacklist_name='condemned host'
     )
 

tests/functional/restart/43-auto-restart-force-override-normal.t

@@ -50,7 +50,10 @@ create_test_global_config '' "
 ${BASE_GLOBAL_CONFIG}
 [scheduler]
     [[run hosts]]
-        available = ${CYLC_TEST_HOST_1}
+        available = ${CYLC_TEST_HOST_1}, ${CYLC_TEST_HOST_2}
+        # ensure the workflow can start if a host is force-condemned
+        # see #6623
+        condemned = ${CYLC_TEST_HOST_2}!
 "
 
 set_test_number 8