Fix platform selection from subshell not re-evaluating each time

changes.d/6836.fix.md

@@ -0,0 +1 @@
+Fix a bug causing the results of `platform = $(subshell commands)` to be cached, and preventing re-evaluation for each task with the same config.

cylc/flow/task_job_mgr.py

@@ -1236,7 +1236,7 @@
                 f"\"{itask.identity}\" the following are not compatible:\n"
             )
 
-        host_n, platform_name = None, None
+        host_n, platform_name, orig_platform_name, orig_host_name = [None] * 4
         try:
             if rtconfig['remote']['host'] is not None:
                 host_n = self.task_remote_mgr.eval_host(
@@ -1269,6 +1269,7 @@
                     f"for task {itask.identity}: platform = "
                     f"{rtconfig['platform']} evaluated as {platform_name}"
                 )
+                orig_platform_name = rtconfig['platform']
                 rtconfig['platform'] = platform_name
             elif (
                 platform_name is None
@@ -1278,6 +1279,7 @@
                     f"[{itask}] host = "
                     f"{rtconfig['remote']['host']} evaluated as {host_n}"
                 )
+                orig_host_name = host_n
                 rtconfig['remote']['host'] = host_n
 
             try:
@@ -1306,6 +1308,13 @@
                 # Retry delays, needed for the try_num
                 self._set_retry_timers(itask, rtconfig)
 
+        # Put the original platform and host names back into the
+        # rtconfig. Prevents storing first evaluation of shell expression.
+        if orig_platform_name:
+            rtconfig['platform'] = orig_platform_name
+        if orig_host_name:
+            rtconfig['remote']['host'] = orig_host_name
+
         try:
             job_conf = self._prep_submit_task_job_impl(
                 workflow,

tests/functional/platforms/12-ping-pong.t

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-----------------------------------------------------------------------------
+
+# If a task has a platform set using a subshell this should be evaluated
+# every time the task is run.
+# https://github.com/cylc/cylc-flow/issues/6808
+export REQUIRE_PLATFORM='loc:remote fs:indep'
+
+. "$(dirname "$0")/test_header"
+
+set_test_number 3
+
+create_test_global_config "" "
+    [platforms]
+        [[${CYLC_TEST_HOST}]]
+            hosts = ${CYLC_TEST_HOST}
+            install target = ${CYLC_TEST_HOST}
+"
+
+install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
+
+cat > var_file <<__HERE__
+REMOTE_HOST="${CYLC_TEST_HOST}"
+__HERE__
+
+workflow_run_ok "${TEST_NAME_BASE}-run" \
+    cylc play "${WORKFLOW_NAME}" --debug --no-detach --set-file var_file
+
+named_grep_ok "1/remote_task submits to ${CYLC_TEST_HOST}" \
+    "\[1/remote_task/01:preparing\] submitted to ${CYLC_TEST_HOST}" \
+    "${WORKFLOW_RUN_DIR}/log/scheduler/log"
+
+named_grep_ok "2/remote_task submits to localhost" \
+    "\[2/remote_task/01:preparing\] submitted to localhost" \
+    "${WORKFLOW_RUN_DIR}/log/scheduler/log"
+

tests/functional/platforms/12-ping-pong/flow.cylc

@@ -0,0 +1,44 @@
+#!jinja2
+[scheduler]
+    [[events]]
+        stall timeout = PT0S
+
+[scheduling]
+    cycling mode = integer
+    final cycle point = 2
+    [[graph]]
+        P1 = remote_task[-P1] => toggler => remote_task
+
+[runtime]
+    [[remote_task]]
+        platform = $(cat ${CYLC_WORKFLOW_RUN_DIR}/pretend-hall-info)
+
+    [[toggler]]
+        script = """
+            # Toggle the platform between localhost and the remote host
+            # using the content of a file, ${TEST_DIR}/pretend-hall-info.
+            # between localhost and the remote.
+
+            A="localhost"
+            B="${REMOTE_HOST}"
+
+            # Create pretend-hall-info file if it does not exist
+            if [[ ! -f ${TEST_DIR}/pretend-hall-info ]]; then
+                echo "localhost" > ${TEST_DIR}/pretend-hall-info
+            fi
+
+            # Read the current platform from the file and toggle it
+            this=$(cat ${TEST_DIR}/pretend-hall-info)
+            echo ${TEST_DIR} $this
+            echo "content of pretent-hall-info: $(cat ${TEST_DIR}/pretend-hall-info)"
+            if [[ $this == "localhost" ]]; then
+                echo ${REMOTE_HOST} > ${TEST_DIR}/pretend-hall-info
+                cylc message -- "WARNING:changing platform to ${REMOTE_HOST}"
+            else
+                echo "localhost" > ${TEST_DIR}/pretend-hall-info
+                cylc message -- "WARNING:changing platform to localhost"
+            fi
+        """
+        [[[environment]]]
+            REMOTE_HOST = {{ CYLC_TEST_HOST }}
+            TEST_DIR = ${CYLC_WORKFLOW_RUN_DIR}