From ec80eb7fbaaf20182047cea66a430a79c6174575 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Fri, 25 Jul 2025 10:01:37 +0100 Subject: [PATCH 1/7] remote-init: add debug logging for symlinks --- cylc/flow/task_remote_cmd.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cylc/flow/task_remote_cmd.py b/cylc/flow/task_remote_cmd.py index 4f66c0f9b0f..39dd11e191d 100644 --- a/cylc/flow/task_remote_cmd.py +++ b/cylc/flow/task_remote_cmd.py @@ -105,6 +105,8 @@ def remote_init(install_target: str, rund: str, *dirs_to_symlink: str) -> None: print(f'Error occurred when symlinking.' f' {target} contains an invalid environment variable.') return + if cylc.flow.flags.verbosity > 1: + print(f'$ ln -s "{target}" "{path}"') make_symlink_dir(path, target) srvd = os.path.join(rund, WorkflowFiles.Service.DIRNAME) os.makedirs(srvd, exist_ok=True) From db78dd2b02ed6ed1e9a07a5ae25286209e42efe6 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Fri, 25 Jul 2025 11:54:30 +0100 Subject: [PATCH 2/7] tests/f: fix remote tests & improve portability * Cylc platforms defined in tests must inherit from the test platform config. * Reference tests only capture triggering, not task outcome (add a downstream task if you need to check outputs). * Fix some timing issues. * Use the platform configured SSH command for any add-hoc SSH'es. * Remove `TMPDIR` dependence (ain't always there!). --- .../functional/cylc-cat-log/09-cat-running.t | 1 + .../cylc-cat-log/09-cat-running/flow.cylc | 25 ++++++++-- .../cylc-cat-log/09-cat-running/reference.log | 1 + .../11-cycle-task-event-job-logs-retrieve.t | 4 +- .../17-task-event-job-logs-retrieve-command.t | 2 + .../intelligent-host-selection/04-kill.t | 43 +++++++++-------- .../04-kill/flow.cylc | 17 +++---- .../platforms/02-host-to-platform-upgrade.t | 7 ++- tests/functional/remote/00-basic.t | 9 +++- tests/functional/remote/04-symlink-dirs.t | 47 ++++++++++++------- .../remote/08-symlink-dir-target-exist.t | 15 ++++-- 11 files changed, 112 insertions(+), 59 deletions(-) diff --git a/tests/functional/cylc-cat-log/09-cat-running.t b/tests/functional/cylc-cat-log/09-cat-running.t index fea12354cbf..ca34f02dc4f 100755 --- a/tests/functional/cylc-cat-log/09-cat-running.t +++ b/tests/functional/cylc-cat-log/09-cat-running.t @@ -19,6 +19,7 @@ export REQUIRE_PLATFORM='loc:remote comms:tcp' . "$(dirname "$0")/test_header" set_test_number 2 +export REFTEST_OPTS="--abort-if-any-task-fails" reftest purge exit diff --git a/tests/functional/cylc-cat-log/09-cat-running/flow.cylc b/tests/functional/cylc-cat-log/09-cat-running/flow.cylc index bcf5b32b8b8..89feae86add 100644 --- a/tests/functional/cylc-cat-log/09-cat-running/flow.cylc +++ b/tests/functional/cylc-cat-log/09-cat-running/flow.cylc @@ -9,7 +9,14 @@ [scheduling] [[graph]] - R1 = local-task:echo & remote-task:echo => cat-log + R1 = """ + # start "local-task" and "remote-task", then run cat-log + local-task:echo & remote-task:echo => cat-log + + # add a downstream task to ensure "local-task" and "remote-task" + # succeed (for the reference test) + local-task & remote-task => fin + """ [runtime] [[ECHO]] @@ -17,7 +24,14 @@ cylc__job__wait_cylc_message_started echo rubbish echo garbage >&2 - cylc message 'echo done' + cylc message -- 'echo done' + + # wait up to PT1M for the cat-log task to succeed + # (the workflow will shut down if cat-log fails) + sleep 60 + + # fail if the task was not orphaned by this point + false """ [[[outputs]]] echo = "echo done" @@ -26,9 +40,9 @@ [[remote-task]] platform = {{ environ['CYLC_TEST_PLATFORM'] }} inherit = ECHO + [[cat-log]] script = """ - cylc__job__wait_cylc_message_started for TASK in '1/local-task' '1/remote-task'; do cylc cat-log --debug -f o \ "${CYLC_WORKFLOW_ID}//${TASK}" \ @@ -36,5 +50,10 @@ cylc cat-log --debug -f e \ "${CYLC_WORKFLOW_ID}//${TASK}" \ | grep 'garbage' + # orphan the sleep command + cylc set "${CYLC_WORKFLOW_ID}//${TASK}" done """ + + [[fin]] + run mode = skip diff --git a/tests/functional/cylc-cat-log/09-cat-running/reference.log b/tests/functional/cylc-cat-log/09-cat-running/reference.log index b0bf51390e6..505d86a90d0 100644 --- a/tests/functional/cylc-cat-log/09-cat-running/reference.log +++ b/tests/functional/cylc-cat-log/09-cat-running/reference.log @@ -3,3 +3,4 @@ Final point: 1 1/local-task -triggered off [] 1/remote-task -triggered off [] 1/cat-log -triggered off ['1/local-task', '1/remote-task'] +1/fin -triggered off ['1/local-task', '1/remote-task'] diff --git a/tests/functional/events/11-cycle-task-event-job-logs-retrieve.t b/tests/functional/events/11-cycle-task-event-job-logs-retrieve.t index 47d4a189c4d..e5b200b0785 100755 --- a/tests/functional/events/11-cycle-task-event-job-logs-retrieve.t +++ b/tests/functional/events/11-cycle-task-event-job-logs-retrieve.t @@ -27,12 +27,12 @@ create_test_global_config '' " [platforms] [[_retrieve]] $(cylc config -i "[platforms][$CYLC_TEST_PLATFORM]") + [[_retrieve]] retrieve job logs = True - install target = $CYLC_TEST_PLATFORM [[_no_retrieve]] $(cylc config -i "[platforms][$CYLC_TEST_PLATFORM]") + [[_no_retrieve]] retrieve job logs = False - install target = $CYLC_TEST_PLATFORM " run_ok "${TEST_NAME_BASE}-validate" \ diff --git a/tests/functional/events/17-task-event-job-logs-retrieve-command.t b/tests/functional/events/17-task-event-job-logs-retrieve-command.t index 76245489735..35853d5c80b 100755 --- a/tests/functional/events/17-task-event-job-logs-retrieve-command.t +++ b/tests/functional/events/17-task-event-job-logs-retrieve-command.t @@ -49,6 +49,8 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \ WORKFLOW_LOG_D="${RUN_DIR}/${WORKFLOW_NAME}/log" sed 's/^.* -v //' "${WORKFLOW_LOG_D}/scheduler/my-rsync.log" >'my-rsync.log.edited' +sed -i -E 's/--max-size=[^ ]* //' 'my-rsync.log.edited' # strip "retrieve job logs max size" arg +sort -u 'my-rsync.log.edited' # strip out duplicates (can result from PBS log file spooling) OPT_HEAD='--include=/1 --include=/1/t1' OPT_TAIL='--exclude=/**' diff --git a/tests/functional/intelligent-host-selection/04-kill.t b/tests/functional/intelligent-host-selection/04-kill.t index 21b467d08da..3a1e5340cf8 100644 --- a/tests/functional/intelligent-host-selection/04-kill.t +++ b/tests/functional/intelligent-host-selection/04-kill.t @@ -15,32 +15,38 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- -# Test job kill. Set the auto clearance of badhosts to be << small time -# so that kill will need to retry, despite 'unreachable_host' being idetified -# as unreachable by job submission. +# Test job kill will retry on a different host if there is a connection failure + export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- -set_test_number 4 - -# Uses a fake background job runner to get around the single host restriction. +set_test_number 6 create_test_global_config "" " [scheduler] [[main loop]] [[[reset bad hosts]]] + # Set the auto clearance of badhosts to be << small time so that + # kill will need to retry, despite 'unreachable_host' being + # idetified as unreachable by job submission. interval = PT5S [platforms] + [[goodhostplatform]] + $(cylc config -i "[platforms][$CYLC_TEST_PLATFORM]") + [[goodhostplatform]] hosts = ${CYLC_TEST_HOST} - install target = ${CYLC_TEST_INSTALL_TARGET} [[mixedhostplatform]] + $(cylc config -i "[platforms][$CYLC_TEST_PLATFORM]") + + [[mixedhostplatform]] + # Use a fake background job runner to get around the + # single host restriction. job runner = my_background hosts = unreachable_host, ${CYLC_TEST_HOST} - install target = ${CYLC_TEST_INSTALL_TARGET} [[[selection]]] method = 'definition order' " @@ -57,18 +63,17 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --debug --no-detach \ "${WORKFLOW_NAME}" -LOGFILE="${WORKFLOW_RUN_DIR}/log/scheduler/log" - -# Check that when a task fail badhosts associated with that task's platform -# are removed from the badhosts set. -named_grep_ok "job kill fails" \ - "unreachable_host has been added to the list of unreachable hosts" \ - "${LOGFILE}" - -named_grep_ok "job kill retries & succeeds" \ - "\[jobs-kill out\] \[TASK JOB SUMMARY\].*1/mixedhosttask/01" \ - "${LOGFILE}" +# job kill for mixedhosttask should have attempted on both hosts +grep_workflow_log_ok "${TEST_NAME_BASE}-kill-failed" \ + 'jobs-kill for mixedhostplatform on unreachable_host' # fail +grep_workflow_log_ok "${TEST_NAME_BASE}-kill-retried" \ + "jobs-kill for mixedhostplatform on $CYLC_TEST_HOST" # retry +# both job kills should succeed +grep_workflow_log_ok "${TEST_NAME_BASE}-kill-succeeded-goodhosttask" \ + '1/goodhosttask/01.* job killed' +grep_workflow_log_ok "${TEST_NAME_BASE}-kill-succeeded-mixedhosttask" \ + '1/mixedhosttask/01.* job killed' purge exit 0 diff --git a/tests/functional/intelligent-host-selection/04-kill/flow.cylc b/tests/functional/intelligent-host-selection/04-kill/flow.cylc index ee90a5ecc69..24cebe57006 100644 --- a/tests/functional/intelligent-host-selection/04-kill/flow.cylc +++ b/tests/functional/intelligent-host-selection/04-kill/flow.cylc @@ -24,8 +24,13 @@ """ [runtime] - [[root]] - script = sleep 120 & echo $! >file; wait + [[goodhosttask]] + script = sleep 60 + platform = goodhostplatform + + [[mixedhosttask]] + script = sleep 60 + platform = mixedhostplatform [[mystop]] script=""" @@ -34,14 +39,6 @@ cylc stop $CYLC_WORKFLOW_ID """ - [[goodhosttask]] - platform = goodhostplatform - - [[mixedhosttask]] - script=sleep 120 & echo $! >file; wait - - platform = mixedhostplatform - [[stop_g]] inherit = mystop [[[environment]]] diff --git a/tests/functional/platforms/02-host-to-platform-upgrade.t b/tests/functional/platforms/02-host-to-platform-upgrade.t index 5ce484a43f7..d561176bec4 100644 --- a/tests/functional/platforms/02-host-to-platform-upgrade.t +++ b/tests/functional/platforms/02-host-to-platform-upgrade.t @@ -19,7 +19,12 @@ # The following scenarios should be covered: # - Task with no settings # - Task with a host setting that should match the test platform -export REQUIRE_PLATFORM='loc:remote' + +# Note: We cannot run this test with job runners that may be configured to +# submit the job to a compute node (e.g. pbs, slurm, etc) as we are unable +# to check the hostname +export REQUIRE_PLATFORM='loc:remote runner:?(background|at)' + . "$(dirname "$0")/test_header" set_test_number 7 diff --git a/tests/functional/remote/00-basic.t b/tests/functional/remote/00-basic.t index 3b6ef667e63..eb288b76432 100644 --- a/tests/functional/remote/00-basic.t +++ b/tests/functional/remote/00-basic.t @@ -15,8 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- -# Test remote host settings. -export REQUIRE_PLATFORM='loc:remote' +# Test job execution on a remote platform +# +# Note: We cannot run this test with job runners that may be configured to +# submit the job to a compute node (e.g. pbs, slurm, etc) as we are unable +# to check the hostname +export REQUIRE_PLATFORM='loc:remote runner:?(background|at)' + . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 4 diff --git a/tests/functional/remote/04-symlink-dirs.t b/tests/functional/remote/04-symlink-dirs.t index 720c2f0fa34..ccb27f3db58 100644 --- a/tests/functional/remote/04-symlink-dirs.t +++ b/tests/functional/remote/04-symlink-dirs.t @@ -24,7 +24,10 @@ if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == "$HOME" ]]; then skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"' fi -set_test_number 14 +set_test_number 15 + +mapfile -d ' ' -t SSH_CMD < <(cylc config -d -i "[platforms][${CYLC_TEST_PLATFORM}]ssh command") +TEST_NAME="$(basename "$0")" create_test_global_config "" " [install] @@ -37,14 +40,20 @@ create_test_global_config "" " share/cycle = \$TMPDIR/\$USER/cylctb_tmp_share_dir work = \$TMPDIR/\$USER [[[$CYLC_TEST_INSTALL_TARGET]]] - run = \$TMPDIR/\$USER/test_cylc_symlink/ctb_tmp_run_dir - share = \$TMPDIR/\$USER/test_cylc_symlink/ - log = \$TMPDIR/\$USER/test_cylc_symlink/ - log/job = \$TMPDIR/\$USER/cylctb_tmp_log_job_dir - share/cycle = \$TMPDIR/\$USER/test_cylc_symlink/ctb_tmp_share_dir - work = \$TMPDIR/\$USER/test_cylc_symlink/ + run = \$HOME/cylctb-symlinks/$TEST_NAME/ctb_tmp_run_dir + share = \$HOME/cylctb-symlinks/$TEST_NAME/ + log = \$HOME/cylctb-symlinks/$TEST_NAME/ + log/job = \$HOME/cylctb_tmp_log_job_dir + share/cycle = \$HOME/cylctb-symlinks/$TEST_NAME/ctb_tmp_share_dir + work = \$HOME/cylctb-symlinks/$TEST_NAME/ " +# shellcheck disable=SC2016 +run_ok \ + "${TEST_NAME_BASE}-mkdir" \ + "${SSH_CMD[@]}" "$CYLC_TEST_HOST" \ + 'mkdir -p $HOME/cylc-symlink-test' + install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" \ @@ -86,43 +95,47 @@ for DIR in 'work' 'share' 'log'; do fi done -SSH="$(cylc config -d -i "[platforms][$CYLC_TEST_PLATFORM]ssh command")" - # shellcheck disable=SC2016 -LINK="$(${SSH} "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"'"')" -if [[ "$LINK" == *"/test_cylc_symlink/ctb_tmp_run_dir/cylc-run/${WORKFLOW_NAME}" ]]; then +LINK="$("${SSH_CMD[@]}" "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"'"')" +if [[ "$LINK" == *"/cylctb-symlinks/$TEST_NAME/ctb_tmp_run_dir/cylc-run/${WORKFLOW_NAME}" ]]; then ok "${TEST_NAME_BASE}-run-symlink-exists-ok.remotehost" else + echo "Link points to $LINK" >&2 fail "${TEST_NAME_BASE}-run-symlink-exists-ok.remotehost" fi # shellcheck disable=SC2016 -LINK="$(${SSH} "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/share/cycle'"')" -if [[ "$LINK" == *"/test_cylc_symlink/ctb_tmp_share_dir/cylc-run/${WORKFLOW_NAME}/share/cycle" ]]; then +LINK="$("${SSH_CMD[@]}" "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/share/cycle'"')" +if [[ "$LINK" == *"/cylctb-symlinks/$TEST_NAME/ctb_tmp_share_dir/cylc-run/${WORKFLOW_NAME}/share/cycle" ]]; then ok "${TEST_NAME_BASE}-share/cycle-symlink-exists-ok.remotehost" else + echo "Link points to $LINK" >&2 fail "${TEST_NAME_BASE}-share/cycle-symlink-exists-ok.remotehost" fi # shellcheck disable=SC2016 -LINK="$(${SSH} "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/log/job'"')" +LINK="$("${SSH_CMD[@]}" "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/log/job'"')" if [[ "$LINK" == *"/cylctb_tmp_log_job_dir/cylc-run/${WORKFLOW_NAME}/log/job" ]]; then ok "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost" else + echo "Link points to $LINK" >&2 fail "${TEST_NAME_BASE}-log/job-symlink-exists-ok.remotehost" fi for DIR in 'work' 'share' 'log'; do # shellcheck disable=SC2016 - LINK="$(${SSH} "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/$DIR'"')" - if [[ "$LINK" == *"/test_cylc_symlink/cylc-run/${WORKFLOW_NAME}/${DIR}" ]]; then + LINK="$("${SSH_CMD[@]}" "${CYLC_TEST_HOST}" 'readlink "$HOME/cylc-run/'"$WORKFLOW_NAME"/$DIR'"')" + if [[ "$LINK" == *"/cylctb-symlinks/$TEST_NAME/cylc-run/${WORKFLOW_NAME}/${DIR}" ]]; then ok "${TEST_NAME_BASE}-${DIR}-symlink-exists-ok.remotehost" else + echo "Link points to $LINK" >&2 fail "${TEST_NAME_BASE}-${DIR}-symlink-exists-ok.remotehost" fi done # clean up remote -${SSH} "${CYLC_TEST_HOST}" rm -rf "${TMPDIR}/${USER}/test_cylc_symlink/" +# shellcheck disable=SC2016 +"${SSH_CMD[@]}" "${CYLC_TEST_HOST}" \ + 'rm -rf "${TMPDIR}/${USER}/cylctb-symlinks/" "${TMPDIR}/${USER}/sym-run" "${HOME}/cylctb-symlinks/$TEST_NAME/"' purge exit diff --git a/tests/functional/remote/08-symlink-dir-target-exist.t b/tests/functional/remote/08-symlink-dir-target-exist.t index 1a7f52fb693..e1f251729c7 100644 --- a/tests/functional/remote/08-symlink-dir-target-exist.t +++ b/tests/functional/remote/08-symlink-dir-target-exist.t @@ -19,25 +19,30 @@ export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp' . "$(dirname "$0")/test_header" -set_test_number 5 +set_test_number 6 -SSH_CMD="$(cylc config -d -i "[platforms][${CYLC_TEST_PLATFORM}]ssh command")" +mapfile -d ' ' -t SSH_CMD < <(cylc config -d -i "[platforms][${CYLC_TEST_PLATFORM}]ssh command") +TEST_NAME="$(basename "$0")" create_test_global_config "" " [install] [[symlink dirs]] [[[${CYLC_TEST_INSTALL_TARGET}]]] - run = \$TMPDIR/\$USER/sym-run + run = \$HOME/cylctb-symlinks/$TEST_NAME/ " install_workflow "${TEST_NAME_BASE}" basic +# shellcheck disable=SC2016 +run_ok "${TEST_NAME_BASE}-mkdir" \ + "${SSH_CMD[@]}" "$CYLC_TEST_HOST" 'mkdir -p $HOME/cylc-symlink-test' + run_ok "${TEST_NAME_BASE}-val" cylc validate "$WORKFLOW_NAME" # Run once to setup symlink dirs on remote install target workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "$WORKFLOW_NAME" # Remove remote run dir symlink (but not its target) -$SSH_CMD "$CYLC_TEST_HOST" "rm -rf ~/cylc-run/${WORKFLOW_NAME}" +"${SSH_CMD[@]}" "$CYLC_TEST_HOST" "rm -rf ~/cylc-run/${WORKFLOW_NAME}" # New run should abort delete_db @@ -49,6 +54,6 @@ grep_ok "WorkflowFilesError: Symlink dir target already exists" "${TEST_NAME}.st # Clean up remote symlink dir target # shellcheck disable=SC2016 -$SSH_CMD "$CYLC_TEST_HOST" 'rm -rf "${TMPDIR}/${USER}/sym-run"' +"${SSH_CMD[@]}" "$CYLC_TEST_HOST" 'rm -rf "${TMPDIR}/${USER}/sym-run" "${HOME}/cylctb-symlinks/$TEST_NAME/"' purge From afe6ff34827acb148428ffd3963e23fa28e129a0 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Tue, 30 Sep 2025 15:54:41 +0100 Subject: [PATCH 3/7] tests/f: fix platforms/13 * Test requires the background job runner. * Test names cannot contain spaces. * Revert an erroneous change made in 15ac2fb which broke test reporting. --- tests/functional/lib/bash/test_header | 2 +- tests/functional/platforms/13-ping-pong-host.t | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header index 07d75a7c4df..d43f55bd791 100644 --- a/tests/functional/lib/bash/test_header +++ b/tests/functional/lib/bash/test_header @@ -463,7 +463,7 @@ named_grep_ok() { shift 3 local OPTS="$*" local TEST_NAME - TEST_NAME="grep-ok: ${NAME}" + TEST_NAME="$(basename "${FILE}")-grep-ok" # shellcheck disable=SC2086 if grep ${OPTS} -q -e "${BRE}" "${FILE}"; then ok "${TEST_NAME}" diff --git a/tests/functional/platforms/13-ping-pong-host.t b/tests/functional/platforms/13-ping-pong-host.t index 1f5dc122f9d..b8f7278c3ca 100644 --- a/tests/functional/platforms/13-ping-pong-host.t +++ b/tests/functional/platforms/13-ping-pong-host.t @@ -30,11 +30,11 @@ install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" --debug --no-detach -named_grep_ok "1/remote_task submits to ${CYLC_TEST_PLATFORM}" \ - "\[1/remote_task/01:preparing\] submitted to ${CYLC_TEST_HOST}" \ +named_grep_ok "${TEST_NAME_BASE}-remote" \ + "\[1/remote_task/01:preparing\] submitted to ${CYLC_TEST_PLATFORM}" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" -named_grep_ok "2/remote_task submits to localhost" \ +named_grep_ok "${TEST_NAME_BASE}-local" \ "\[2/remote_task/01:preparing\] submitted to localhost" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" From 4ba34ddac2599386f6dbe59d297819664d513373 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Tue, 30 Sep 2025 17:19:59 +0100 Subject: [PATCH 4/7] tests/f: fix cylc-poll/04 for PBS * Tasks needed to wait for their started messages. * Give the job runner a few seconds grace time for the job to exit the queue. * Document the test. --- .../cylc-poll/04-poll-multi-hosts/flow.cylc | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/functional/cylc-poll/04-poll-multi-hosts/flow.cylc b/tests/functional/cylc-poll/04-poll-multi-hosts/flow.cylc index 10b36b8677c..88879642bcd 100644 --- a/tests/functional/cylc-poll/04-poll-multi-hosts/flow.cylc +++ b/tests/functional/cylc-poll/04-poll-multi-hosts/flow.cylc @@ -1,20 +1,29 @@ #!Jinja2 + [scheduler] UTC mode = True [[events]] expected task failures = 1/local-fail-1, 1/local-fail-2, 1/remote-fail-1 + [scheduling] [[graph]] R1 = """ POLLABLE:start-all => poller POLLABLE:succeed-any # (make member success optional) """ + [runtime] + # turn off error trapping to disable "cylc message" calls + # (this prevents the "succeeded" / "failed" messages getting through) [[POLLABLE]] init-script = cylc__job__disable_fail_signals ERR EXIT + + # these tasks will fail (but not send the failed message or write it to the + # job.status file) [[FAIL]] inherit = POLLABLE script = """ + cylc__job__wait_cylc_message_started echo 'I am failing...' >&2 exit 1 """ @@ -23,9 +32,13 @@ [[remote-fail-1]] inherit = FAIL platform = {{ environ['CYLC_TEST_PLATFORM'] }} + + # these tasks will succeed (but not send the succeeded message or write it + # to the job.status file) [[SUCCESS]] inherit = POLLABLE script = """ + cylc__job__wait_cylc_message_started echo 'I am OK.' { echo 'CYLC_JOB_EXIT=SUCCEEDED' @@ -38,8 +51,18 @@ [[remote-success-1, remote-success-2]] inherit = SUCCESS platform = {{ environ['CYLC_TEST_PLATFORM'] }} + + # this task will poll the active jobs and hopefully detect their + # success / failure [[poller]] script = """ + # give a bit of grace time for the job to leave the job runner queue + sleep 5 + + # poll the job (it should change to a final state) cylc poll "${CYLC_WORKFLOW_ID}//*/POLLABLE" + + # shut down the scheduler (there should not be any active tasks + # left to delay shutdown) cylc stop "${CYLC_WORKFLOW_ID}" """ From e9457ed5ab775a2e47fb528887c4f0f4a89752b9 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Wed, 1 Oct 2025 11:28:56 +0100 Subject: [PATCH 5/7] test/f: fix intelligent-host-selection/04 * Test works by overriding the background job runner so is not compatible with other job runners. --- tests/functional/intelligent-host-selection/04-kill.t | 6 ++++-- .../functional/intelligent-host-selection/04-kill/flow.cylc | 2 -- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/intelligent-host-selection/04-kill.t b/tests/functional/intelligent-host-selection/04-kill.t index 3a1e5340cf8..a4a22b19a15 100644 --- a/tests/functional/intelligent-host-selection/04-kill.t +++ b/tests/functional/intelligent-host-selection/04-kill.t @@ -17,7 +17,7 @@ #------------------------------------------------------------------------------- # Test job kill will retry on a different host if there is a connection failure -export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp' +export REQUIRE_PLATFORM='loc:remote fs:indep comms:tcp runner:background' . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- @@ -29,7 +29,7 @@ create_test_global_config "" " [[[reset bad hosts]]] # Set the auto clearance of badhosts to be << small time so that # kill will need to retry, despite 'unreachable_host' being - # idetified as unreachable by job submission. + # identified as unreachable by job submission. interval = PT5S [platforms] @@ -38,6 +38,7 @@ create_test_global_config "" " [[goodhostplatform]] hosts = ${CYLC_TEST_HOST} + install target = ${CYLC_TEST_HOST} [[mixedhostplatform]] $(cylc config -i "[platforms][$CYLC_TEST_PLATFORM]") @@ -47,6 +48,7 @@ create_test_global_config "" " # single host restriction. job runner = my_background hosts = unreachable_host, ${CYLC_TEST_HOST} + install target = ${CYLC_TEST_HOST} [[[selection]]] method = 'definition order' " diff --git a/tests/functional/intelligent-host-selection/04-kill/flow.cylc b/tests/functional/intelligent-host-selection/04-kill/flow.cylc index 24cebe57006..c45a37654a2 100644 --- a/tests/functional/intelligent-host-selection/04-kill/flow.cylc +++ b/tests/functional/intelligent-host-selection/04-kill/flow.cylc @@ -3,7 +3,6 @@ description = """ Tasks - goodhost: a control to check that everything works - - badhost is always going to fail - mixedhost contains some hosts that will and won't fail """ @@ -18,7 +17,6 @@ [[graph]] # Run good and mixed as controls R1 = """ - goodhosttask & mixedhosttask goodhosttask:start => stop_g mixedhosttask:start => stop_m """ From 883f0e1a34e39723cb246427a7ce66c83989ccc1 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Wed, 1 Oct 2025 12:09:25 +0100 Subject: [PATCH 6/7] tests/f: fix platforms/13 * Test is only compatible with background platforms. --- tests/functional/platforms/13-ping-pong-host.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/platforms/13-ping-pong-host.t b/tests/functional/platforms/13-ping-pong-host.t index b8f7278c3ca..2f75681f3ee 100644 --- a/tests/functional/platforms/13-ping-pong-host.t +++ b/tests/functional/platforms/13-ping-pong-host.t @@ -19,7 +19,7 @@ # If a task has [remote]host=$(subshell) this should be evaluated # every time the task is run. # https://github.com/cylc/cylc-flow/issues/6808 -export REQUIRE_PLATFORM='loc:remote' +export REQUIRE_PLATFORM='loc:remote runner:background' . "$(dirname "$0")/test_header" From 5860116886ce01f541b8b24ede183d3fb7568a30 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Wed, 1 Oct 2025 13:10:46 +0100 Subject: [PATCH 7/7] tests/f: fix test IDs * Test IDs should be prefixed with "$TEST_NAME_BASE". * Test IDs should be valid for use as file names (no ":", "/", " ", etc). * `named_grep_ok` was broken by default :( --- .../10-CYLC_WORKFLOW_SRC_DIR.t | 6 +++--- tests/functional/cylc-lint/00.lint.t | 14 +++++++------- tests/functional/cylc-lint/01.lint-toml.t | 14 +++++++------- .../intelligent-host-selection/00-mixedhost.t | 2 +- .../intelligent-host-selection/02-badhosts.t | 8 ++++---- .../intelligent-host-selection/03-polling.t | 5 +++-- .../06-from-platform-group-fails.t | 4 ++-- tests/functional/job-kill/00-local.t | 2 +- tests/functional/lib/bash/test_header | 4 +--- tests/functional/platforms/12-ping-pong.t | 6 ++++-- tests/functional/run_modes/06-run-mode-overrides.t | 14 ++++++++------ tests/functional/shutdown/21-stop-kill.t | 10 ++++++++-- 12 files changed, 49 insertions(+), 40 deletions(-) diff --git a/tests/functional/cylc-combination-scripts/10-CYLC_WORKFLOW_SRC_DIR.t b/tests/functional/cylc-combination-scripts/10-CYLC_WORKFLOW_SRC_DIR.t index a9300cb0121..3c404de1fac 100644 --- a/tests/functional/cylc-combination-scripts/10-CYLC_WORKFLOW_SRC_DIR.t +++ b/tests/functional/cylc-combination-scripts/10-CYLC_WORKFLOW_SRC_DIR.t @@ -36,7 +36,7 @@ __HERE__ run_ok "${TEST_NAME_BASE}-view" cylc view -p . -named_grep_ok "src-path-in-view-p" "TEST: $PWD" \ +named_grep_ok "${TEST_NAME_BASE}-src-path-in-view-p" "TEST: $PWD" \ "${TEST_NAME_BASE}-view.stdout" # It starts playing: @@ -47,7 +47,7 @@ run_ok "${TEST_NAME_BASE}-vip" \ --workflow-name "${WORKFLOW_NAME}" # It can get CYLC_WORKFLOW_SRC_DIR -named_grep_ok "src-path-available" \ +named_grep_ok "${TEST_NAME_BASE}-src-path-available" \ "TEST: $PWD" "${RUN_DIR}/${WORKFLOW_NAME}/log/config/flow-processed.cylc" # It can be updated with Cylc VR @@ -57,7 +57,7 @@ run_ok "${TEST_NAME_BASE}-vr" \ poll_grep "meta" "${RUN_DIR}/${WORKFLOW_NAME}/log/config/flow-processed.cylc" # It can get CYLC_WORKFLOW_SRC_DIR -named_grep_ok "src-path-available" \ +named_grep_ok "${TEST_NAME_BASE}-src-path-available" \ "TEST: $PWD" "${RUN_DIR}/${WORKFLOW_NAME}/log/config/flow-processed.cylc" cylc stop "${WORKFLOW_NAME}" diff --git a/tests/functional/cylc-lint/00.lint.t b/tests/functional/cylc-lint/00.lint.t index 1882fe36906..3bace63cdf0 100644 --- a/tests/functional/cylc-lint/00.lint.t +++ b/tests/functional/cylc-lint/00.lint.t @@ -30,15 +30,15 @@ rm etc/global.cylc TEST_NAME="${TEST_NAME_BASE}.vanilla" run_fail "${TEST_NAME}" cylc lint . -named_grep_ok "check-for-error-code" "S004" "${TEST_NAME}.stdout" +named_grep_ok "${TEST_NAME_BASE}-check-for-error-code" "S004" "${TEST_NAME}.stdout" TEST_NAME="${TEST_NAME_BASE}.pick-a-ruleset" run_fail "${TEST_NAME}" cylc lint . -r 728 -named_grep_ok "check-for-error-code" "U998" "${TEST_NAME}.stdout" +named_grep_ok "${TEST_NAME_BASE}-check-for-error-code" "U998" "${TEST_NAME}.stdout" TEST_NAME="${TEST_NAME_BASE}.inplace" run_fail "${TEST_NAME}" cylc lint . -i -named_grep_ok "check-for-error-code-in-file" "U998" flow.cylc +named_grep_ok "${TEST_NAME_BASE}-check-for-error-code-in-file" "U998" flow.cylc rm flow.cylc @@ -49,7 +49,7 @@ __HERE__ TEST_NAME="${TEST_NAME_BASE}.pick-a-ruleset-728" run_fail "${TEST_NAME}" cylc lint . -r 728 -named_grep_ok "do-not-upgrade-check-if-compat-mode" "Lint after renaming" "${TEST_NAME}.stderr" +named_grep_ok "${TEST_NAME_BASE}-do-not-upgrade-check-if-compat-mode" "Lint after renaming" "${TEST_NAME}.stderr" TEST_NAME="${TEST_NAME_BASE}.pick-a-ruleset-728-exit-zero" run_ok "${TEST_NAME}" cylc lint . -r 728 --exit-zero @@ -69,15 +69,15 @@ __HERE__ TEST_NAME="${TEST_NAME_BASE}.zero-issues" run_ok "${TEST_NAME}" cylc lint . -named_grep_ok "message on no errors" "found no issues" "${TEST_NAME}.stdout" +named_grep_ok "${TEST_NAME_BASE}-message-on-no-errors" "found no issues" "${TEST_NAME}.stdout" # It returns an error message if you attempt to lint a non-existant location TEST_NAME="it-fails-if-not-target" -run_fail ${TEST_NAME} cylc lint "a-$(uuidgen)" +run_fail "${TEST_NAME}" cylc lint "a-$(uuidgen)" grep_ok "Workflow ID not found" "${TEST_NAME}.stderr" # It returns a reference in reference mode -TEST_NAME="it-returns-a-reference" +TEST_NAME="${TEST_NAME_BASE}-it-returns-a-reference" run_ok "${TEST_NAME}" cylc lint --list-codes named_grep_ok "${TEST_NAME}-contains-style-codes" "^S001:" "${TEST_NAME}.stdout" TEST_NAME="it-returns-a-reference-style" diff --git a/tests/functional/cylc-lint/01.lint-toml.t b/tests/functional/cylc-lint/01.lint-toml.t index fbdd243e859..8e3a4058453 100644 --- a/tests/functional/cylc-lint/01.lint-toml.t +++ b/tests/functional/cylc-lint/01.lint-toml.t @@ -48,12 +48,12 @@ __HERE__ # Control tests -TEST_NAME="it lints without toml file" +TEST_NAME='it-lints-without-toml-file' run_fail "${TEST_NAME}" cylc lint TESTOUT="${TEST_NAME}.stdout" -named_grep_ok "it returns error code" "S004" "${TESTOUT}" -named_grep_ok "it returns error from subdirectory" "niwa.cylc" "${TESTOUT}" -named_grep_ok "it returns a 728 upgrade code" "^\[U" "${TESTOUT}" +named_grep_ok "${TEST_NAME_BASE}-returns-error-code" "S004" "${TESTOUT}" +named_grep_ok "${TEST_NAME_BASE}-returns-error-from-subdirectory" "niwa.cylc" "${TESTOUT}" +named_grep_ok "${TEST_NAME_BASE}-returns-728-upgrade-code" "^\[U" "${TESTOUT}" # Add a pyproject.toml file @@ -75,7 +75,7 @@ exclude = [ __HERE__ # Test that results are different: -TEST_NAME="it_lints_with_toml_file" +TEST_NAME='it-lints-with-toml-file' run_fail "${TEST_NAME}" cylc lint TESTOUT="${TEST_NAME}.stdout" grep_fail "S004" "${TESTOUT}" @@ -93,13 +93,13 @@ cat > flow.cylc <<__HERE__ """ __HERE__ -TEST_NAME="it_fails_if_max-line-length_set" +TEST_NAME="${TEST_NAME_BASE}-it-fails-if-max-line-length-set" run_fail "${TEST_NAME}" cylc lint named_grep_ok "${TEST_NAME}-line-too-long-message" \ "\[${LINE_LEN_NO}\] flow.cylc:2: line > 4 characters." \ "${TEST_NAME}.stdout" -TEST_NAME="it_does_not_fail_if_max-line-length_set_but_ignored" +TEST_NAME='it-does-not-fail-if-max-line-length-set-but-ignored' cat > pyproject.toml <<__HERE__ [tool.cylc.lint] # Check against these rules diff --git a/tests/functional/intelligent-host-selection/00-mixedhost.t b/tests/functional/intelligent-host-selection/00-mixedhost.t index 91e5aef3963..6f27f998628 100644 --- a/tests/functional/intelligent-host-selection/00-mixedhost.t +++ b/tests/functional/intelligent-host-selection/00-mixedhost.t @@ -56,7 +56,7 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \ # Run a bunch of tests on the workflow logs to ensure that warning messages # produced by Intelligent Host Selection Logic have happened. -named_grep_ok "unreachable host warning" \ +named_grep_ok "${TEST_NAME_BASE}-unreachable-host-warning" \ 'unreachable_host has been added to the list of unreachable hosts' \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" diff --git a/tests/functional/intelligent-host-selection/02-badhosts.t b/tests/functional/intelligent-host-selection/02-badhosts.t index 2d12c62f4c9..88284b116ea 100644 --- a/tests/functional/intelligent-host-selection/02-badhosts.t +++ b/tests/functional/intelligent-host-selection/02-badhosts.t @@ -67,16 +67,16 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \ LOGFILE="${WORKFLOW_RUN_DIR}/log/scheduler/log" # Check that badhosttask has submit failed, but not good or mixed -named_grep_ok "badhost task submit failed" \ +named_grep_ok "${TEST_NAME_BASE}-badhost-task-submit-failed" \ "1/badhosttask.* submit-failed" "${LOGFILE}" -named_grep_ok "goodhost suceeded" \ +named_grep_ok "${TEST_NAME_BASE}-goodhost-suceeded" \ "1/mixedhosttask.* succeeded" "${LOGFILE}" -named_grep_ok "mixedhost task suceeded" \ +named_grep_ok "${TEST_NAME_BASE}-mixedhost-task-suceeded" \ "1/goodhosttask.* succeeded" "${LOGFILE}" # Check that when a task fail badhosts associated with that task's platform # are removed from the badhosts set. -named_grep_ok "remove task platform bad hosts after submit-fail" \ +named_grep_ok "${TEST_NAME_BASE}-remove-task-platform-bad-hosts-after-submit-fail" \ "initialisation did not complete (no hosts were reachable)" \ "${LOGFILE}" diff --git a/tests/functional/intelligent-host-selection/03-polling.t b/tests/functional/intelligent-host-selection/03-polling.t index 5d36c8b97ae..fb2c65209e6 100644 --- a/tests/functional/intelligent-host-selection/03-polling.t +++ b/tests/functional/intelligent-host-selection/03-polling.t @@ -75,11 +75,12 @@ LOGFILE="${WORKFLOW_RUN_DIR}/log/scheduler/log" # Check that when a task fail badhosts associated with that task's platform # are removed from the badhosts set. named_grep_ok \ - "job poll fails" \ + "${TEST_NAME_BASE}-job-poll-fails" \ "unreachable_host has been added to the list of unreachable hosts" \ "${LOGFILE}" -named_grep_ok "job poll retries & succeeds" \ +named_grep_ok \ + "${TEST_NAME_BASE}-job-poll-retries-and-succeeds" \ "\[jobs-poll out\] \[TASK JOB SUMMARY\].*1/mixedhosttask/01" \ "${LOGFILE}" diff --git a/tests/functional/intelligent-host-selection/06-from-platform-group-fails.t b/tests/functional/intelligent-host-selection/06-from-platform-group-fails.t index 32eaf01d1e2..4080810b03e 100644 --- a/tests/functional/intelligent-host-selection/06-from-platform-group-fails.t +++ b/tests/functional/intelligent-host-selection/06-from-platform-group-fails.t @@ -53,7 +53,7 @@ logfile="${WORKFLOW_RUN_DIR}/log/scheduler/log" # Check workflow fails for the reason we want it to fail named_grep_ok \ - "Workflow stalled with 1/bad (submit-failed)" \ + "${TEST_NAME_BASE}-workflow-stalled-with-bad-submit-failed" \ "1/bad did not complete the required outputs" \ "$logfile" @@ -80,7 +80,7 @@ done # Look for message indicating that remote init has failed. named_grep_ok \ - "platform: badhostplatform. - initialisation did not complete (no hosts were reachable)" \ + "${TEST_NAME_BASE}-initialisation-did-not-complete" \ "platform: badhostplatform. - initialisation did not complete (no hosts were reachable)" \ "${logfile}" diff --git a/tests/functional/job-kill/00-local.t b/tests/functional/job-kill/00-local.t index 24c9f6bd186..8f59768ab47 100755 --- a/tests/functional/job-kill/00-local.t +++ b/tests/functional/job-kill/00-local.t @@ -29,7 +29,7 @@ TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" \ cylc play --reference-test --debug --no-detach "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- -TEST_NAME=${TEST_NAME_BASE}-ps +TEST_NAME="${TEST_NAME_BASE}-ps" for DIR in "${WORKFLOW_RUN_DIR}"/work/*/t*; do run_fail "${TEST_NAME}.$(basename "$DIR")" ps "$(cat "${DIR}/file")" done diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header index d43f55bd791..4b948d503f1 100644 --- a/tests/functional/lib/bash/test_header +++ b/tests/functional/lib/bash/test_header @@ -457,13 +457,11 @@ __ERR__ named_grep_ok() { # (Put extra grep options like '-E' at end of the command line) - local NAME="$1" + local TEST_NAME="$1" local BRE="$2" local FILE="$3" shift 3 local OPTS="$*" - local TEST_NAME - TEST_NAME="$(basename "${FILE}")-grep-ok" # shellcheck disable=SC2086 if grep ${OPTS} -q -e "${BRE}" "${FILE}"; then ok "${TEST_NAME}" diff --git a/tests/functional/platforms/12-ping-pong.t b/tests/functional/platforms/12-ping-pong.t index 012fc37d0b9..786466fb1f9 100644 --- a/tests/functional/platforms/12-ping-pong.t +++ b/tests/functional/platforms/12-ping-pong.t @@ -30,11 +30,13 @@ install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play "${WORKFLOW_NAME}" --debug --no-detach -named_grep_ok "1/remote_task submits to ${CYLC_TEST_PLATFORM}" \ +named_grep_ok \ + "${TEST_NAME_BASE}-remote_task-submits-to-${CYLC_TEST_PLATFORM}" \ "\[1/remote_task/01:preparing\] submitted to ${CYLC_TEST_PLATFORM}" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" -named_grep_ok "2/remote_task submits to localhost" \ +named_grep_ok \ + "${TEST_NAME_BASE}-remote_task-submits-to-localhost" \ "\[2/remote_task/01:preparing\] submitted to localhost" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" diff --git a/tests/functional/run_modes/06-run-mode-overrides.t b/tests/functional/run_modes/06-run-mode-overrides.t index c7fc3325b8f..e078acb169e 100644 --- a/tests/functional/run_modes/06-run-mode-overrides.t +++ b/tests/functional/run_modes/06-run-mode-overrides.t @@ -23,23 +23,25 @@ set_test_number 6 # Install and run the workflow in live mode (default). # Check that tasks with run mode unset and run mode = live # leave log files, and that skip mode tasks don't. -TEST_NAME="${TEST_NAME_BASE}:live-workflow" install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" -run_ok "${TEST_NAME}:validate" cylc validate "${WORKFLOW_NAME}" -workflow_run_ok "${TEST_NAME}:play" \ +run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" +workflow_run_ok "${TEST_NAME_BASE}-play" \ cylc play "${WORKFLOW_NAME}" \ --no-detach JOB_LOGS="${WORKFLOW_RUN_DIR}/log/job/1000" -run_fail "${TEST_NAME}:config run mode=skip" ls "${JOB_LOGS}/skip_" +run_fail "${TEST_NAME_BASE}-config run mode=skip" ls "${JOB_LOGS}/skip_" for MODE in default live; do - named_grep_ok "${TEST_NAME}:config run mode=${MODE}" "===.*===" "${JOB_LOGS}/${MODE}_/NN/job.out" + named_grep_ok \ + "${TEST_NAME_BASE}-mode=${MODE}" \ + "===.*===" \ + "${JOB_LOGS}/${MODE}_/NN/job.out" done # After broadcasting a change in run_mode to task default_ it now runs # in skip mode and fails to produce a log file: JOB_LOGS="${WORKFLOW_RUN_DIR}/log/job/1001" -run_fail "${TEST_NAME}:broadcast run mode=skip" ls "${JOB_LOGS}/default_/" +run_fail "${TEST_NAME_BASE}-broadcast-run-mode=skip" ls "${JOB_LOGS}/default_/" purge exit 0 diff --git a/tests/functional/shutdown/21-stop-kill.t b/tests/functional/shutdown/21-stop-kill.t index 126a312cd1e..6b82292261a 100644 --- a/tests/functional/shutdown/21-stop-kill.t +++ b/tests/functional/shutdown/21-stop-kill.t @@ -42,7 +42,13 @@ run_ok "${TEST_NAME_BASE}" cylc play --no-detach "${WORKFLOW_NAME}" --debug WORKFLOW_LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" -named_grep_ok 'jobs kill succeeded' "jobs-kill ret_code\] 0" "${WORKFLOW_LOG}" -named_grep_ok 'jobs kill killed 1/foo' "jobs-kill out.*1/foo/01" "${WORKFLOW_LOG}" +named_grep_ok \ + "${TEST_NAME_BASE}-jobs-kill-succeeded" \ + "jobs-kill ret_code\] 0" \ + "${WORKFLOW_LOG}" +named_grep_ok \ + "${TEST_NAME_BASE}-killed-foo" \ + "jobs-kill out.*1/foo/01" \ + "${WORKFLOW_LOG}" purge