cat-log: infer --force-remote for --mode=list-dir if job.out not present

cylc/flow/scripts/cat_log.py

@@ -92,6 +92,7 @@
     JOB_LOG_OUT, JOB_LOG_ERR, JOB_LOG_OPTS, NN, JOB_LOG_ACTIVITY)
 from cylc.flow.terminal import cli_function
 from cylc.flow.platforms import get_platform
+from cylc.flow import LOG
 
 
 if TYPE_CHECKING:
@@ -569,13 +570,18 @@ def _main(
             workflow_id, point, task, submit_num
         )
 
+        job_log_present = (Path(local_log_dir) / "job.out").exists()
+
         log_is_remote = (is_remote_platform(platform)
                          and (options.filename != JOB_LOG_ACTIVITY))
         log_is_retrieved = (platform['retrieve job logs']
                             and live_job_id is None)
         if (
+            # if the log file is not present locally
+            # (e.g. job is running, or job logs not retrieved)
+            (not job_log_present and log_is_remote)
             # only go remote for log files we can't get locally
-            log_is_remote
+            or log_is_remote
             # don't look for remote log files for submit-failed tasks
             # (there might not be any at all)
             and not submit_failed
@@ -602,6 +608,9 @@ def _main(
                 # (Ctrl-C while tailing)
                 # NOTE: This will raise NoHostsError if the platform is not
                 # contactable
+                # For testing purposes
+                if not job_log_present:
+                    LOG.debug("job.out not present, getting job log remotely")
                 proc = remote_cylc_cmd(
                     cmd,
                     platform,

tests/functional/cylc-cat-log/15-remote_when_no_local_log.t

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+# Test "cylc cat-log" for a specific circumstance that caused cat-log to
+# not work properly. This tests simulates small window of time where a
+# workflow has finished but the logs have not yet been retrieved. In this
+# situation cat-log should remote log retrieval
+export REQUIRE_PLATFORM='loc:remote'
+. "$(dirname "$0")/test_header"
+#-------------------------------------------------------------------------------
+set_test_number 5
+create_test_global_config "" "
+[platforms]
+   [[${CYLC_TEST_PLATFORM}]]
+       retrieve job logs = True"
+install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
+#-------------------------------------------------------------------------------
+TEST_NAME="${TEST_NAME_BASE}-run"
+workflow_run_ok "${TEST_NAME}" \
+    cylc play --debug --no-detach \
+        -s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'" "${WORKFLOW_NAME}"
+#-------------------------------------------------------------------------------
+# remote
+TEST_NAME=${TEST_NAME_BASE}-no_log_remote
+run_ok "$TEST_NAME" cylc cat-log --debug -f j "${WORKFLOW_NAME}//1/a-task"
+grep_ok "job.out not present, getting job log remotely" "${TEST_NAME}.stderr"
+
+# remote
+TEST_NAME=${TEST_NAME_BASE}-no_log_remote_list_dir
+run_ok "$TEST_NAME" cylc cat-log --debug --mode=list-dir "${WORKFLOW_NAME}//1/a-task"
+grep_ok "job.out not present, getting job log remotely" "${TEST_NAME}.stderr"
+
+# Clean up the task host.
+purge
+exit

tests/functional/cylc-cat-log/15-remote_when_no_local_log/flow.cylc

@@ -0,0 +1,19 @@
+#!Jinja2
+[scheduler]
+   [[events]]
+       abort on stall timeout = True
+       stall timeout = PT1M
+[scheduling]
+    [[graph]]
+        R1 = a-task => b-task
+[runtime]
+    [[a-task]]
+        script = """
+# Write to task stdout log
+echo "Master, Master! Where are those dreams that I've been after?"
+"""
+        platform = {{CYLC_TEST_PLATFORM | default("localhost")}}
+    [[b-task]]
+        script = """
+        cylc__job__poll_grep_workflow_log 'TaskJobLogsRetrieveContext.*ret_code.*0'
+        rm ${CYLC_WORKFLOW_RUN_DIR}/log/job/1/a-task/NN/job.out"""

tests/integration/scripts/test_cat_log.py

@@ -97,7 +97,7 @@ def test_bad_workflow2(run_dir, brokendir, capsys):
 
 
 def test_bad_task_dir(run_dir, brokendir, capsys):
-    """Check a non existent job log dir in a valid workflow results in error.
+    """Check a non-existent job log dir in a valid workflow results in error.
     """
     parser = cat_log_gop()
     with pytest.raises(SystemExit, match='1'):