#6598 cat-log: new auto cat/tail mode

cylc/flow/scripts/cat_log.py

@@ -152,12 +152,20 @@
 #                  and kills its tail subprocess, then exits as finished
 
 
+PRINT = 'print'
+LISTDIR = 'list-dir'
+PRINTDIR = 'print-dir'
+CAT = 'cat'
+TAIL = 'tail'
+AUTO = 'auto'
+
 MODES = {
-    'p': 'print',
-    'l': 'list-dir',
-    'd': 'print-dir',
-    'c': 'cat',
-    't': 'tail',
+    'p': PRINT,
+    'l': LISTDIR,
+    'd': PRINTDIR,
+    'c': CAT,
+    't': TAIL,
+    'a': AUTO,
 }
 
 
@@ -253,14 +261,14 @@ def view_log(
 
     """
     # The log file path may contain '$USER' to be evaluated on the job host.
-    if mode == 'print':
+    if mode == PRINT:
         # Print location even if the workflow does not exist yet.
         print(logpath)
         return 0
-    if mode == 'print-dir':
+    if mode == PRINTDIR:
         print(os.path.dirname(logpath))
         return 0
-    if mode == 'list-dir':
+    if mode == LISTDIR:
         dirname = os.path.dirname(logpath)
         if not os.path.exists(dirname):
             sys.stderr.write(f"Directory not found: {dirname}\n")
@@ -276,12 +284,12 @@ def view_log(
     if prepend_path:
         from cylc.flow.hostuserutil import get_host
         print(f'# {get_host()}:{logpath}')
-    if mode == 'cat':
+    if mode == CAT:
         # print file contents to stdout.
         if batchview_cmd is not None:
             cmd = shlex.split(batchview_cmd)
         else:
-            cmd = ['cat', logpath]
+            cmd = [CAT, logpath]
         proc1 = Popen(  # nosec
             cmd,
             stdin=DEVNULL,
@@ -290,7 +298,7 @@ def view_log(
         # * batchview command is user configurable
         colorise_cat_log(proc1, color=color)
         return 0
-    if mode == 'tail':
+    if mode == TAIL:
         if batchview_cmd is not None:
             cmd = batchview_cmd
         else:
@@ -367,7 +375,7 @@ def get_task_job_attrs(workflow_id, point, task, submit_num):
     """Retrieve job info from the database.
 
     * live_job_id is the job ID if job is running, else None.
-    * submit_failed is True if the the submission failed.
+    * submit_failed is True if the submission failed.
 
     Returns:
         tuple - (platform, job_runner_name, live_job_id, submit_failed)
@@ -466,7 +474,11 @@ def _main(
         file_name: str = options.filename or 's'
         log_file_path: Path
 
-        if mode == 'list-dir':
+        # auto mode only applies to task logs. Default to cat mode.
+        if mode == AUTO:
+            mode = CAT
+
+        if mode == LISTDIR:
             # list workflow logs
             print('\n'.join(sorted(
                 str(path.relative_to(log_dir))
@@ -540,21 +552,25 @@ def _main(
                 # KeyError: Is already long form (standard log, or custom).
         platform_name, _, live_job_id, submit_failed = get_task_job_attrs(
             workflow_id, point, task, submit_num)
+        if mode == AUTO and live_job_id is None:
+            mode = CAT
+        elif mode == AUTO and live_job_id:
+            mode = TAIL
         platform = get_platform(platform_name)
         batchview_cmd = None
         if live_job_id is not None:
             # Job is currently running. Get special job runner log view
             # command (e.g. qcat) if one exists, and the log is out or err.
             conf_key = None
             if options.filename == JOB_LOG_OUT:
-                if mode == 'cat':
+                if mode == CAT:
                     conf_key = "out viewer"
-                elif mode == 'tail':
+                elif mode == TAIL:
                     conf_key = "out tailer"
             elif options.filename == JOB_LOG_ERR:
-                if mode == 'cat':
+                if mode == CAT:
                     conf_key = "err viewer"
-                elif mode == 'tail':
+                elif mode == TAIL:
                     conf_key = "err tailer"
             if conf_key is not None:
                 batchview_cmd_tmpl = None
@@ -604,14 +620,14 @@ def _main(
                 proc = remote_cylc_cmd(
                     cmd,
                     platform,
-                    capture_process=(mode == 'list-dir'),
-                    manage=(mode == 'tail'),
-                    text=(mode == 'list-dir'),
+                    capture_process=(mode == LISTDIR),
+                    manage=(mode == TAIL),
+                    text=(mode == LISTDIR),
                 )
 
             # add and missing items to file listing results
             if isinstance(proc, Popen):
-                # i.e: if mode=='list-dir' and ctrl+c not pressed
+                # i.e: if mode == LISTDIR and ctrl+c not pressed
                 out, err = proc.communicate()
                 files = out.splitlines()
 

tests/functional/cylc-cat-log/14-auto-mode.t

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+# Test "cylc cat-log" using auto mode with a custom tail /command
+. "$(dirname "$0")/test_header"
+#-------------------------------------------------------------------------------
+set_test_number 10
+install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
+create_test_global_config "" "
+[platforms]
+   [[localhost]]
+        tail command template = $PWD/bin/my-tailer.sh %(filename)s
+"
+#-------------------------------------------------------------------------------
+TEST_NAME="${TEST_NAME_BASE}-validate"
+run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}"
+workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}"
+cylc workflow-state "${WORKFLOW_NAME}//1/foo:started" --interval=1 --messages
+# Test that the custom tail command is used when the task is live
+TEST_NAME=${TEST_NAME_BASE}-cat-log-auto-tail
+run_ok "$TEST_NAME" cylc cat-log "${WORKFLOW_NAME}//1/foo" -f o -m a
+grep_ok "HELLO from foo 1" "${TEST_NAME}.stdout"
+#-------------------------------------------------------------------------------
+cylc stop --kill --max-polls=20 --interval=1 "${WORKFLOW_NAME}"
+#-------------------------------------------------------------------------------
+# Test that the custom tail command is not used when the task is not live
+# (i.e., auto mode should not use the custom tail command)
+TEST_NAME=${TEST_NAME_BASE}-cat-log-auto-cat
+run_ok "$TEST_NAME" cylc cat-log "${WORKFLOW_NAME}//1/foo" -f o -m a
+grep_fail "HELLO from foo 1" "${TEST_NAME}.stdout"
+grep_ok "from foo 1" "${TEST_NAME}.stdout"
+#-------------------------------------------------------------------------------
+# Test that the custom tail command is not used when workflow logs are requested
+# (i.e., auto mode should not use the custom tail command)
+TEST_NAME=${TEST_NAME_BASE}-cat-log-auto-workflow
+# Use the workflow name without the task ID to get the workflow logs
+run_ok "$TEST_NAME" cylc cat-log "${WORKFLOW_NAME}" -m a
+grep_fail "HELLO" "${TEST_NAME}.stdout"
+grep_ok "Workflow:.*/f.*/cylc-cat-log/14-auto-mode" "${TEST_NAME}.stdout"
+purge

tests/functional/cylc-cat-log/14-auto-mode/bin/my-tailer.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Modify 'tail' output to prove that cylc used the custom tailer.
+# Exit immediately, for the test (i.e. don't 'tail -F')
+FILE="$1"
+tail -n +1 "${FILE}" | awk '{print "HELLO", $0; fflush() }'

tests/functional/cylc-cat-log/14-auto-mode/flow.cylc

@@ -0,0 +1,15 @@
+[scheduler]
+   [[events]]
+       abort on stall timeout = True
+       stall timeout = PT2M
+[scheduling]
+    [[graph]]
+        R1 = foo
+[runtime]
+    [[foo]]
+        script = """
+            for I in $(seq 1 100); do
+                echo "from $CYLC_TASK_NAME $I"
+                sleep 1
+            done
+        """