Test should error out if pulling from guest fails

When the guest becomes unresponsive, make sure the test
errors out instead of staying in `pending` state.

Resolves #3647

Signed-off-by: Miroslav Vadkerti <[email protected]>

Author: thrix

docs/releases.rst

@@ -4,6 +4,12 @@
     Releases
 ======================
 
+tmt-1.47.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Fixed a bug that caused executed tests to remain in the
+``pending`` state when the machine became unresponsive. Tests will
+now correctly transition to the ``error`` state.
 
 tmt-1.46.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

tests/execute/unresponsive/data/.fmf/version

@@ -0,0 +1 @@
+1

tests/execute/unresponsive/data/unresponsive.fmf

@@ -0,0 +1,12 @@
+/test:
+    test: shutdown -h now && sleep 60
+    duration: 5s
+
+    /error:
+    /pending:
+
+/plan:
+    discover:
+        how: fmf
+    execute:
+        how: tmt

tests/execute/unresponsive/main.fmf

@@ -0,0 +1,5 @@
+summary: Verify that test is marked as errored if machine becomes unresponsive.
+duration: 5m
+tag+:
+  - provision-only
+  - provision-virtual

tests/execute/unresponsive/test.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+rlJournalStart
+    rlPhaseStartSetup
+        rlRun "tmp=\$(mktemp -d)" 0 "Creating tmp directory"
+        rlRun "export TMT_WORKDIR_ROOT=$tmp"
+        rlRun "pushd data"
+    rlPhaseEnd
+
+    rlPhaseStartTest "Test must error out if machine becomes unresponsive during execution (#3630)."
+        rlRun -s "tmt run -vv -a provision -h $PROVISION_HOW" "2"
+        rlAssertGrep 'fail: Failed to pull workdir from the guest.' $rlRun_LOG '-F'
+        rlAssertGrep 'errr /unresponsive/test/error' $rlRun_LOG '-F'
+        rlAssertGrep 'pending /unresponsive/test/pending' $rlRun_LOG '-F'
+    rlPhaseEnd
+
+    rlPhaseStartCleanup
+        rlRun "popd"
+        rlRun "rm -rf $tmp" 0 "Removing tmp directory"
+    rlPhaseEnd
+rlJournalEnd

tmt/steps/execute/internal.py

@@ -631,20 +631,33 @@ def _save_process(
         # overwrite it.
         self.write(invocation.path / TEST_OUTPUT_FILENAME, stdout or '', mode='a', level=3)
 
+        def pull_from_guest() -> None:
+            if not invocation.is_guest_healthy:
+                return
+
+            try:
+                guest.pull(
+                    source=invocation.path,
+                    extend_options=[
+                        *test.test_framework.get_pull_options(invocation, logger),
+                        '--exclude',
+                        str(invocation.path / TEST_OUTPUT_FILENAME),
+                    ],
+                )
+
+                # Fetch plan data content as well in order to prevent
+                # losing logs if the guest becomes later unresponsive.
+                guest.pull(source=self.step.plan.data_directory)
+
+            # Handle failing to pull test artifacts after guest becoming
+            # unresponsive. If not handled test would stay in 'pending' state.
+            # See issue https://github.com/teemtee/tmt/issues/3647.
+            except tmt.utils.RunError:
+                pass
+
         # Fetch #1: we need logs and everything the test produced so we could
         # collect its results.
-        if invocation.is_guest_healthy:
-            guest.pull(
-                source=invocation.path,
-                extend_options=[
-                    *test.test_framework.get_pull_options(invocation, logger),
-                    '--exclude',
-                    str(invocation.path / TEST_OUTPUT_FILENAME),
-                ],
-            )
-            # Fetch plan data content as well in order to prevent
-            # losing logs if the guest becomes later unresponsive.
-            guest.pull(source=self.step.plan.data_directory)
+        pull_from_guest()
 
         # Run after-test checks before extracting results
         invocation.check_results += self.run_checks_after_test(
@@ -656,20 +669,9 @@ def _save_process(
         # results after a successful reboot in the middle of a test.
         invocation.results = self.extract_results(invocation, logger)
 
-        if invocation.is_guest_healthy:
-            # Fetch #2: after-test checks might have produced remote files as well,
-            # we need to fetch them too.
-            guest.pull(
-                source=invocation.path,
-                extend_options=[
-                    *test.test_framework.get_pull_options(invocation, logger),
-                    '--exclude',
-                    str(invocation.path / TEST_OUTPUT_FILENAME),
-                ],
-            )
-            # Fetch plan data content as well in order to prevent
-            # losing logs if the guest becomes later unresponsive.
-            guest.pull(source=self.step.plan.data_directory)
+        # Fetch #2: after-test checks might have produced remote files as well,
+        # we need to fetch them too.
+        pull_from_guest()
 
         # Attach check results to every test result. There might be more than one,
         # and it's hard to pick the main one, who knows what custom results might