Fix: Add validations to suspend_trigger (Bugfix)

 * Run "systemctl list-jobs suspend" to check if there are
   suspend jobs running before running one more.

 * If "systemctl list-jobs suspend" detects suspend
   jobs running wait for a while before proceeding.

 * Set "set -o pipefail" on the Checkbox job definition
   for the job to be marked as "failed" when exiting on error.

Author: EstebanVg15

providers/base/bin/suspend_trigger.py

@@ -67,6 +67,23 @@ def main(args=sys.argv[1:]):
             str(args.sleep_delay),
         ]
         suspend_cmd = ["systemctl", "suspend"]
+        list_jobs_cmd = ["systemctl", "list-jobs", "*suspend*"]
+        timeout = 10
+        while timeout > 0:
+            output = subprocess.check_output(
+                list_jobs_cmd,
+                stderr=subprocess.STDOUT,
+                universal_newlines=True,
+            ).strip()
+            print(output)
+            if "No jobs running." in output or "No jobs listed." in output:
+                break
+            print("Suspend jobs ongoing, waiting...")
+            time.sleep(1)
+            timeout -= 1
+        else:
+            print("Timed out waiting for suspend jobs to finish")
+            return 1
         print("Running: {}".format(" ".join(rtcwake_cmd)))
         subprocess.check_call(rtcwake_cmd)
         print(
@@ -80,6 +97,8 @@ def main(args=sys.argv[1:]):
         print("Removing {}...".format(log_path))
         os.remove(log_path)
 
+    return 0
+
 
 if __name__ == "__main__":
     sys.exit(main())

providers/base/tests/test_suspend_trigger.py

@@ -74,6 +74,7 @@ def test_fwts_path_on_i386_with_defaults(
 
 
 @patch("suspend_trigger.fwts_test")
+@patch("suspend_trigger.subprocess.check_output")
 @patch("suspend_trigger.subprocess.check_call")
 @patch("suspend_trigger.platform.machine")
 @patch("os.remove")
@@ -85,12 +86,14 @@ def test_rtcwake_path_success_with_args(
         mock_remove,
         mock_machine,
         mock_check_call,
+        mock_check_output,
         mock_fwts_test,
     ):
         """
         Tests the rtcwake/systemctl path on aarch64 with custom arguments.
         """
         mock_machine.return_value = "aarch64"
+        mock_check_output.return_value = "No jobs listed."
 
         suspend_trigger.main(
             ["--sleep-delay", "25", "--rtc-device", "/dev/my_rtc"]
@@ -108,25 +111,34 @@ def test_rtcwake_path_success_with_args(
             "25",
         ]
         expected_suspend_cmd = ["systemctl", "suspend"]
+        expected_list_cmd = ["systemctl", "list-jobs", "*suspend*"]
+        mock_check_output.assert_called_with(
+            expected_list_cmd,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+        )
         subprocess_calls = [
             call(expected_rtcwake_cmd),
             call(expected_suspend_cmd),
         ]
         mock_check_call.assert_has_calls(subprocess_calls)
         self.assertEqual(mock_check_call.call_count, 2)
+        self.assertEqual(mock_check_output.call_count, 1)
 
     def test_rtcwake_path_with_defaults(
         self,
         mock_exists,
         mock_remove,
         mock_machine,
         mock_check_call,
+        mock_check_output,
         mock_fwts_test,
     ):
         """
         Tests the rtcwake/systemctl path without any argument.
         """
         mock_machine.return_value = "riscv64"
+        mock_check_output.return_value = "No jobs listed."
 
         suspend_trigger.main([])
 
@@ -141,6 +153,12 @@ def test_rtcwake_path_with_defaults(
             "30",
         ]
         expected_suspend_cmd = ["systemctl", "suspend"]
+        expected_list_cmd = ["systemctl", "list-jobs", "*suspend*"]
+        mock_check_output.assert_called_with(
+            expected_list_cmd,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+        )
         subprocess_calls = [
             call(expected_rtcwake_cmd),
             call(expected_suspend_cmd),
@@ -153,12 +171,14 @@ def test_rtcwake_command_failure(
         mock_remove,
         mock_machine,
         mock_check_call,
+        mock_check_output,
         mock_fwts_test,
     ):
         """
         Tests the case where the rtcwake command fails.
         """
         mock_machine.return_value = "aarch64"
+        mock_check_output.return_value = "No jobs listed."
         # Simulate a command failure
         error = subprocess.CalledProcessError(
             returncode=1, cmd="rtcwake", output="Error from rtcwake"
@@ -169,7 +189,8 @@ def test_rtcwake_command_failure(
         with self.assertRaises(subprocess.CalledProcessError):
             suspend_trigger.main([])
 
-        # Verify that only the first command (rtcwake) was attempted
+        # Verify that only the first 2 commands were attempted
+        self.assertTrue(mock_check_output.called)
         self.assertTrue(mock_check_call.called)
         self.assertIn("rtcwake", mock_check_call.call_args[0][0])
 
@@ -179,12 +200,14 @@ def test_suspend_command_failure(
         mock_remove,
         mock_machine,
         mock_check_call,
+        mock_check_output,
         mock_fwts_test,
     ):
         """
         Tests the case where the systemctl suspend command fails.
         """
         mock_machine.return_value = "aarch64"
+        mock_check_output.return_value = "No jobs listed."
         suspend_error = subprocess.CalledProcessError(
             returncode=1, cmd="systemctl suspend", output="Error from suspend"
         )
@@ -195,6 +218,7 @@ def test_suspend_command_failure(
             suspend_trigger.main([])
 
         # Verify both commands were attempted
+        self.assertTrue(mock_check_output.called)
         self.assertEqual(mock_check_call.call_count, 2)
         self.assertIn("rtcwake", mock_check_call.call_args_list[0][0][0])
         self.assertIn("systemctl", mock_check_call.call_args_list[1][0][0])
@@ -205,6 +229,7 @@ def test_log_file_removed_if_exists(
         mock_remove,
         mock_machine,
         mock_check_call,
+        mock_check_output,
         mock_fwts_test,
     ):
         """
@@ -214,10 +239,17 @@ def test_log_file_removed_if_exists(
         mock_machine.return_value = "aarch64"
         mock_exists.return_value = True  # Simulate file exists
         mock_check_call.side_effect = [None, None]
+        mock_check_output.return_value = "No jobs listed."
 
         suspend_trigger.main([])
 
         # Verify commands were called
+        expected_list_cmd = ["systemctl", "list-jobs", "*suspend*"]
+        mock_check_output.assert_called_with(
+            expected_list_cmd,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+        )
         expected_rtcwake_cmd = [
             "rtcwake",
             "--verbose",
@@ -246,6 +278,7 @@ def test_log_file_not_removed_if_missing(
         mock_remove,
         mock_machine,
         mock_check_call,
+        mock_check_output,
         mock_fwts_test,
     ):
         """
@@ -254,6 +287,7 @@ def test_log_file_not_removed_if_missing(
         mock_machine.return_value = "aarch64"
         mock_exists.return_value = False  # Simulate file missing
         mock_check_call.side_effect = [None, None]
+        mock_check_output.return_value = "No jobs listed."
 
         suspend_trigger.main([])
         # Verify remove was never called

providers/base/units/stress/suspend_cycles_reboot.pxu

@@ -79,6 +79,7 @@ estimated_duration: 75.0
 environ: PLAINBOX_SESSION_SHARE STRESS_S3_INIT_DELAY STRESS_S3_SLEEP_DELAY STRESS_S3_WAIT_DELAY LD_LIBRARY_PATH RTC_DEVICE_FILE
 user: root
 command:
+   set -o pipefail
    echo "Current boot ID is: $(tr -d - < /proc/sys/kernel/random/boot_id)"
    suspend_trigger.py --wait "${STRESS_S3_INIT_DELAY:-120}" --check-delay "${STRESS_S3_WAIT_DELAY:-45}" --sleep-delay "${STRESS_S3_SLEEP_DELAY:-30}" --rtc-device "${RTC_DEVICE_FILE:-/dev/rtc0}" 2>&1 | tee -a "$PLAINBOX_SESSION_SHARE"/suspend_cycles_with_reboot_total.log
 summary:
@@ -105,6 +106,7 @@ environ: PLAINBOX_SESSION_SHARE STRESS_S3_INIT_DELAY STRESS_S3_SLEEP_DELAY STRES
 after: stress-tests/suspend_cycles_reboot{{suspend_reboot_previous}}
 user: root
 command:
+ set -o pipefail
  echo "Current boot ID is: $(tr -d - < /proc/sys/kernel/random/boot_id)"
  suspend_trigger.py --wait "${STRESS_S3_INIT_DELAY:-120}" --check-delay "${STRESS_S3_WAIT_DELAY:-45}" --sleep-delay "${STRESS_S3_SLEEP_DELAY:-30}" --rtc-device "${RTC_DEVICE_FILE:-/dev/rtc0}" 2>&1 | tee -a "$PLAINBOX_SESSION_SHARE"/suspend_cycles_with_reboot_total.log
 summary:
@@ -129,6 +131,7 @@ environ: PLAINBOX_SESSION_SHARE STRESS_S3_INIT_DELAY STRESS_S3_SLEEP_DELAY STRES
 after: stress-tests/suspend_cycles_{{suspend_id_previous}}_reboot{{suspend_reboot_id}}
 user: root
 command:
+ set -o pipefail
  echo "Current boot ID is: $(tr -d - < /proc/sys/kernel/random/boot_id)"
  suspend_trigger.py --check-delay "${STRESS_S3_WAIT_DELAY:-45}" --sleep-delay "${STRESS_S3_SLEEP_DELAY:-30}" --rtc-device "${RTC_DEVICE_FILE:-/dev/rtc0}" 2>&1 | tee -a "$PLAINBOX_SESSION_SHARE"/suspend_cycles_with_reboot_total.log
 summary: