Fix: Add validations to suspend_trigger

 * Run "systemctl list-jobs suspend" to check if there are
   suspend jobs running before running one more.

 * If "systemctl list-jobs suspend" detects suspend
   jobs running wait for a while before proceeding.

 * Check the return code of "systemctl suspend" and
   fail with error when needed.

 * Set "set -o pipefail" on the Checkbox job definition
   for the job to be marked as "failed" when exiting on error.

Author: EstebanVg15

providers/base/bin/suspend_trigger.py

@@ -67,19 +67,38 @@ def main(args=sys.argv[1:]):
             str(args.sleep_delay),
         ]
         suspend_cmd = ["systemctl", "suspend"]
+        list_jobs_cmd = ["systemctl", "list-jobs", "*suspend*"]
         print("Running: {}".format(" ".join(rtcwake_cmd)))
         subprocess.check_call(rtcwake_cmd)
+        timeout = 100
+        while timeout > 0:
+            ret_list = subprocess.run(list_jobs_cmd, capture_output=True, text=True, check=True)
+            if "No jobs running" in ret_list.stdout:
+                break
+            print(f"Suspend jobs ongoing, waiting... ({timeout} left)")
+            time.sleep(1)
+            timeout -= 1
+        else:
+            print("Timed out waiting for suspend jobs to finish")
+            return 1
         print(
             "Running: {} to suspend the system".format(" ".join(suspend_cmd))
         )
-        subprocess.check_call(suspend_cmd)
+        ret_suspend = subprocess.run(suspend_cmd)
+        if ret_suspend.returncode != 0:
+            print(
+                "Issues running:{} returned:{}".format(" ".join(suspend_cmd), ret_suspend.returncode)
+            )
+            return ret_suspend.returncode
 
     # Clean up the FWTS log file from its default path.
     log_path = "/tmp/fwts_results.log"
     if os.path.exists(log_path):
         print("Removing {}...".format(log_path))
         os.remove(log_path)
 
+    return 0
+
 
 if __name__ == "__main__":
     sys.exit(main())

providers/base/units/stress/suspend_cycles_reboot.pxu

@@ -79,6 +79,7 @@ estimated_duration: 75.0
 environ: PLAINBOX_SESSION_SHARE STRESS_S3_INIT_DELAY STRESS_S3_SLEEP_DELAY STRESS_S3_WAIT_DELAY LD_LIBRARY_PATH RTC_DEVICE_FILE
 user: root
 command:
+   set -o pipefail
    echo "Current boot ID is: $(tr -d - < /proc/sys/kernel/random/boot_id)"
    suspend_trigger.py --wait "${STRESS_S3_INIT_DELAY:-120}" --check-delay "${STRESS_S3_WAIT_DELAY:-45}" --sleep-delay "${STRESS_S3_SLEEP_DELAY:-30}" --rtc-device "${RTC_DEVICE_FILE:-/dev/rtc0}" 2>&1 | tee -a "$PLAINBOX_SESSION_SHARE"/suspend_cycles_with_reboot_total.log
 summary:
@@ -105,6 +106,7 @@ environ: PLAINBOX_SESSION_SHARE STRESS_S3_INIT_DELAY STRESS_S3_SLEEP_DELAY STRES
 after: stress-tests/suspend_cycles_reboot{{suspend_reboot_previous}}
 user: root
 command:
+ set -o pipefail
  echo "Current boot ID is: $(tr -d - < /proc/sys/kernel/random/boot_id)"
  suspend_trigger.py --wait "${STRESS_S3_INIT_DELAY:-120}" --check-delay "${STRESS_S3_WAIT_DELAY:-45}" --sleep-delay "${STRESS_S3_SLEEP_DELAY:-30}" --rtc-device "${RTC_DEVICE_FILE:-/dev/rtc0}" 2>&1 | tee -a "$PLAINBOX_SESSION_SHARE"/suspend_cycles_with_reboot_total.log
 summary:
@@ -129,6 +131,7 @@ environ: PLAINBOX_SESSION_SHARE STRESS_S3_INIT_DELAY STRESS_S3_SLEEP_DELAY STRES
 after: stress-tests/suspend_cycles_{{suspend_id_previous}}_reboot{{suspend_reboot_id}}
 user: root
 command:
+ set -o pipefail
  echo "Current boot ID is: $(tr -d - < /proc/sys/kernel/random/boot_id)"
  suspend_trigger.py --check-delay "${STRESS_S3_WAIT_DELAY:-45}" --sleep-delay "${STRESS_S3_SLEEP_DELAY:-30}" --rtc-device "${RTC_DEVICE_FILE:-/dev/rtc0}" 2>&1 | tee -a "$PLAINBOX_SESSION_SHARE"/suspend_cycles_with_reboot_total.log
 summary: