Resiliency test automation when platform under stress (#11925)

Signed-off-by: Parag Kamble <[email protected]>

Author: paraggit

ocs_ci/ocs/ocp.py

@@ -238,7 +238,9 @@ def exec_oc_cmd(
             config.switch_ctx(original_context)
         return out
 
-    def exec_oc_debug_cmd(self, node, cmd_list, timeout=300, namespace=None):
+    def exec_oc_debug_cmd(
+        self, node, cmd_list, timeout=300, namespace=None, use_root=True
+    ):
         """
         Function to execute "oc debug" command on OCP node
 
@@ -258,11 +260,15 @@ def exec_oc_debug_cmd(self, node, cmd_list, timeout=300, namespace=None):
         create_cmd_list = copy.deepcopy(cmd_list)
         create_cmd_list.append(" ")
         err_msg = "CMD FAILED"
+        if use_root:
+            root_option = " chroot /host /bin/bash -c "
+        else:
+            root_option = " /bin/bash -c "
         cmd = f" || echo '{err_msg}';".join(create_cmd_list)
         namespace = namespace or config.ENV_DATA["cluster_namespace"]
         debug_cmd = (
             f"debug nodes/{node} --to-namespace={namespace} "
-            f' -- chroot /host /bin/bash -c "{cmd}"'
+            f' -- {root_option} "{cmd}"'
         )
         out = str(
             self.exec_oc_cmd(command=debug_cmd, out_yaml_format=False, timeout=timeout)

ocs_ci/ocs/resources/pod.py

@@ -4119,3 +4119,45 @@ def get_pods_pvcs(pod_objs, namespace=None):
     namespace = namespace or config.ENV_DATA["cluster_namespace"]
     pvc_names = [get_pvc_name(p) for p in pod_objs]
     return get_pvc_objs(pvc_names, namespace)
+
+
+def delete_pod_by_phase(
+    pod_phase,
+    namespace=config.ENV_DATA["cluster_namespace"],
+):
+    """
+    Delete the pods in a specific phase
+    Args:
+        pod_status (str): The pod status to delete
+        namespace (str): Name of cluster namespace(default: config.ENV_DATA["cluster_namespace"])
+    Returns:
+        bool: True, if the pods deleted successfully. False, otherwise
+    """
+    logger.info(f"Delete all the pods in the status '{pod_phase}'")
+    if pod_phase.lower() == "succeeded":
+        phase = "Succeeded"
+    elif pod_phase.lower() == "failed":
+        phase = "Failed"
+    elif pod_phase.lower() == "Pending":
+        phase = "Pending"
+    elif pod_phase.lower() == "running":
+        phase = "Running"
+    else:
+        raise ValueError(
+            f"Invalid pod status '{pod_phase}'. "
+            f"Valid options are 'succeeded' or 'failed'"
+        )
+
+    cmd = f"oc delete pod --field-selector=status.phase={phase} -n {namespace}"
+    logger.info(cmd)
+    try:
+        run_cmd(cmd=cmd)
+    except CommandFailed as ex:
+        logger.warning(
+            f"Failed to delete the pods in the status '{pod_phase}' due to the error: {ex}"
+        )
+        return False
+
+    logger.info(f"All '{pod_phase}' pods deleted successfully.")
+
+    return True

ocs_ci/resiliency/network_faults.py

@@ -80,7 +80,7 @@ def _get_all_node_network_interfaces(self, interface_types):
                         iface.strip() for iface in output.splitlines() if iface.strip()
                     ]
                     interfaces.extend(ifaces)
-                except CommandFailed as e:
+                except (CommandFailed, subprocess.TimeoutExpired) as e:
                     log.error(f"Error retrieving interfaces from node {node.name}: {e}")
                     continue
 
@@ -160,7 +160,7 @@ def _apply_fault(self, description, netem_command):
                     cmd = f"tc qdisc del dev {iface} root"
                     try:
                         self.ocp_obj.exec_oc_debug_cmd(node=node.name, cmd_list=[cmd])
-                    except CommandFailed as e:
+                    except (CommandFailed, subprocess.TimeoutExpired) as e:
                         log.error(
                             f"Failed to remove fault from {node.name}/{iface}: {e}"
                         )
@@ -188,7 +188,7 @@ def _remove_faults_all_nodes(self):
                 try:
                     self.ocp_obj.exec_oc_debug_cmd(node=node.name, cmd_list=[cmd_del])
                     log.debug(f"Deleted qdisc on {node.name}/{iface}")
-                except CommandFailed as e:
+                except (CommandFailed, subprocess.TimeoutExpired) as e:
                     log.warning(f"Could not delete qdisc on {node.name}/{iface}: {e}")
                     continue
 
@@ -206,7 +206,7 @@ def _remove_faults_all_nodes(self):
                         log.info(
                             f"Verified: netem successfully removed from {node.name}/{iface}"
                         )
-                except CommandFailed as e:
+                except (CommandFailed, subprocess.TimeoutExpired) as e:
                     log.warning(
                         f"Could not verify qdisc status on {node.name}/{iface}: {e}"
                     )

ocs_ci/resiliency/node_stats.py

@@ -13,27 +13,63 @@ class NodeStats:
     """
 
     @staticmethod
-    def cpu_stats(node_obj):
+    def cpu_stats(node_obj, interval=1, count=2, format="json"):
         """
         Get CPU statistics for a given node using `mpstat`.
 
         Args:
-            node_obj (OCSNode): Node object to query.
+            node_obj (OCSNode): The node object to fetch stats from.
+            interval (int): Interval in seconds between samples. Default 1.
+            count (int): Number of samples to take. Default 2.
+            format (str): Output format - "json" or "text". Default "json".
 
         Returns:
-            list: List of CPU stats dictionaries parsed from mpstat JSON output.
+            list or str or None: Parsed statistics list (JSON), raw output (text),
+                                or None on failure.
         """
-        ocp_obj = ocp.OCP(kind="node")
-        cmd = f"debug nodes/{node_obj.name} -- mpstat 1 2 -o JSON"
+        log.info(
+            f"Running mpstat on node '{node_obj.name}' with interval={interval}, count={count}, format={format}"
+        )
+
+        if format not in ("json", "text"):
+            log.error(f"Unsupported format '{format}'. Use 'json' or 'text'.")
+            return None
 
+        cmd = f"mpstat {interval} {count}"
+        if format == "json":
+            cmd += " -o JSON"
+        elif format == "text":
+            cmd += " > /tmp/mpstat.txt && cat /tmp/mpstat.txt"
+            log.warning("Text format selected. Output will be raw and unparsed.")
+
+        ocp_obj = ocp.OCP(kind="node")
         try:
-            log.info(f"Running mpstat on node: {node_obj.name}")
-            cmd_output = ocp_obj.exec_oc_cmd(command=cmd, out_yaml_format=False)
-            output = json.loads(cmd_output)
-            return output.get("sysstat", {}).get("hosts", [{}])[0].get("statistics", [])
+            cmd_output = ocp_obj.exec_oc_debug_cmd(
+                node=node_obj.name, cmd_list=[cmd], use_root=False
+            )
+
+            if not cmd_output:
+                log.warning("Empty response received from mpstat command")
+                return None
+
+            if format == "json":
+                try:
+                    output = json.loads(cmd_output)
+                    return (
+                        output.get("sysstat", {})
+                        .get("hosts", [{}])[0]
+                        .get("statistics", [])
+                    )
+                except json.JSONDecodeError as e:
+                    log.error(
+                        f"Failed to parse JSON from mpstat on node '{node_obj.name}': {e}"
+                    )
+                    return None
+            return cmd_output.splitlines()
+
         except CommandFailed as e:
-            log.error(f"Failed to fetch CPU stats for node {node_obj.name}: {e}")
-            return []
+            log.error(f"Failed to fetch CPU stats from node '{node_obj.name}': {e}")
+            return None
 
     @staticmethod
     def memory_usage_percent(node_obj):
@@ -47,15 +83,20 @@ def memory_usage_percent(node_obj):
             float: Used memory percentage.
         """
         ocp_obj = ocp.OCP(kind="node")
-        cmd = f"debug nodes/{node_obj.name} -- cat /proc/meminfo"
+        cmd = "cat /proc/meminfo"
 
         try:
-            output = ocp_obj.exec_oc_cmd(command=cmd, out_yaml_format=False)
+            output = ocp_obj.exec_oc_debug_cmd(
+                node=node_obj.name, cmd_list=[cmd], use_root=False, timeout=30
+            )
 
             meminfo = {}
             for line in output.splitlines():
-                key, value = line.strip().split(":", 1)
-                meminfo[key] = int(value.strip().split()[0])  # in KB
+                try:
+                    key, value = line.strip().split(":", 1)
+                    meminfo[key] = int(value.strip().split()[0])  # in KB
+                except (ValueError, IndexError):
+                    continue
 
             mem_total = meminfo.get("MemTotal")
             mem_available = meminfo.get("MemAvailable")
@@ -64,57 +105,88 @@ def memory_usage_percent(node_obj):
                 used_percent = ((mem_total - mem_available) / mem_total) * 100
                 log.info(f"Memory usage on node {node_obj.name}: {used_percent:.2f}%")
                 return round(used_percent, 2)
-            else:
-                log.warning("Missing MemTotal or MemAvailable in /proc/meminfo")
-                return 0.0
+
+            log.warning("Missing MemTotal or MemAvailable in /proc/meminfo")
+            return 0.0
 
         except CommandFailed as e:
-            log.error(f"Failed to compute memory usage on node {node_obj.name}: {e}")
+            log.error(
+                f"Failed to compute memory usage on node {node_obj.name}:  {str(e)}"
+            )
             return 0.0
 
     @staticmethod
-    def disk_stats(node_obj):
+    def disk_stats(node_obj, format="json", interval=1, count=2):
         """
         Get disk I/O statistics using `iostat`.
 
         Args:
             node_obj (OCSNode): Node object to query.
+            format (str): Output format ("json" or "text"). Default "json".
+            interval (int): Interval in seconds between samples. Default 1.
+            count (int): Number of samples to take. Default 2.
 
         Returns:
-            dict: Latest disk statistics.
+            dict or list: Latest disk statistics (dict for JSON, list for text).
         """
         ocp_obj = ocp.OCP(kind="node")
-        cmd = f"debug nodes/{node_obj.name} -- iostat -xt -o JSON 1 2"
+        if format not in ("json", "text"):
+            log.error(f"Unsupported format '{format}'. Use 'json' or 'text'.")
+            return {}
+
+        cmd = f"iostat -xt {interval} {count}"
+        if format == "json":
+            cmd += " -o JSON"
 
         try:
-            log.info(f"Running disk stats command on node: {node_obj.name}")
-            cmd_output = ocp_obj.exec_oc_cmd(command=cmd, out_yaml_format=False)
-            output = json.loads(cmd_output)
-            stats = (
-                output.get("sysstat", {}).get("hosts", [{}])[0].get("statistics", [])
+            output = ocp_obj.exec_oc_debug_cmd(
+                node=node_obj.name, cmd_list=[cmd], use_root=False, timeout=30
             )
-            return stats[-1] if stats else {}
+
+            if format == "json":
+                try:
+                    output = json.loads(output)
+                    stats = (
+                        output.get("sysstat", {})
+                        .get("hosts", [{}])[0]
+                        .get("statistics", [])
+                    )
+                    return stats[-1] if stats else {}
+                except json.JSONDecodeError as e:
+                    log.error(
+                        f"Failed to parse JSON from iostat on node '{node_obj.name}': {e}"
+                    )
+                    return {}
+
+            return output.splitlines()
+
         except CommandFailed as e:
-            log.error(f"Failed to fetch disk stats for node {node_obj.name}: {e}")
+            log.error(f"Failed to fetch disk stats from node '{node_obj.name}': {e}")
             return {}
 
     @staticmethod
-    def network_stats(node_obj, interface="ovn-k8s-mp0"):
+    def network_stats(node_obj, interface="ovn-k8s-mp0", interval=1, count=2):
         """
         Get network interface statistics using `sar`.
 
         Args:
             node_obj (OCSNode): Node object to query.
+            interface (str): Network interface to monitor. Default "ovn-k8s-mp0".
+            interval (int): Interval in seconds between samples. Default 1.
+            count (int): Number of samples to take. Default 2.
 
         Returns:
-            str: Network interface statistics as text.
+            list: Network interface statistics as text lines.
         """
         ocp_obj = ocp.OCP(kind="node")
-        cmd = f"debug nodes/{node_obj.name} -- sar -n DEV 1 1"
+        cmd = f"sar -n DEV {interval} {count}"
 
         try:
+            output = ocp_obj.exec_oc_debug_cmd(
+                node=node_obj.name, cmd_list=[cmd], use_root=False
+            )
             log.info(f"Running network stats command on node: {node_obj.name}")
-            return ocp_obj.exec_oc_cmd(command=cmd, out_yaml_format=False)
+            return output.splitlines()
         except CommandFailed as e:
             log.error(f"Failed to fetch network stats for node {node_obj.name}: {e}")
-            return ""
+            return []

ocs_ci/resiliency/platform_failures.py

@@ -17,6 +17,7 @@
 
 import logging
 import random
+import subprocess
 
 from ocs_ci.ocs.platform_nodes import PlatformNodesFactory
 from ocs_ci.resiliency.network_faults import NetworkFaults
@@ -148,7 +149,7 @@ def run_fault_simulation(nodes, interfaces, label):
                 nf = NetworkFaults(nodes, interface_types=interfaces)
                 nf.run()
                 log.info(f"[{label}] Completed fault simulation.")
-            except (ValueError, CommandFailed) as e:
+            except (ValueError, CommandFailed, subprocess.TimeoutExpired) as e:
                 log.error(f"[{label}] Error during simulation: {e}")
 
         # Helper to get a random subset of any list