change: wait for the pod to be in the correct status before to read the logs

Author: fege

tests/model_registry/negative_tests/test_db_migration.py

@@ -7,7 +7,7 @@
 from utilities.constants import DscComponents
 from tests.model_registry.constants import MR_INSTANCE_NAME
 from kubernetes.dynamic.client import DynamicClient
-from utilities.general import wait_for_pods_by_labels
+from utilities.general import wait_for_pods_by_labels, wait_for_container_status
 
 
 LOGGER = get_logger(name=__name__)
@@ -51,6 +51,9 @@ def test_db_migration_negative(
             expected_num_pods=1,
         )
         mr_pod = mr_pods[0]
+        LOGGER.info("Waiting for model registry pod to crash")
+        assert wait_for_container_status(mr_pod, "rest-container", Pod.Status.CRASH_LOOPBACK_OFF)
+
         LOGGER.info("Checking the logs for the expected error")
 
         log_output = mr_pod.log(container="rest-container")

utilities/general.py

@@ -11,8 +11,8 @@
 from simple_logger.logger import get_logger
 
 import utilities.infra
-from utilities.constants import Annotations, KServeDeploymentType, MODELMESH_SERVING
-from utilities.exceptions import UnexpectedResourceCountError
+from utilities.constants import Annotations, KServeDeploymentType, MODELMESH_SERVING, Timeout
+from utilities.exceptions import UnexpectedResourceCountError, ResourceValueMismatch
 from ocp_resources.resource import Resource
 from timeout_sampler import retry
 
@@ -331,3 +331,27 @@ def generate_random_name(prefix: str = "", length: int = 8) -> str:
     # random_uuid.hex is 32 characters long.
     suffix = random_uuid.hex[:length]
     return f"{prefix}-{suffix}" if prefix else suffix
+
+
+@retry(wait_timeout=Timeout.TIMEOUT_15_SEC, sleep=1)
+def wait_for_container_status(pod: Pod, container_name: str, expected_status: str) -> bool:
+    """
+    Wait for a container to be in the expected status.
+
+    Args:
+        pod: The pod to wait for
+        container_name: The name of the container to wait for
+        expected_status: The expected status
+
+    Returns:
+        bool: True if the container is in the expected status, False otherwise
+
+    Raises:
+        ResourceValueMismatch: If the container is not in the expected status
+    """
+
+    for cs in pod.instance.status.get("containerStatuses", []):
+        if cs.name == container_name and cs.state.waiting.reason == expected_status:
+            LOGGER.info(f"Pod is in the expected status {expected_status}")
+            return True
+    raise ResourceValueMismatch(f"Pod is not in the expected status {expected_status}")