Wait for MC pod before api call (#580)

Author: dbasunag

tests/model_registry/conftest.py

@@ -203,6 +203,11 @@ def updated_dsc_component_state_scope_session(
                 namespace_name=py_config["applications_namespace"],
                 number_of_consecutive_checks=6,
             )
+            wait_for_pods_running(
+                admin_client=admin_client,
+                namespace_name=py_config["model_registry_namespace"],
+                number_of_consecutive_checks=6,
+            )
             yield dsc_resource
 
         for component_name, value in component_patch.items():

tests/model_registry/model_catalog/utils.py

@@ -15,7 +15,7 @@
     CATALOG_TYPE,
     DEFAULT_CATALOG_FILE,
 )
-from tests.model_registry.utils import get_model_catalog_pod
+from tests.model_registry.utils import get_model_catalog_pod, wait_for_pods_running
 
 LOGGER = get_logger(name=__name__)
 
@@ -49,24 +49,31 @@ def validate_model_catalog_enabled(pod: Pod) -> bool:
     return False
 
 
-def is_model_catalog_ready(client: DynamicClient, model_registry_namespace: str, consecutive_try: int = 3):
+def is_model_catalog_ready(client: DynamicClient, model_registry_namespace: str, consecutive_try: int = 6):
     model_catalog_pods = get_model_catalog_pod(client=client, model_registry_namespace=model_registry_namespace)
     # We can wait for the pods to reflect updated catalog, however, deleting them ensures the updated config is
     # applied immediately.
     for pod in model_catalog_pods:
         pod.delete()
     # After the deletion, we need to wait for the pod to be spinned up and get to ready state.
-    for _ in range(consecutive_try):
-        wait_for_model_catalog_update(client=client, model_registry_namespace=model_registry_namespace)
+    assert wait_for_model_catalog_pod_created(client=client, model_registry_namespace=model_registry_namespace)
+    wait_for_pods_running(
+        admin_client=client, namespace_name=model_registry_namespace, number_of_consecutive_checks=consecutive_try
+    )
 
 
-@retry(wait_timeout=30, sleep=5)
-def wait_for_model_catalog_update(client: DynamicClient, model_registry_namespace: str):
+class PodNotFound(Exception):
+    """Pod not found"""
+
+    pass
+
+
+@retry(wait_timeout=30, sleep=5, exceptions_dict={PodNotFound: []})
+def wait_for_model_catalog_pod_created(client: DynamicClient, model_registry_namespace: str) -> bool:
     pods = get_model_catalog_pod(client=client, model_registry_namespace=model_registry_namespace)
     if pods:
-        pods[0].wait_for_status(status=Pod.Status.RUNNING)
         return True
-    return False
+    raise PodNotFound("Model catalog pod not found")
 
 
 def validate_model_catalog_resource(kind: Any, admin_client: DynamicClient, namespace: str) -> None: