Add pod readiness check after LLMInferenceService deployment (#1203)

* Add pod readiness check after LLMInferenceService deployment 
* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: threcc

tests/model_serving/model_server/llmd/conftest.py

@@ -18,7 +18,7 @@
 from simple_logger.logger import get_logger
 
 from tests.model_serving.model_server.llmd.llmd_configs import TinyLlamaOciConfig
-from tests.model_serving.model_server.llmd.utils import wait_for_llmisvc
+from tests.model_serving.model_server.llmd.utils import wait_for_llmisvc, wait_for_llmisvc_pods_ready
 from utilities.constants import Timeout
 from utilities.infra import create_inference_token, s3_endpoint_secret, update_configmap_data
 from utilities.llmd_constants import LLMDGateway
@@ -296,4 +296,5 @@ def _create_llmisvc_from_config(
 
     with LLMInferenceService(**svc_kwargs) as llm_service:
         wait_for_llmisvc(llmisvc=llm_service, timeout=config_cls.wait_timeout)
+        wait_for_llmisvc_pods_ready(client=client, llmisvc=llm_service)
         yield llm_service

tests/model_serving/model_server/llmd/utils.py

@@ -41,6 +41,28 @@ def wait_for_llmisvc(llmisvc: LLMInferenceService, timeout: int = Timeout.TIMEOU
     LOGGER.info(f"LLMInferenceService {llmisvc.name} is Ready in namespace {llmisvc.namespace}")
 
 
+def wait_for_llmisvc_pods_ready(
+    client: DynamicClient,
+    llmisvc: LLMInferenceService,
+    timeout: int = 30,
+) -> None:
+    """Wait for all LLMISVC pods (workload + router-scheduler) to be Ready."""
+    pods = list(
+        Pod.get(
+            client=client,
+            namespace=llmisvc.namespace,
+            label_selector=(
+                f"{Pod.ApiGroup.APP_KUBERNETES_IO}/part-of=llminferenceservice,"
+                f"{Pod.ApiGroup.APP_KUBERNETES_IO}/name={llmisvc.name}"
+            ),
+        )
+    )
+    LOGGER.info(f"Waiting for {len(pods)} pod(s) to be Ready for {llmisvc.name}")
+    for pod in pods:
+        pod.wait_for_condition(condition="Ready", status="True", timeout=timeout)
+        LOGGER.info(f"Pod {pod.name} is Ready")
+
+
 def _get_inference_url(llmisvc: LLMInferenceService) -> str:
     """Extract inference URL from LLMISVC status."""
     status = llmisvc.instance.status