From ae603ffdb09d9c169bbddeba7e2dd1c95ed072e7 Mon Sep 17 00:00:00 2001 From: Brett Thompson <196701379+brettmthompson@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:32:29 -0400 Subject: [PATCH] now yielding TimeoutSampler get_pods_by_isvc_label func output and handling raised ResourceNotFoundError Signed-off-by: Brett Thompson <196701379+brettmthompson@users.noreply.github.com> --- .../model_server/serverless/utils.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/model_serving/model_server/serverless/utils.py b/tests/model_serving/model_server/serverless/utils.py index fcb93e1dc..b2a9265fb 100644 --- a/tests/model_serving/model_server/serverless/utils.py +++ b/tests/model_serving/model_server/serverless/utils.py @@ -3,7 +3,9 @@ from typing import Any from kubernetes.dynamic import DynamicClient +from kubernetes.dynamic.exceptions import ResourceNotFoundError from ocp_resources.inference_service import InferenceService +from ocp_resources.utils.constants import DEFAULT_CLUSTER_RETRY_EXCEPTIONS from simple_logger.logger import get_logger from timeout_sampler import TimeoutSampler from timeout_sampler import TimeoutExpiredError @@ -32,17 +34,20 @@ def verify_no_inference_pods(client: DynamicClient, isvc: InferenceService) -> N pods = [] try: - pods = TimeoutSampler( + for pods in TimeoutSampler( wait_timeout=Timeout.TIMEOUT_4MIN, sleep=5, + exceptions_dict=DEFAULT_CLUSTER_RETRY_EXCEPTIONS, func=get_pods_by_isvc_label, client=client, isvc=isvc, - ) - if not pods: - return + ): + if not pods: + return - except TimeoutError: + except TimeoutExpiredError as e: + if isinstance(e.last_exp, ResourceNotFoundError): + return LOGGER.error(f"{[pod.name for pod in pods]} were not deleted") raise