diff --git a/tests/model_serving/model_server/serverless/utils.py b/tests/model_serving/model_server/serverless/utils.py index 7f997e262..c5f6dfc2b 100644 --- a/tests/model_serving/model_server/serverless/utils.py +++ b/tests/model_serving/model_server/serverless/utils.py @@ -1,7 +1,9 @@ from typing import Any from kubernetes.dynamic import DynamicClient +from kubernetes.dynamic.exceptions import ResourceNotFoundError from ocp_resources.inference_service import InferenceService +from ocp_resources.utils.constants import DEFAULT_CLUSTER_RETRY_EXCEPTIONS from simple_logger.logger import get_logger from timeout_sampler import TimeoutSampler from timeout_sampler import TimeoutExpiredError @@ -30,17 +32,20 @@ def verify_no_inference_pods(client: DynamicClient, isvc: InferenceService) -> N pods = [] try: - pods = TimeoutSampler( + for pods in TimeoutSampler( wait_timeout=Timeout.TIMEOUT_4MIN, sleep=5, + exceptions_dict=DEFAULT_CLUSTER_RETRY_EXCEPTIONS, func=get_pods_by_isvc_label, client=client, isvc=isvc, - ) - if not pods: - return + ): + if not pods: + return - except TimeoutError: + except TimeoutExpiredError as e: + if isinstance(e.last_exp, ResourceNotFoundError): + return LOGGER.error(f"{[pod.name for pod in pods]} were not deleted") raise