now yielding TimeoutSampler get_pods_by_isvc_label func output and handling raised ResourceNotFoundError

Signed-off-by: Brett Thompson <196701379+brettmthompson@users.noreply.github.com>

Author: brettmthompson

tests/model_serving/model_server/serverless/utils.py

@@ -3,7 +3,9 @@
 from typing import Any
 
 from kubernetes.dynamic import DynamicClient
+from kubernetes.dynamic.exceptions import ResourceNotFoundError
 from ocp_resources.inference_service import InferenceService
+from ocp_resources.utils.constants import DEFAULT_CLUSTER_RETRY_EXCEPTIONS
 from simple_logger.logger import get_logger
 from timeout_sampler import TimeoutSampler
 from timeout_sampler import TimeoutExpiredError
@@ -32,17 +34,20 @@ def verify_no_inference_pods(client: DynamicClient, isvc: InferenceService) -> N
     pods = []
 
     try:
-        pods = TimeoutSampler(
+        for pods in TimeoutSampler(
             wait_timeout=Timeout.TIMEOUT_4MIN,
             sleep=5,
+            exceptions_dict=DEFAULT_CLUSTER_RETRY_EXCEPTIONS,
             func=get_pods_by_isvc_label,
             client=client,
             isvc=isvc,
-        )
-        if not pods:
-            return
+        ):
+            if not pods:
+                return
 
-    except TimeoutError:
+    except TimeoutExpiredError as e:
+        if type(e.last_exp) == ResourceNotFoundError:
+            return
         LOGGER.error(f"{[pod.name for pod in pods]} were not deleted")
         raise