now yielding TimeoutSampler get_pods_by_isvc_label func output and handling raised ResourceNotFoundError (opendatahub-io#237)

Signed-off-by: Brett Thompson <196701379+brettmthompson@users.noreply.github.com>

Author: brettmthompson

tests/model_serving/model_server/serverless/utils.py

@@ -1,7 +1,9 @@
 from typing import Any
 
 from kubernetes.dynamic import DynamicClient
+from kubernetes.dynamic.exceptions import ResourceNotFoundError
 from ocp_resources.inference_service import InferenceService
+from ocp_resources.utils.constants import DEFAULT_CLUSTER_RETRY_EXCEPTIONS
 from simple_logger.logger import get_logger
 from timeout_sampler import TimeoutSampler
 from timeout_sampler import TimeoutExpiredError
@@ -30,17 +32,20 @@ def verify_no_inference_pods(client: DynamicClient, isvc: InferenceService) -> N
     pods = []
 
     try:
-        pods = TimeoutSampler(
+        for pods in TimeoutSampler(
             wait_timeout=Timeout.TIMEOUT_4MIN,
             sleep=5,
+            exceptions_dict=DEFAULT_CLUSTER_RETRY_EXCEPTIONS,
             func=get_pods_by_isvc_label,
             client=client,
             isvc=isvc,
-        )
-        if not pods:
-            return
+        ):
+            if not pods:
+                return
 
-    except TimeoutError:
+    except TimeoutExpiredError as e:
+        if isinstance(e.last_exp, ResourceNotFoundError):
+            return
         LOGGER.error(f"{[pod.name for pod in pods]} were not deleted")
         raise