Add new test for maistra error (#181)

* Add new test for maistra

Signed-off-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add new test for maistra

Signed-off-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add new test for maistra

Signed-off-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add new test for maistra

Signed-off-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add new test for maistra

Signed-off-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add new test for maistra

Signed-off-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>

---------

Signed-off-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>
Co-authored-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 3 people

tests/model_serving/model_server/serverless/conftest.py

@@ -87,3 +87,8 @@ def s3_flan_small_hf_caikit_serverless_inference_service(
         external_route=True,
     ) as isvc:
         yield isvc
+
+
+@pytest.fixture(scope="class")
+def deleted_isvc(ovms_serverless_inference_service: InferenceService) -> None:
+    ovms_serverless_inference_service.clean_up()

tests/model_serving/model_server/serverless/test_odh_model_controller_failing_while_deleted_isvc.py

@@ -0,0 +1,81 @@
+import pytest
+import re
+from ocp_utilities.infra import get_pods_by_name_prefix
+from pytest_testconfig import py_config
+from simple_logger.logger import get_logger
+from timeout_sampler import TimeoutSampler, TimeoutExpiredError
+
+from tests.model_serving.model_server.utils import verify_inference_response
+from utilities.constants import (
+    ModelFormat,
+    ModelInferenceRuntime,
+    ModelVersion,
+    Protocols,
+    Timeout,
+)
+from utilities.exceptions import PodLogMissMatchError
+from utilities.inference_utils import Inference
+from utilities.manifests.onnx import ONNX_INFERENCE_CONFIG
+
+pytestmark = [
+    pytest.mark.serverless,
+    pytest.mark.sanity,
+    pytest.mark.usefixtures("valid_aws_config"),
+]
+
+LOGGER = get_logger(name=__name__)
+
+
+@pytest.mark.parametrize(
+    "model_namespace, openvino_kserve_serving_runtime, ovms_serverless_inference_service",
+    [
+        pytest.param(
+            {"name": "serverless-maistra"},
+            {
+                "runtime-name": ModelInferenceRuntime.ONNX_RUNTIME,
+                "model-format": {ModelFormat.ONNX: ModelVersion.OPSET13},
+            },
+            {
+                "name": ModelFormat.ONNX,
+                "model-version": ModelVersion.OPSET13,
+                "model-dir": "test-dir",
+            },
+        )
+    ],
+    indirect=True,
+)
+class TestNoMaistraErrorInLogs:
+    def test_inference_before_isvc_deletion(self, ovms_serverless_inference_service):
+        """Verify model can be queried"""
+        verify_inference_response(
+            inference_service=ovms_serverless_inference_service,
+            inference_config=ONNX_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
+            protocol=Protocols.HTTPS,
+            use_default_query=True,
+        )
+
+    def test_pod_logs_do_not_contain_maistra_error(self, deleted_isvc, admin_client):
+        """Delete isvc and check pod logs for 'ServiceMeshMemberRoll' error"""
+        regex_pattern = r'"error":\s*"no matches for kind \\"ServiceMeshMemberRoll\\" in version \\"maistra\.io/v1\\""'
+        pod = get_pods_by_name_prefix(
+            client=admin_client, namespace=py_config["applications_namespace"], pod_prefix="odh-model-controller"
+        )[0]
+
+        try:
+            log_sampler = TimeoutSampler(
+                wait_timeout=Timeout.TIMEOUT_4MIN,
+                sleep=5,
+                func=pod.log,
+                container="manager",
+                tail_lines=500,
+                timestamps=True,
+            )
+
+            for log_output in log_sampler:
+                LOGGER.info("Log output fetched during sampling:")
+                if re.search(regex_pattern, log_output):
+                    raise PodLogMissMatchError("ServiceMeshMemberRoll error found in pod logs")
+
+        except TimeoutExpiredError:
+            LOGGER.info(f"Error {regex_pattern} not found in {pod.name} logs")

utilities/exceptions.py

@@ -76,3 +76,7 @@ class DeploymentValidationError(Exception):
 
 class InferenceCanaryTrafficError(Exception):
     pass
+
+
+class PodLogMissMatchError(Exception):
+    pass