diff --git a/tests/model_serving/model_server/serverless/conftest.py b/tests/model_serving/model_server/serverless/conftest.py index 1a2a520b2..1fea403fd 100644 --- a/tests/model_serving/model_server/serverless/conftest.py +++ b/tests/model_serving/model_server/serverless/conftest.py @@ -87,3 +87,8 @@ def s3_flan_small_hf_caikit_serverless_inference_service( external_route=True, ) as isvc: yield isvc + + +@pytest.fixture(scope="class") +def deleted_isvc(ovms_serverless_inference_service: InferenceService) -> None: + ovms_serverless_inference_service.clean_up() diff --git a/tests/model_serving/model_server/serverless/test_odh_model_controller_failing_while_deleted_isvc.py b/tests/model_serving/model_server/serverless/test_odh_model_controller_failing_while_deleted_isvc.py new file mode 100644 index 000000000..9728ae527 --- /dev/null +++ b/tests/model_serving/model_server/serverless/test_odh_model_controller_failing_while_deleted_isvc.py @@ -0,0 +1,81 @@ +import pytest +import re +from ocp_utilities.infra import get_pods_by_name_prefix +from pytest_testconfig import py_config +from simple_logger.logger import get_logger +from timeout_sampler import TimeoutSampler, TimeoutExpiredError + +from tests.model_serving.model_server.utils import verify_inference_response +from utilities.constants import ( + ModelFormat, + ModelInferenceRuntime, + ModelVersion, + Protocols, + Timeout, +) +from utilities.exceptions import PodLogMissMatchError +from utilities.inference_utils import Inference +from utilities.manifests.onnx import ONNX_INFERENCE_CONFIG + +pytestmark = [ + pytest.mark.serverless, + pytest.mark.sanity, + pytest.mark.usefixtures("valid_aws_config"), +] + +LOGGER = get_logger(name=__name__) + + +@pytest.mark.parametrize( + "model_namespace, openvino_kserve_serving_runtime, ovms_serverless_inference_service", + [ + pytest.param( + {"name": "serverless-maistra"}, + { + "runtime-name": ModelInferenceRuntime.ONNX_RUNTIME, + "model-format": {ModelFormat.ONNX: ModelVersion.OPSET13}, + }, + { + "name": ModelFormat.ONNX, + "model-version": ModelVersion.OPSET13, + "model-dir": "test-dir", + }, + ) + ], + indirect=True, +) +class TestNoMaistraErrorInLogs: + def test_inference_before_isvc_deletion(self, ovms_serverless_inference_service): + """Verify model can be queried""" + verify_inference_response( + inference_service=ovms_serverless_inference_service, + inference_config=ONNX_INFERENCE_CONFIG, + inference_type=Inference.INFER, + protocol=Protocols.HTTPS, + use_default_query=True, + ) + + def test_pod_logs_do_not_contain_maistra_error(self, deleted_isvc, admin_client): + """Delete isvc and check pod logs for 'ServiceMeshMemberRoll' error""" + regex_pattern = r'"error":\s*"no matches for kind \\"ServiceMeshMemberRoll\\" in version \\"maistra\.io/v1\\""' + pod = get_pods_by_name_prefix( + client=admin_client, namespace=py_config["applications_namespace"], pod_prefix="odh-model-controller" + )[0] + + try: + log_sampler = TimeoutSampler( + wait_timeout=Timeout.TIMEOUT_4MIN, + sleep=5, + func=pod.log, + container="manager", + tail_lines=500, + timestamps=True, + ) + + for log_output in log_sampler: + LOGGER.info("Log output fetched during sampling:") + if re.search(regex_pattern, log_output): + raise PodLogMissMatchError("ServiceMeshMemberRoll error found in pod logs") + + except TimeoutExpiredError: + LOGGER.info(f"Error {regex_pattern} not found in {pod.name} logs") diff --git a/utilities/exceptions.py b/utilities/exceptions.py index 8c36616f2..cc9d24ecb 100644 --- a/utilities/exceptions.py +++ b/utilities/exceptions.py @@ -76,3 +76,7 @@ class DeploymentValidationError(Exception): class InferenceCanaryTrafficError(Exception): pass + + +class PodLogMissMatchError(Exception): + pass