test: Add test for kserve-raw model car (#337)

* test: Add test for kserve-raw model car

Signed-off-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>

* test: Add test for kserve-raw model car

Signed-off-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>

* test: Add test for kserve-raw model car

Signed-off-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>

---------

Signed-off-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>
Co-authored-by: Milind Waykole <mwaykole@mwaykole-thinkpadp1gen4i.bengluru.csb>

Author: mwaykole

tests/model_serving/model_server/model_car/conftest.py

@@ -12,20 +12,22 @@
 
 
 @pytest.fixture(scope="class")
-def model_car_serverless_inference_service(
+def model_car_inference_service(
     request: FixtureRequest,
     unprivileged_client: DynamicClient,
     unprivileged_model_namespace: Namespace,
     serving_runtime_from_template: ServingRuntime,
 ) -> Generator[InferenceService, Any, Any]:
+    deployment_mode = request.param.get("deployment-mode", KServeDeploymentType.SERVERLESS)
     with create_isvc(
         client=unprivileged_client,
-        name="serverless-model-car",
+        name=f"model-car-{deployment_mode.lower()}",
         namespace=unprivileged_model_namespace.name,
         runtime=serving_runtime_from_template.name,
         storage_uri=request.param["storage-uri"],
         model_format=serving_runtime_from_template.instance.spec.supportedModelFormats[0].name,
-        deployment_mode=KServeDeploymentType.SERVERLESS,
+        deployment_mode=deployment_mode,
+        external_route=request.param.get("external-route", True),
         wait_for_predictor_pods=False,  # Until modelcar initContainer completed, other containers may have Error status
     ) as isvc:
         yield isvc

tests/model_serving/model_server/model_car/test_oci_image.py

@@ -2,15 +2,15 @@
 
 from tests.model_serving.model_server.utils import verify_inference_response
 from utilities.infra import get_pods_by_isvc_label
-from utilities.constants import ModelFormat, ModelName, Protocols, RuntimeTemplates
+from utilities.constants import ModelCarImage, ModelFormat, ModelName, Protocols, RuntimeTemplates, KServeDeploymentType
 from utilities.inference_utils import Inference
 from utilities.manifests.onnx import ONNX_INFERENCE_CONFIG
 
-pytestmark = pytest.mark.serverless
+pytestmark = [pytest.mark.serverless, pytest.mark.rawdeployment]
 
 
 @pytest.mark.parametrize(
-    "unprivileged_model_namespace, serving_runtime_from_template, model_car_serverless_inference_service",
+    "unprivileged_model_namespace, serving_runtime_from_template, model_car_inference_service",
     [
         pytest.param(
             {"name": f"{ModelFormat.OPENVINO}-model-car"},
@@ -21,33 +21,49 @@
             },
             {
                 # Using mnist-8-1 model from OCI image
-                "storage-uri": "oci://quay.io/mwaykole/test@sha256:8a3217bcfa2cc5fa3d07496cff8b234acdf2c9725dd307dc0a80401f55e1a11c"  # noqa: E501
+                "storage-uri": ModelCarImage.MNIST_8_1,
+                "deployment-mode": KServeDeploymentType.SERVERLESS,
             },
-        )
+            marks=[pytest.mark.serverless],
+        ),
+        pytest.param(
+            {"name": f"{ModelFormat.OPENVINO}-model-car"},
+            {
+                "name": f"{ModelName.MNIST}-runtime",
+                "template-name": RuntimeTemplates.OVMS_KSERVE,
+                "multi-model": False,
+            },
+            {
+                # Using mnist-8-1 model from OCI image
+                "storage-uri": ModelCarImage.MNIST_8_1,
+                "deployment-mode": KServeDeploymentType.RAW_DEPLOYMENT,
+            },
+            marks=[pytest.mark.rawdeployment],
+        ),
     ],
     indirect=True,
 )
 class TestKserveModelCar:
     @pytest.mark.smoke
     @pytest.mark.jira("RHOAIENG-13465")
-    def test_model_car_no_restarts(self, model_car_serverless_inference_service):
+    def test_model_car_no_restarts(self, model_car_inference_service):
         """Verify that model pod doesn't restart"""
         pod = get_pods_by_isvc_label(
-            client=model_car_serverless_inference_service.client,
-            isvc=model_car_serverless_inference_service,
+            client=model_car_inference_service.client,
+            isvc=model_car_inference_service,
         )[0]
         restarted_containers = [
-            container.name for container in pod.instance.status.containerStatuses if container.restartCount > 1
+            container.name for container in pod.instance.status.containerStatuses if container.restartCount > 2
         ]
         assert not restarted_containers, f"Containers {restarted_containers} restarted"
 
     @pytest.mark.smoke
     @pytest.mark.ocp_interop
     @pytest.mark.jira("RHOAIENG-12306")
-    def test_model_car_using_rest(self, model_car_serverless_inference_service):
+    def test_model_car_using_rest(self, model_car_inference_service):
         """Verify model query with token using REST"""
         verify_inference_response(
-            inference_service=model_car_serverless_inference_service,
+            inference_service=model_car_inference_service,
             inference_config=ONNX_INFERENCE_CONFIG,
             inference_type=Inference.INFER,
             protocol=Protocols.HTTPS,

utilities/constants.py

@@ -211,6 +211,12 @@ class RunTimeConfigs:
     }
 
 
+class ModelCarImage:
+    MNIST_8_1: str = (
+        "oci://quay.io/mwaykole/test@sha256:8a3217bcfa2cc5fa3d07496cff8b234acdf2c9725dd307dc0a80401f55e1a11c"  # noqa: E501
+    )
+
+
 class MinIo:
     class Metadata:
         NAME: str = "minio"