changes to fix model server Test (#1271)

* changes to fix model server Test

Signed-off-by: Milind waykole <mwaykole@redhat.com>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Signed-off-by: Milind waykole <mwaykole@redhat.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: mwaykole

.gitignore

@@ -165,3 +165,4 @@ cython_debug/
 
 # VSCode config
 .vscode/
+.cursor/

pyproject.toml

@@ -56,7 +56,7 @@ dependencies = [
     "syrupy",
     "protobuf",
     "grpcio-reflection",
-    "portforward>=0.7.1",
+    "portforward>=0.7.6",
     "pytest-testconfig>=0.2.0",
     "pygithub>=2.5.0",
     "timeout-sampler>=1.0.6",

tests/model_serving/model_server/upgrade/conftest.py

@@ -30,10 +30,10 @@
 UPGRADE_RESOURCES: str = (
     f"{{Namespace: {{{UPGRADE_NAMESPACE:}}},"
     f"ServingRuntime: {{onnx-serverless: {UPGRADE_NAMESPACE},"
-    f"caikit-raw: {UPGRADE_NAMESPACE},ovms-model-mesh: {UPGRADE_NAMESPACE}}},"
+    f"onnx-raw: {UPGRADE_NAMESPACE},ovms-model-mesh: {UPGRADE_NAMESPACE}}},"
     f"InferenceService: {{onnx-serverless: {UPGRADE_NAMESPACE},"
-    f"caikit-raw: {UPGRADE_NAMESPACE}, ovms-model-mesh: {UPGRADE_NAMESPACE}}},"
-    f"Secret: {{ci-bucket-secret: {UPGRADE_NAMESPACE}, models-bucket-secret: {UPGRADE_NAMESPACE}}},"
+    f"onnx-raw: {UPGRADE_NAMESPACE}, ovms-model-mesh: {UPGRADE_NAMESPACE}}},"
+    f"Secret: {{ci-bucket-secret: {UPGRADE_NAMESPACE}}},"
     f"ServiceAccount: {{models-bucket-sa: {UPGRADE_NAMESPACE}}}}}"
 )
 
@@ -63,29 +63,6 @@ def model_namespace_scope_session(
         yield ns
 
 
-@pytest.fixture(scope="session")
-def models_endpoint_s3_secret_scope_session(
-    admin_client: DynamicClient,
-    model_namespace_scope_session: Namespace,
-    aws_access_key_id: str,
-    aws_secret_access_key: str,
-    models_s3_bucket_name: str,
-    models_s3_bucket_region: str,
-    models_s3_bucket_endpoint: str,
-) -> Generator[Secret, Any, Any]:
-    with s3_endpoint_secret(
-        admin_client=admin_client,
-        name="models-bucket-secret",
-        namespace=model_namespace_scope_session.name,
-        aws_access_key=aws_access_key_id,
-        aws_secret_access_key=aws_secret_access_key,
-        aws_s3_region=models_s3_bucket_region,
-        aws_s3_bucket=models_s3_bucket_name,
-        aws_s3_endpoint=models_s3_bucket_endpoint,
-    ) as secret:
-        yield secret
-
-
 @pytest.fixture(scope="session")
 def ci_endpoint_s3_secret_scope_session(
     admin_client: DynamicClient,
@@ -177,49 +154,54 @@ def ovms_serverless_inference_service_scope_session(
 
 
 @pytest.fixture(scope="session")
-def caikit_raw_serving_runtime_scope_session(
+def ovms_raw_serving_runtime_scope_session(
     admin_client: DynamicClient,
     model_namespace_scope_session: Namespace,
 ) -> Generator[ServingRuntime, Any, Any]:
     with ServingRuntimeFromTemplate(
         client=admin_client,
-        name="caikit-raw",
+        name="onnx-raw",
         namespace=model_namespace_scope_session.name,
-        template_name=RuntimeTemplates.CAIKIT_STANDALONE_SERVING,
+        template_name=RuntimeTemplates.OVMS_KSERVE,
         multi_model=False,
-        enable_http=True,
+        resources={
+            ModelFormat.OVMS: {
+                "requests": {"cpu": "1", "memory": "4Gi"},
+                "limits": {"cpu": "2", "memory": "8Gi"},
+            }
+        },
+        model_format_name={ModelFormat.ONNX: ModelVersion.OPSET13},
     ) as model_runtime:
         yield model_runtime
 
 
 @pytest.fixture(scope="session")
-def caikit_raw_inference_service_scope_session(
+def ovms_raw_inference_service_scope_session(
     pytestconfig: pytest.Config,
     admin_client: DynamicClient,
-    caikit_raw_serving_runtime_scope_session: ServingRuntime,
-    models_endpoint_s3_secret_scope_session: Secret,
+    ovms_raw_serving_runtime_scope_session: ServingRuntime,
+    ci_endpoint_s3_secret_scope_session: Secret,
 ) -> Generator[InferenceService, Any, Any]:
     isvc_kwargs = {
         "client": admin_client,
-        "name": caikit_raw_serving_runtime_scope_session.name,
-        "namespace": caikit_raw_serving_runtime_scope_session.namespace,
+        "name": ovms_raw_serving_runtime_scope_session.name,
+        "namespace": ovms_raw_serving_runtime_scope_session.namespace,
     }
 
     isvc = InferenceService(**isvc_kwargs)
 
     if pytestconfig.option.post_upgrade:
         yield isvc
-
         isvc.clean_up()
 
     else:
         with create_isvc(
-            runtime=caikit_raw_serving_runtime_scope_session.name,
-            model_format=caikit_raw_serving_runtime_scope_session.instance.spec.supportedModelFormats[0].name,
+            runtime=ovms_raw_serving_runtime_scope_session.name,
+            storage_path="test-dir",
+            storage_key=ci_endpoint_s3_secret_scope_session.name,
+            model_format=ModelAndFormat.OPENVINO_IR,
             deployment_mode=KServeDeploymentType.RAW_DEPLOYMENT,
-            storage_key=models_endpoint_s3_secret_scope_session.name,
-            storage_path=ModelStoragePath.EMBEDDING_MODEL,
-            external_route=True,
+            model_version=ModelVersion.OPSET13,
             **isvc_kwargs,
         ) as isvc:
             yield isvc

tests/model_serving/model_server/upgrade/test_upgrade.py

@@ -1,9 +1,8 @@
 import pytest
 
 from tests.model_serving.model_server.utils import verify_inference_response
-from utilities.constants import ModelName, Protocols
+from utilities.constants import Protocols
 from utilities.inference_utils import Inference
-from utilities.manifests.caikit_standalone import CAIKIT_STANDALONE_INFERENCE_CONFIG
 from utilities.manifests.onnx import ONNX_INFERENCE_CONFIG
 from utilities.manifests.openvino import OPENVINO_INFERENCE_CONFIG
 
@@ -27,14 +26,13 @@ def test_serverless_onnx_pre_upgrade_inference(self, ovms_serverless_inference_s
         )
 
     @pytest.mark.pre_upgrade
-    def test_raw_caikit_bge_pre_upgrade_inference(self, caikit_raw_inference_service_scope_session):
-        """Test Caikit bge-large-en embedding model inference using internal route before upgrade"""
+    def test_raw_ovms_onnx_pre_upgrade_inference(self, ovms_raw_inference_service_scope_session):
+        """Verify that kserve Raw OVMS ONNX model can be queried using REST before upgrade"""
         verify_inference_response(
-            inference_service=caikit_raw_inference_service_scope_session,
-            inference_config=CAIKIT_STANDALONE_INFERENCE_CONFIG,
-            inference_type="embedding",
-            protocol=Protocols.HTTPS,
-            model_name=ModelName.CAIKIT_BGE_LARGE_EN,
+            inference_service=ovms_raw_inference_service_scope_session,
+            inference_config=ONNX_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
+            protocol=Protocols.HTTP,
             use_default_query=True,
         )
 
@@ -73,21 +71,20 @@ def test_serverless_onnx_post_upgrade_inference(self, ovms_serverless_inference_
         )
 
     @pytest.mark.post_upgrade
-    @pytest.mark.dependency(name="test_raw_caikit_bge_post_upgrade_inference_exists")
-    def test_raw_caikit_bge_post_upgrade_inference_exists(self, caikit_raw_inference_service_scope_session):
-        """Test that raw deployment inference service exists after upgrade"""
-        assert caikit_raw_inference_service_scope_session.exists
+    @pytest.mark.dependency(name="test_raw_ovms_onnx_post_upgrade_inference_exists")
+    def test_raw_ovms_onnx_post_upgrade_inference_exists(self, ovms_raw_inference_service_scope_session):
+        """Test that raw OVMS inference service exists after upgrade"""
+        assert ovms_raw_inference_service_scope_session.exists
 
     @pytest.mark.post_upgrade
-    @pytest.mark.dependency(depends=["test_raw_caikit_bge_post_upgrade_inference_exists"])
-    def test_raw_caikit_bge_post_upgrade_inference(self, caikit_raw_inference_service_scope_session):
-        """Test Caikit bge-large-en embedding model inference using internal route after upgrade"""
+    @pytest.mark.dependency(depends=["test_raw_ovms_onnx_post_upgrade_inference_exists"])
+    def test_raw_ovms_onnx_post_upgrade_inference(self, ovms_raw_inference_service_scope_session):
+        """Verify that kserve Raw OVMS ONNX model can be queried using REST after upgrade"""
         verify_inference_response(
-            inference_service=caikit_raw_inference_service_scope_session,
-            inference_config=CAIKIT_STANDALONE_INFERENCE_CONFIG,
-            inference_type="embedding",
-            protocol=Protocols.HTTPS,
-            model_name=ModelName.CAIKIT_BGE_LARGE_EN,
+            inference_service=ovms_raw_inference_service_scope_session,
+            inference_config=ONNX_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
+            protocol=Protocols.HTTP,
             use_default_query=True,
         )