RHOAIENG-36714 - Model server upgrade tests (#811)

* update model server ugrade tests to use kserve-ovms

* fix bug

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: threcc

tests/model_serving/model_server/upgrade/conftest.py

@@ -7,10 +7,12 @@
 from ocp_resources.secret import Secret
 from ocp_resources.serving_runtime import ServingRuntime
 from simple_logger.logger import get_logger
-
 from utilities.constants import (
     KServeDeploymentType,
+    ModelAndFormat,
+    ModelFormat,
     ModelStoragePath,
+    ModelVersion,
     RuntimeTemplates,
 )
 from utilities.inference_utils import create_isvc
@@ -20,15 +22,17 @@
 
 LOGGER = get_logger(name=__name__)
 
+UPGRADE_NAMESPACE = "upgrade-model-server"
+S3_CONNECTION = "upgrade-connection"
+
 
 @pytest.fixture(scope="session")
-def model_namespace_scope_session(
+def namespace_fixture(
     pytestconfig: pytest.Config,
     admin_client: DynamicClient,
     teardown_resources: bool,
 ) -> Generator[Namespace, Any, Any]:
-    name = "upgrade-model-server"
-    ns = Namespace(client=admin_client, name=name)
+    ns = Namespace(client=admin_client, name=UPGRADE_NAMESPACE)
 
     if pytestconfig.option.post_upgrade:
         yield ns
@@ -37,7 +41,7 @@ def model_namespace_scope_session(
     else:
         with create_ns(
             admin_client=admin_client,
-            name=name,
+            name=UPGRADE_NAMESPACE,
             model_mesh_enabled=False,
             add_dashboard_label=True,
             teardown=teardown_resources,
@@ -46,10 +50,10 @@ def model_namespace_scope_session(
 
 
 @pytest.fixture(scope="session")
-def models_endpoint_s3_secret_scope_session(
+def s3_connection_fixture(
     pytestconfig: pytest.Config,
     admin_client: DynamicClient,
-    model_namespace_scope_session: Namespace,
+    namespace_fixture: Namespace,
     aws_access_key_id: str,
     aws_secret_access_key: str,
     models_s3_bucket_name: str,
@@ -59,8 +63,8 @@ def models_endpoint_s3_secret_scope_session(
 ) -> Generator[Secret, Any, Any]:
     secret_kwargs = {
         "client": admin_client,
-        "name": "models-bucket-secret",
-        "namespace": model_namespace_scope_session.name,
+        "name": S3_CONNECTION,
+        "namespace": namespace_fixture.name,
     }
 
     secret = Secret(**secret_kwargs)
@@ -83,16 +87,16 @@ def models_endpoint_s3_secret_scope_session(
 
 
 @pytest.fixture(scope="session")
-def caikit_raw_serving_runtime_scope_session(
+def serving_runtime_fixture(
     pytestconfig: pytest.Config,
     admin_client: DynamicClient,
-    model_namespace_scope_session: Namespace,
+    namespace_fixture: Namespace,
     teardown_resources: bool,
 ) -> Generator[ServingRuntime, Any, Any]:
     runtime_kwargs = {
         "client": admin_client,
-        "name": "caikit-raw",
-        "namespace": model_namespace_scope_session.name,
+        "name": "upgrade-runtime",
+        "namespace": namespace_fixture.name,
     }
 
     model_runtime = ServingRuntime(**runtime_kwargs)
@@ -104,26 +108,32 @@ def caikit_raw_serving_runtime_scope_session(
     else:
         with ServingRuntimeFromTemplate(
             **runtime_kwargs,
-            template_name=RuntimeTemplates.CAIKIT_STANDALONE_SERVING,
+            template_name=RuntimeTemplates.OVMS_KSERVE,
             multi_model=False,
             enable_http=True,
             teardown=teardown_resources,
+            resources={
+                ModelFormat.OVMS: {
+                    "requests": {"cpu": "1", "memory": "4Gi"},
+                    "limits": {"cpu": "2", "memory": "8Gi"},
+                }
+            },
         ) as model_runtime:
             yield model_runtime
 
 
 @pytest.fixture(scope="session")
-def caikit_raw_inference_service_scope_session(
+def inference_service_fixture(
     pytestconfig: pytest.Config,
     admin_client: DynamicClient,
-    caikit_raw_serving_runtime_scope_session: ServingRuntime,
-    models_endpoint_s3_secret_scope_session: Secret,
+    serving_runtime_fixture: ServingRuntime,
+    s3_connection_fixture: Secret,
     teardown_resources: bool,
 ) -> Generator[InferenceService, Any, Any]:
     isvc_kwargs = {
         "client": admin_client,
-        "name": caikit_raw_serving_runtime_scope_session.name,
-        "namespace": caikit_raw_serving_runtime_scope_session.namespace,
+        "name": "upgrade-isvc",
+        "namespace": serving_runtime_fixture.namespace,
     }
 
     isvc = InferenceService(**isvc_kwargs)
@@ -135,11 +145,12 @@ def caikit_raw_inference_service_scope_session(
 
     else:
         with create_isvc(
-            runtime=caikit_raw_serving_runtime_scope_session.name,
-            model_format=caikit_raw_serving_runtime_scope_session.instance.spec.supportedModelFormats[0].name,
+            runtime=serving_runtime_fixture.name,
+            model_format=ModelAndFormat.OPENVINO_IR,
             deployment_mode=KServeDeploymentType.RAW_DEPLOYMENT,
-            storage_key=models_endpoint_s3_secret_scope_session.name,
-            storage_path=ModelStoragePath.EMBEDDING_MODEL,
+            storage_key=s3_connection_fixture.name,
+            storage_path=ModelStoragePath.OPENVINO_EXAMPLE_MODEL,
+            model_version=ModelVersion.OPSET13,
             external_route=False,
             teardown=teardown_resources,
             **isvc_kwargs,

tests/model_serving/model_server/upgrade/test_upgrade.py

@@ -1,72 +1,63 @@
 import pytest
 
+from utilities.manifests.openvino import OPENVINO_KSERVE_INFERENCE_CONFIG
 from tests.model_serving.model_server.upgrade.utils import (
     verify_inference_generation,
     verify_pod_containers_not_restarted,
     verify_serving_runtime_generation,
 )
 from tests.model_serving.model_server.utils import verify_inference_response
-from utilities.constants import ModelName, Protocols
-from utilities.manifests.caikit_standalone import CAIKIT_STANDALONE_INFERENCE_CONFIG
+from utilities.constants import Protocols
 
 
-TEST_RAW_CAIKIT_BGE_POST_UPGRADE_INFERENCE_EXISTS: str = "test_raw_caikit_bge_post_upgrade_inference_exists"
+pytestmark = [pytest.mark.rawdeployment, pytest.mark.usefixtures("valid_aws_config")]
 
 
-@pytest.mark.usefixtures("valid_aws_config")
 class TestPreUpgradeModelServer:
     @pytest.mark.pre_upgrade
-    @pytest.mark.rawdeployment
-    def test_raw_caikit_bge_pre_upgrade_inference(self, caikit_raw_inference_service_scope_session):
-        """Test Caikit bge-large-en embedding model inference using internal route before upgrade"""
+    def test_raw_deployment_pre_upgrade_inference(self, inference_service_fixture):
+        """Test raw deployment model inference using internal route before upgrade"""
         verify_inference_response(
-            inference_service=caikit_raw_inference_service_scope_session,
-            inference_config=CAIKIT_STANDALONE_INFERENCE_CONFIG,
-            inference_type="embedding",
+            inference_service=inference_service_fixture,
+            inference_config=OPENVINO_KSERVE_INFERENCE_CONFIG,
+            inference_type="infer",
             protocol=Protocols.HTTP,
-            model_name=ModelName.CAIKIT_BGE_LARGE_EN,
             use_default_query=True,
         )
 
 
 class TestPostUpgradeModelServer:
     @pytest.mark.post_upgrade
-    @pytest.mark.rawdeployment
-    @pytest.mark.dependency(name=TEST_RAW_CAIKIT_BGE_POST_UPGRADE_INFERENCE_EXISTS)
-    def test_raw_caikit_bge_post_upgrade_inference_exists(self, caikit_raw_inference_service_scope_session):
+    @pytest.mark.dependency(name="isvc_exists")
+    def test_raw_deployment_post_upgrade_inference_exists(self, inference_service_fixture):
         """Test that raw deployment inference service exists after upgrade"""
-        assert caikit_raw_inference_service_scope_session.exists
+        assert inference_service_fixture.exists
 
     @pytest.mark.post_upgrade
-    @pytest.mark.rawdeployment
-    @pytest.mark.dependency(depends=[TEST_RAW_CAIKIT_BGE_POST_UPGRADE_INFERENCE_EXISTS])
-    def test_raw_caikit_bge_post_upgrade_inference_not_modified(self, caikit_raw_inference_service_scope_session):
+    @pytest.mark.dependency(depends=["isvc_exists"])
+    def test_raw_deployment_post_upgrade_inference_not_modified(self, inference_service_fixture):
         """Test that the raw deployment inference service is not modified in upgrade"""
-        verify_inference_generation(isvc=caikit_raw_inference_service_scope_session, expected_generation=1)
+        verify_inference_generation(isvc=inference_service_fixture, expected_generation=1)
 
     @pytest.mark.post_upgrade
-    @pytest.mark.rawdeployment
-    @pytest.mark.dependency(depends=[TEST_RAW_CAIKIT_BGE_POST_UPGRADE_INFERENCE_EXISTS])
-    def test_raw_caikit_bge_post_upgrade_runtime_not_modified(self, caikit_raw_inference_service_scope_session):
+    @pytest.mark.dependency(depends=["isvc_exists"])
+    def test_raw_deployment_post_upgrade_runtime_not_modified(self, inference_service_fixture):
         """Test that the raw deployment runtime is not modified in upgrade"""
-        verify_serving_runtime_generation(isvc=caikit_raw_inference_service_scope_session, expected_generation=1)
+        verify_serving_runtime_generation(isvc=inference_service_fixture, expected_generation=1)
 
     @pytest.mark.post_upgrade
-    @pytest.mark.rawdeployment
-    @pytest.mark.dependency(depends=[TEST_RAW_CAIKIT_BGE_POST_UPGRADE_INFERENCE_EXISTS])
-    def test_raw_caikit_bge_post_upgrade_inference(self, caikit_raw_inference_service_scope_session):
-        """Test Caikit bge-large-en embedding model inference using internal route after upgrade"""
+    @pytest.mark.dependency(depends=["isvc_exists"])
+    def test_raw_deployment_post_upgrade_inference(self, inference_service_fixture):
+        """Test raw deployment model inference using internal route after upgrade"""
         verify_inference_response(
-            inference_service=caikit_raw_inference_service_scope_session,
-            inference_config=CAIKIT_STANDALONE_INFERENCE_CONFIG,
-            inference_type="embedding",
+            inference_service=inference_service_fixture,
+            inference_config=OPENVINO_KSERVE_INFERENCE_CONFIG,
+            inference_type="infer",
             protocol=Protocols.HTTP,
-            model_name=ModelName.CAIKIT_BGE_LARGE_EN,
             use_default_query=True,
         )
 
     @pytest.mark.post_upgrade
-    @pytest.mark.rawdeployment
     def test_verify_odh_model_controller_pod_not_restarted_post_upgrade(self, admin_client):
         """Verify that ODH Model Controller pod is not restarted after upgrade"""
         verify_pod_containers_not_restarted(
@@ -75,7 +66,6 @@ def test_verify_odh_model_controller_pod_not_restarted_post_upgrade(self, admin_
         )
 
     @pytest.mark.post_upgrade
-    @pytest.mark.rawdeployment
     def test_verify_kserve_pod_not_restarted_post_upgrade(self, admin_client):
         """Verify that KServe pod is not restarted after upgrade"""
         verify_pod_containers_not_restarted(

tests/model_serving/model_server/upgrade/utils.py

@@ -47,7 +47,7 @@ def verify_inference_generation(isvc: InferenceService, expected_generation: int
         ResourceMismatch: If inference generation is not equal to expected generation
     """
     if isvc.instance.status.observedGeneration != expected_generation:
-        ResourceMismatchError(f"Inference service {isvc.name} was modified")
+        raise ResourceMismatchError(f"Inference service {isvc.name} was modified")
 
 
 def verify_serving_runtime_generation(isvc: InferenceService, expected_generation: int) -> None:
@@ -62,4 +62,4 @@ def verify_serving_runtime_generation(isvc: InferenceService, expected_generatio
     """
     runtime = get_inference_serving_runtime(isvc=isvc)
     if runtime.instance.metadata.generation != expected_generation:
-        ResourceMismatchError(f"Serving runtime {runtime.name} was modified")
+        raise ResourceMismatchError(f"Serving runtime {runtime.name} was modified")