feat: add upgrade tests (opendatahub-io#258)

Author: rnetser

tests/model_serving/model_server/upgrade/test_upgrade.py

@@ -1,5 +1,10 @@
 import pytest
 
+from tests.model_serving.model_server.upgrade.utils import (
+    verify_inference_generation,
+    verify_pod_containers_not_restarted,
+    verify_serving_runtime_generation,
+)
 from tests.model_serving.model_server.utils import verify_inference_response
 from utilities.constants import ModelName, Protocols
 from utilities.inference_utils import Inference
@@ -8,7 +13,11 @@
 from utilities.manifests.openvino import OPENVINO_INFERENCE_CONFIG
 
 
-# TODO: add auth, external route and grpc tests
+TEST_SERVERLESS_ONNX_POST_UPGRADE_INFERENCE_SERVICE_EXISTS: str = (
+    "test_serverless_onnx_post_upgrade_inference_service_exists"
+)
+TEST_RAW_CAIKIT_BGE_POST_UPGRADE_INFERENCE_EXISTS: str = "test_raw_caikit_bge_post_upgrade_inference_exists"
+TEST_MODEL_MESH_OPENVINO_POST_UPGRADE_INFERENCE_EXISTS: str = "test_model_mesh_openvino_post_upgrade_inference_exists"
 
 
 @pytest.mark.usefixtures("valid_aws_config")
@@ -53,7 +62,9 @@ def test_model_mesh_openvino_pre_upgrade_inference(self, openvino_model_mesh_inf
     @pytest.mark.pre_upgrade
     @pytest.mark.serverless
     def test_serverless_authenticated_onnx_pre_upgrade_inference(
-        self, ovms_authenticated_serverless_inference_service_scope_session, http_inference_token_scope_session
+        self,
+        ovms_authenticated_serverless_inference_service_scope_session,
+        http_inference_token_scope_session,
     ):
         """Verify that kserve Serverless with auth ONNX model can be queried using REST before upgrade"""
         verify_inference_response(
@@ -69,7 +80,7 @@ def test_serverless_authenticated_onnx_pre_upgrade_inference(
 class TestPostUpgradeModelServer:
     @pytest.mark.post_upgrade
     @pytest.mark.serverless
-    @pytest.mark.dependency(name="test_serverless_onnx_post_upgrade_inference_service_exists")
+    @pytest.mark.dependency(name=TEST_SERVERLESS_ONNX_POST_UPGRADE_INFERENCE_SERVICE_EXISTS)
     def test_serverless_onnx_post_upgrade_inference_service_exists(
         self, ovms_serverless_inference_service_scope_session
     ):
@@ -78,7 +89,23 @@ def test_serverless_onnx_post_upgrade_inference_service_exists(
 
     @pytest.mark.post_upgrade
     @pytest.mark.serverless
-    @pytest.mark.dependency(depends=["test_serverless_onnx_post_upgrade_inference_service_exists"])
+    @pytest.mark.dependency(depends=[TEST_SERVERLESS_ONNX_POST_UPGRADE_INFERENCE_SERVICE_EXISTS])
+    def test_serverless_onnx_post_upgrade_inference_service_not_modified(
+        self, ovms_serverless_inference_service_scope_session
+    ):
+        """Test that the serverless inference service is not modified in upgrade"""
+        verify_inference_generation(isvc=ovms_serverless_inference_service_scope_session, expected_generation=1)
+
+    @pytest.mark.post_upgrade
+    @pytest.mark.serverless
+    @pytest.mark.dependency(depends=[TEST_SERVERLESS_ONNX_POST_UPGRADE_INFERENCE_SERVICE_EXISTS])
+    def test_serverless_onnx_post_upgrade_runtime_not_modified(self, ovms_serverless_inference_service_scope_session):
+        """Test that the serverless runtime is not modified in upgrade"""
+        verify_serving_runtime_generation(isvc=ovms_serverless_inference_service_scope_session, expected_generation=1)
+
+    @pytest.mark.post_upgrade
+    @pytest.mark.serverless
+    @pytest.mark.dependency(depends=[TEST_SERVERLESS_ONNX_POST_UPGRADE_INFERENCE_SERVICE_EXISTS])
     def test_serverless_onnx_post_upgrade_inference(self, ovms_serverless_inference_service_scope_session):
         """Verify that kserve Serverless ONNX model can be queried using REST after upgrade"""
         verify_inference_response(
@@ -91,14 +118,28 @@ def test_serverless_onnx_post_upgrade_inference(self, ovms_serverless_inference_
 
     @pytest.mark.post_upgrade
     @pytest.mark.rawdeployment
-    @pytest.mark.dependency(name="test_raw_caikit_bge_post_upgrade_inference_exists")
+    @pytest.mark.dependency(name=TEST_RAW_CAIKIT_BGE_POST_UPGRADE_INFERENCE_EXISTS)
     def test_raw_caikit_bge_post_upgrade_inference_exists(self, caikit_raw_inference_service_scope_session):
         """Test that raw deployment inference service exists after upgrade"""
         assert caikit_raw_inference_service_scope_session.exists
 
     @pytest.mark.post_upgrade
     @pytest.mark.rawdeployment
-    @pytest.mark.dependency(depends=["test_raw_caikit_bge_post_upgrade_inference_exists"])
+    @pytest.mark.dependency(depends=[TEST_RAW_CAIKIT_BGE_POST_UPGRADE_INFERENCE_EXISTS])
+    def test_raw_caikit_bge_post_upgrade_inference_not_modified(self, caikit_raw_inference_service_scope_session):
+        """Test that the raw deployment inference service is not modified in upgrade"""
+        verify_inference_generation(isvc=caikit_raw_inference_service_scope_session, expected_generation=1)
+
+    @pytest.mark.post_upgrade
+    @pytest.mark.rawdeployment
+    @pytest.mark.dependency(depends=[TEST_RAW_CAIKIT_BGE_POST_UPGRADE_INFERENCE_EXISTS])
+    def test_raw_caikit_bge_post_upgrade_runtime_not_modified(self, caikit_raw_inference_service_scope_session):
+        """Test that the raw deployment runtime is not modified in upgrade"""
+        verify_serving_runtime_generation(isvc=caikit_raw_inference_service_scope_session, expected_generation=1)
+
+    @pytest.mark.post_upgrade
+    @pytest.mark.rawdeployment
+    @pytest.mark.dependency(depends=[TEST_RAW_CAIKIT_BGE_POST_UPGRADE_INFERENCE_EXISTS])
     def test_raw_caikit_bge_post_upgrade_inference(self, caikit_raw_inference_service_scope_session):
         """Test Caikit bge-large-en embedding model inference using internal route after upgrade"""
         verify_inference_response(
@@ -112,7 +153,7 @@ def test_raw_caikit_bge_post_upgrade_inference(self, caikit_raw_inference_servic
 
     @pytest.mark.post_upgrade
     @pytest.mark.modelmesh
-    @pytest.mark.dependency(name="test_model_mesh_openvino_post_upgrade_inference_exists")
+    @pytest.mark.dependency(name=TEST_MODEL_MESH_OPENVINO_POST_UPGRADE_INFERENCE_EXISTS)
     def test_model_mesh_openvino_post_upgrade_inference_exists(
         self, openvino_model_mesh_inference_service_scope_session
     ):
@@ -121,7 +162,31 @@ def test_model_mesh_openvino_post_upgrade_inference_exists(
 
     @pytest.mark.post_upgrade
     @pytest.mark.modelmesh
-    @pytest.mark.dependency(depends=["test_model_mesh_openvino_post_upgrade_inference_exists"])
+    @pytest.mark.dependency(depends=[TEST_MODEL_MESH_OPENVINO_POST_UPGRADE_INFERENCE_EXISTS])
+    def test_model_mesh_openvino_post_upgrade_inference_not_modified(
+        self, openvino_model_mesh_inference_service_scope_session
+    ):
+        """Test that the model mesh deployment inference service is not modified in upgrade"""
+        verify_inference_generation(
+            isvc=openvino_model_mesh_inference_service_scope_session,
+            expected_generation=1,
+        )
+
+    @pytest.mark.post_upgrade
+    @pytest.mark.modelmesh
+    @pytest.mark.dependency(depends=[TEST_MODEL_MESH_OPENVINO_POST_UPGRADE_INFERENCE_EXISTS])
+    def test_model_mesh_openvino_post_upgrade_runtime_not_modified(
+        self, openvino_model_mesh_inference_service_scope_session
+    ):
+        """Test that the model mesh deployment runtime is not modified in upgrade"""
+        verify_serving_runtime_generation(
+            isvc=openvino_model_mesh_inference_service_scope_session,
+            expected_generation=1,
+        )
+
+    @pytest.mark.post_upgrade
+    @pytest.mark.modelmesh
+    @pytest.mark.dependency(depends=[TEST_MODEL_MESH_OPENVINO_POST_UPGRADE_INFERENCE_EXISTS])
     def test_model_mesh_openvino_post_upgrade_inference(self, openvino_model_mesh_inference_service_scope_session):
         """Test OpenVINO ModelMesh inference with internal route after upgrade"""
         verify_inference_response(
@@ -145,7 +210,9 @@ def test_serverless_authenticated_onnx_post_upgrade_inference_service_exists(
     @pytest.mark.serverless
     @pytest.mark.dependency(depends=["test_serverless_authenticated_onnx_post_upgrade_inference_service_exists"])
     def test_serverless_authenticated_onnx_post_upgrade_inference(
-        self, ovms_authenticated_serverless_inference_service_scope_session, http_inference_token_scope_session
+        self,
+        ovms_authenticated_serverless_inference_service_scope_session,
+        http_inference_token_scope_session,
     ):
         """Verify that kserve Serverless with auth ONNX model can be queried using REST before upgrade"""
         verify_inference_response(
@@ -156,3 +223,36 @@ def test_serverless_authenticated_onnx_post_upgrade_inference(
             token=http_inference_token_scope_session,
             use_default_query=True,
         )
+
+    @pytest.mark.post_upgrade
+    @pytest.mark.serverless
+    @pytest.mark.rawdeployment
+    @pytest.mark.modelmesh
+    def test_verify_odh_model_controller_pod_not_restarted_post_upgrade(self, admin_client):
+        """Verify that ODH Model Controller pod is not restarted after upgrade"""
+        verify_pod_containers_not_restarted(
+            client=admin_client,
+            component_name="odh-model-controller",
+        )
+
+    @pytest.mark.post_upgrade
+    @pytest.mark.serverless
+    @pytest.mark.rawdeployment
+    def test_verify_kserve_pod_not_restarted_post_upgrade(self, admin_client):
+        """Verify that KServe pod is not restarted after upgrade"""
+        verify_pod_containers_not_restarted(
+            client=admin_client,
+            component_name="kserve",
+        )
+
+    @pytest.mark.post_upgrade
+    @pytest.mark.modelmesh
+    def test_verify_model_mesh_pod_not_restarted_post_upgrade(
+        self,
+        admin_client,
+    ):
+        """Verify that Model Mesh pods are not restarted after upgrade"""
+        verify_pod_containers_not_restarted(
+            client=admin_client,
+            component_name="model-mesh",
+        )

tests/model_serving/model_server/upgrade/utils.py

@@ -0,0 +1,65 @@
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.inference_service import InferenceService
+from ocp_resources.pod import Pod
+from pytest_testconfig import config as py_config
+
+from utilities.exceptions import PodContainersRestartError, ResourceMismatchError
+from utilities.infra import get_inference_serving_runtime
+
+
+def verify_pod_containers_not_restarted(client: DynamicClient, component_name: str) -> None:
+    """
+    Verify pod containers not restarted.
+
+    Args:
+        client (DynamicClient): DynamicClient instance
+        component_name (str): Name of the component
+
+    Raises:
+        AssertionError: If pod containers are restarted
+
+    """
+    restarted_containers = {}
+
+    for pod in Pod.get(
+        dyn_client=client,
+        namespace=py_config["applications_namespace"],
+        label_selector=f"{Pod.ApiGroup.APP_KUBERNETES_IO}/part-of={component_name}",
+    ):
+        if _restarted_containers := [
+            container.name for container in pod.instance.status.containerStatuses if container.restartCount > 0
+        ]:
+            restarted_containers[pod.name] = _restarted_containers
+
+    if restarted_containers:
+        raise PodContainersRestartError(f"Containers {restarted_containers} restarted")
+
+
+def verify_inference_generation(isvc: InferenceService, expected_generation: int) -> None:
+    """
+    Verify that inference generation is equal to expected generation.
+
+    Args:
+        isvc (InferenceService): InferenceService instance
+        expected_generation (int): Expected generation
+
+    Raises:
+        ResourceMismatch: If inference generation is not equal to expected generation
+    """
+    if isvc.instance.status.observedGeneration != expected_generation:
+        ResourceMismatchError(f"Inference service {isvc.name} was modified")
+
+
+def verify_serving_runtime_generation(isvc: InferenceService, expected_generation: int) -> None:
+    """
+    Verify that serving runtime generation is equal to expected generation.
+    Args:
+        isvc (InferenceService): InferenceService instance
+        expected_generation (int): Expected generation
+
+    Raises:
+        ResourceMismatch: If serving runtime generation is not equal to expected generation
+    """
+    runtime = get_inference_serving_runtime(isvc=isvc)
+    if runtime.instance.metadata.generation != expected_generation:
+        ResourceMismatchError(f"Serving runtime {runtime.name} was modified")

utilities/exceptions.py

@@ -84,7 +84,7 @@ class PodLogMissMatchError(Exception):
     pass
 
 
-class ResourceMismatch(Exception):
+class ResourceMismatchError(Exception):
     pass
 
 
@@ -117,3 +117,7 @@ class UnexpectedFailureError(Exception):
 
 class ResourceNotReadyError(Exception):
     pass
+
+
+class PodContainersRestartError(Exception):
+    pass