Add post upgrade scenarios for MLMD removal

Author: dbasunag

tests/model_registry/conftest.py

@@ -482,3 +482,23 @@ def model_registry_rest_headers(current_client_token: str) -> dict[str, str]:
         "accept": "application/json",
         "Content-Type": "application/json",
     }
+
+
+@pytest.fixture(scope="class")
+def model_registry_deployment_containers(model_registry_namespace: str) -> list[dict[str, Any]]:
+    return Deployment(
+        name=MR_INSTANCE_NAME, namespace=model_registry_namespace, ensure_exists=True
+    ).instance.spec.template.spec.containers
+
+
+@pytest.fixture(scope="class")
+def model_registry_pod(admin_client: DynamicClient, model_registry_namespace: str) -> Pod:
+    mr_pod = list(
+        Pod.get(
+            dyn_client=admin_client,
+            namespace=model_registry_namespace,
+            label_selector=f"app={MR_INSTANCE_NAME}",
+        )
+    )
+    assert len(mr_pod) == 1
+    return mr_pod[0]

tests/model_registry/python_client/conftest.py

@@ -5,7 +5,11 @@
 
 from ocp_resources.pod import Pod
 from ocp_resources.namespace import Namespace
-from tests.model_registry.utils import execute_model_registry_get_command
+from tests.model_registry.utils import (
+    execute_model_registry_get_command,
+    validate_no_grpc_container,
+    validate_mlmd_removal_in_model_registry_pod_log,
+)
 from utilities.constants import DscComponents
 from tests.model_registry.constants import MODEL_NAME, MODEL_DICT
 from model_registry import ModelRegistry as ModelRegistryClient
@@ -87,18 +91,29 @@ def test_model_registry_operator_env(
         if not namespace_env:
             pytest.fail("Missing environment variable REGISTRIES_NAMESPACE")
 
-    def test_model_registry_grpc_container_removal(self, model_registry_deployment_containers: dict[str, Any]):
+    def test_model_registry_grpc_container_removal(self, model_registry_deployment_containers: list[dict[str, Any]]):
         """
         RHOAIENG-26239: Test to ensure removal of grpc container from model registry deployment
         Steps:
             Create metadata database
             Deploys model registry using the same
             Check model registry deployment for grpc container. It should not be present
         """
+        validate_no_grpc_container(deployment_containers=model_registry_deployment_containers)
 
-        for container in model_registry_deployment_containers:
-            if "grpc" in container["name"]:
-                pytest.fail(f"GRPC container found: {container}")
+    def test_model_registry_pod_log_mlmd_removal(
+        self, model_registry_deployment_containers: list[dict[str, Any]], model_registry_pod: Pod
+    ):
+        """
+        RHOAIENG-26239: Test to ensure removal of grpc container from model registry deployment
+        Steps:
+            Create metadata database
+            Deploys model registry using the same
+            Check model registry deployment for grpc container. It should not be present
+        """
+        validate_mlmd_removal_in_model_registry_pod_log(
+            deployment_containers=model_registry_deployment_containers, pod_object=model_registry_pod
+        )
 
     @pytest.mark.parametrize(
         "endpoint",

tests/model_registry/python_client/test_model_registry_creation.py

@@ -1,12 +1,18 @@
 import pytest
-from typing import Self
+from typing import Self, Any
+
+from ocp_resources.pod import Pod
 from tests.model_registry.constants import MODEL_NAME, MODEL_DICT
 from model_registry.types import RegisteredModel
 from model_registry import ModelRegistry as ModelRegistryClient
 from ocp_resources.model_registry_modelregistry_opendatahub_io import ModelRegistry
 from simple_logger.logger import get_logger
 from tests.model_registry.rest_api.utils import ModelRegistryV1Alpha1
-from tests.model_registry.utils import get_and_validate_registered_model
+from tests.model_registry.utils import (
+    get_and_validate_registered_model,
+    validate_no_grpc_container,
+    validate_mlmd_removal_in_model_registry_pod_log,
+)
 
 LOGGER = get_logger(name=__name__)
 
@@ -85,3 +91,29 @@ def test_model_registry_instance_status_conversion_post_upgrade(
         LOGGER.info(f"Validating MR status {status}")
         if not status:
             pytest.fail(f"Empty status found for {mr_instance}")
+
+    @pytest.mark.post_upgrade
+    def test_model_registry_grpc_container_removal_post_upgrade(
+        self, model_registry_deployment_containers: list[dict[str, Any]]
+    ):
+        """
+        RHOAIENG-29161: Test to ensure removal of grpc container from model registry deployment post upgrade
+        Steps:
+            Check model registry deployment for grpc container. It should not be present
+        """
+        validate_no_grpc_container(deployment_containers=model_registry_deployment_containers)
+
+    @pytest.mark.post_upgrade
+    def test_model_registry_pod_log_mlmd_removal(
+        self, model_registry_deployment_containers: list[dict[str, Any]], model_registry_pod: Pod
+    ):
+        """
+        RHOAIENG-29161: Test to ensure removal of grpc container from model registry deployment
+        Steps:
+            Create metadata database
+            Deploys model registry using the same
+            Check model registry deployment for grpc container. It should not be present
+        """
+        validate_mlmd_removal_in_model_registry_pod_log(
+            deployment_containers=model_registry_deployment_containers, pod_object=model_registry_pod
+        )

tests/model_registry/upgrade/test_model_registry_upgrade.py

@@ -362,3 +362,28 @@ def execute_model_registry_get_command(url: str, headers: dict[str, str], json_o
             raise
     else:
         return {"raw_output": resp.text}
+
+
+def validate_no_grpc_container(deployment_containers: list[dict[str, Any]]) -> None:
+    grpc_container = None
+    for container in deployment_containers:
+        if "grpc" in container["name"]:
+            grpc_container = container
+    assert not grpc_container, f"GRPC container found: {grpc_container}"
+
+
+def validate_mlmd_removal_in_model_registry_pod_log(
+    deployment_containers: list[dict[str, Any]], pod_object: Pod
+) -> None:
+    errors = []
+    embedmd_message = "EmbedMD service connected"
+    for container in deployment_containers:
+        container_name = container["name"]
+        LOGGER.info(f"Checking {container_name}")
+        log = pod_object.log(container=container_name)
+        if "rest" in container_name:
+            if embedmd_message not in log:
+                errors.append(f"Missing {embedmd_message} in {container_name} log")
+        if "MLMD" in log:
+            errors.append(f"MLMD reference found in {container_name} log")
+    assert not errors, f"Log validation failed with error(s): {errors}"