Updates to image validation tests to ensure all ai hub resources are considered (#788)

Author: dbasunag

tests/model_registry/constants.py

@@ -80,3 +80,4 @@ class ModelRegistryEndpoints:
     Deployment: f"{MR_INSTANCE_NAME}-postgres",
     Secret: f"{MR_INSTANCE_NAME}-postgres-credentials",
 }
+MR_POSTGRES_DEPLOYMENT_NAME_STR = f"{MR_INSTANCE_NAME}-postgres"

tests/model_registry/image_validation/conftest.py

@@ -2,19 +2,33 @@
 
 import pytest
 from kubernetes.dynamic import DynamicClient
-from pytest_testconfig import config as py_config
 
 from ocp_resources.pod import Pod
-from tests.model_registry.constants import MODEL_REGISTRY_POD_FILTER
 from utilities.general import wait_for_pods_by_labels
+from pytest import FixtureRequest
 
 
-@pytest.fixture(scope="function")
-def model_registry_instance_pod_by_label(admin_client: DynamicClient) -> Generator[Pod, Any, Any]:
+@pytest.fixture(scope="class")
+def model_registry_instance_pods_by_label(
+    request: FixtureRequest, admin_client: DynamicClient, model_registry_namespace: str
+) -> Generator[list[Pod], Any, Any]:
     """Get the model registry instance pod."""
-    yield wait_for_pods_by_labels(
-        admin_client=admin_client,
-        namespace=py_config["model_registry_namespace"],
-        label_selector=MODEL_REGISTRY_POD_FILTER,
-        expected_num_pods=1,
-    )[0]
+    pods = []
+    for label in request.param["label_selectors"]:
+        pods.append(
+            wait_for_pods_by_labels(
+                admin_client=admin_client,
+                namespace=model_registry_namespace,
+                label_selector=label,
+                expected_num_pods=1,
+            )[0]
+        )
+    yield pods
+
+
+@pytest.fixture(scope="function")
+def resource_pods(request: FixtureRequest, admin_client: DynamicClient) -> list[Pod]:
+    namespace = request.param.get("namespace")
+    label_selector = request.param.get("label_selector")
+    assert namespace
+    return list(Pod.get(namespace=namespace, label_selector=label_selector, dyn_client=admin_client))

tests/model_registry/image_validation/test_verify_rhoai_images.py

@@ -1,53 +1,78 @@
+"""
+Tests to verify that all Model Registry component images (operator and instance container images)
+meet the requirements:
+1. Images are hosted in registry.redhat.io
+2. Images use sha256 digest instead of tags
+3. Images are listed in the CSV's relatedImages section
+"""
+
 import pytest
 from typing import Self, Set
 from simple_logger.logger import get_logger
 from kubernetes.dynamic import DynamicClient
 
-from utilities.general import (
-    validate_container_images,
-)
+from tests.model_registry.constants import MR_INSTANCE_NAME, MR_POSTGRES_DEPLOYMENT_NAME_STR, MR_OPERATOR_NAME
+from tests.model_registry.image_validation.utils import validate_images
+from utilities.constants import Labels
 from ocp_resources.pod import Pod
-from tests.model_registry.utils import get_model_catalog_pod
+from pytest_testconfig import config as py_config
 
 LOGGER = get_logger(name=__name__)
+pytestmark = [pytest.mark.downstream_only, pytest.mark.skip_must_gather, pytest.mark.smoke]
 
 
+class TestAIHubResourcesImages:
+    @pytest.mark.parametrize(
+        "resource_pods",
+        [
+            pytest.param(
+                {"namespace": py_config["model_registry_namespace"], "label_selector": "component=model-catalog"},
+                marks=pytest.mark.smoke,
+                id="test_model_catalog_pods_images",
+            ),
+            pytest.param(
+                {
+                    "namespace": py_config["applications_namespace"],
+                    "label_selector": f"{Labels.OpenDataHubIo.NAME}={MR_OPERATOR_NAME}",
+                },
+                marks=pytest.mark.smoke,
+                id="test_model_registry_operator_pods_images",
+            ),
+        ],
+        indirect=True,
+    )
+    def test_verify_pod_images(
+        self: Self,
+        admin_client: DynamicClient,
+        resource_pods: list[Pod],
+        related_images_refs: Set[str],
+    ):
+        validate_images(pods_to_validate=resource_pods, related_images_refs=related_images_refs)
+
+
+@pytest.mark.parametrize(
+    "model_registry_metadata_db_resources, model_registry_instance, model_registry_instance_pods_by_label",
+    [
+        pytest.param({}, {}, {"label_selectors": [f"app={MR_INSTANCE_NAME}"]}),
+        pytest.param(
+            {"db_name": "default"},
+            {"db_name": "default"},
+            {"label_selectors": [f"app={MR_INSTANCE_NAME}", f"app={MR_POSTGRES_DEPLOYMENT_NAME_STR}"]},
+        ),
+    ],
+    indirect=True,
+)
 @pytest.mark.usefixtures(
     "updated_dsc_component_state_scope_session",
-    "model_registry_namespace",
     "model_registry_metadata_db_resources",
     "model_registry_instance",
+    "model_registry_instance_pods_by_label",
 )
-@pytest.mark.downstream_only
 class TestModelRegistryImages:
-    """
-    Tests to verify that all Model Registry component images (operator and instance container images)
-    meet the requirements:
-    1. Images are hosted in registry.redhat.io
-    2. Images use sha256 digest instead of tags
-    3. Images are listed in the CSV's relatedImages section
-    """
-
-    @pytest.mark.smoke
-    @pytest.mark.skip_must_gather
-    def test_verify_model_registry_images(
+    def test_verify_model_registry_pod_images(
         self: Self,
         admin_client: DynamicClient,
-        model_registry_operator_pod: Pod,
-        model_registry_instance_pod_by_label: Pod,
-        model_registry_namespace: str,
+        model_registry_instance_pods_by_label: list[Pod],
         related_images_refs: Set[str],
     ):
-        model_catalog_pod = get_model_catalog_pod(
-            client=admin_client, model_registry_namespace=model_registry_namespace
-        )[0]
-        validation_errors = []
-        for pod in [model_registry_operator_pod, model_registry_instance_pod_by_label, model_catalog_pod]:
-            validation_errors.extend(
-                validate_container_images(
-                    pod=pod, valid_image_refs=related_images_refs, skip_patterns=["openshift-service-mesh"]
-                )
-            )
-
-        if validation_errors:
-            pytest.fail("\n".join(validation_errors))
+        validate_images(pods_to_validate=model_registry_instance_pods_by_label, related_images_refs=related_images_refs)

tests/model_registry/image_validation/utils.py

@@ -0,0 +1,22 @@
+from typing import Set
+from simple_logger.logger import get_logger
+import pytest
+from ocp_resources.pod import Pod
+from utilities.general import validate_container_images
+
+LOGGER = get_logger(name=__name__)
+
+
+def validate_images(pods_to_validate: list[Pod], related_images_refs: Set[str]):
+    validation_errors = []
+    for pod in pods_to_validate:
+        LOGGER.info(f"Validating {pod.name} in {pod.namespace}")
+        validation_errors.extend(
+            validate_container_images(
+                pod=pod,
+                valid_image_refs=related_images_refs,
+            )
+        )
+
+    if validation_errors:
+        pytest.fail("\n".join(validation_errors))

tests/model_registry/scc/conftest.py

@@ -4,9 +4,8 @@
 from ocp_resources.namespace import Namespace
 from ocp_resources.pod import Pod
 from ocp_resources.deployment import Deployment
-from tests.model_registry.scc.constants import MR_POSTGRES_DEPLOYMENT_NAME_STR
 from tests.model_registry.scc.utils import get_pod_by_deployment_name
-from tests.model_registry.constants import MR_INSTANCE_NAME
+from tests.model_registry.constants import MR_INSTANCE_NAME, MR_POSTGRES_DEPLOYMENT_NAME_STR
 
 from kubernetes.dynamic import DynamicClient
 from simple_logger.logger import get_logger

tests/model_registry/scc/constants.py

@@ -1,4 +1 @@
-from tests.model_registry.constants import MR_INSTANCE_NAME
-
 MODEL_CATALOG_STR = "model-catalog"
-MR_POSTGRES_DEPLOYMENT_NAME_STR = f"{MR_INSTANCE_NAME}-postgres"

tests/model_registry/scc/test_model_registry_scc.py

@@ -5,13 +5,11 @@
 
 from ocp_resources.pod import Pod
 from ocp_resources.deployment import Deployment
-from tests.model_registry.scc.constants import MR_POSTGRES_DEPLOYMENT_NAME_STR
 from tests.model_registry.scc.utils import (
     validate_deployment_scc,
     validate_pod_scc,
 )
-from tests.model_registry.constants import MR_INSTANCE_NAME
-
+from tests.model_registry.constants import MR_INSTANCE_NAME, MR_POSTGRES_DEPLOYMENT_NAME_STR
 
 LOGGER = get_logger(name=__name__)
 

utilities/general.py

@@ -289,11 +289,15 @@ def validate_container_images(
         # Validate image format
         is_valid, error_msg = validate_image_format(image=image)
         if not is_valid:
-            validation_errors.append(f"Pod {pod.name} image validation failed: {error_msg}")
+            validation_errors.append(
+                f"Pod {pod.name} in namespace: {pod.namespace} image validation failed: {error_msg}"
+            )
 
         # Check if image is in valid references
         if image not in valid_image_refs:
-            validation_errors.append(f"Pod {pod.name} image {image} is not in valid image references")
+            validation_errors.append(
+                f"Pod {pod.name}, namespace: {pod.namespace} image {image} is not in valid image references"
+            )
 
     return validation_errors