Added tests to Validated Model Runtimes Component Images (#1148)

* Added tests to Validated Model Runtimes Component Images

* Fixed failures

Author: Raghul-M

tests/model_serving/model_runtime/image_validation/__init__.py

@@ -0,0 +1 @@
+"""Serving runtime image validation tests."""

tests/model_serving/model_runtime/image_validation/conftest.py

@@ -0,0 +1,90 @@
+"""
+Fixtures for serving runtime image validation tests.
+
+Creates minimal ServingRuntime + InferenceService so that deployments/pods
+are created and their spec.containers[*].image can be validated against
+the CSV relatedImages (registry.redhat.io, sha256 digest).
+"""
+
+from collections.abc import Generator
+from typing import Any
+
+import pytest
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.namespace import Namespace
+from ocp_resources.pod import Pod
+from timeout_sampler import TimeoutSampler
+
+from tests.model_serving.model_runtime.image_validation.constant import PLACEHOLDER_STORAGE_URI
+from utilities.constants import KServeDeploymentType
+from utilities.inference_utils import create_isvc
+from utilities.infra import create_ns, wait_for_isvc_pods
+from utilities.serving_runtime import ServingRuntimeFromTemplate
+
+
+@pytest.fixture(scope="class")
+def serving_runtime_image_validation_namespace(
+    admin_client: DynamicClient,
+) -> Generator[Namespace, Any, Any]:
+    """
+    A dedicated namespace for serving runtime image validation.
+
+    Ensures deployments/pods created by the test have a clean namespace
+    that is torn down after the test.
+    """
+    name = "runtime-verification"
+    with create_ns(admin_client=admin_client, name=name, teardown=True) as ns:
+        yield ns
+
+
+@pytest.fixture(scope="function")
+def serving_runtime_pods_for_runtime(
+    request: pytest.FixtureRequest,
+    admin_client: DynamicClient,
+    serving_runtime_image_validation_namespace: Namespace,
+) -> Generator[tuple[list[Pod], str], Any, Any]:
+    """
+    For a given runtime config (parametrized), create ServingRuntime + InferenceService,
+    wait for pods, yield (pods, display_name) for validation. Teardown after test.
+    """
+    config = request.param
+    display_name = config["name"]
+    name_slug = display_name.replace("_", "-")
+    namespace_name = serving_runtime_image_validation_namespace.name
+    runtime_name = f"{name_slug}-runtime"
+    isvc_name = f"{name_slug}-isvc"
+
+    with ServingRuntimeFromTemplate(
+        client=admin_client,
+        name=runtime_name,
+        namespace=namespace_name,
+        template_name=config["template"],
+        deployment_type="raw",
+    ) as serving_runtime:
+        # Get model format from the runtime for the InferenceService spec.
+        model_format = serving_runtime.instance.spec.supportedModelFormats[0].name
+        with create_isvc(
+            client=admin_client,
+            name=isvc_name,
+            namespace=namespace_name,
+            model_format=model_format,
+            runtime=runtime_name,
+            storage_uri=PLACEHOLDER_STORAGE_URI,
+            deployment_mode=KServeDeploymentType.RAW_DEPLOYMENT,
+            wait=False,
+            wait_for_predictor_pods=False,
+            timeout=120,
+            teardown=True,
+        ) as isvc:
+            # Wait for pods to be created (300 seconds timeout)
+            for pods in TimeoutSampler(
+                wait_timeout=300,
+                sleep=5,
+                func=wait_for_isvc_pods,
+                client=admin_client,
+                isvc=isvc,
+                runtime_name=runtime_name,
+            ):
+                if pods:
+                    yield (pods, display_name)
+                    return

tests/model_serving/model_runtime/image_validation/constant.py

@@ -0,0 +1,19 @@
+"""Constants for serving runtime image validation tests."""
+
+from utilities.constants import RuntimeTemplates
+
+# Placeholder storage URI so the controller creates Deployment/Pod with runtime image.
+# No actual model or inference is required; pod phase does not need to be Ready.
+PLACEHOLDER_STORAGE_URI = "s3://dummy-bucket/dummy/"
+
+# Runtime configs: display name (for "name : passed") and template name.
+# For each we create ServingRuntime + InferenceService, wait for pod(s), validate, then teardown.
+RUNTIME_CONFIGS = [
+    {"name": "odh_openvino_model_server_image", "template": RuntimeTemplates.OVMS_KSERVE},
+    {"name": "odh_vllm_cpu_image", "template": RuntimeTemplates.VLLM_CPU_x86},
+    {"name": "odh_vllm_gaudi_image", "template": RuntimeTemplates.VLLM_GAUDI},
+    {"name": "odh_mlserver_image", "template": RuntimeTemplates.MLSERVER},
+    {"name": "rhaiis_vllm_cuda_image", "template": RuntimeTemplates.VLLM_CUDA},
+    {"name": "rhaiis_vllm_rocm_image", "template": RuntimeTemplates.VLLM_ROCM},
+    {"name": "rhaiis_vllm_spyre_image", "template": RuntimeTemplates.VLLM_SPYRE},
+]

tests/model_serving/model_runtime/image_validation/test_verify_serving_runtime_images.py

@@ -0,0 +1,58 @@
+"""
+Tests to verify that serving runtime component images meet the requirements:
+1. Images are hosted in registry.redhat.io
+2. Images use sha256 digest instead of tags
+3. Images are listed in the CSV's relatedImages section
+
+For each runtime template we create ServingRuntime + InferenceService, wait for pod(s),
+then validate the pod's container images against the cluster CSV (relatedImages) at runtime.
+No hardcoded image SHAs—validation uses whatever CSV is installed (e.g. rhods-operator.3.3.0).
+"""
+
+from typing import Self
+
+import pytest
+from ocp_resources.pod import Pod
+from simple_logger.logger import get_logger
+
+from tests.model_serving.model_runtime.image_validation.constant import RUNTIME_CONFIGS
+from utilities.general import validate_container_images
+
+LOGGER = get_logger(name=__name__)
+
+pytestmark = [
+    pytest.mark.downstream_only,
+    pytest.mark.skip_must_gather,
+    pytest.mark.smoke,
+]
+
+
+@pytest.mark.parametrize("serving_runtime_pods_for_runtime", RUNTIME_CONFIGS, indirect=True)
+class TestServingRuntimeImagesPerTemplate:
+    """
+    For each runtime template: create ServingRuntime + InferenceService, wait for pod(s),
+    validate pod images (registry.redhat.io, sha256, CSV), output runtimename : passed, then teardown.
+    """
+
+    def test_verify_serving_runtime_pod_images_from_template(
+        self: Self,
+        serving_runtime_pods_for_runtime: tuple[list[Pod], str],
+        related_images_refs: set[str],
+    ) -> None:
+        """
+        For the parametrized runtime: create SR+ISVC from template, validate pod images, report name : passed.
+        """
+        pods, runtime_name = serving_runtime_pods_for_runtime
+        validation_errors = []
+        for pod in pods:
+            LOGGER.info(f"Validating {pod.name} in {pod.namespace}")
+            validation_errors.extend(
+                validate_container_images(
+                    pod=pod,
+                    valid_image_refs=related_images_refs,
+                )
+            )
+
+        if validation_errors:
+            pytest.fail("\n".join(validation_errors))
+        LOGGER.info(f"{runtime_name} : passed")