Test LMEval and Guardrails orchestrator images (opendatahub-io#350)

* feat: add image validation for trustyai operator and service

* feat: add image validation for trustyai operator and service

* feat: add fixture for trustyai operator deployment

* feat: add fixture for trustyai operator deployment

* docs: add docs for service image validation

* feat: add lmeval image test

* feat: refactor code and add test for guardrails orch

* fix: format and copy

* fix: method name for validator and remove unnecessary layer calls

* fix: add smoke tag and add docstring

Author: sheltoncyril

tests/model_explainability/guardrails/conftest.py

@@ -9,6 +9,7 @@
 from ocp_resources.inference_service import InferenceService
 from ocp_resources.namespace import Namespace
 from ocp_resources.persistent_volume_claim import PersistentVolumeClaim
+from ocp_resources.pod import Pod
 from ocp_resources.route import Route
 from ocp_resources.secret import Secret
 from ocp_resources.service import Service
@@ -25,6 +26,8 @@
 from utilities.inference_utils import create_isvc
 from utilities.serving_runtime import ServingRuntimeFromTemplate
 
+GORCH_NAME = "gorch-test"
+
 USER_ONE: str = "user-one"
 GUARDRAILS_ORCHESTRATOR_PORT: int = 8032
 
@@ -53,7 +56,7 @@ def guardrails_orchestrator(
 ) -> Generator[GuardrailsOrchestrator, Any, Any]:
     with GuardrailsOrchestrator(
         client=admin_client,
-        name="gorch-test",
+        name=GORCH_NAME,
         namespace=model_namespace.name,
         enable_built_in_detectors=True,
         enable_guardrails_gateway=True,
@@ -67,6 +70,13 @@ def guardrails_orchestrator(
         yield gorch
 
 
+@pytest.fixture(scope="class")
+def guardrails_orchestrator_pod(
+    admin_client: DynamicClient, model_namespace: Namespace, guardrails_orchestrator: GuardrailsOrchestrator
+) -> Pod:
+    return list(Pod.get(namespace=model_namespace.name, label_selector=f"app.kubernetes.io/instance={GORCH_NAME}"))[0]
+
+
 @pytest.fixture(scope="class")
 def qwen_llm_model(
     admin_client: DynamicClient,

tests/model_explainability/guardrails/test_guardrails.py

@@ -4,6 +4,7 @@
 import requests
 from timeout_sampler import retry
 
+from tests.model_explainability.utils import validate_tai_component_images
 from utilities.constants import Timeout
 
 
@@ -39,3 +40,12 @@ def test_guardrails_info_endpoint(self, admin_client, qwen_llm_model, guardrails
         response_data = response.json()
         assert response_data["services"]["chat_generation"]["status"] == healthy_status
         assert response_data["services"]["regex"]["status"] == healthy_status
+
+    @pytest.mark.smoke
+    def test_validate_guardrails_orchestrator_images(self, guardrails_orchestrator_pod, trustyai_operator_configmap):
+        """Test to verify Guardrails pod images.
+        Checks if the image tag from the ConfigMap is used within the Pod and if it's pinned using a sha256 digest.
+        """
+        validate_tai_component_images(
+            pod=guardrails_orchestrator_pod, tai_operator_configmap=trustyai_operator_configmap
+        )

tests/model_explainability/lm_eval/test_lm_eval.py

@@ -1,5 +1,6 @@
 import pytest
 
+from tests.model_explainability.utils import validate_tai_component_images
 from utilities.constants import Timeout
 
 LMEVALJOB_COMPLETE_STATE: str = "Complete"
@@ -160,3 +161,27 @@ def test_lmeval_s3_storage(
     lmevaljob_s3_offline_pod.wait_for_status(
         status=lmevaljob_s3_offline_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_20MIN
     )
+
+
+@pytest.mark.parametrize(
+    "model_namespace, minio_data_connection",
+    [
+        pytest.param(
+            {"name": "test-lmeval-images"},
+            {"bucket": "models"},
+        )
+    ],
+    indirect=True,
+)
+@pytest.mark.smoke
+def test_verify_lmeval_pod_images(lmevaljob_s3_offline_pod, trustyai_operator_configmap) -> None:
+    """Test to verify LMEval pod images.
+    Checks if the image tag from the ConfigMap is used within the Pod and if it's pinned using a sha256 digest.
+
+    Verifies:
+        - lmeval driver image
+        - lmeval job runner image
+    """
+    validate_tai_component_images(
+        pod=lmevaljob_s3_offline_pod, tai_operator_configmap=trustyai_operator_configmap, include_init_containers=True
+    )

tests/model_explainability/utils.py

@@ -0,0 +1,34 @@
+import re
+from ocp_resources.config_map import ConfigMap
+from ocp_resources.pod import Pod
+
+from utilities.general import SHA256_DIGEST_PATTERN
+
+
+def validate_tai_component_images(
+    pod: Pod, tai_operator_configmap: ConfigMap, include_init_containers: bool = False
+) -> None:
+    """Validate pod image against tai configmap images and check image for sha256 digest.
+
+    Args:
+        pod: Pod
+        tai_operator_configmap: ConfigMap
+        include_init_containers: bool
+
+    Returns:
+        None
+
+    Raises:
+        AssertionError: If validation fails.
+    """
+    tai_configmap_values = tai_operator_configmap.instance.data.values()
+    containers = list(pod.instance.spec.containers)
+    if include_init_containers:
+        containers.extend(pod.instance.spec.initContainers)
+    for container in containers:
+        assert re.search(SHA256_DIGEST_PATTERN, container.image), (
+            f"{container.name} : {container.image} does not have a valid SHA256 digest."
+        )
+        assert container.image in tai_configmap_values, (
+            f"{container.name} : {container.image} not present in TrustyAI operator configmap."
+        )