feat: add image validation for trustyai operator and service (opendatahub-io#348)

* feat: add image validation for trustyai operator and service

* feat: add image validation for trustyai operator and service

* feat: add fixture for trustyai operator deployment

* feat: add fixture for trustyai operator deployment

* docs: add docs for service image validation

Author: sheltoncyril

tests/conftest.py

@@ -42,6 +42,7 @@
 )
 from utilities.infra import update_configmap_data
 from utilities.minio import create_minio_data_connection_secret
+from utilities.operator_utils import get_csv_related_images
 
 LOGGER = get_logger(name=__name__)
 
@@ -517,3 +518,10 @@ def prometheus(admin_client: DynamicClient) -> Prometheus:
         ),  # TODO: Verify SSL with appropriate certs
         bearer_token=get_openshift_token(),
     )
+
+
+@pytest.fixture(scope="session")
+def related_images_refs(admin_client: DynamicClient) -> set[str]:
+    related_images = get_csv_related_images(admin_client=admin_client)
+    related_images_refs = {img["image"] for img in related_images}
+    return related_images_refs

tests/model_explainability/conftest.py

@@ -2,8 +2,12 @@
 
 import pytest
 from kubernetes.dynamic import DynamicClient
+from ocp_resources.config_map import ConfigMap
 from ocp_resources.namespace import Namespace
 from ocp_resources.persistent_volume_claim import PersistentVolumeClaim
+from pytest_testconfig import config as py_config
+
+from tests.model_explainability.trustyai_service.trustyai_service_utils import TRUSTYAI_SERVICE_NAME
 
 
 @pytest.fixture(scope="class")
@@ -19,3 +23,15 @@ def pvc_minio_namespace(
         size="10Gi",
     ) as pvc:
         yield pvc
+
+
+@pytest.fixture(scope="session")
+def trustyai_operator_configmap(
+    admin_client: DynamicClient,
+) -> ConfigMap:
+    return ConfigMap(
+        client=admin_client,
+        namespace=py_config["applications_namespace"],
+        name=f"{TRUSTYAI_SERVICE_NAME}-operator-config",
+        ensure_exists=True,
+    )

tests/model_explainability/trustyai_operator/__init__.py

@@ -0,0 +1,16 @@
+import pytest
+from kubernetes.dynamic import DynamicClient
+from pytest_testconfig import config as py_config
+from ocp_resources.deployment import Deployment
+
+from tests.model_explainability.trustyai_service.trustyai_service_utils import TRUSTYAI_SERVICE_NAME
+
+
+@pytest.fixture(scope="class")
+def trustyai_operator_deployment(admin_client: DynamicClient) -> Deployment:
+    return Deployment(
+        client=admin_client,
+        name=f"{TRUSTYAI_SERVICE_NAME}-operator-controller-manager",
+        namespace=py_config["applications_namespace"],
+        ensure_exists=True,
+    )

tests/model_explainability/trustyai_operator/conftest.py

@@ -0,0 +1,20 @@
+import pytest
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.config_map import ConfigMap
+from ocp_resources.deployment import Deployment
+
+from tests.model_explainability.trustyai_operator.utils import validate_trustyai_operator_image
+
+
+@pytest.mark.smoke
+def test_validate_trustyai_operator_image(
+    admin_client: DynamicClient,
+    related_images_refs: set[str],
+    trustyai_operator_configmap: ConfigMap,
+    trustyai_operator_deployment: Deployment,
+):
+    return validate_trustyai_operator_image(
+        related_images_refs=related_images_refs,
+        tai_operator_configmap_data=trustyai_operator_configmap.instance.data,
+        tai_operator_deployment=trustyai_operator_deployment,
+    )

tests/model_explainability/trustyai_operator/test_trustyai_operator.py

@@ -0,0 +1,30 @@
+from ocp_resources.deployment import Deployment
+
+from utilities.general import validate_image_format
+
+
+def validate_trustyai_operator_image(
+    related_images_refs: set[str], tai_operator_configmap_data: dict[str, str], tai_operator_deployment: Deployment
+) -> None:
+    """Validates the TrustyAI operator image.
+    Checks if:
+        - container image matches that of the operator configmap.
+        - image is present in relatedImages of CSV.
+        - image complies with OpenShift AI requirements i.e. sourced from registry.redhat.io and pinned w/o tags.
+
+        Args:
+            related_images_refs (set[str]): set of related image refs from the RHOAI CSV
+            tai_operator_configmap_data (dict[str, str]): TrustyAI configmap data
+            tai_operator_deployment (Deployment): TrustyAI deployment object
+
+        Returns:
+            None
+
+        Raises:
+            AssertionError: If any of the related images references are not present or invalid.
+    """
+    tai_operator_image = tai_operator_deployment.instance.spec.template.spec.containers[0].image
+    assert tai_operator_image == tai_operator_configmap_data["trustyaiOperatorImage"]
+    assert tai_operator_image in related_images_refs
+    image_valid, error_message = validate_image_format(image=tai_operator_image)
+    assert image_valid, error_message

tests/model_explainability/trustyai_operator/utils.py

@@ -1,5 +1,6 @@
 import pytest
 from ocp_resources.namespace import Namespace
+from ocp_resources.trustyai_service import TrustyAIService
 
 from tests.model_explainability.trustyai_service.constants import DRIFT_BASE_DATA_PATH
 from tests.model_explainability.trustyai_service.trustyai_service_utils import (
@@ -9,7 +10,10 @@
     verify_trustyai_service_metric_scheduling_request,
     verify_trustyai_service_metric_delete_request,
 )
-from tests.model_explainability.trustyai_service.utils import validate_trustyai_service_db_conn_failure
+from tests.model_explainability.trustyai_service.utils import (
+    validate_trustyai_service_db_conn_failure,
+    validate_trustyai_service_images,
+)
 from utilities.constants import MinIo
 from utilities.manifests.openvino import OPENVINO_KSERVE_INFERENCE_CONFIG
 
@@ -146,3 +150,29 @@ def test_drift_metric_delete_multiple_ns(
                 token=current_client_token,
                 metric_name=TrustyAIServiceMetrics.Drift.MEANSHIFT,
             )
+
+
+@pytest.mark.parametrize(
+    "model_namespace",
+    [
+        pytest.param(
+            {"name": "test-validate-trustyai-service-images"},
+        )
+    ],
+    indirect=True,
+)
+@pytest.mark.smoke
+def test_validate_trustyai_service_image(
+    admin_client,
+    model_namespace: Namespace,
+    related_images_refs: set[str],
+    trustyai_service_with_pvc_storage: TrustyAIService,
+    trustyai_operator_configmap,
+):
+    return validate_trustyai_service_images(
+        client=admin_client,
+        related_images_refs=related_images_refs,
+        model_namespace=model_namespace,
+        label_selector=f"app.kubernetes.io/instance={trustyai_service_with_pvc_storage.name}",
+        trustyai_operator_configmap=trustyai_operator_configmap,
+    )

tests/model_explainability/trustyai_service/service/test_trustyai_service.py

@@ -2,6 +2,7 @@
 import re
 
 from kubernetes.dynamic import DynamicClient
+from ocp_resources.config_map import ConfigMap
 from ocp_resources.deployment import Deployment
 from ocp_resources.mariadb_operator import MariadbOperator
 from ocp_resources.maria_db import MariaDB
@@ -19,6 +20,7 @@
 from timeout_sampler import retry
 
 from utilities.exceptions import TooManyPodsError, UnexpectedFailureError
+from utilities.general import wait_for_pods_by_labels, validate_container_images
 
 LOGGER = get_logger(name=__name__)
 
@@ -245,3 +247,38 @@ def create_isvc_getter_token_secret(
         type="kubernetes.io/service-account-token",
     ) as secret:
         yield secret
+
+
+def validate_trustyai_service_images(
+    client: DynamicClient,
+    related_images_refs: set[str],
+    model_namespace: Namespace,
+    label_selector: str,
+    trustyai_operator_configmap: ConfigMap,
+) -> None:
+    """Validates trustyai service images against a set of related images.
+
+    Args:
+        client: DynamicClient: The Kubernetes dynamic client.
+        related_images_refs: list[str]: Related images references from RHOAI CSV.
+        model_namespace: Namespace: namespace to run the test against.
+        label_selector: str: Label selector string to get the trustyai pod.
+        trustyai_operator_configmap: ConfigMap: The trustyai operator configmap.
+
+    Returns:
+        None
+
+    Raises:
+        AssertionError: If any of the related images references are not present or invalid.
+    """
+    tai_image_refs = set(
+        v
+        for k, v in trustyai_operator_configmap.instance.data.items()
+        if k in ["oauthProxyImage", "trustyaiServiceImage"]
+    )
+    trustyai_service_pod = wait_for_pods_by_labels(
+        admin_client=client, namespace=model_namespace.name, label_selector=label_selector, expected_num_pods=1
+    )[0]
+    validation_errors = validate_container_images(pod=trustyai_service_pod, valid_image_refs=tai_image_refs)
+    assert len(validation_errors) == 0, validation_errors
+    assert tai_image_refs.issubset(related_images_refs), "TrustyAI service container images are not present in CSV."