From 5f414a0e2d0abf70c9ffbdca6e0ef88fc713aa55 Mon Sep 17 00:00:00 2001 From: Shelton Cyril Date: Wed, 4 Jun 2025 12:34:20 +0100 Subject: [PATCH 1/6] feat: add validation for TrustyAI Operator image --- tests/conftest.py | 10 ++++- .../trustyai_service/conftest.py | 26 ++++++++---- .../trustyai_service/test_trustyai_service.py | 40 ++++++++++++++++++- .../trustyai_service/utils.py | 24 ++++++++++- .../image_validation/conftest.py | 11 ----- 5 files changed, 89 insertions(+), 22 deletions(-) delete mode 100644 tests/model_registry/image_validation/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py index 35c82064d..40512b0c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,7 @@ import os import shutil from ast import literal_eval -from typing import Any, Callable, Generator +from typing import Any, Callable, Generator, Set import pytest import shortuuid @@ -45,6 +45,7 @@ ) from utilities.infra import update_configmap_data from utilities.minio import create_minio_data_connection_secret +from utilities.operator_utils import get_csv_related_images LOGGER = get_logger(name=__name__) @@ -537,3 +538,10 @@ def prometheus(admin_client: DynamicClient) -> Prometheus: ), # TODO: Verify SSL with appropriate certs bearer_token=get_openshift_token(), ) + + +@pytest.fixture(scope="session") +def related_images_refs(admin_client: DynamicClient) -> Set[str]: + related_images = get_csv_related_images(admin_client=admin_client) + related_images_refs = {img["image"] for img in related_images} + return related_images_refs diff --git a/tests/model_explainability/trustyai_service/conftest.py b/tests/model_explainability/trustyai_service/conftest.py index 4b6b29fb7..4b0e34ae3 100644 --- a/tests/model_explainability/trustyai_service/conftest.py +++ b/tests/model_explainability/trustyai_service/conftest.py @@ -1,4 +1,4 @@ -from typing import Generator, Any +from typing import Any, Generator import pytest import yaml @@ -21,21 +21,21 @@ from ocp_resources.subscription import Subscription from ocp_resources.trustyai_service import TrustyAIService from ocp_utilities.operators import install_operator, uninstall_operator +from pytest_testconfig import config as py_config from tests.model_explainability.trustyai_service.trustyai_service_utils import ( wait_for_isvc_deployment_registered_by_trustyai_service, ) from tests.model_explainability.trustyai_service.utils import ( - wait_for_mariadb_operator_deployments, + TRUSTYAI_SERVICE_NAME, create_trustyai_service, + wait_for_mariadb_operator_deployments, wait_for_mariadb_pods, - TRUSTYAI_SERVICE_NAME, ) -from utilities.operator_utils import get_cluster_service_version - -from utilities.constants import Timeout, KServeDeploymentType, ApiGroups, Labels, Ports +from utilities.constants import ApiGroups, KServeDeploymentType, Labels, Ports, Timeout from utilities.inference_utils import create_isvc -from utilities.infra import update_configmap_data, create_inference_token +from utilities.infra import create_inference_token, update_configmap_data +from utilities.operator_utils import get_cluster_service_version OPENSHIFT_OPERATORS: str = "openshift-operators" @@ -455,3 +455,15 @@ def isvc_getter_token_secret( @pytest.fixture(scope="class") def isvc_getter_token(isvc_getter_service_account: ServiceAccount, isvc_getter_token_secret: Secret) -> str: return create_inference_token(model_service_account=isvc_getter_service_account) + + +@pytest.fixture(scope="class") +def trustyai_operator_configmap( + admin_client: DynamicClient, +) -> ConfigMap: + return ConfigMap( + client=admin_client, + namespace=py_config["applications_namespace"], + name=f"{TRUSTYAI_SERVICE_NAME}-operator-config", + ensure_exists=True, + ) diff --git a/tests/model_explainability/trustyai_service/test_trustyai_service.py b/tests/model_explainability/trustyai_service/test_trustyai_service.py index e37ba3c32..1081b2ef3 100644 --- a/tests/model_explainability/trustyai_service/test_trustyai_service.py +++ b/tests/model_explainability/trustyai_service/test_trustyai_service.py @@ -1,7 +1,14 @@ +from typing import Set + import pytest +from ocp_resources.config_map import ConfigMap from ocp_resources.namespace import Namespace +from ocp_resources.trustyai_service import TrustyAIService -from tests.model_explainability.trustyai_service.utils import validate_trustyai_service_db_conn_failure +from tests.model_explainability.trustyai_service.utils import ( + validate_trustyai_service_db_conn_failure, + validate_trustyai_operator_image, +) @pytest.mark.parametrize( @@ -25,3 +32,34 @@ def test_trustyai_service_with_invalid_db_cert( namespace=model_namespace, label_selector=f"app.kubernetes.io/instance={trustyai_service_with_invalid_db_cert.name}", ) + + +@pytest.mark.parametrize( + "model_namespace", + [ + pytest.param( + {"name": "test-validate-trustyai-images"}, + ) + ], + indirect=True, +) +@pytest.mark.smoke +class TestValidateTrustyAIImages: + """Test to validate if operator and service image for TrustyAIService matches related images in the CSV. + Also validate if the image is pinned using a digest and is sourced from registry.redhat.io. + """ + + def test_validate_trustyai_operator_image( + self, + admin_client, + current_client_token, + model_namespace: Namespace, + related_images_refs: Set[str], + trustyai_service_with_pvc_storage: TrustyAIService, + trustyai_operator_configmap: ConfigMap, + ): + return validate_trustyai_operator_image( + client=admin_client, + related_images_refs=related_images_refs, + tai_operator_configmap_data=trustyai_operator_configmap.instance.data, + ) diff --git a/tests/model_explainability/trustyai_service/utils.py b/tests/model_explainability/trustyai_service/utils.py index 23151e7dd..cedb0522d 100644 --- a/tests/model_explainability/trustyai_service/utils.py +++ b/tests/model_explainability/trustyai_service/utils.py @@ -1,4 +1,4 @@ -from typing import Generator, Any, Optional +from typing import Generator, Any, Optional, Set import re from kubernetes.dynamic import DynamicClient @@ -13,11 +13,16 @@ from tests.model_explainability.trustyai_service.trustyai_service_utils import TRUSTYAI_SERVICE_NAME from utilities.constants import Timeout from timeout_sampler import retry - from utilities.exceptions import TooManyPodsError, UnexpectedFailureError +from pytest_testconfig import config as py_config + +from utilities.general import validate_image_format + LOGGER = get_logger(name=__name__) +TAI_OPERATOR_DEPLOYMENT_NAME = f"{TRUSTYAI_SERVICE_NAME}-operator-controller-manager" + def wait_for_mariadb_operator_deployments(mariadb_operator: MariadbOperator) -> None: expected_deployment_names: list[str] = [ @@ -147,3 +152,18 @@ def create_trustyai_service( if wait_for_replicas: trustyai_deployment.wait_for_replicas() yield trustyai_service + + +def validate_trustyai_operator_image( + client: DynamicClient, related_images_refs: Set[str], tai_operator_configmap_data: dict[str, str] +) -> None: + tai_operator_deployment = Deployment( + client=client, + name=TAI_OPERATOR_DEPLOYMENT_NAME, + namespace=py_config["applications_namespace"], + wait_for_resource=True, + ) + tai_operator_image = tai_operator_deployment.instance.spec.template.spec.containers[0].image + assert tai_operator_image == tai_operator_configmap_data["trustyaiOperatorImage"] + assert tai_operator_image in related_images_refs + assert validate_image_format(image=tai_operator_image) diff --git a/tests/model_registry/image_validation/conftest.py b/tests/model_registry/image_validation/conftest.py deleted file mode 100644 index a08ece849..000000000 --- a/tests/model_registry/image_validation/conftest.py +++ /dev/null @@ -1,11 +0,0 @@ -import pytest -from typing import Set -from kubernetes.dynamic import DynamicClient -from utilities.operator_utils import get_csv_related_images - - -@pytest.fixture() -def related_images_refs(admin_client: DynamicClient) -> Set[str]: - related_images = get_csv_related_images(admin_client=admin_client) - related_images_refs = {img["image"] for img in related_images} - return related_images_refs From 09f9a9ef67228f82e21071627a9f3c87e99a1159 Mon Sep 17 00:00:00 2001 From: Shelton Cyril Date: Wed, 4 Jun 2025 12:42:35 +0100 Subject: [PATCH 2/6] docs: added docstring for validate method --- .../trustyai_service/utils.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/model_explainability/trustyai_service/utils.py b/tests/model_explainability/trustyai_service/utils.py index cedb0522d..8080aa7b4 100644 --- a/tests/model_explainability/trustyai_service/utils.py +++ b/tests/model_explainability/trustyai_service/utils.py @@ -1,22 +1,20 @@ -from typing import Generator, Any, Optional, Set import re +from typing import Any, Generator, Optional, Set from kubernetes.dynamic import DynamicClient from ocp_resources.deployment import Deployment -from ocp_resources.mariadb_operator import MariadbOperator from ocp_resources.maria_db import MariaDB +from ocp_resources.mariadb_operator import MariadbOperator from ocp_resources.namespace import Namespace from ocp_resources.pod import Pod from ocp_resources.trustyai_service import TrustyAIService +from pytest_testconfig import config as py_config from simple_logger.logger import get_logger -from timeout_sampler import TimeoutSampler +from timeout_sampler import TimeoutSampler, retry + from tests.model_explainability.trustyai_service.trustyai_service_utils import TRUSTYAI_SERVICE_NAME from utilities.constants import Timeout -from timeout_sampler import retry from utilities.exceptions import TooManyPodsError, UnexpectedFailureError - -from pytest_testconfig import config as py_config - from utilities.general import validate_image_format LOGGER = get_logger(name=__name__) @@ -157,6 +155,12 @@ def create_trustyai_service( def validate_trustyai_operator_image( client: DynamicClient, related_images_refs: Set[str], tai_operator_configmap_data: dict[str, str] ) -> None: + """Validates the TrustyAI operator image. + Checks if: + - container image matches that of the operator configmap. + - image is present in relatedImages of CSV. + - image complies with model registry requirements i.e. sourced from registry.redhat.io and pinned w/o tags. + """ tai_operator_deployment = Deployment( client=client, name=TAI_OPERATOR_DEPLOYMENT_NAME, From d36c1d05dfcb0feb45daf55d09238e2e63c8ecd9 Mon Sep 17 00:00:00 2001 From: Shelton Cyril Date: Wed, 4 Jun 2025 15:52:12 +0100 Subject: [PATCH 3/6] Update tests/model_explainability/trustyai_service/utils.py Co-authored-by: Adolfo Aguirrezabal --- tests/model_explainability/trustyai_service/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/model_explainability/trustyai_service/utils.py b/tests/model_explainability/trustyai_service/utils.py index 8080aa7b4..5e79c6b4b 100644 --- a/tests/model_explainability/trustyai_service/utils.py +++ b/tests/model_explainability/trustyai_service/utils.py @@ -159,7 +159,7 @@ def validate_trustyai_operator_image( Checks if: - container image matches that of the operator configmap. - image is present in relatedImages of CSV. - - image complies with model registry requirements i.e. sourced from registry.redhat.io and pinned w/o tags. + - image complies with OpenShift AI requirements i.e. sourced from registry.redhat.io and pinned w/o tags. """ tai_operator_deployment = Deployment( client=client, From f9a68acc48411c04388113a61967cdfe52586ca0 Mon Sep 17 00:00:00 2001 From: Shelton Cyril Date: Thu, 12 Jun 2025 01:23:38 +0100 Subject: [PATCH 4/6] feat: add service and operator image validation with MR checks --- .../trustyai_service/test_trustyai_service.py | 17 +++++++++++++++-- .../trustyai_service/utils.py | 19 +++++++++++++++++-- utilities/general.py | 10 +++++----- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/tests/model_explainability/trustyai_service/test_trustyai_service.py b/tests/model_explainability/trustyai_service/test_trustyai_service.py index 1081b2ef3..014a4639b 100644 --- a/tests/model_explainability/trustyai_service/test_trustyai_service.py +++ b/tests/model_explainability/trustyai_service/test_trustyai_service.py @@ -8,6 +8,7 @@ from tests.model_explainability.trustyai_service.utils import ( validate_trustyai_service_db_conn_failure, validate_trustyai_operator_image, + validate_trustyai_service_images, ) @@ -52,10 +53,8 @@ class TestValidateTrustyAIImages: def test_validate_trustyai_operator_image( self, admin_client, - current_client_token, model_namespace: Namespace, related_images_refs: Set[str], - trustyai_service_with_pvc_storage: TrustyAIService, trustyai_operator_configmap: ConfigMap, ): return validate_trustyai_operator_image( @@ -63,3 +62,17 @@ def test_validate_trustyai_operator_image( related_images_refs=related_images_refs, tai_operator_configmap_data=trustyai_operator_configmap.instance.data, ) + + def test_validate_trustyai_service_image( + self, + admin_client, + model_namespace: Namespace, + related_images_refs: Set[str], + trustyai_service_with_pvc_storage: TrustyAIService, + ): + return validate_trustyai_service_images( + client=admin_client, + related_images_refs=related_images_refs, + model_namespace=model_namespace, + label_selector=f"app.kubernetes.io/instance={trustyai_service_with_pvc_storage.name}", + ) diff --git a/tests/model_explainability/trustyai_service/utils.py b/tests/model_explainability/trustyai_service/utils.py index 5e79c6b4b..6244ea52f 100644 --- a/tests/model_explainability/trustyai_service/utils.py +++ b/tests/model_explainability/trustyai_service/utils.py @@ -15,7 +15,7 @@ from tests.model_explainability.trustyai_service.trustyai_service_utils import TRUSTYAI_SERVICE_NAME from utilities.constants import Timeout from utilities.exceptions import TooManyPodsError, UnexpectedFailureError -from utilities.general import validate_image_format +from utilities.general import validate_image_format, wait_for_pods_by_labels, validate_container_images LOGGER = get_logger(name=__name__) @@ -170,4 +170,19 @@ def validate_trustyai_operator_image( tai_operator_image = tai_operator_deployment.instance.spec.template.spec.containers[0].image assert tai_operator_image == tai_operator_configmap_data["trustyaiOperatorImage"] assert tai_operator_image in related_images_refs - assert validate_image_format(image=tai_operator_image) + image_valid, error_message = validate_image_format(image=tai_operator_image) + assert image_valid, error_message + + +def validate_trustyai_service_images( + client: DynamicClient, + related_images_refs: Set[str], + model_namespace: Namespace, + label_selector: str, +) -> None: + """Validates the TrustyAI service container images.""" + trustyai_service_pod = wait_for_pods_by_labels( + admin_client=client, namespace=model_namespace.name, label_selector=label_selector, expected_num_pods=1 + ) + validation_errors = validate_container_images(pod=trustyai_service_pod, valid_image_refs=related_images_refs) + assert len(validation_errors) == 0, validation_errors diff --git a/utilities/general.py b/utilities/general.py index 4369675af..53e237178 100644 --- a/utilities/general.py +++ b/utilities/general.py @@ -222,9 +222,9 @@ def wait_for_pods_by_labels( namespace: str, label_selector: str, expected_num_pods: int, -) -> list[Pod]: +) -> list[Pod] | Pod: """ - Get pods by label selector in a namespace. + Get pods or a pod by label selector in a namespace. Args: admin_client: The admin client to use for pod retrieval @@ -232,7 +232,7 @@ def wait_for_pods_by_labels( label_selector: The label selector to filter pods expected_num_pods: The expected number of pods to be found Returns: - List of matching pods + List of matching pods or a single pod if expected_num_pods is 1 Raises: ResourceNotFoundError: If no pods are found @@ -247,8 +247,8 @@ def wait_for_pods_by_labels( if not pods: raise ResourceNotFoundError(f"No pods found with label selector {label_selector} in namespace {namespace}") if len(pods) != expected_num_pods: - raise UnexpectedResourceCountError(f"Expected {expected_num_pods} pods, found {len(pods)}") - return pods + raise UnexpectedResourceCountError(f"Expected {expected_num_pods} pod(s), found {len(pods)}") + return pods[0] if expected_num_pods and expected_num_pods == 1 else pods def validate_container_images( From 84497faf3548b05a83cdd62a8ef28ed347e09170 Mon Sep 17 00:00:00 2001 From: Shelton Cyril Date: Thu, 12 Jun 2025 01:25:46 +0100 Subject: [PATCH 5/6] feat: remove unnecessary check for none --- utilities/general.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/general.py b/utilities/general.py index 53e237178..2bcde245e 100644 --- a/utilities/general.py +++ b/utilities/general.py @@ -248,7 +248,7 @@ def wait_for_pods_by_labels( raise ResourceNotFoundError(f"No pods found with label selector {label_selector} in namespace {namespace}") if len(pods) != expected_num_pods: raise UnexpectedResourceCountError(f"Expected {expected_num_pods} pod(s), found {len(pods)}") - return pods[0] if expected_num_pods and expected_num_pods == 1 else pods + return pods[0] if expected_num_pods == 1 else pods def validate_container_images( From e683c01d863522dbfff0c2a8fdccaa6d288d34a5 Mon Sep 17 00:00:00 2001 From: Shelton Cyril Date: Thu, 12 Jun 2025 01:28:15 +0100 Subject: [PATCH 6/6] feat: fix model registry wait_for_pods_by_labels --- tests/model_registry/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/model_registry/conftest.py b/tests/model_registry/conftest.py index 7195e55d2..a8196af6c 100644 --- a/tests/model_registry/conftest.py +++ b/tests/model_registry/conftest.py @@ -297,7 +297,7 @@ def model_registry_operator_pod(admin_client: DynamicClient) -> Generator[Pod, A namespace=py_config["applications_namespace"], label_selector=f"{Labels.OpenDataHubIo.NAME}={MR_OPERATOR_NAME}", expected_num_pods=1, - )[0] + ) @pytest.fixture() @@ -308,7 +308,7 @@ def model_registry_instance_pod(admin_client: DynamicClient) -> Generator[Pod, A namespace=py_config["model_registry_namespace"], label_selector=f"app={MR_INSTANCE_NAME}", expected_num_pods=1, - )[0] + ) @pytest.fixture(scope="package", autouse=True)