diff --git a/tests/model_explainability/trustyai_service/service/conftest.py b/tests/model_explainability/trustyai_service/service/conftest.py index e773cc4ab..30bd3b36e 100644 --- a/tests/model_explainability/trustyai_service/service/conftest.py +++ b/tests/model_explainability/trustyai_service/service/conftest.py @@ -1,52 +1,24 @@ from typing import Generator, Any import pytest -from _pytest.fixtures import FixtureRequest from kubernetes.dynamic import DynamicClient from ocp_resources.config_map import ConfigMap -from ocp_resources.inference_service import InferenceService from ocp_resources.maria_db import MariaDB from ocp_resources.namespace import Namespace -from ocp_resources.pod import Pod -from ocp_resources.role import Role -from ocp_resources.role_binding import RoleBinding from ocp_resources.secret import Secret -from ocp_resources.service import Service -from ocp_resources.service_account import ServiceAccount -from ocp_resources.serving_runtime import ServingRuntime from ocp_resources.trustyai_service import TrustyAIService from tests.model_explainability.trustyai_service.constants import ( TAI_METRICS_CONFIG, - TAI_DATA_CONFIG, - TAI_PVC_STORAGE_CONFIG, - GAUSSIAN_CREDIT_MODEL, - GAUSSIAN_CREDIT_MODEL_STORAGE_PATH, - GAUSSIAN_CREDIT_MODEL_RESOURCES, - KSERVE_MLSERVER, - KSERVE_MLSERVER_CONTAINERS, - KSERVE_MLSERVER_SUPPORTED_MODEL_FORMATS, - KSERVE_MLSERVER_ANNOTATIONS, - XGBOOST, TAI_DB_STORAGE_CONFIG, - ISVC_GETTER, ) from tests.model_explainability.trustyai_service.trustyai_service_utils import ( TRUSTYAI_SERVICE_NAME, - wait_for_isvc_deployment_registered_by_trustyai_service, ) from tests.model_explainability.trustyai_service.utils import ( create_trustyai_service, - create_isvc_getter_service_account, - create_isvc_getter_role, - create_isvc_getter_role_binding, - create_isvc_getter_token_secret, ) -from utilities.constants import KServeDeploymentType, Labels -from utilities.inference_utils import create_isvc -from utilities.infra import create_ns, create_inference_token -from utilities.minio import create_minio_data_connection_secret INVALID_TLS_CERTIFICATE: str = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJnRENDQVNlZ0F3SUJBZ0lRRGtTcXVuUWRzRmZwdi8zSm\ @@ -59,150 +31,6 @@ LS0t" # pragma: allowlist secret -@pytest.fixture(scope="class") -def model_namespace_2( - request: FixtureRequest, - pytestconfig: pytest.Config, - admin_client: DynamicClient, - teardown_resources: bool, -) -> Generator[Namespace, Any, Any]: - if request.param.get("modelmesh-enabled"): - request.getfixturevalue(argname="enabled_modelmesh_in_dsc") - - if pytestconfig.option.post_upgrade: - ns = Namespace(client=admin_client, name=request.param["name"]) - yield ns - ns.clean_up() - else: - with create_ns( - admin_client=admin_client, - pytest_request=request, - teardown=teardown_resources, - ) as ns: - yield ns - - -@pytest.fixture(scope="class") -def minio_data_connection_2( - request: FixtureRequest, - admin_client: DynamicClient, - model_namespace_2: Namespace, - minio_service: Service, -) -> Generator[Secret, Any, Any]: - with create_minio_data_connection_secret( - minio_service=minio_service, - model_namespace=model_namespace_2.name, - aws_s3_bucket=request.param["bucket"], - client=admin_client, - ) as secret: - yield secret - - -@pytest.fixture(scope="class") -def trustyai_service_with_pvc_storage_2( - pytestconfig: pytest.Config, - admin_client: DynamicClient, - model_namespace_2: Namespace, - cluster_monitoring_config: ConfigMap, - user_workload_monitoring_config: ConfigMap, - teardown_resources: bool, -) -> Generator[TrustyAIService, Any, Any]: - trustyai_service_kwargs = { - "client": admin_client, - "namespace": model_namespace_2.name, - "name": TRUSTYAI_SERVICE_NAME, - } - - if pytestconfig.option.post_upgrade: - trustyai_service = TrustyAIService(**trustyai_service_kwargs) - yield trustyai_service - trustyai_service.clean_up() - - else: - yield from create_trustyai_service( - **trustyai_service_kwargs, - storage=TAI_PVC_STORAGE_CONFIG, - metrics=TAI_METRICS_CONFIG, - data=TAI_DATA_CONFIG, - wait_for_replicas=True, - teardown=teardown_resources, - ) - - -@pytest.fixture(scope="class") -def gaussian_credit_model_2( - pytestconfig: pytest.Config, - admin_client: DynamicClient, - model_namespace_2: Namespace, - minio_pod: Pod, - minio_service: Service, - minio_data_connection_2: Secret, - mlserver_runtime_2: ServingRuntime, - trustyai_service_with_pvc_storage_2: TrustyAIService, - teardown_resources: bool, -) -> Generator[InferenceService, Any, Any]: - gaussian_credit_model_kwargs = { - "client": admin_client, - "namespace": model_namespace_2.name, - "name": GAUSSIAN_CREDIT_MODEL, - } - - if pytestconfig.option.post_upgrade: - isvc = InferenceService(**gaussian_credit_model_kwargs) - yield isvc - isvc.clean_up() - else: - with create_isvc( - deployment_mode=KServeDeploymentType.SERVERLESS, - model_format=XGBOOST, - runtime=mlserver_runtime_2.name, - storage_key=minio_data_connection_2.name, - storage_path=GAUSSIAN_CREDIT_MODEL_STORAGE_PATH, - enable_auth=True, - wait_for_predictor_pods=False, - resources=GAUSSIAN_CREDIT_MODEL_RESOURCES, - teardown=teardown_resources, - **gaussian_credit_model_kwargs, - ) as isvc: - wait_for_isvc_deployment_registered_by_trustyai_service( - client=admin_client, - isvc=isvc, - runtime_name=mlserver_runtime_2.name, - ) - yield isvc - - -@pytest.fixture(scope="class") -def mlserver_runtime_2( - pytestconfig: pytest.Config, - admin_client: DynamicClient, - minio_data_connection: Secret, - model_namespace_2: Namespace, - teardown_resources: bool, -) -> Generator[ServingRuntime, Any, Any]: - mlserver_runtime_kwargs = { - "client": admin_client, - "namespace": model_namespace_2.name, - "name": KSERVE_MLSERVER, - } - - if pytestconfig.option.post_upgrade: - serving_runtime = ServingRuntime(**mlserver_runtime_kwargs) - yield serving_runtime - serving_runtime.clean_up() - else: - with ServingRuntime( - containers=KSERVE_MLSERVER_CONTAINERS, - supported_model_formats=KSERVE_MLSERVER_SUPPORTED_MODEL_FORMATS, - protocol_versions=["v2"], - annotations=KSERVE_MLSERVER_ANNOTATIONS, - label={Labels.OpenDataHub.DASHBOARD: "true"}, - teardown=teardown_resources, - **mlserver_runtime_kwargs, - ) as mlserver: - yield mlserver - - @pytest.fixture(scope="class") def trustyai_service_with_invalid_db_cert( admin_client: DynamicClient, @@ -236,51 +64,3 @@ def trustyai_invalid_db_ca_secret( data_dict={"ca.crt": INVALID_TLS_CERTIFICATE}, ) as secret: yield secret - - -@pytest.fixture(scope="class") -def isvc_getter_service_account_2( - admin_client: DynamicClient, model_namespace_2: Namespace -) -> Generator[ServiceAccount, Any, Any]: - yield from create_isvc_getter_service_account(client=admin_client, namespace=model_namespace_2, name=ISVC_GETTER) - - -@pytest.fixture(scope="class") -def isvc_getter_role_2(admin_client: DynamicClient, model_namespace_2: Namespace) -> Generator[Role, Any, Any]: - yield from create_isvc_getter_role(client=admin_client, namespace=model_namespace_2, name=ISVC_GETTER) - - -@pytest.fixture(scope="class") -def isvc_getter_role_binding_2( - admin_client: DynamicClient, - model_namespace_2: Namespace, - isvc_getter_role_2: Role, - isvc_getter_service_account_2: ServiceAccount, -) -> Generator[RoleBinding, Any, Any]: - yield from create_isvc_getter_role_binding( - client=admin_client, - namespace=model_namespace_2, - role=isvc_getter_role_2, - service_account=isvc_getter_service_account_2, - name=ISVC_GETTER, - ) - - -@pytest.fixture(scope="class") -def isvc_getter_token_secret_2( - admin_client: DynamicClient, - model_namespace_2: Namespace, - isvc_getter_service_account_2: ServiceAccount, - isvc_getter_role_binding_2: RoleBinding, -) -> Generator[Secret, Any, Any]: - yield from create_isvc_getter_token_secret( - client=admin_client, - name="sa-token", - namespace=model_namespace_2, - service_account=isvc_getter_service_account_2, - ) - - -@pytest.fixture(scope="class") -def isvc_getter_token_2(isvc_getter_service_account_2: ServiceAccount, isvc_getter_token_secret_2: Secret) -> str: - return create_inference_token(model_service_account=isvc_getter_service_account_2) diff --git a/tests/model_explainability/trustyai_service/service/multi_ns/conftest.py b/tests/model_explainability/trustyai_service/service/multi_ns/conftest.py new file mode 100644 index 000000000..5d25dbad0 --- /dev/null +++ b/tests/model_explainability/trustyai_service/service/multi_ns/conftest.py @@ -0,0 +1,225 @@ +from contextlib import ExitStack +from typing import Generator, Any, List +import pytest +from ocp_resources.namespace import Namespace +from ocp_resources.secret import Secret +from ocp_resources.serving_runtime import ServingRuntime +from ocp_resources.inference_service import InferenceService +from ocp_resources.trustyai_service import TrustyAIService +from ocp_resources.service_account import ServiceAccount +from ocp_resources.role import Role +from ocp_resources.role_binding import RoleBinding +from tests.model_explainability.trustyai_service.constants import ( + TAI_METRICS_CONFIG, + TAI_DATA_CONFIG, + TAI_PVC_STORAGE_CONFIG, + GAUSSIAN_CREDIT_MODEL_STORAGE_PATH, + GAUSSIAN_CREDIT_MODEL_RESOURCES, + KSERVE_MLSERVER, + KSERVE_MLSERVER_CONTAINERS, + KSERVE_MLSERVER_SUPPORTED_MODEL_FORMATS, + KSERVE_MLSERVER_ANNOTATIONS, + XGBOOST, + ISVC_GETTER, + GAUSSIAN_CREDIT_MODEL, +) +from tests.model_explainability.trustyai_service.trustyai_service_utils import ( + TRUSTYAI_SERVICE_NAME, + wait_for_isvc_deployment_registered_by_trustyai_service, +) +from tests.model_explainability.trustyai_service.utils import ( + create_trustyai_service, + create_isvc_getter_service_account, + create_isvc_getter_role, + create_isvc_getter_role_binding, + create_isvc_getter_token_secret, +) +from utilities.constants import KServeDeploymentType +from utilities.inference_utils import create_isvc +from utilities.infra import create_inference_token, create_ns +from utilities.minio import create_minio_data_connection_secret + + +@pytest.fixture(scope="class") +def model_namespaces(request, admin_client) -> Generator[List[Namespace], Any, None]: + with ExitStack() as stack: + namespaces = [ + stack.enter_context(create_ns(client=admin_client, name=param["name"])) for param in request.param + ] + yield namespaces + + +@pytest.fixture(scope="class") +def minio_data_connection_multi_ns( + request, admin_client, model_namespaces, minio_service +) -> Generator[List[Secret], Any, None]: + with ExitStack() as stack: + secrets = [ + stack.enter_context( + create_minio_data_connection_secret( + minio_service=minio_service, + model_namespace=ns.name, + aws_s3_bucket=param["bucket"], + client=admin_client, + ) + ) + for ns, param in zip(model_namespaces, request.param) + ] + yield secrets + + +@pytest.fixture(scope="class") +def trustyai_service_with_pvc_storage_multi_ns( + admin_client, model_namespaces, cluster_monitoring_config, user_workload_monitoring_config +) -> Generator[List[TrustyAIService], Any, None]: + with ExitStack() as stack: + services = [ + stack.enter_context( + create_trustyai_service( + client=admin_client, + namespace=ns.name, + name=TRUSTYAI_SERVICE_NAME, + storage=TAI_PVC_STORAGE_CONFIG, + metrics=TAI_METRICS_CONFIG, + data=TAI_DATA_CONFIG, + wait_for_replicas=True, + teardown=False, + ) + ) + for ns in model_namespaces + ] + yield services + + +@pytest.fixture(scope="class") +def mlserver_runtime_multi_ns(admin_client, model_namespaces) -> Generator[List[ServingRuntime], Any, None]: + with ExitStack() as stack: + runtimes = [ + stack.enter_context( + ServingRuntime( + client=admin_client, + namespace=ns.name, + name=KSERVE_MLSERVER, + containers=KSERVE_MLSERVER_CONTAINERS, + supported_model_formats=KSERVE_MLSERVER_SUPPORTED_MODEL_FORMATS, + protocol_versions=["v2"], + annotations=KSERVE_MLSERVER_ANNOTATIONS, + label={"opendatahub.io/dashboard": "true"}, + teardown=False, + ) + ) + for ns in model_namespaces + ] + yield runtimes + + +@pytest.fixture(scope="class") +def gaussian_credit_model_multi_ns( + admin_client, + model_namespaces, + minio_pod, + minio_service, + minio_data_connection_multi_ns, + mlserver_runtime_multi_ns, + trustyai_service_with_pvc_storage_multi_ns, +) -> Generator[List[InferenceService], Any, None]: + with ExitStack() as stack: + models = [] + for ns, secret, runtime in zip(model_namespaces, minio_data_connection_multi_ns, mlserver_runtime_multi_ns): + isvc_context = create_isvc( + client=admin_client, + namespace=ns.name, + name=GAUSSIAN_CREDIT_MODEL, + deployment_mode=KServeDeploymentType.SERVERLESS, + model_format=XGBOOST, + runtime=runtime.name, + storage_key=secret.name, + storage_path=GAUSSIAN_CREDIT_MODEL_STORAGE_PATH, + enable_auth=True, + wait_for_predictor_pods=False, + resources=GAUSSIAN_CREDIT_MODEL_RESOURCES, + ) + isvc = stack.enter_context(isvc_context) # noqa: FCN001 + + wait_for_isvc_deployment_registered_by_trustyai_service( + client=admin_client, + isvc=isvc, + runtime_name=runtime.name, + ) + + models.append(isvc) + + yield models + + +@pytest.fixture(scope="class") +def isvc_getter_service_account_multi_ns(admin_client, model_namespaces) -> Generator[List[ServiceAccount], None, None]: + with ExitStack() as stack: + sas = [ + stack.enter_context(create_isvc_getter_service_account(admin_client, ns, ISVC_GETTER)) + for ns in model_namespaces + ] + yield sas + + +@pytest.fixture(scope="class") +def isvc_getter_role_multi_ns(admin_client, model_namespaces) -> Generator[List[Role], None, None]: + with ExitStack() as stack: + roles = [ + stack.enter_context(create_isvc_getter_role(admin_client, ns, f"isvc-getter-{ns.name}")) + for ns in model_namespaces + ] + yield roles + + +@pytest.fixture(scope="class") +def isvc_getter_role_binding_multi_ns( + admin_client, + model_namespaces, + isvc_getter_role_multi_ns, + isvc_getter_service_account_multi_ns, +) -> Generator[List[RoleBinding], None, None]: + with ExitStack() as stack: + bindings = [ + stack.enter_context( + create_isvc_getter_role_binding( + client=admin_client, + namespace=ns, + role=role, + service_account=sa, + name=ISVC_GETTER, + ) + ) + for ns, role, sa in zip(model_namespaces, isvc_getter_role_multi_ns, isvc_getter_service_account_multi_ns) + ] + yield bindings + + +@pytest.fixture(scope="class") +def isvc_getter_token_secret_multi_ns( + admin_client, + model_namespaces, + isvc_getter_service_account_multi_ns, + isvc_getter_role_binding_multi_ns, +) -> Generator[List[Secret], None, None]: + with ExitStack() as stack: + secrets = [ + stack.enter_context( + create_isvc_getter_token_secret( + client=admin_client, + namespace=ns, + name=f"sa-token-{ns.name}", + service_account=sa, + ) + ) + for ns, sa in zip(model_namespaces, isvc_getter_service_account_multi_ns) + ] + yield secrets + + +@pytest.fixture(scope="class") +def isvc_getter_token_multi_ns( + isvc_getter_service_account_multi_ns, + isvc_getter_token_secret_multi_ns, +) -> List[str]: + return [create_inference_token(model_service_account=sa) for sa in isvc_getter_service_account_multi_ns] diff --git a/tests/model_explainability/trustyai_service/service/multi_ns/test_trustyai_service_multi_ns.py b/tests/model_explainability/trustyai_service/service/multi_ns/test_trustyai_service_multi_ns.py new file mode 100644 index 000000000..37e660e54 --- /dev/null +++ b/tests/model_explainability/trustyai_service/service/multi_ns/test_trustyai_service_multi_ns.py @@ -0,0 +1,107 @@ +import pytest +from tests.model_explainability.trustyai_service.constants import DRIFT_BASE_DATA_PATH +from tests.model_explainability.trustyai_service.trustyai_service_utils import ( + send_inferences_and_verify_trustyai_service_registered, + verify_upload_data_to_trustyai_service, + TrustyAIServiceMetrics, + verify_trustyai_service_metric_scheduling_request, + verify_trustyai_service_metric_delete_request, +) +from utilities.constants import MinIo +from utilities.manifests.openvino import OPENVINO_KSERVE_INFERENCE_CONFIG + + +@pytest.mark.usefixtures("minio_pod") +@pytest.mark.parametrize( + "model_namespaces, minio_pod, minio_data_connection_multi_ns", + [ + pytest.param( + [ + {"name": "test-trustyaiservice-multins-1"}, + {"name": "test-trustyaiservice-multins-2"}, + ], + MinIo.PodConfig.MODEL_MESH_MINIO_CONFIG, + [ + {"bucket": MinIo.Buckets.MODELMESH_EXAMPLE_MODELS}, + {"bucket": MinIo.Buckets.MODELMESH_EXAMPLE_MODELS}, + ], + ), + ], + indirect=True, +) +class TestTrustyAIServiceMultipleNS: + """Verifies TrustyAIService operations across multiple namespaces, + i.e. registering inference requests, uploading data, scheduling and deleting metrics, + that can be performed with a TrustyAIService metric(drift, in this case)""" + + def test_drift_send_inference_and_verify_trustyai_service_multiple_ns( + self, + admin_client, + current_client_token, + trustyai_service_with_pvc_storage_multi_ns, + gaussian_credit_model_multi_ns, + isvc_getter_token_multi_ns, + ): + for tai, model, token in zip( + trustyai_service_with_pvc_storage_multi_ns, gaussian_credit_model_multi_ns, isvc_getter_token_multi_ns + ): + send_inferences_and_verify_trustyai_service_registered( + client=admin_client, + token=current_client_token, + data_path=f"{DRIFT_BASE_DATA_PATH}/data_batches", + trustyai_service=tai, + inference_service=model, + inference_config=OPENVINO_KSERVE_INFERENCE_CONFIG, + inference_token=token, + ) + + def test_upload_data_to_trustyai_service_multiple_ns( + self, + admin_client, + current_client_token, + trustyai_service_with_pvc_storage_multi_ns, + gaussian_credit_model_multi_ns, + isvc_getter_token_multi_ns, + model_namespaces, + ) -> None: + for tai in trustyai_service_with_pvc_storage_multi_ns: + verify_upload_data_to_trustyai_service( + client=admin_client, + trustyai_service=tai, + token=current_client_token, + data_path=f"{DRIFT_BASE_DATA_PATH}/training_data.json", + ) + + def test_drift_metric_schedule_meanshift_multiple_ns( + self, + admin_client, + current_client_token, + trustyai_service_with_pvc_storage_multi_ns, + gaussian_credit_model_multi_ns, + ): + for tai, inference_model in zip(trustyai_service_with_pvc_storage_multi_ns, gaussian_credit_model_multi_ns): + verify_trustyai_service_metric_scheduling_request( + client=admin_client, + trustyai_service=tai, + token=current_client_token, + metric_name=TrustyAIServiceMetrics.Drift.MEANSHIFT, + json_data={ + "modelId": inference_model.name, + "referenceTag": "TRAINING", + }, + ) + + def test_drift_metric_delete_multiple_ns( + self, + admin_client, + current_client_token, + minio_data_connection_multi_ns, + trustyai_service_with_pvc_storage_multi_ns, + ): + for tai in trustyai_service_with_pvc_storage_multi_ns: + verify_trustyai_service_metric_delete_request( + client=admin_client, + trustyai_service=tai, + token=current_client_token, + metric_name=TrustyAIServiceMetrics.Drift.MEANSHIFT, + ) diff --git a/tests/model_explainability/trustyai_service/service/test_trustyai_service.py b/tests/model_explainability/trustyai_service/service/test_trustyai_service.py index 09a64140d..3cec55b8a 100644 --- a/tests/model_explainability/trustyai_service/service/test_trustyai_service.py +++ b/tests/model_explainability/trustyai_service/service/test_trustyai_service.py @@ -7,11 +7,9 @@ TRUSTYAI_DB_MIGRATION_PATCH, ) from tests.model_explainability.trustyai_service.trustyai_service_utils import ( - send_inferences_and_verify_trustyai_service_registered, verify_upload_data_to_trustyai_service, TrustyAIServiceMetrics, verify_trustyai_service_metric_scheduling_request, - verify_trustyai_service_metric_delete_request, ) from tests.model_explainability.trustyai_service.utils import ( validate_trustyai_service_db_conn_failure, @@ -22,7 +20,6 @@ patch_trustyai_service_cr, ) from utilities.constants import MinIo -from utilities.manifests.openvino import OPENVINO_KSERVE_INFERENCE_CONFIG @pytest.mark.parametrize( @@ -48,117 +45,6 @@ def test_trustyai_service_with_invalid_db_cert( ) -@pytest.mark.parametrize( - "model_namespace, model_namespace_2, minio_pod, minio_data_connection, minio_data_connection_2", - [ - pytest.param( - {"name": "test-trustyaiservice-multins-1"}, - {"name": "test-trustyaiservice-multins-2"}, - MinIo.PodConfig.MODEL_MESH_MINIO_CONFIG, - {"bucket": MinIo.Buckets.MODELMESH_EXAMPLE_MODELS}, - {"bucket": MinIo.Buckets.MODELMESH_EXAMPLE_MODELS}, - ) - ], - indirect=True, -) -@pytest.mark.usefixtures("minio_pod") -class TestTrustyAIServiceMultipleNS: - """Verifies that all the basic operations, - i.e. registering inference requests, uploading data, scheduling and deleting metrics, - that can be performed with a TrustyAIService metric - (drift, in this case), work as expected when having more than one namespace.""" - - def test_drift_send_inference_and_verify_trustyai_service_multiple_ns( - self, - admin_client, - current_client_token, - trustyai_service_with_pvc_storage, - trustyai_service_with_pvc_storage_2, - gaussian_credit_model, - gaussian_credit_model_2, - isvc_getter_token, - isvc_getter_token_2, - ) -> None: - for trustyaiservice, inference_model, inference_token in zip( - [trustyai_service_with_pvc_storage, trustyai_service_with_pvc_storage_2], - [gaussian_credit_model, gaussian_credit_model_2], - [isvc_getter_token, isvc_getter_token_2], - ): - send_inferences_and_verify_trustyai_service_registered( - client=admin_client, - token=current_client_token, - data_path=f"{DRIFT_BASE_DATA_PATH}/data_batches", - trustyai_service=trustyaiservice, - inference_service=inference_model, - inference_config=OPENVINO_KSERVE_INFERENCE_CONFIG, - inference_token=inference_token, - ) - - def test_upload_data_to_trustyai_service_multiple_ns( - self, - admin_client, - current_client_token, - minio_data_connection, - minio_data_connection_2, - trustyai_service_with_pvc_storage, - trustyai_service_with_pvc_storage_2, - ) -> None: - for trustyaiservice in [ - trustyai_service_with_pvc_storage, - trustyai_service_with_pvc_storage_2, - ]: - verify_upload_data_to_trustyai_service( - client=admin_client, - trustyai_service=trustyaiservice, - token=current_client_token, - data_path=f"{DRIFT_BASE_DATA_PATH}/training_data.json", - ) - - def test_drift_metric_schedule_meanshift_multiple_ns( - self, - admin_client, - current_client_token, - trustyai_service_with_pvc_storage, - trustyai_service_with_pvc_storage_2, - gaussian_credit_model, - gaussian_credit_model_2, - ): - for trustyaiservice, inference_model in zip( - [trustyai_service_with_pvc_storage, trustyai_service_with_pvc_storage_2], - [gaussian_credit_model, gaussian_credit_model_2], - ): - verify_trustyai_service_metric_scheduling_request( - client=admin_client, - trustyai_service=trustyaiservice, - token=current_client_token, - metric_name=TrustyAIServiceMetrics.Drift.MEANSHIFT, - json_data={ - "modelId": inference_model.name, - "referenceTag": "TRAINING", - }, - ) - - def test_drift_metric_delete_multiple_ns( - self, - admin_client, - current_client_token, - minio_data_connection, - minio_data_connection_2, - trustyai_service_with_pvc_storage, - trustyai_service_with_pvc_storage_2, - ): - for trustyaiservice in [ - trustyai_service_with_pvc_storage, - trustyai_service_with_pvc_storage_2, - ]: - verify_trustyai_service_metric_delete_request( - client=admin_client, - trustyai_service=trustyaiservice, - token=current_client_token, - metric_name=TrustyAIServiceMetrics.Drift.MEANSHIFT, - ) - - @pytest.mark.parametrize( "model_namespace", [ diff --git a/tests/model_explainability/trustyai_service/utils.py b/tests/model_explainability/trustyai_service/utils.py index a8319a32b..457d85f0d 100644 --- a/tests/model_explainability/trustyai_service/utils.py +++ b/tests/model_explainability/trustyai_service/utils.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager from typing import Generator, Any, Optional import re @@ -115,6 +116,7 @@ def validate_trustyai_service_db_conn_failure( return False +@contextmanager def create_trustyai_service( client: DynamicClient, namespace: str, @@ -155,6 +157,7 @@ def create_trustyai_service( yield trustyai_service +@contextmanager def create_isvc_getter_service_account( client: DynamicClient, namespace: Namespace, name: str ) -> Generator[ServiceAccount, Any, Any]: @@ -172,6 +175,7 @@ def create_isvc_getter_service_account( yield sa +@contextmanager def create_isvc_getter_role(client: DynamicClient, namespace: Namespace, name: str) -> Generator[Role, Any, Any]: """Creates a Role with permissions to get, list, and watch InferenceServices. @@ -198,6 +202,7 @@ def create_isvc_getter_role(client: DynamicClient, namespace: Namespace, name: s yield role +@contextmanager def create_isvc_getter_role_binding( client: DynamicClient, namespace: Namespace, role: Role, service_account: ServiceAccount, name: str ) -> Generator[RoleBinding, Any, Any]: @@ -225,6 +230,7 @@ def create_isvc_getter_role_binding( yield rb +@contextmanager def create_isvc_getter_token_secret( client: DynamicClient, namespace: Namespace, service_account: ServiceAccount, name: str ) -> Generator[Secret, Any, Any]: