From e43cae7cf89c7500fa3ebfb6d2fd1073dd62d624 Mon Sep 17 00:00:00 2001 From: aaguirre Date: Tue, 18 Mar 2025 15:31:09 +0100 Subject: [PATCH 1/3] [Explainability] Add tests for GuardrailsOrchestrator --- pyproject.toml | 2 +- tests/conftest.py | 11 + tests/model_explainability/conftest.py | 54 ++++ tests/model_explainability/constants.py | 17 ++ .../guardrails/conftest.py | 287 ++++++++++++++++++ .../guardrails/test_guardrails.py | 46 +++ .../trustyai_service/drift/conftest.py | 4 + .../trustyai_service/drift/test_drift.py | 9 +- .../trustyai_service/fairness/conftest.py | 4 + .../fairness/test_fairness.py | 7 +- tests/model_serving/model_server/conftest.py | 15 +- utilities/serving_runtime.py | 35 ++- uv.lock | 6 +- 13 files changed, 472 insertions(+), 25 deletions(-) create mode 100644 tests/model_explainability/conftest.py create mode 100644 tests/model_explainability/constants.py create mode 100644 tests/model_explainability/guardrails/conftest.py create mode 100644 tests/model_explainability/guardrails/test_guardrails.py diff --git a/pyproject.toml b/pyproject.toml index 3a54abb67..7d7093765 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ dependencies = [ "timeout-sampler>=1.0.6", "shortuuid>=1.0.13", "jira>=3.8.0", - "openshift-python-wrapper>=11.0.26", + "openshift-python-wrapper>=11.0.38", "semver>=3.0.4", ] diff --git a/tests/conftest.py b/tests/conftest.py index 3f6b69ea0..ef90e178b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -296,6 +296,17 @@ def enabled_modelmesh_in_dsc(dsc_resource: DataScienceCluster) -> Generator[Data yield dsc +@pytest.fixture(scope="package") +def enabled_kserve_in_dsc( + dsc_resource: DataScienceCluster, +) -> Generator[DataScienceCluster, Any, Any]: + with update_components_in_dsc( + dsc=dsc_resource, + components={DscComponents.KSERVE: DscComponents.ManagementState.MANAGED}, + ) as dsc: + yield dsc + + @pytest.fixture(scope="session") def cluster_monitoring_config(admin_client: DynamicClient) -> Generator[ConfigMap, Any, Any]: data = {"config.yaml": yaml.dump({"enableUserWorkload": True})} diff --git a/tests/model_explainability/conftest.py b/tests/model_explainability/conftest.py new file mode 100644 index 000000000..a82aa5694 --- /dev/null +++ b/tests/model_explainability/conftest.py @@ -0,0 +1,54 @@ +from typing import Generator, Any + +import pytest +from _pytest.fixtures import FixtureRequest +from kubernetes.dynamic import DynamicClient +from ocp_resources.namespace import Namespace +from ocp_resources.secret import Secret +from ocp_resources.service import Service + +from tests.model_explainability.constants import MINIO, MINIO_PORT + +OPENDATAHUB_IO: str = "opendatahub.io" + + +@pytest.fixture(scope="class") +def minio_service(admin_client: DynamicClient, model_namespace: Namespace) -> Generator[Service, Any, Any]: + with Service( + client=admin_client, + name=MINIO, + namespace=model_namespace.name, + ports=[ + { + "name": "minio-client-port", + "port": MINIO_PORT, + "protocol": "TCP", + "targetPort": MINIO_PORT, + } + ], + selector={ + "app": MINIO, + }, + ) as minio_service: + yield minio_service + + +@pytest.fixture(scope="class") +def minio_data_connection( + request: FixtureRequest, admin_client: DynamicClient, model_namespace: Namespace, minio_service: Service +) -> Generator[Secret, Any, Any]: + with Secret( + client=admin_client, + name="aws-connection-minio-data-connection", + namespace=model_namespace.name, + data_dict=request.param["data-dict"], + label={ + f"{OPENDATAHUB_IO}/dashboard": "true", + f"{OPENDATAHUB_IO}/managed": "true", + }, + annotations={ + f"{OPENDATAHUB_IO}/connection-type": "s3", + "openshift.io/display-name": "Minio Data Connection", + }, + ) as minio_secret: + yield minio_secret diff --git a/tests/model_explainability/constants.py b/tests/model_explainability/constants.py new file mode 100644 index 000000000..3d9b5304b --- /dev/null +++ b/tests/model_explainability/constants.py @@ -0,0 +1,17 @@ +from utilities.general import get_s3_secret_dict + +MINIO: str = "minio" +MINIO_PORT: int = 9000 + +MINIO_ACCESS_KEY: str = "MINIO_ACCESS_KEY" +MINIO_ACCESS_KEY_VALUE: str = "THEACCESSKEY" +MINIO_SECRET_KEY: str = "MINIO_SECRET_KEY" +MINIO_SECRET_KEY_VALUE: str = "THESECRETKEY" + +MINIO_DATA_DICT: dict[str, str] = get_s3_secret_dict( + aws_access_key=MINIO_ACCESS_KEY_VALUE, + aws_secret_access_key=MINIO_SECRET_KEY_VALUE, # pragma: allowlist secret + aws_s3_bucket="modelmesh-example-models", + aws_s3_endpoint=f"http://minio:{str(MINIO_PORT)}", + aws_s3_region="us-south", +) diff --git a/tests/model_explainability/guardrails/conftest.py b/tests/model_explainability/guardrails/conftest.py new file mode 100644 index 000000000..3737f9813 --- /dev/null +++ b/tests/model_explainability/guardrails/conftest.py @@ -0,0 +1,287 @@ +from typing import Generator, Any + +import pytest +import yaml +from kubernetes.dynamic import DynamicClient +from ocp_resources.config_map import ConfigMap +from ocp_resources.deployment import Deployment +from ocp_resources.guardrails_orchestrator import GuardrailsOrchestrator +from ocp_resources.inference_service import InferenceService +from ocp_resources.namespace import Namespace +from ocp_resources.persistent_volume_claim import PersistentVolumeClaim +from ocp_resources.role_binding import RoleBinding +from ocp_resources.route import Route +from ocp_resources.secret import Secret +from ocp_resources.service import Service +from ocp_resources.service_account import ServiceAccount +from ocp_resources.serving_runtime import ServingRuntime + +from tests.model_explainability.constants import ( + MINIO, + MINIO_ACCESS_KEY, + MINIO_SECRET_KEY, + MINIO_ACCESS_KEY_VALUE, + MINIO_SECRET_KEY_VALUE, +) +from utilities.constants import KServeDeploymentType, Timeout, Ports +from utilities.inference_utils import create_isvc +from utilities.serving_runtime import ServingRuntimeFromTemplate + + +USER_ONE: str = "user-one" +GUARDRAILS_ORCHESTRATOR_PORT: int = 8032 + + +@pytest.fixture(scope="class") +def guardrails_orchestrator_health_route( + admin_client: DynamicClient, model_namespace: Namespace, guardrails_orchestrator: GuardrailsOrchestrator +) -> Generator[Route, Any, Any]: + route = Route( + name=f"{guardrails_orchestrator.name}-health", + namespace=guardrails_orchestrator.namespace, + wait_for_resource=True, + ensure_exists=True, + ) + yield route + + +@pytest.fixture(scope="class") +def guardrails_orchestrator( + admin_client: DynamicClient, + model_namespace: Namespace, + orchestrator_configmap: ConfigMap, + vllm_gateway_config: ConfigMap, + vllm_images_configmap: ConfigMap, +) -> Generator[GuardrailsOrchestrator, Any, Any]: + with GuardrailsOrchestrator( + client=admin_client, + name="gorch-test", + namespace=model_namespace.name, + orchestrator_config=orchestrator_configmap.name, + vllm_gateway_config=vllm_gateway_config.name, + replicas=1, + wait_for_resource=True, + ) as gorch: + orchestrator_deployment = Deployment(name=gorch.name, namespace=gorch.namespace, wait_for_resource=True) + orchestrator_deployment.wait_for_replicas() + yield gorch + + +@pytest.fixture(scope="class") +def qwen_llm_model( + admin_client: DynamicClient, + model_namespace: Namespace, + minio_data_connection: Secret, + vllm_runtime: ServingRuntime, +) -> Generator[InferenceService, Any, Any]: + with create_isvc( + client=admin_client, + name="llm", + namespace=model_namespace.name, + deployment_mode=KServeDeploymentType.RAW_DEPLOYMENT, + model_format="vLLM", + runtime=vllm_runtime.name, + storage_key=minio_data_connection.name, + storage_path="Qwen2.5-0.5B-Instruct", + wait_for_predictor_pods=False, + enable_auth=True, + resources={"requests": {"cpu": "1", "memory": "8Gi"}, "limits": {"cpu": "2", "memory": "10Gi"}}, + ) as isvc: + yield isvc + + +@pytest.fixture(scope="class") +def vllm_runtime( + admin_client: DynamicClient, + model_namespace: Namespace, + minio_llm_deployment: Deployment, + minio_service: Service, + minio_data_connection: Secret, +) -> Generator[ServingRuntime, Any, Any]: + with ServingRuntimeFromTemplate( + client=admin_client, + name="vllm-runtime-cpu-fp16", + namespace=model_namespace.name, + template_name="vllm-runtime-template", + deployment_type=KServeDeploymentType.RAW_DEPLOYMENT, + runtime_image="quay.io/rh-aiservices-bu/vllm-cpu-openai-ubi9" + "@sha256:d680ff8becb6bbaf83dfee7b2d9b8a2beb130db7fd5aa7f9a6d8286a58cebbfd", + containers={ + "kserve-container": { + "args": [ + f"--port={str(GUARDRAILS_ORCHESTRATOR_PORT)}", + "--model=/mnt/models", + ], + "ports": [{"containerPort": GUARDRAILS_ORCHESTRATOR_PORT, "protocol": "TCP"}], + "volumeMounts": [{"mountPath": "/dev/shm", "name": "shm"}], + } + }, + volumes=[{"emptyDir": {"medium": "Memory", "sizeLimit": "2Gi"}, "name": "shm"}], + ) as serving_runtime: + yield serving_runtime + + +@pytest.fixture(scope="class") +def vllm_images_configmap(admin_client: DynamicClient, model_namespace: Namespace) -> Generator[ConfigMap, Any, Any]: + with ConfigMap( + client=admin_client, + name="gorch-test-config", + namespace=model_namespace.name, + data={ + "regexDetectorImage": "quay.io/trustyai_testing/regex-detector" + "@sha256:e9df9f7e7429e29da9b8d9920d80cdc85a496e7961f6edb19132d604a914049b", + "vllmGatewayImage": "quay.io/trustyai_testing/vllm-orchestrator-gateway" + "@sha256:d0bbf2de95c69f76215a016820f294202c48721dee452b3939e36133697d5b1d", + }, + ) as cm: + yield cm + + +@pytest.fixture(scope="class") +def orchestrator_configmap( + admin_client: DynamicClient, model_namespace: Namespace, qwen_llm_model: InferenceService +) -> Generator[ConfigMap, Any, Any]: + with ConfigMap( + client=admin_client, + name="fms-orchestr8-config-nlp", + namespace=model_namespace.name, + data={ + "config.yaml": yaml.dump({ + "chat_generation": { + "service": { + "hostname": f"{qwen_llm_model.name}-predictor.{model_namespace.name}.svc.cluster.local", + "port": GUARDRAILS_ORCHESTRATOR_PORT, + } + }, + "detectors": { + "regex": { + "type": "text_contents", + "service": {"hostname": "127.0.0.1", "port": Ports.REST_PORT}, + "chunker_id": "whole_doc_chunker", + "default_threshold": 0.5, + } + }, + }) + }, + ) as cm: + yield cm + + +@pytest.fixture(scope="class") +def vllm_gateway_config(admin_client: DynamicClient, model_namespace: Namespace) -> Generator[ConfigMap, Any, Any]: + with ConfigMap( + client=admin_client, + name="fms-orchestr8-config-gateway", + namespace=model_namespace.name, + label={"app": "fmstack-nlp"}, + data={ + "config.yaml": yaml.dump({ + "orchestrator": {"host": "localhost", "port": GUARDRAILS_ORCHESTRATOR_PORT}, + "detectors": [ + {"name": "regex", "detector_params": {"regex": ["email", "ssn"]}}, + {"name": "other_detector"}, + ], + "routes": [{"name": "pii", "detectors": ["regex"]}, {"name": "passthrough", "detectors": []}], + }) + }, + ) as cm: + yield cm + + +@pytest.fixture(scope="class") +def minio_llm_deployment( + admin_client: DynamicClient, + model_namespace: Namespace, + llm_models_pvc: PersistentVolumeClaim, +) -> Generator[Deployment, Any, Any]: + with Deployment( + client=admin_client, + name="llm-container-deployment", + namespace=model_namespace.name, + replicas=1, + selector={"matchLabels": {"app": MINIO}}, + template={ + "metadata": {"labels": {"app": MINIO, "maistra.io/expose-route": "true"}, "name": MINIO}, + "spec": { + "volumes": [{"name": "model-volume", "persistentVolumeClaim": {"claimName": "llm-models-claim"}}], + "initContainers": [ + { + "name": "download-model", + "image": "quay.io/trustyai_testing/llm-downloader-bootstrap" + "@sha256:d3211cc581fe69ca9a1cb75f84e5d08cacd1854cb2d63591439910323b0cbb57", + "securityContext": {"fsGroup": 1001}, + "command": [ + "bash", + "-c", + 'model="Qwen/Qwen2.5-0.5B-Instruct"' + '\necho "starting download"' + "\n/tmp/venv/bin/huggingface-cli download $model " + "--local-dir /mnt/models/llms/$(basename $model)" + '\necho "Done!"', + ], + "resources": {"limits": {"memory": "5Gi", "cpu": "2"}}, + "volumeMounts": [{"mountPath": "/mnt/models/", "name": "model-volume"}], + } + ], + "containers": [ + { + "args": ["server", "/models"], + "env": [ + {"name": MINIO_ACCESS_KEY, "value": MINIO_ACCESS_KEY_VALUE}, + {"name": MINIO_SECRET_KEY, "value": MINIO_SECRET_KEY_VALUE}, + ], + "image": "quay.io/trustyai/modelmesh-minio-examples" + "@sha256:65cb22335574b89af15d7409f62feffcc52cc0e870e9419d63586f37706321a5", + "name": MINIO, + "securityContext": { + "allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}, + "seccompProfile": {"type": "RuntimeDefault"}, + }, + "volumeMounts": [{"mountPath": "/models/", "name": "model-volume"}], + } + ], + }, + }, + label={"app": MINIO}, + wait_for_resource=True, + ) as deployment: + deployment.wait_for_replicas(timeout=Timeout.TIMEOUT_10MIN) + yield deployment + + +@pytest.fixture(scope="class") +def llm_models_pvc( + admin_client: DynamicClient, model_namespace: Namespace +) -> Generator[PersistentVolumeClaim, Any, Any]: + with PersistentVolumeClaim( + client=admin_client, + name="llm-models-claim", + namespace=model_namespace.name, + accessmodes=PersistentVolumeClaim.AccessMode.RWO, + volume_mode=PersistentVolumeClaim.VolumeMode.FILE, + size="10Gi", + ) as pvc: + yield pvc + + +@pytest.fixture(scope="class") +def user_one_service_account( + admin_client: DynamicClient, model_namespace: Namespace +) -> Generator[ServiceAccount, Any, Any]: + with ServiceAccount(client=admin_client, name=USER_ONE, namespace=model_namespace.name) as service_account: + yield service_account + + +@pytest.fixture(scope="class") +def user_one_rolebinding( + admin_client: DynamicClient, model_namespace: Namespace, user_one_service_account: ServiceAccount +) -> Generator[RoleBinding, Any, Any]: + with RoleBinding( + client=admin_client, + name=f"{user_one_service_account.name}-view", + namespace=model_namespace.name, + subjects=[{"kind": "ServiceAccount", "name": user_one_service_account.name}], + role_ref={"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": "view"}, + ) as role_binding: + yield role_binding diff --git a/tests/model_explainability/guardrails/test_guardrails.py b/tests/model_explainability/guardrails/test_guardrails.py new file mode 100644 index 000000000..a592d8387 --- /dev/null +++ b/tests/model_explainability/guardrails/test_guardrails.py @@ -0,0 +1,46 @@ +import http + +import pytest +import requests +from timeout_sampler import retry + +from tests.model_explainability.constants import MINIO_DATA_DICT +from utilities.constants import Timeout +from utilities.general import b64_encoded_string + +DATA_DICT: dict[str, str] = MINIO_DATA_DICT +DATA_DICT["AWS_S3_BUCKET"] = b64_encoded_string(string_to_encode="llms") + + +@pytest.mark.parametrize( + "model_namespace, minio_data_connection", + [ + pytest.param( + {"name": "test-guardrails"}, + {"data-dict": DATA_DICT}, + ) + ], + indirect=True, +) +@pytest.mark.rawdeployment +class TestGuardrails: + def test_guardrails_health_endpoint(self, admin_client, qwen_llm_model, guardrails_orchestrator_health_route): + # It takes a bit for the endpoint to come online, so we retry for a brief period of time + @retry(wait_timeout=Timeout.TIMEOUT_1MIN, sleep=1) + def check_health_endpoint(): + response = requests.get(url=f"https://{guardrails_orchestrator_health_route.host}/health", verify=False) + if response.status_code == http.HTTPStatus.OK: + return response + return False + + response = check_health_endpoint() + assert "fms-guardrails-orchestr8" in response.text + + def test_guardrails_info_endpoint(self, admin_client, qwen_llm_model, guardrails_orchestrator_health_route): + response = requests.get(url=f"https://{guardrails_orchestrator_health_route.host}/info", verify=False) + assert response.status_code == http.HTTPStatus.OK + + healthy_status = "HEALTHY" + response_data = response.json() + assert response_data["services"]["chat_generation"]["status"] == healthy_status + assert response_data["services"]["regex"]["status"] == healthy_status diff --git a/tests/model_explainability/trustyai_service/drift/conftest.py b/tests/model_explainability/trustyai_service/drift/conftest.py index b62e3fac0..54a4d9128 100644 --- a/tests/model_explainability/trustyai_service/drift/conftest.py +++ b/tests/model_explainability/trustyai_service/drift/conftest.py @@ -4,7 +4,9 @@ from kubernetes.dynamic import DynamicClient from ocp_resources.inference_service import InferenceService from ocp_resources.namespace import Namespace +from ocp_resources.pod import Pod from ocp_resources.secret import Secret +from ocp_resources.service import Service from ocp_resources.serving_runtime import ServingRuntime from ocp_resources.trustyai_service import TrustyAIService @@ -76,6 +78,8 @@ def mlserver_runtime( def gaussian_credit_model( admin_client: DynamicClient, model_namespace: Namespace, + minio_pod: Pod, + minio_service: Service, minio_data_connection: Secret, mlserver_runtime: ServingRuntime, trustyai_service_with_pvc_storage: TrustyAIService, diff --git a/tests/model_explainability/trustyai_service/drift/test_drift.py b/tests/model_explainability/trustyai_service/drift/test_drift.py index 7b686db1f..7684fa17e 100644 --- a/tests/model_explainability/trustyai_service/drift/test_drift.py +++ b/tests/model_explainability/trustyai_service/drift/test_drift.py @@ -1,5 +1,6 @@ import pytest +from tests.model_explainability.constants import MINIO_DATA_DICT from tests.model_explainability.trustyai_service.trustyai_service_utils import ( send_inferences_and_verify_trustyai_service_registered, verify_upload_data_to_trustyai_service, @@ -14,10 +15,11 @@ @pytest.mark.parametrize( - "model_namespace", + "model_namespace, minio_data_connection", [ pytest.param( {"name": "test-drift"}, + {"data-dict": MINIO_DATA_DICT}, ) ], indirect=True, @@ -53,6 +55,7 @@ def test_drift_send_inference_and_verify_trustyai_service( def test_upload_data_to_trustyai_service( self, admin_client, + minio_data_connection, current_client_token, trustyai_service_with_pvc_storage, ) -> None: @@ -85,7 +88,9 @@ def test_drift_metric_schedule_meanshift( json_data={"modelId": gaussian_credit_model.name, "referenceTag": "TRAINING"}, ) - def test_drift_metric_delete(self, admin_client, current_client_token, trustyai_service_with_pvc_storage): + def test_drift_metric_delete( + self, admin_client, minio_data_connection, current_client_token, trustyai_service_with_pvc_storage + ): verify_trustyai_service_metric_delete_request( client=admin_client, trustyai_service=trustyai_service_with_pvc_storage, diff --git a/tests/model_explainability/trustyai_service/fairness/conftest.py b/tests/model_explainability/trustyai_service/fairness/conftest.py index c3ae8ae0a..842b42be5 100644 --- a/tests/model_explainability/trustyai_service/fairness/conftest.py +++ b/tests/model_explainability/trustyai_service/fairness/conftest.py @@ -4,7 +4,9 @@ from kubernetes.dynamic import DynamicClient from ocp_resources.inference_service import InferenceService from ocp_resources.namespace import Namespace +from ocp_resources.pod import Pod from ocp_resources.secret import Secret +from ocp_resources.service import Service from ocp_resources.serving_runtime import ServingRuntime from tests.model_explainability.trustyai_service.trustyai_service_utils import ( @@ -39,6 +41,8 @@ def ovms_runtime( def onnx_loan_model( admin_client: DynamicClient, model_namespace: Namespace, + minio_pod: Pod, + minio_service: Service, minio_data_connection: Secret, ovms_runtime: ServingRuntime, ) -> Generator[InferenceService, Any, Any]: diff --git a/tests/model_explainability/trustyai_service/fairness/test_fairness.py b/tests/model_explainability/trustyai_service/fairness/test_fairness.py index 4a054db94..02f1d777f 100644 --- a/tests/model_explainability/trustyai_service/fairness/test_fairness.py +++ b/tests/model_explainability/trustyai_service/fairness/test_fairness.py @@ -3,6 +3,7 @@ import pytest from ocp_resources.inference_service import InferenceService +from tests.model_explainability.constants import MINIO_DATA_DICT from tests.model_explainability.trustyai_service.trustyai_service_utils import ( send_inferences_and_verify_trustyai_service_registered, verify_trustyai_service_name_mappings, @@ -45,10 +46,11 @@ def get_fairness_request_json_data(isvc: InferenceService) -> dict[str, Any]: @pytest.mark.parametrize( - "model_namespace", + "model_namespace, minio_data_connection", [ pytest.param( {"name": "test-fairness-pvc"}, + {"data-dict": MINIO_DATA_DICT}, ) ], indirect=True, @@ -122,10 +124,11 @@ def test_fairness_metric_delete_with_pvc_storage( @pytest.mark.parametrize( - "model_namespace", + "model_namespace, minio_data_connection", [ pytest.param( {"name": "test-fairness-db"}, + {"data-dict": MINIO_DATA_DICT}, ) ], indirect=True, diff --git a/tests/model_serving/model_server/conftest.py b/tests/model_serving/model_server/conftest.py index d25c13cb7..3a887d63c 100644 --- a/tests/model_serving/model_server/conftest.py +++ b/tests/model_serving/model_server/conftest.py @@ -7,7 +7,6 @@ from ocp_resources.authorino import Authorino from ocp_resources.cluster_service_version import ClusterServiceVersion from ocp_resources.config_map import ConfigMap -from ocp_resources.data_science_cluster import DataScienceCluster from ocp_resources.inference_service import InferenceService from ocp_resources.namespace import Namespace from ocp_resources.persistent_volume_claim import PersistentVolumeClaim @@ -19,7 +18,7 @@ from ocp_utilities.monitoring import Prometheus from pytest_testconfig import config as py_config -from utilities.constants import DscComponents, StorageClassName +from utilities.constants import StorageClassName from utilities.constants import ( KServeDeploymentType, ModelFormat, @@ -37,7 +36,6 @@ s3_endpoint_secret, update_configmap_data, ) -from utilities.data_science_cluster_utils import update_components_in_dsc from utilities.serving_runtime import ServingRuntimeFromTemplate @@ -203,17 +201,6 @@ def skip_if_no_deployed_redhat_authorino_operator(admin_client: DynamicClient) - pytest.skip(f"Authorino {name} CR is missing from {namespace} namespace") -@pytest.fixture(scope="package") -def enabled_kserve_in_dsc( - dsc_resource: DataScienceCluster, -) -> Generator[DataScienceCluster, Any, Any]: - with update_components_in_dsc( - dsc=dsc_resource, - components={DscComponents.KSERVE: DscComponents.ManagementState.MANAGED}, - ) as dsc: - yield dsc - - @pytest.fixture(scope="package") def skip_if_no_deployed_openshift_service_mesh(admin_client: DynamicClient) -> None: smcp = ServiceMeshControlPlane(client=admin_client, name="data-science-smcp", namespace="istio-system") diff --git a/utilities/serving_runtime.py b/utilities/serving_runtime.py index 88ee2aab2..632a2ace3 100644 --- a/utilities/serving_runtime.py +++ b/utilities/serving_runtime.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy from typing import Any from kubernetes.dynamic import DynamicClient from kubernetes.dynamic.exceptions import ResourceNotFoundError @@ -29,6 +30,8 @@ def __init__( runtime_image: str | None = None, models_priorities: dict[str, str] | None = None, supported_model_formats: dict[str, list[dict[str, str]]] | None = None, + volumes: list[dict[str, Any]] | None = None, + containers: dict[str, dict[str, Any]] | None = None, support_tgis_open_ai_endpoints: bool = False, ): """ @@ -51,6 +54,9 @@ def __init__( models_priorities (dict[str, str]): Model priority to be used for the serving runtime supported_model_formats (dict[str, list[dict[str, str]]]): Model formats; overwrites template's `supportedModelFormats` + volumes (list[dict[str, Any]]): Volumes to be used with the serving runtime + containers (dict[str, dict[str, Any]]): Containers configurations to override or add + to the serving runtime support_tgis_open_ai_endpoints (bool): Whether to support TGIS and OpenAI endpoints using a single entry point """ @@ -69,6 +75,8 @@ def __init__( self.runtime_image = runtime_image self.models_priorities = models_priorities self.supported_model_formats = supported_model_formats + self.volumes = volumes + self.containers = containers self.support_tgis_open_ai_endpoints = support_tgis_open_ai_endpoints # model mesh attributes @@ -145,7 +153,13 @@ def update_model_dict(self) -> dict[str, Any]: if self.protocol is not None: _model_metadata.setdefault("annotations", {})["opendatahub.io/apiProtocol"] = self.protocol - for container in _model_spec["containers"]: + template_containers = _model_spec.get("containers", []) + + containers_to_add = copy.deepcopy(self.containers) if self.containers else {} + + for container in template_containers: + container_name = container.get("name") + for env in container.get("env", []): if env["name"] == "RUNTIME_HTTP_ENABLED" and self.enable_http is not None: env["value"] = str(self.enable_http).lower() @@ -160,7 +174,7 @@ def update_model_dict(self) -> dict[str, Any]: "protocol": Protocols.TCP, } - if self.resources is not None and (resource_dict := self.resources.get(container["name"])): + if self.resources is not None and (resource_dict := self.resources.get(container_name)): container["resources"] = resource_dict if self.runtime_image is not None: @@ -182,9 +196,21 @@ def update_model_dict(self) -> dict[str, Any]: elif is_raw: container["ports"] = vLLM_CONFIG["port_configurations"]["raw"] + if containers_to_add and container_name in containers_to_add: + container_config = containers_to_add.pop(container_name) + for key, value in container_config.items(): + container[key] = value + + if containers_to_add: + for container_name, container_config in containers_to_add.items(): + new_container = {"name": container_name} + new_container.update(container_config) + template_containers.append(new_container) + + _model_spec["containers"] = template_containers + if self.supported_model_formats: _model_spec_supported_formats = self.supported_model_formats - else: if self.model_format_name is not None: for model in _model_spec_supported_formats: @@ -197,4 +223,7 @@ def update_model_dict(self) -> dict[str, Any]: if _model_name in self.models_priorities: _model["priority"] = self.models_priorities[_model_name] + if self.volumes: + _model_spec["volumes"] = self.volumes + return _model_dict diff --git a/uv.lock b/uv.lock index 6c572a601..1a5fdd6e2 100644 --- a/uv.lock +++ b/uv.lock @@ -1539,7 +1539,7 @@ requires-dist = [ { name = "jira", specifier = ">=3.8.0" }, { name = "model-registry", specifier = ">=0.2.13" }, { name = "openshift-python-utilities", specifier = ">=5.0.71" }, - { name = "openshift-python-wrapper", specifier = ">=11.0.26" }, + { name = "openshift-python-wrapper", specifier = ">=11.0.38" }, { name = "portforward", specifier = ">=0.7.1" }, { name = "protobuf" }, { name = "pygithub", specifier = ">=2.5.0" }, @@ -1587,7 +1587,7 @@ sdist = { url = "https://files.pythonhosted.org/packages/6f/a5/4027502ea98beaef4 [[package]] name = "openshift-python-wrapper" -version = "11.0.37" +version = "11.0.38" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -1609,7 +1609,7 @@ dependencies = [ { name = "timeout-sampler" }, { name = "xmltodict" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0f/5e/100233fd73a7381f08c948ad799e1352aa6499d468109b92f39b9547aa05/openshift_python_wrapper-11.0.37.tar.gz", hash = "sha256:c4ae3acbb8889d63e15ea7503066e442d6caf311021d3d2c850cd87da22abe06", size = 7040375 } +sdist = { url = "https://files.pythonhosted.org/packages/f4/0a/0f8ee1a42a1e68de0bf900d2a428d4d9f13cbdff2f699acd6444097c95f5/openshift_python_wrapper-11.0.38.tar.gz", hash = "sha256:8a954355a6c90e9a68db13ffb7daed0092be401cf5d652047fc90cd82cb42f1a", size = 7074191 } [[package]] name = "openshift-python-wrapper-data-collector" From 4483ac83653bc06b9d3e9f2bd31c0e94ef251792 Mon Sep 17 00:00:00 2001 From: aaguirre Date: Tue, 18 Mar 2025 15:55:02 +0100 Subject: [PATCH 2/3] Remove duplicate fixtures --- .../trustyai_service/conftest.py | 49 ------------------- 1 file changed, 49 deletions(-) diff --git a/tests/model_explainability/trustyai_service/conftest.py b/tests/model_explainability/trustyai_service/conftest.py index 3ceded939..b83572dea 100644 --- a/tests/model_explainability/trustyai_service/conftest.py +++ b/tests/model_explainability/trustyai_service/conftest.py @@ -12,7 +12,6 @@ from ocp_resources.namespace import Namespace from ocp_resources.pod import Pod from ocp_resources.secret import Secret -from ocp_resources.service import Service from ocp_resources.subscription import Subscription from ocp_resources.trustyai_service import TrustyAIService from ocp_utilities.operators import install_operator, uninstall_operator @@ -127,54 +126,6 @@ def minio_pod(admin_client: DynamicClient, model_namespace: Namespace) -> Genera yield minio_pod -@pytest.fixture(scope="class") -def minio_service(admin_client: DynamicClient, model_namespace: Namespace) -> Generator[Service, Any, Any]: - with Service( - client=admin_client, - name=MINIO, - namespace=model_namespace.name, - ports=[ - { - "name": "minio-client-port", - "port": 9000, - "protocol": "TCP", - "targetPort": 9000, - } - ], - selector={ - "app": MINIO, - }, - ) as minio_service: - yield minio_service - - -@pytest.fixture(scope="class") -def minio_data_connection( - admin_client: DynamicClient, model_namespace: Namespace, minio_pod: Pod, minio_service: Service -) -> Generator[Secret, Any, Any]: - with Secret( - client=admin_client, - name="aws-connection-minio-data-connection", - namespace=model_namespace.name, - data_dict={ - "AWS_ACCESS_KEY_ID": "VEhFQUNDRVNTS0VZ", - "AWS_DEFAULT_REGION": "dXMtc291dGg=", - "AWS_S3_BUCKET": "bW9kZWxtZXNoLWV4YW1wbGUtbW9kZWxz", - "AWS_S3_ENDPOINT": "aHR0cDovL21pbmlvOjkwMDA=", - "AWS_SECRET_ACCESS_KEY": "VEhFU0VDUkVUS0VZ", # pragma: allowlist secret - }, - label={ - f"{OPENDATAHUB_IO}/dashboard": "true", - f"{OPENDATAHUB_IO}/managed": "true", - }, - annotations={ - f"{OPENDATAHUB_IO}/connection-type": "s3", - "openshift.io/display-name": "Minio Data Connection", - }, - ) as minio_secret: - yield minio_secret - - @pytest.fixture(scope="class") def db_credentials_secret(admin_client: DynamicClient, model_namespace: Namespace) -> Generator[Secret, Any, Any]: with Secret( From a8e747e19b55ffadc3c73f0fcda60de95b4a5505 Mon Sep 17 00:00:00 2001 From: aaguirre Date: Thu, 20 Mar 2025 11:47:23 +0100 Subject: [PATCH 3/3] Remove unused fixtures --- .../guardrails/conftest.py | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/tests/model_explainability/guardrails/conftest.py b/tests/model_explainability/guardrails/conftest.py index 3737f9813..94531139f 100644 --- a/tests/model_explainability/guardrails/conftest.py +++ b/tests/model_explainability/guardrails/conftest.py @@ -9,11 +9,9 @@ from ocp_resources.inference_service import InferenceService from ocp_resources.namespace import Namespace from ocp_resources.persistent_volume_claim import PersistentVolumeClaim -from ocp_resources.role_binding import RoleBinding from ocp_resources.route import Route from ocp_resources.secret import Secret from ocp_resources.service import Service -from ocp_resources.service_account import ServiceAccount from ocp_resources.serving_runtime import ServingRuntime from tests.model_explainability.constants import ( @@ -263,25 +261,3 @@ def llm_models_pvc( size="10Gi", ) as pvc: yield pvc - - -@pytest.fixture(scope="class") -def user_one_service_account( - admin_client: DynamicClient, model_namespace: Namespace -) -> Generator[ServiceAccount, Any, Any]: - with ServiceAccount(client=admin_client, name=USER_ONE, namespace=model_namespace.name) as service_account: - yield service_account - - -@pytest.fixture(scope="class") -def user_one_rolebinding( - admin_client: DynamicClient, model_namespace: Namespace, user_one_service_account: ServiceAccount -) -> Generator[RoleBinding, Any, Any]: - with RoleBinding( - client=admin_client, - name=f"{user_one_service_account.name}-view", - namespace=model_namespace.name, - subjects=[{"kind": "ServiceAccount", "name": user_one_service_account.name}], - role_ref={"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": "view"}, - ) as role_binding: - yield role_binding