[Draft] SHAP test

Author: adolfo-ab

tests/model_explainability/conftest.py

@@ -83,7 +83,8 @@ def minio_pod(admin_client: DynamicClient, model_namespace: Namespace) -> Genera
                         "value": "THESECRETKEY",
                     },
                 ],
-                "image": "quay.io/rh-ee-mmisiura/modelmesh-minio-examples:latest",
+                "image": "quay.io/trustyai_testing/modelmesh-minio-examples"
+                "@sha256:d2ccbe92abf9aa5085b594b2cae6c65de2bf06306c30ff5207956eb949bb49da",
                 "name": MINIO,
             }
         ],

tests/model_explainability/lm_eval/test_lm_eval.py

@@ -27,7 +27,10 @@ def test_lmeval_huggingface_model(admin_client, model_namespace, lmevaljob_hf):
     [
         pytest.param(
             {"name": "lmevaljob-local-offline-builtin"},
-            {"image": "quay.io/trustyai_testing/lmeval-assets-flan-arceasy:latest"},
+            {
+                "image": "quay.io/trustyai_testing/lmeval-assets-flan-arceasy"
+                "@sha256:11cc9c2f38ac9cc26c4fab1a01a8c02db81c8f4801b5d2b2b90f90f91b97ac98"
+            },
             {"task_list": {"taskNames": ["arc_easy"]}},
         )
     ],
@@ -48,7 +51,10 @@ def test_lmeval_local_offline_builtin_tasks_flan_arceasy(
     [
         pytest.param(
             {"name": "lmevaljob-local-offline-unitxt"},
-            {"image": "quay.io/trustyai_testing/lmeval-assets-flan-20newsgroups:latest"},
+            {
+                "image": "quay.io/trustyai_testing/lmeval-assets-flan-20newsgroups"
+                "@sha256:3778c15079f11ef338a82ee35ae1aa43d6db52bac7bbfdeab343ccabe2608a0c"
+            },
             {
                 "task_list": {
                     "taskRecipes": [

tests/model_explainability/trustyai_service/conftest.py

@@ -0,0 +1,66 @@
+from typing import Any, Generator
+
+import pytest
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.inference_service import InferenceService
+from ocp_resources.namespace import Namespace
+from ocp_resources.secret import Secret
+from ocp_resources.serving_runtime import ServingRuntime
+from ocp_resources.trustyai_service import TrustyAIService
+
+from tests.model_explainability.trustyai_service.utils import wait_for_isvc_deployment_registered_by_trustyaiservice
+from utilities.constants import ModelFormat, KServeDeploymentType, RuntimeTemplates
+from utilities.inference_utils import create_isvc
+from utilities.serving_runtime import ServingRuntimeFromTemplate
+
+
+@pytest.fixture(scope="class")
+def ovms_runtime(
+    admin_client: DynamicClient, minio_data_connection: Secret, model_namespace: Namespace
+) -> Generator[ServingRuntime, Any, Any]:
+    with ServingRuntimeFromTemplate(
+        client=admin_client,
+        name=f"{ModelFormat.OVMS}-1.x",
+        namespace=model_namespace.name,
+        template_name=RuntimeTemplates.OVMS_KSERVE,
+        multi_model=False,
+        enable_http=False,
+        enable_grpc=True,
+        model_format_name={"name": ModelFormat.ONNX, "version": "1"},
+        runtime_image="quay.io/opendatahub/openvino_model_server"
+        "@sha256:564664371d3a21b9e732a5c1b4b40bacad714a5144c0a9aaf675baec4a04b148",
+    ) as sr:
+        yield sr
+
+
+@pytest.fixture(scope="class")
+def onnx_loan_model(
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    minio_data_connection: Secret,
+    ovms_runtime: ServingRuntime,
+    trustyai_service_with_pvc_storage: TrustyAIService,
+) -> Generator[InferenceService, Any, Any]:
+    with create_isvc(
+        client=admin_client,
+        name="demo-loan-nn-onnx-alpha",
+        namespace=model_namespace.name,
+        deployment_mode=KServeDeploymentType.SERVERLESS,
+        model_format=ModelFormat.ONNX,
+        runtime=ovms_runtime.name,
+        storage_key=minio_data_connection.name,
+        storage_path="ovms/loan_model_alpha",
+        min_replicas=1,
+        resources={"limits": {"cpu": "2", "memory": "8Gi"}, "requests": {"cpu": "1", "memory": "4Gi"}},
+        enable_auth=True,
+        model_version="1",
+        wait=True,
+        wait_for_predictor_pods=False,
+    ) as isvc:
+        wait_for_isvc_deployment_registered_by_trustyaiservice(
+            client=admin_client,
+            isvc=isvc,
+            trustyai_service=trustyai_service_with_pvc_storage,
+            runtime_name=ovms_runtime.name,
+        )
+        yield isvc

tests/model_explainability/trustyai_service/constants.py

@@ -0,0 +1 @@
+MODEL_DATA_PATH: str = "./tests/model_explainability/trustyai_service/model_data"

tests/model_explainability/trustyai_service/drift/conftest.py

@@ -38,7 +38,8 @@ def mlserver_runtime(
     containers = [
         {
             "name": "kserve-container",
-            "image": "quay.io/rh-ee-mmisiura/mlserver:1.6.1",
+            "image": "quay.io/trustyai_testing/mlserver"
+            "@sha256:68a4cd74fff40a3c4f29caddbdbdc9e54888aba54bf3c5f78c8ffd577c3a1c89",
             "env": [
                 {"name": "MLSERVER_MODEL_IMPLEMENTATION", "value": "{{.Labels.modelClass}}"},
                 {"name": "MLSERVER_HTTP_PORT", "value": "8080"},

tests/model_explainability/trustyai_service/drift/test_drift.py

@@ -1,5 +1,6 @@
 import pytest
 
+from tests.model_explainability.trustyai_service.constants import MODEL_DATA_PATH
 from tests.model_explainability.trustyai_service.utils import (
     send_inference_requests_and_verify_trustyai_service,
     verify_upload_data_to_trustyai_service,
@@ -10,7 +11,7 @@
 )
 from utilities.manifests.openvino import OPENVINO_KSERVE_INFERENCE_CONFIG
 
-BASE_DATA_PATH: str = "./tests/model_explainability/trustyai_service/drift/model_data"
+BASE_DATA_PATH: str = f"{MODEL_DATA_PATH}/drift/"
 
 
 @pytest.mark.parametrize(

tests/model_explainability/trustyai_service/explainers/test_explainers.py

@@ -0,0 +1,37 @@
+import pytest
+
+from tests.model_explainability.trustyai_service.constants import MODEL_DATA_PATH
+from tests.model_explainability.trustyai_service.explainers.utils import verify_shap_explanation
+from tests.model_explainability.trustyai_service.utils import send_inference_requests_and_verify_trustyai_service
+from utilities.manifests.openvino import OPENVINO_KSERVE_INFERENCE_CONFIG
+
+BASE_DATA_PATH: str = f"{MODEL_DATA_PATH}/fairness/"
+
+
+@pytest.mark.parametrize(
+    "model_namespace",
+    [
+        pytest.param(
+            {"name": "test-shap"},
+        )
+    ],
+    indirect=True,
+)
+def test_explainers_shap(
+    admin_client, current_client_token, model_namespace, trustyai_service_with_pvc_storage, onnx_loan_model
+):
+    send_inference_requests_and_verify_trustyai_service(
+        client=admin_client,
+        token=current_client_token,
+        data_path=f"{BASE_DATA_PATH}",
+        trustyai_service=trustyai_service_with_pvc_storage,
+        inference_service=onnx_loan_model,
+        inference_config=OPENVINO_KSERVE_INFERENCE_CONFIG,
+    )
+
+    verify_shap_explanation(
+        client=admin_client,
+        token=current_client_token,
+        trustyai_service=trustyai_service_with_pvc_storage,
+        inference_service=onnx_loan_model,
+    )

tests/model_explainability/trustyai_service/explainers/utils.py

@@ -0,0 +1,34 @@
+import json
+from typing import Any
+
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.inference_service import InferenceService
+from ocp_resources.trustyai_service import TrustyAIService
+from requests import Response
+from simple_logger.logger import get_logger
+
+from tests.model_explainability.trustyai_service.utils import TrustyAIServiceClient
+
+LOGGER = get_logger(name=__name__)
+
+
+def verify_shap_explanation(
+    client: DynamicClient, token: str, trustyai_service: TrustyAIService, inference_service: InferenceService
+) -> None:
+    tas_client: TrustyAIServiceClient = TrustyAIServiceClient(token=token, service=trustyai_service, client=client)
+
+    response: Response = tas_client.get_inference_ids(model_name=inference_service.name)
+    response_data: Any = json.loads(response.text)
+
+    if len(response_data) < 2:
+        raise ValueError(f"Not enough inferences available. Found only {len(response_data)}")
+
+    inference_id: str = response_data[-1]["id"]
+
+    response = tas_client.request_shap_explanation(
+        model_name=inference_service.name,
+        inference_id=inference_id,
+        target="knative-local-gateway.istio-system.svc.cluster.local:80",
+        n_samples=75,
+    )
+    LOGGER.error(response.text)

tests/model_explainability/trustyai_service/fairness/conftest.py

@@ -3,6 +3,7 @@
 import pytest
 from ocp_resources.inference_service import InferenceService
 
+from tests.model_explainability.trustyai_service.constants import MODEL_DATA_PATH
 from tests.model_explainability.trustyai_service.utils import (
     send_inference_requests_and_verify_trustyai_service,
     verify_name_mappings,
@@ -13,7 +14,7 @@
 )
 from utilities.manifests.openvino import OPENVINO_KSERVE_INFERENCE_CONFIG
 
-BASE_DATA_PATH: str = "./tests/model_explainability/trustyai_service/fairness/model_data"
+BASE_DATA_PATH: str = f"{MODEL_DATA_PATH}/fairness/"
 IS_MALE_IDENTIFYING: str = "Is Male-Identifying?"
 WILL_DEFAULT: str = "Will Default?"