[AI Explainability] TrustyAI DSC based LMEval configuration (#672) (#673)

* refactor: replace TrustyAI ConfigMap patching with DataScienceCluster config

* Apply suggestion from @adolfo-ab



* refactor: remove utils file and fn

* refactor: remove unused file and code

---------

Co-authored-by: Shelton Cyril <sheltoncyril@gmail.com>
Co-authored-by: Adolfo Aguirrezabal <aaguirre@redhat.com>

Author: 3 people

tests/fixtures/trustyai.py

@@ -1,14 +1,15 @@
 import pytest
 from kubernetes.dynamic import DynamicClient
+from ocp_resources.data_science_cluster import DataScienceCluster
 from ocp_resources.deployment import Deployment
 
-from typing import Generator, Any
+from typing import Generator
 
-from ocp_resources.config_map import ConfigMap
 from ocp_resources.resource import ResourceEditor
 from pytest_testconfig import py_config
 
-from utilities.constants import Annotations, TRUSTYAI_SERVICE_NAME
+from utilities.constants import TRUSTYAI_SERVICE_NAME
+from utilities.infra import get_data_science_cluster
 
 
 @pytest.fixture(scope="class")
@@ -21,36 +22,32 @@ def trustyai_operator_deployment(admin_client: DynamicClient) -> Deployment:
     )
 
 
-@pytest.fixture(scope="function")
-def patched_trustyai_configmap_allow_online(
-    admin_client: DynamicClient, trustyai_operator_deployment: Deployment
-) -> Generator[ConfigMap, Any, Any]:
-    """
-    Patches the TrustyAI Operator ConfigMap in order to set allowOnline and allowCodeExecution to true.
-    These options are needed to run some LMEval tasks, which rely on having access to the internet
-    and running arbitrary code. The deployment needs to be restarted in order for these changes to be applied.
-    """
-    trustyai_service_operator: str = "trustyai-service-operator"
-
-    configmap: ConfigMap = ConfigMap(
-        client=admin_client,
-        name=f"{trustyai_service_operator}-config",
-        namespace=py_config["applications_namespace"],
-        ensure_exists=True,
-    )
+@pytest.fixture(scope="class")
+def patched_dsc_lmeval_allow_all(
+    admin_client, trustyai_operator_deployment: Deployment
+) -> Generator[DataScienceCluster, None, None]:
+    """Enable LMEval PermitOnline and PermitCodeExecution flags in the Datascience cluster."""
+    dsc = get_data_science_cluster(client=admin_client)
     with ResourceEditor(
         patches={
-            configmap: {
-                "metadata": {"annotations": {Annotations.OpenDataHubIo.MANAGED: "false"}},
-                "data": {
-                    "lmes-allow-online": "true",
-                    "lmes-allow-code-execution": "true",
-                },
+            dsc: {
+                "spec": {
+                    "components": {
+                        "trustyai": {
+                            "eval": {
+                                "lmeval": {
+                                    "permitCodeExecution": "allow",
+                                    "permitOnline": "allow",
+                                }
+                            }
+                        }
+                    }
+                }
             }
         }
     ):
         num_replicas: int = trustyai_operator_deployment.replicas
         trustyai_operator_deployment.scale_replicas(replica_count=0)
         trustyai_operator_deployment.scale_replicas(replica_count=num_replicas)
         trustyai_operator_deployment.wait_for_replicas()
-        yield configmap
+        yield dsc

tests/llama_stack/eval/test_lmeval_provider.py

@@ -52,7 +52,7 @@ def test_lmeval_register_benchmark(self, minio_pod, minio_data_connection, llama
         assert benchmarks[0].provider_id == LlamaStackProviders.Eval.TRUSTYAI_LMEVAL
 
     def test_llamastack_run_eval(
-        self, minio_pod, minio_data_connection, patched_trustyai_configmap_allow_online, llama_stack_client
+        self, minio_pod, minio_data_connection, patched_dsc_lmeval_allow_all, llama_stack_client
     ):
         job = llama_stack_client.eval.run_eval(
             benchmark_id=TRUSTYAI_LMEVAL_ARCEASY,

tests/model_explainability/lm_eval/conftest.py

@@ -2,7 +2,7 @@
 
 import pytest
 from kubernetes.dynamic import DynamicClient
-from ocp_resources.config_map import ConfigMap
+from ocp_resources.data_science_cluster import DataScienceCluster
 from ocp_resources.deployment import Deployment
 from ocp_resources.lm_eval_job import LMEvalJob
 from ocp_resources.namespace import Namespace
@@ -27,7 +27,7 @@ def lmevaljob_hf(
     request: FixtureRequest,
     admin_client: DynamicClient,
     model_namespace: Namespace,
-    patched_trustyai_configmap_allow_online: ConfigMap,
+    patched_dsc_lmeval_allow_all: DataScienceCluster,
     lmeval_hf_access_token: Secret,
 ) -> Generator[LMEvalJob, None, None]:
     with LMEvalJob(
@@ -74,7 +74,7 @@ def lmevaljob_local_offline(
     request: FixtureRequest,
     admin_client: DynamicClient,
     model_namespace: Namespace,
-    patched_trustyai_configmap_allow_online: ConfigMap,
+    patched_dsc_lmeval_allow_all: DataScienceCluster,
     lmeval_data_downloader_pod: Pod,
 ) -> Generator[LMEvalJob, Any, Any]:
     with LMEvalJob(
@@ -104,7 +104,7 @@ def lmevaljob_local_offline(
 def lmevaljob_vllm_emulator(
     admin_client: DynamicClient,
     model_namespace: Namespace,
-    patched_trustyai_configmap_allow_online: ConfigMap,
+    patched_dsc_lmeval_allow_all: DataScienceCluster,
     vllm_emulator_deployment: Deployment,
     vllm_emulator_service: Service,
     vllm_emulator_route: Route,