Merge remote-tracking branch 'origin' into cherry-pick

Author: dbasunag

tests/conftest.py

@@ -336,5 +336,8 @@ def unprivileged_model_namespace(
         request.getfixturevalue(argname="enabled_modelmesh_in_dsc")
         ns_kwargs["model_mesh_enabled"] = True
 
+    if (_dashboard_label := request.param.get("dashboard-label")) is not None:
+        ns_kwargs["add_dashboard_label"] = _dashboard_label
+
     with create_ns(**ns_kwargs) as ns:
         yield ns

tests/model_explainability/lm_eval/conftest.py

@@ -1,6 +1,8 @@
-from typing import Generator
+from typing import Generator, Any
 
 import pytest
+from ocp_resources.route import Route
+from ocp_resources.service import Service
 from pytest import FixtureRequest
 from kubernetes.dynamic import DynamicClient
 from ocp_resources.config_map import ConfigMap
@@ -12,7 +14,25 @@
 from ocp_resources.resource import ResourceEditor
 from pytest_testconfig import py_config
 
-from utilities.constants import Labels, Timeout, Annotations
+from utilities.constants import Labels, Timeout, Annotations, Protocols
+
+VLLM_EMULATOR: str = "vllm-emulator"
+VLLM_EMULATOR_PORT: int = 8000
+LMEVALJOB_NAME: str = "lmeval-test-job"
+
+
+@pytest.fixture(scope="function")
+def lmevaljob_hf_pod(admin_client: DynamicClient, lmevaljob_hf: LMEvalJob) -> Generator[Pod, Any, Any]:
+    lmeval_pod = Pod(
+        client=admin_client,
+        namespace=lmevaljob_hf.namespace,
+        name=lmevaljob_hf.name,
+    )
+
+    # TODO: Check if we can rely on LMEvalJob instead of pod
+    lmeval_pod.wait(timeout=Timeout.TIMEOUT_2MIN)
+
+    yield lmeval_pod
 
 
 @pytest.fixture(scope="function")
@@ -26,9 +46,29 @@ def lmevaljob_hf(
         model="hf",
         model_args=[{"name": "pretrained", "value": "google/flan-t5-base"}],
         task_list={
+            "custom": {
+                "systemPrompts": [
+                    {"name": "sp_0", "value": "Be concise. At every point give the shortest acceptable answer."}
+                ],
+                "templates": [
+                    {
+                        "name": "tp_0",
+                        "value": '{ "__type__": "input_output_template", '
+                        '"input_format": "{text_a_type}: {text_a}\\n'
+                        '{text_b_type}: {text_b}", '
+                        '"output_format": "{label}", '
+                        '"target_prefix": '
+                        '"The {type_of_relation} class is ", '
+                        '"instruction": "Given a {text_a_type} and {text_b_type} '
+                        'classify the {type_of_relation} of the {text_b_type} to one of {classes}.",'
+                        ' "postprocessors": [ "processors.take_first_non_empty_line",'
+                        ' "processors.lower_case_till_punc" ] }',
+                    }
+                ],
+            },
             "taskRecipes": [
-                {"card": {"name": "cards.wnli"}, "template": "templates.classification.multi_class.relation.default"}
-            ]
+                {"card": {"name": "cards.wnli"}, "systemPrompt": {"ref": "sp_0"}, "template": {"ref": "tp_0"}}
+            ],
         },
         log_samples=True,
         allow_online=True,
@@ -44,10 +84,10 @@ def lmevaljob_local_offline(
     model_namespace: Namespace,
     patched_trustyai_operator_configmap_allow_online: ConfigMap,
     lmeval_data_downloader_pod: Pod,
-) -> Generator[LMEvalJob, None, None]:
+) -> Generator[LMEvalJob, Any, Any]:
     with LMEvalJob(
         client=admin_client,
-        name="lmeval-test",
+        name=LMEVALJOB_NAME,
         namespace=model_namespace.name,
         model="hf",
         model_args=[{"name": "pretrained", "value": "/opt/app-root/src/hf_home/flan"}],
@@ -68,7 +108,58 @@ def lmevaljob_local_offline(
 
 
 @pytest.fixture(scope="function")
-def patched_trustyai_operator_configmap_allow_online(admin_client: DynamicClient) -> Generator[ConfigMap, None, None]:
+def lmevaljob_vllm_emulator_pod(
+    admin_client: DynamicClient, lmevaljob_vllm_emulator: LMEvalJob
+) -> Generator[Pod, Any, Any]:
+    lmeval_pod = Pod(
+        client=admin_client,
+        namespace=lmevaljob_vllm_emulator.namespace,
+        name=lmevaljob_vllm_emulator.name,
+    )
+
+    # TODO: Check if we can rely on LMEvalJob instead of pod
+    lmeval_pod.wait(timeout=Timeout.TIMEOUT_2MIN)
+
+    yield lmeval_pod
+
+
+@pytest.fixture(scope="function")
+def lmevaljob_vllm_emulator(
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    patched_trustyai_operator_configmap_allow_online: ConfigMap,
+    vllm_emulator_deployment: Deployment,
+    vllm_emulator_service: Service,
+    vllm_emulator_route: Route,
+) -> Generator[LMEvalJob, Any, Any]:
+    with LMEvalJob(
+        client=admin_client,
+        namespace=model_namespace.name,
+        name=LMEVALJOB_NAME,
+        model="local-completions",
+        task_list={"taskNames": ["arc_easy"]},
+        log_samples=True,
+        batch_size="1",
+        allow_online=True,
+        allow_code_execution=False,
+        outputs={"pvcManaged": {"size": "5Gi"}},
+        model_args=[
+            {"name": "model", "value": "emulatedModel"},
+            {
+                "name": "base_url",
+                "value": f"http://{vllm_emulator_service.name}:{str(VLLM_EMULATOR_PORT)}/v1/completions",
+            },
+            {"name": "num_concurrent", "value": "1"},
+            {"name": "max_retries", "value": "3"},
+            {"name": "tokenized_requests", "value": "False"},
+            {"name": "tokenizer", "value": "ibm-granite/granite-guardian-3.1-8b"},
+        ],
+    ) as job:
+        yield job
+
+
+@pytest.fixture(scope="function")
+def patched_trustyai_operator_configmap_allow_online(admin_client: DynamicClient) -> Generator[ConfigMap, Any, Any]:
     namespace: str = py_config["applications_namespace"]
     trustyai_service_operator: str = "trustyai-service-operator"
 
@@ -99,7 +190,7 @@ def patched_trustyai_operator_configmap_allow_online(admin_client: DynamicClient
 @pytest.fixture(scope="function")
 def lmeval_data_pvc(
     admin_client: DynamicClient, model_namespace: Namespace
-) -> Generator[PersistentVolumeClaim, None, None]:
+) -> Generator[PersistentVolumeClaim, Any, Any]:
     with PersistentVolumeClaim(
         client=admin_client,
         name="lmeval-data",
@@ -117,7 +208,7 @@ def lmeval_data_downloader_pod(
     admin_client: DynamicClient,
     model_namespace: Namespace,
     lmeval_data_pvc: PersistentVolumeClaim,
-) -> Generator[Pod, None, None]:
+) -> Generator[Pod, Any, Any]:
     with Pod(
         client=admin_client,
         namespace=model_namespace.name,
@@ -143,3 +234,76 @@ def lmeval_data_downloader_pod(
     ) as pod:
         pod.wait_for_status(status=Pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_10MIN)
         yield pod
+
+
+@pytest.fixture(scope="function")
+def vllm_emulator_deployment(
+    admin_client: DynamicClient, model_namespace: Namespace
+) -> Generator[Deployment, Any, Any]:
+    label = {"app": VLLM_EMULATOR}
+    with Deployment(
+        client=admin_client,
+        namespace=model_namespace.name,
+        name=VLLM_EMULATOR,
+        label=label,
+        selector={"matchLabels": label},
+        template={
+            "metadata": {
+                "labels": {
+                    "app": VLLM_EMULATOR,
+                    "maistra.io/expose-route": "true",
+                },
+                "name": VLLM_EMULATOR,
+            },
+            "spec": {
+                "containers": [
+                    {
+                        "image": "quay.io/trustyai_testing/vllm_emulator"
+                        "@sha256:4214f31bff9de6cc723da23324fb8974cea8abadcab621d85a97a3503cabbdc6",
+                        "name": "vllm-emulator",
+                        "securityContext": {
+                            "allowPrivilegeEscalation": False,
+                            "capabilities": {"drop": ["ALL"]},
+                            "seccompProfile": {"type": "RuntimeDefault"},
+                        },
+                    }
+                ]
+            },
+        },
+        replicas=1,
+    ) as deployment:
+        yield deployment
+
+
+@pytest.fixture(scope="function")
+def vllm_emulator_service(
+    admin_client: DynamicClient, model_namespace: Namespace, vllm_emulator_deployment: Deployment
+) -> Generator[Service, Any, Any]:
+    with Service(
+        client=admin_client,
+        namespace=vllm_emulator_deployment.namespace,
+        name=f"{VLLM_EMULATOR}-service",
+        ports=[
+            {
+                "name": f"{VLLM_EMULATOR}-endpoint",
+                "port": VLLM_EMULATOR_PORT,
+                "protocol": Protocols.TCP,
+                "targetPort": VLLM_EMULATOR_PORT,
+            }
+        ],
+        selector={"app": VLLM_EMULATOR},
+    ) as service:
+        yield service
+
+
+@pytest.fixture(scope="function")
+def vllm_emulator_route(
+    admin_client: DynamicClient, model_namespace: Namespace, vllm_emulator_service: Service
+) -> Generator[Route, Any, Any]:
+    with Route(
+        client=admin_client,
+        namespace=vllm_emulator_service.namespace,
+        name=VLLM_EMULATOR,
+        service=vllm_emulator_service.name,
+    ) as route:
+        yield route

tests/model_explainability/lm_eval/test_lm_eval.py

@@ -1,5 +1,4 @@
 import pytest
-from ocp_resources.pod import Pod
 
 from tests.model_explainability.lm_eval.utils import verify_lmevaljob_running
 from utilities.constants import Timeout
@@ -9,25 +8,22 @@
     "model_namespace",
     [
         pytest.param(
-            {"name": "lmevaljob-hf"},
+            {"name": "test-lmeval-huggingface"},
         )
     ],
     indirect=True,
 )
 @pytest.mark.smoke
-def test_lmeval_huggingface_model(admin_client, model_namespace, lmevaljob_hf):
+def test_lmeval_huggingface_model(admin_client, model_namespace, lmevaljob_hf_pod):
     """Basic test that verifies that LMEval can run successfully pulling a model from HuggingFace."""
-    lmevaljob_pod = Pod(
-        client=admin_client, name=lmevaljob_hf.name, namespace=lmevaljob_hf.namespace, wait_for_resource=True
-    )
-    lmevaljob_pod.wait_for_status(status=lmevaljob_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_10MIN)
+    lmevaljob_hf_pod.wait_for_status(status=lmevaljob_hf_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_10MIN)
 
 
 @pytest.mark.parametrize(
     "model_namespace, lmeval_data_downloader_pod, lmevaljob_local_offline",
     [
         pytest.param(
-            {"name": "lmevaljob-local-offline-builtin"},
+            {"name": "test-lmeval-local-offline-builtin"},
             {
                 "image": "quay.io/trustyai_testing/lmeval-assets-flan-arceasy"
                 "@sha256:11cc9c2f38ac9cc26c4fab1a01a8c02db81c8f4801b5d2b2b90f90f91b97ac98"
@@ -51,7 +47,7 @@ def test_lmeval_local_offline_builtin_tasks_flan_arceasy(
     "model_namespace, lmeval_data_downloader_pod, lmevaljob_local_offline",
     [
         pytest.param(
-            {"name": "lmevaljob-local-offline-unitxt"},
+            {"name": "test-lmeval-local-offline-unitxt"},
             {
                 "image": "quay.io/trustyai_testing/lmeval-assets-flan-20newsgroups"
                 "@sha256:3778c15079f11ef338a82ee35ae1aa43d6db52bac7bbfdeab343ccabe2608a0c"
@@ -61,7 +57,7 @@ def test_lmeval_local_offline_builtin_tasks_flan_arceasy(
                     "taskRecipes": [
                         {
                             "card": {"name": "cards.20_newsgroups_short"},
-                            "template": "templates.classification.multi_class.title",
+                            "template": {"name": "templates.classification.multi_class.title"},
                         }
                     ]
                 }
@@ -78,3 +74,19 @@ def test_lmeval_local_offline_unitxt_tasks_flan_20newsgroups(
 ):
     """Test that verifies that LMEval can run successfully in local, offline mode using unitxt"""
     verify_lmevaljob_running(client=admin_client, lmevaljob=lmevaljob_local_offline)
+
+
+@pytest.mark.parametrize(
+    "model_namespace",
+    [
+        pytest.param(
+            {"name": "test-lmeval-vllm"},
+        )
+    ],
+    indirect=True,
+)
+def test_lmeval_vllm_emulator(admin_client, model_namespace, lmevaljob_vllm_emulator_pod):
+    """Basic test that verifies LMEval works with vLLM using a vLLM emulator for more efficient evaluation"""
+    lmevaljob_vllm_emulator_pod.wait_for_status(
+        status=lmevaljob_vllm_emulator_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_10MIN
+    )

tests/model_explainability/trustyai_service/conftest.py

@@ -66,6 +66,7 @@ def trustyai_service_with_db_storage(
     cluster_monitoring_config: ConfigMap,
     user_workload_monitoring_config: ConfigMap,
     mariadb: MariaDB,
+    trustyai_db_ca_secret: None,
 ) -> Generator[TrustyAIService, Any, Any]:
     with TrustyAIService(
         client=admin_client,
@@ -226,12 +227,28 @@ def mariadb(
     mariadb_dict["spec"]["replicas"] = 1
     mariadb_dict["spec"]["galera"]["enabled"] = False
     mariadb_dict["spec"]["metrics"]["enabled"] = False
+    mariadb_dict["spec"]["tls"] = {"enabled": True, "required": True}
 
     password_secret_key_ref = {"generate": False, "key": "databasePassword", "name": DB_CREDENTIALS_SECRET_NAME}
 
     mariadb_dict["spec"]["rootPasswordSecretKeyRef"] = password_secret_key_ref
     mariadb_dict["spec"]["passwordSecretKeyRef"] = password_secret_key_ref
-
     with MariaDB(kind_dict=mariadb_dict) as mariadb:
         wait_for_mariadb_pods(client=admin_client, mariadb=mariadb)
         yield mariadb
+
+
+@pytest.fixture(scope="class")
+def trustyai_db_ca_secret(
+    admin_client: DynamicClient, model_namespace: Namespace, mariadb: MariaDB
+) -> Generator[None, Any, None]:
+    mariadb_ca_secret = Secret(
+        client=admin_client, name=f"{mariadb.name}-ca", namespace=model_namespace.name, ensure_exists=True
+    )
+    with Secret(
+        client=admin_client,
+        name=f"{TRUSTYAI_SERVICE_NAME}-db-ca",
+        namespace=model_namespace.name,
+        data_dict={"ca.crt": mariadb_ca_secret.instance.data["ca.crt"]},
+    ):
+        yield