Merge branch 'main' into oom

Author: dbasunag

.pre-commit-config.yaml

@@ -36,7 +36,7 @@ repos:
         exclude: .*/__snapshots__/.*|.*-input\.json$
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.13.0
+    rev: v0.13.2
     hooks:
       - id: ruff
       - id: ruff-format
@@ -55,7 +55,7 @@ repos:
       - id: gitleaks
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.18.1
+    rev: v1.18.2
     hooks:
       - id: mypy
         additional_dependencies: ["types-PyYAML", "types-requests"]

conftest.py

@@ -19,6 +19,7 @@
     Config,
     CollectReport,
 )
+from _pytest.nodes import Node
 from _pytest.terminal import TerminalReporter
 from typing import Optional, Any
 from pytest_testconfig import config as py_config
@@ -434,9 +435,17 @@ def calculate_must_gather_timer(test_start_time: int) -> int:
         return default_duration
 
 
+def get_all_node_markers(node: Node) -> list[str]:
+    return [mark.name for mark in list(node.iter_markers())]
+
+
+def is_skip_must_gather(node: Node) -> bool:
+    return "skip_must_gather" in get_all_node_markers(node=node)
+
+
 def pytest_exception_interact(node: Item | Collector, call: CallInfo[Any], report: TestReport | CollectReport) -> None:
     LOGGER.error(report.longreprtext)
-    if node.config.getoption("--collect-must-gather"):
+    if node.config.getoption("--collect-must-gather") and not is_skip_must_gather(node=node):
         test_name = f"{node.fspath}::{node.name}"
         LOGGER.info(f"Must-gather collection is enabled for {test_name}.")
 

pyproject.toml

@@ -69,8 +69,9 @@ dependencies = [
     "marshmallow==3.26.1,<4", # this version is needed for pytest-jira
     "pytest-html>=4.1.1",
     "fire",
-    "llama_stack_client==0.2.21",
+    "llama_stack_client==0.2.23",
     "pytest-xdist==3.8.0",
+    "dictdiffer>=0.9.0",
 ]
 
 [project.urls]

pytest.ini

@@ -20,6 +20,7 @@ markers =
     ocp_interop: Interop testing with Openshift.
     downstream_only: Tests that are specific to downstream
     cluster_health: Tests that verifies that cluster is healthy to begin testing
+    skip_must_gather: Tests that does not require must-gather for triaging
 
     # Model server
     modelmesh: Mark tests which are model mesh tests
@@ -33,6 +34,7 @@ markers =
     gpu: Mark tests which require GPU resources
     multinode: Mark tests which require multiple nodes
     keda: Mark tests which are testing KEDA scaling
+    llmd_cpu: Mark tests which are testing LLMD (LLM Deployment) with CPU resources
 
     # Model Registry:
     custom_namespace: mark tests that are to be run with custom namespace

tests/conftest.py

@@ -517,6 +517,7 @@ def minio_pod(
         label=pod_labels,
         annotations=request.param.get("annotations"),
     ) as minio_pod:
+        minio_pod.wait_for_status(status=Pod.Status.RUNNING)
         yield minio_pod
 
 

tests/fixtures/trustyai.py

@@ -1,14 +1,15 @@
 import pytest
 from kubernetes.dynamic import DynamicClient
+from ocp_resources.data_science_cluster import DataScienceCluster
 from ocp_resources.deployment import Deployment
 
-from typing import Generator, Any
+from typing import Generator
 
-from ocp_resources.config_map import ConfigMap
 from ocp_resources.resource import ResourceEditor
 from pytest_testconfig import py_config
 
-from utilities.constants import Annotations, TRUSTYAI_SERVICE_NAME
+from utilities.constants import TRUSTYAI_SERVICE_NAME
+from utilities.infra import get_data_science_cluster
 
 
 @pytest.fixture(scope="class")
@@ -21,36 +22,32 @@ def trustyai_operator_deployment(admin_client: DynamicClient) -> Deployment:
     )
 
 
-@pytest.fixture(scope="function")
-def patched_trustyai_configmap_allow_online(
-    admin_client: DynamicClient, trustyai_operator_deployment: Deployment
-) -> Generator[ConfigMap, Any, Any]:
-    """
-    Patches the TrustyAI Operator ConfigMap in order to set allowOnline and allowCodeExecution to true.
-    These options are needed to run some LMEval tasks, which rely on having access to the internet
-    and running arbitrary code. The deployment needs to be restarted in order for these changes to be applied.
-    """
-    trustyai_service_operator: str = "trustyai-service-operator"
-
-    configmap: ConfigMap = ConfigMap(
-        client=admin_client,
-        name=f"{trustyai_service_operator}-config",
-        namespace=py_config["applications_namespace"],
-        ensure_exists=True,
-    )
+@pytest.fixture(scope="class")
+def patched_dsc_lmeval_allow_all(
+    admin_client, trustyai_operator_deployment: Deployment
+) -> Generator[DataScienceCluster, None, None]:
+    """Enable LMEval PermitOnline and PermitCodeExecution flags in the Datascience cluster."""
+    dsc = get_data_science_cluster(client=admin_client)
     with ResourceEditor(
         patches={
-            configmap: {
-                "metadata": {"annotations": {Annotations.OpenDataHubIo.MANAGED: "false"}},
-                "data": {
-                    "lmes-allow-online": "true",
-                    "lmes-allow-code-execution": "true",
-                },
+            dsc: {
+                "spec": {
+                    "components": {
+                        "trustyai": {
+                            "eval": {
+                                "lmeval": {
+                                    "permitCodeExecution": "allow",
+                                    "permitOnline": "allow",
+                                }
+                            }
+                        }
+                    }
+                }
             }
         }
     ):
         num_replicas: int = trustyai_operator_deployment.replicas
         trustyai_operator_deployment.scale_replicas(replica_count=0)
         trustyai_operator_deployment.scale_replicas(replica_count=num_replicas)
         trustyai_operator_deployment.wait_for_replicas()
-        yield configmap
+        yield dsc

tests/llama_stack/conftest.py

@@ -40,8 +40,7 @@ def enabled_llama_stack_operator(dsc_resource: DataScienceCluster) -> Generator[
 @pytest.fixture(scope="class")
 def llama_stack_server_config(
     request: FixtureRequest,
-    admin_client: DynamicClient,
-    model_namespace: Namespace,
+    unprivileged_model_namespace: Namespace,
 ) -> Dict[str, Any]:
     fms_orchestrator_url = "http://localhost"
     inference_model = os.getenv("LLS_CORE_INFERENCE_MODEL", "")
@@ -100,15 +99,15 @@ def llama_stack_server_config(
 
 @pytest.fixture(scope="class")
 def llama_stack_distribution(
-    admin_client: DynamicClient,
-    model_namespace: Namespace,
+    unprivileged_client: DynamicClient,
+    unprivileged_model_namespace: Namespace,
     enabled_llama_stack_operator: DataScienceCluster,
     llama_stack_server_config: Dict[str, Any],
 ) -> Generator[LlamaStackDistribution, None, None]:
     with create_llama_stack_distribution(
-        client=admin_client,
-        name="llama-stack-distribution",
-        namespace=model_namespace.name,
+        client=unprivileged_client,
+        name="test-lama-stack-distribution",
+        namespace=unprivileged_model_namespace.name,
         replicas=1,
         server=llama_stack_server_config,
     ) as lls_dist:
@@ -118,11 +117,11 @@ def llama_stack_distribution(
 
 @pytest.fixture(scope="class")
 def llama_stack_distribution_deployment(
-    admin_client: DynamicClient,
+    unprivileged_client: DynamicClient,
     llama_stack_distribution: LlamaStackDistribution,
 ) -> Generator[Deployment, Any, Any]:
     deployment = Deployment(
-        client=admin_client,
+        client=unprivileged_client,
         namespace=llama_stack_distribution.namespace,
         name=llama_stack_distribution.name,
     )
@@ -133,15 +132,13 @@ def llama_stack_distribution_deployment(
 
 @pytest.fixture(scope="class")
 def llama_stack_client(
-    admin_client: DynamicClient,
     llama_stack_distribution_deployment: Deployment,
 ) -> Generator[LlamaStackClient, Any, Any]:
     """
     Returns a ready to use LlamaStackClient,  enabling port forwarding
     from the llama-stack-server service:8321 to localhost:8321
 
     Args:
-        admin_client (DynamicClient): Kubernetes dynamic client for cluster operations
         llama_stack_distribution_deployment (Deployment): LlamaStack distribution deployment resource
 
     Yields:

tests/llama_stack/constants.py

@@ -16,6 +16,9 @@ class Eval(str, Enum):
         TRUSTYAI_LMEVAL = "trustyai_lmeval"
 
 
+LLS_CORE_POD_FILTER: str = "app=llama-stack"
+
+
 @dataclass
 class TorchTuneTestExpectation:
     """Test expectation for TorchTune documentation questions."""

tests/llama_stack/core/test_llamastack_core.py

@@ -17,7 +17,7 @@
             {
                 "vllm_url_fixture": "qwen_isvc_url",
                 "inference_model": QWEN_MODEL_NAME,
-                "llama_stack_storage_size": "10Gi",
+                "llama_stack_storage_size": "2Gi",
             },
         )
     ],

tests/llama_stack/eval/test_lmeval_provider.py

@@ -52,7 +52,7 @@ def test_lmeval_register_benchmark(self, minio_pod, minio_data_connection, llama
         assert benchmarks[0].provider_id == LlamaStackProviders.Eval.TRUSTYAI_LMEVAL
 
     def test_llamastack_run_eval(
-        self, minio_pod, minio_data_connection, patched_trustyai_configmap_allow_online, llama_stack_client
+        self, minio_pod, minio_data_connection, patched_dsc_lmeval_allow_all, llama_stack_client
     ):
         job = llama_stack_client.eval.run_eval(
             benchmark_id=TRUSTYAI_LMEVAL_ARCEASY,