Merge branch 'main' into refactor-llmd

Author: threcc

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ minimum_pre_commit_version: 3.3.0
 default_install_hook_types: [pre-commit, commit-msg]
 
 default_language_version:
-  python: python3.13
+  python: python3.14
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -36,7 +36,7 @@ repos:
         exclude: .*/__snapshots__/.*|.*-input\.json$|^semgrep\.yaml$
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.4
+    rev: v0.15.5
     hooks:
       - id: ruff
       - id: ruff-format

Dockerfile

@@ -1,19 +1,19 @@
-FROM fedora:42
+FROM fedora:43
 
 ARG USER=odh
 ARG HOME=/home/$USER
 ARG TESTS_DIR=$HOME/opendatahub-tests/
-ENV UV_PYTHON=python3.13
+ENV UV_PYTHON=python3.14
 ENV UV_COMPILE_BYTECODE=1
 ENV UV_NO_SYNC=1
 ENV UV_NO_CACHE=1
 
 ENV BIN_DIR="$HOME_DIR/.local/bin"
 ENV PATH="$PATH:$BIN_DIR"
 
-# Install Python 3.13 and other dependencies using dnf
+# Install system dependencies using dnf
 RUN dnf update -y \
-    && dnf install -y python3.13 python3.13-pip ssh gnupg curl gpg wget vim httpd-tools rsync openssl openssl-devel\
+    && dnf install -y python3 python3-pip ssh gnupg curl gpg wget vim httpd-tools rsync openssl openssl-devel\
     && dnf clean all \
     && rm -rf /var/cache/dnf
 

pyproject.toml

@@ -31,7 +31,7 @@ dev = [
 ]
 
 [project]
-requires-python = "==3.13.*"
+requires-python = "==3.14.*"
 name = "opendatahub-tests"
 version = "0.1.0"
 description = "Tests repository for Open Data Hub (ODH)"

pytest.ini

@@ -10,10 +10,11 @@ markers =
     parallel: marks tests that can run in parallel along with pytest-xdist
 
     # CI
-    smoke: Mark tests as smoke tests; covers core functionality of the product. Aims to ensure that the build is stable enough for further testing.
-    sanity: Mark tests as sanity tests. Aims to verify that specific functionality is working as expected.
-    tier1: Mark tests as tier1. Aims to cover frequently used functionality of the product and basic user flows.
-    tier2: Mark tests as tier2. Aims to cover more advanced functionality of the product.
+    smoke: Mark tests as smoke tests; very high critical priority tests. Covers core functionality of the product. Aims to ensure that the build is stable enough for further testing.
+    sanity: <<DEPRECATION WARNING: to be superseded by tier1>> Mark tests as sanity tests. Aims to verify that specific functionality is working as expected.
+    tier1: Mark tests as tier1. High-priority tests.
+    tier2: Mark tests as tier2. Medium/low-priority positive tests.
+    tier3: Mark tests as tier3. Negative and destructive tests.
     slow: Mark tests which take more than 10 minutes as slow tests.
     pre_upgrade: Mark tests which should be run before upgrading the product.
     post_upgrade: Mark tests which should be run after upgrading the product.

tests/conftest.py

@@ -676,6 +676,12 @@ def cluster_sanity_scope_session(
     dsc_resource: DataScienceCluster,
     junitxml_plugin: Callable[[str, object], None],
 ) -> None:
+    # Skip cluster sanity check when running tests that have cluster_health or operator_health markers
+    selected_markers = {mark.name for item in request.session.items for mark in item.iter_markers()}
+    if {"cluster_health", "operator_health"} & selected_markers:
+        LOGGER.info("Skipping cluster sanity check because selected tests include cluster/operator health")
+        return
+
     verify_cluster_sanity(
         request=request,
         nodes=nodes,
@@ -829,7 +835,7 @@ def gpu_count_on_cluster(nodes: list[Any]) -> int:
             if key in allowed_exact or any(key.startswith(p) for p in allowed_prefixes):
                 try:
                     total_gpus += int(val)
-                except (ValueError, TypeError):
+                except ValueError, TypeError:
                     LOGGER.debug(f"Skipping non-integer allocatable for {key} on {node.name}: {val!r}")
                     continue
     return total_gpus

tests/llama_stack/safety/test_trustyai_fms_provider.py

@@ -9,6 +9,7 @@
 SECURE_SHIELD_ID: str = "secure_shield"
 
 
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace, minio_pod, minio_data_connection, "
     "orchestrator_config, guardrails_orchestrator, llama_stack_server_config",

tests/model_explainability/guardrails/test_guardrails.py

@@ -38,6 +38,7 @@
 LOGGER = get_logger(name=__name__)
 
 
+@pytest.mark.smoke
 @pytest.mark.parametrize(
     "model_namespace, orchestrator_config, guardrails_orchestrator",
     [
@@ -56,7 +57,6 @@
     ],
     indirect=True,
 )
-@pytest.mark.smoke
 def test_validate_guardrails_orchestrator_images(
     model_namespace,
     orchestrator_config,
@@ -70,6 +70,7 @@ def test_validate_guardrails_orchestrator_images(
     validate_tai_component_images(pod=guardrails_orchestrator_pod, tai_operator_configmap=trustyai_operator_configmap)
 
 
+@pytest.mark.smoke
 @pytest.mark.parametrize(
     "model_namespace, orchestrator_config, guardrails_gateway_config, guardrails_orchestrator",
     [
@@ -115,7 +116,6 @@ def test_validate_guardrails_orchestrator_images(
     ],
     indirect=True,
 )
-@pytest.mark.smoke
 @pytest.mark.rawdeployment
 @pytest.mark.usefixtures("patched_dsc_kserve_headed", "guardrails_gateway_config")
 class TestGuardrailsOrchestratorWithBuiltInDetectors:
@@ -212,6 +212,7 @@ def test_guardrails_builtin_detectors_negative_detection(
         )
 
 
+@pytest.mark.smoke
 @pytest.mark.parametrize(
     "model_namespace, orchestrator_config, guardrails_gateway_config,guardrails_orchestrator",
     [
@@ -421,6 +422,7 @@ def check_traces():
     indirect=True,
 )
 @pytest.mark.usefixtures("patched_dsc_kserve_headed")
+@pytest.mark.tier1
 @pytest.mark.rawdeployment
 class TestGuardrailsOrchestratorAutoConfig:
     """
@@ -495,6 +497,7 @@ def test_guardrails_autoconfig_negative_detection(
     indirect=True,
 )
 @pytest.mark.usefixtures("patched_dsc_kserve_headed")
+@pytest.mark.tier2
 @pytest.mark.rawdeployment
 class TestGuardrailsOrchestratorAutoConfigWithGateway:
     """

tests/model_explainability/lm_eval/test_lm_eval.py

@@ -28,17 +28,14 @@
 
 
 @pytest.mark.skip_on_disconnected
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace, lmevaljob_hf",
     [
         pytest.param(
             {"name": "test-lmeval-hf-tier1"},
             {"task_list": {"taskNames": TIER1_LMEVAL_TASKS}},
         ),
-        pytest.param(
-            {"name": "test-lmeval-hf-tier2"},
-            {"task_list": {"taskNames": TIER2_LMEVAL_TASKS}},
-        ),
         pytest.param(
             {"name": "test-lmeval-hf-custom-task"},
             CUSTOM_UNITXT_TASK_DATA,
@@ -58,6 +55,24 @@ def test_lmeval_huggingface_model(admin_client, model_namespace, lmevaljob_hf_po
     validate_lmeval_job_pod_and_logs(lmevaljob_pod=lmevaljob_hf_pod)
 
 
+@pytest.mark.skip_on_disconnected
+@pytest.mark.tier2
+@pytest.mark.parametrize(
+    "model_namespace, lmevaljob_hf",
+    [
+        pytest.param(
+            {"name": "test-lmeval-hf-tier2"},
+            {"task_list": {"taskNames": TIER2_LMEVAL_TASKS}},
+        ),
+    ],
+    indirect=True,
+)
+def test_lmeval_huggingface_model_tier2(admin_client, model_namespace, lmevaljob_hf_pod):
+    """Tests that verify running common evaluations (and a custom one) on a model pulled directly from HuggingFace.
+    On each test we run a different evaluation task, limiting it to 0.5% of the questions on each eval."""
+    validate_lmeval_job_pod_and_logs(lmevaljob_pod=lmevaljob_hf_pod)
+
+
 @pytest.mark.parametrize(
     "model_namespace, lmeval_data_downloader_pod, lmevaljob_local_offline",
     [
@@ -80,6 +95,7 @@ def test_lmeval_local_offline_builtin_tasks_flan_arceasy(
     validate_lmeval_job_pod_and_logs(lmevaljob_pod=lmevaljob_local_offline_pod)
 
 
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace",
     [
@@ -95,6 +111,7 @@ def test_lmeval_vllm_emulator(admin_client, model_namespace, lmevaljob_vllm_emul
     validate_lmeval_job_pod_and_logs(lmevaljob_pod=lmevaljob_vllm_emulator_pod)
 
 
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace, minio_data_connection",
     [
@@ -138,6 +155,7 @@ def test_verify_lmeval_pod_images(lmevaljob_s3_offline_pod, trustyai_operator_co
     )
 
 
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace, oci_registry_pod_with_minio, lmeval_data_downloader_pod, lmevaljob_local_offline_oci",
     [

tests/model_explainability/trustyai_service/service/test_trustyai_service.py

@@ -22,6 +22,7 @@
 from utilities.constants import MinIo
 
 
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace",
     [
@@ -45,6 +46,7 @@ def test_trustyai_service_with_invalid_db_cert(
     )
 
 
+@pytest.mark.smoke
 @pytest.mark.parametrize(
     "model_namespace, trustyai_service",
     [
@@ -55,7 +57,6 @@ def test_trustyai_service_with_invalid_db_cert(
     ],
     indirect=True,
 )
-@pytest.mark.smoke
 def test_validate_trustyai_service_image(
     admin_client,
     model_namespace: Namespace,
@@ -72,6 +73,7 @@ def test_validate_trustyai_service_image(
     )
 
 
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace, minio_pod, minio_data_connection, trustyai_service",
     [

tests/model_registry/conftest.py

@@ -4,6 +4,7 @@
 from typing import Any
 
 import pytest
+import yaml
 from kubernetes.dynamic import DynamicClient
 from ocp_resources.config_map import ConfigMap
 from ocp_resources.data_science_cluster import DataScienceCluster
@@ -25,18 +26,26 @@
 from tests.model_registry.constants import (
     DB_BASE_RESOURCES_NAME,
     DB_RESOURCE_NAME,
+    DEFAULT_CUSTOM_MODEL_CATALOG,
     KUBERBACPROXY_STR,
     MR_INSTANCE_BASE_NAME,
     MR_INSTANCE_NAME,
     MR_OPERATOR_NAME,
 )
+from tests.model_registry.mcp_servers.constants import (
+    MCP_CATALOG_API_PATH,
+    MCP_CATALOG_SOURCE,
+    MCP_SERVERS_YAML,
+)
 from tests.model_registry.utils import (
     generate_namespace_name,
     get_byoidc_user_credentials,
     get_model_registry_metadata_resources,
     get_model_registry_objects,
     get_rest_headers,
     wait_for_default_resource_cleanedup,
+    wait_for_mcp_catalog_api,
+    wait_for_model_catalog_pod_ready_after_deletion,
 )
 from utilities.constants import DscComponents, Labels
 from utilities.general import (
@@ -466,3 +475,54 @@ def model_catalog_routes(admin_client: DynamicClient, model_registry_namespace:
     return list(
         Route.get(namespace=model_registry_namespace, label_selector="component=model-catalog", client=admin_client)
     )
+
+
+@pytest.fixture(scope="class")
+def mcp_catalog_rest_urls(model_registry_namespace: str, model_catalog_routes: list[Route]) -> list[str]:
+    """Build MCP catalog REST URL from existing model catalog routes."""
+    assert model_catalog_routes, f"Model catalog routes do not exist in {model_registry_namespace}"
+    return [f"https://{route.instance.spec.host}:443{MCP_CATALOG_API_PATH}" for route in model_catalog_routes]
+
+
+@pytest.fixture(scope="class")
+def mcp_servers_configmap_patch(
+    admin_client: DynamicClient,
+    model_registry_namespace: str,
+    mcp_catalog_rest_urls: list[str],
+    model_registry_rest_headers: dict[str, str],
+) -> Generator[None]:
+    """
+    Class-scoped fixture that patches the model-catalog-sources ConfigMap
+
+    Sets two keys in the ConfigMap data:
+    - sources.yaml: catalog source definition pointing to the MCP servers YAML
+    - mcp-servers.yaml: the actual MCP server definitions
+    """
+    catalog_config_map = ConfigMap(
+        name=DEFAULT_CUSTOM_MODEL_CATALOG,
+        client=admin_client,
+        namespace=model_registry_namespace,
+    )
+
+    current_data = yaml.safe_load(catalog_config_map.instance.data.get("sources.yaml", "{}") or "{}")
+    if "mcp_catalogs" not in current_data:
+        current_data["mcp_catalogs"] = []
+    current_data["mcp_catalogs"].append(MCP_CATALOG_SOURCE)
+
+    patches = {
+        "data": {
+            "sources.yaml": yaml.dump(current_data, default_flow_style=False),
+            "mcp-servers.yaml": MCP_SERVERS_YAML,
+        }
+    }
+
+    with ResourceEditor(patches={catalog_config_map: patches}):
+        wait_for_model_catalog_pod_ready_after_deletion(
+            client=admin_client, model_registry_namespace=model_registry_namespace
+        )
+        wait_for_mcp_catalog_api(url=mcp_catalog_rest_urls[0], headers=model_registry_rest_headers)
+        yield
+
+    wait_for_model_catalog_pod_ready_after_deletion(
+        client=admin_client, model_registry_namespace=model_registry_namespace
+    )