Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions tests/model_registry/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from typing import Any

import pytest
import yaml
from kubernetes.dynamic import DynamicClient
from ocp_resources.config_map import ConfigMap
from ocp_resources.data_science_cluster import DataScienceCluster
Expand All @@ -25,18 +26,26 @@
from tests.model_registry.constants import (
DB_BASE_RESOURCES_NAME,
DB_RESOURCE_NAME,
DEFAULT_CUSTOM_MODEL_CATALOG,
KUBERBACPROXY_STR,
MR_INSTANCE_BASE_NAME,
MR_INSTANCE_NAME,
MR_OPERATOR_NAME,
)
from tests.model_registry.mcp_servers.constants import (
MCP_CATALOG_API_PATH,
MCP_CATALOG_SOURCE,
MCP_SERVERS_YAML,
)
from tests.model_registry.utils import (
generate_namespace_name,
get_byoidc_user_credentials,
get_model_registry_metadata_resources,
get_model_registry_objects,
get_rest_headers,
wait_for_default_resource_cleanedup,
wait_for_mcp_catalog_api,
wait_for_model_catalog_pod_ready_after_deletion,
)
from utilities.constants import DscComponents, Labels
from utilities.general import (
Expand Down Expand Up @@ -466,3 +475,54 @@ def model_catalog_routes(admin_client: DynamicClient, model_registry_namespace:
return list(
Route.get(namespace=model_registry_namespace, label_selector="component=model-catalog", client=admin_client)
)


@pytest.fixture(scope="class")
def mcp_catalog_rest_urls(model_registry_namespace: str, model_catalog_routes: list[Route]) -> list[str]:
"""Build MCP catalog REST URL from existing model catalog routes."""
assert model_catalog_routes, f"Model catalog routes do not exist in {model_registry_namespace}"
return [f"https://{route.instance.spec.host}:443{MCP_CATALOG_API_PATH}" for route in model_catalog_routes]


@pytest.fixture(scope="class")
def mcp_servers_configmap_patch(
Comment thread
fege marked this conversation as resolved.
admin_client: DynamicClient,
model_registry_namespace: str,
mcp_catalog_rest_urls: list[str],
model_registry_rest_headers: dict[str, str],
) -> Generator[None]:
"""
Class-scoped fixture that patches the model-catalog-sources ConfigMap

Sets two keys in the ConfigMap data:
- sources.yaml: catalog source definition pointing to the MCP servers YAML
- mcp-servers.yaml: the actual MCP server definitions
"""
catalog_config_map = ConfigMap(
Comment thread
fege marked this conversation as resolved.
name=DEFAULT_CUSTOM_MODEL_CATALOG,
client=admin_client,
namespace=model_registry_namespace,
)

current_data = yaml.safe_load(catalog_config_map.instance.data.get("sources.yaml", "{}") or "{}")
if "mcp_catalogs" not in current_data:
current_data["mcp_catalogs"] = []
current_data["mcp_catalogs"].append(MCP_CATALOG_SOURCE)

patches = {
"data": {
"sources.yaml": yaml.dump(current_data, default_flow_style=False),
"mcp-servers.yaml": MCP_SERVERS_YAML,
}
}

with ResourceEditor(patches={catalog_config_map: patches}):
wait_for_model_catalog_pod_ready_after_deletion(
client=admin_client, model_registry_namespace=model_registry_namespace
)
wait_for_mcp_catalog_api(url=mcp_catalog_rest_urls[0], headers=model_registry_rest_headers)
yield

wait_for_model_catalog_pod_ready_after_deletion(
client=admin_client, model_registry_namespace=model_registry_namespace
)
Comment thread
fege marked this conversation as resolved.
74 changes: 1 addition & 73 deletions tests/model_registry/mcp_servers/conftest.py
Original file line number Diff line number Diff line change
@@ -1,57 +1,29 @@
from collections.abc import Generator

import pytest
import requests
import yaml
from kubernetes.dynamic import DynamicClient
from kubernetes.dynamic.exceptions import ResourceNotFoundError
from ocp_resources.config_map import ConfigMap
from ocp_resources.resource import ResourceEditor
from ocp_resources.route import Route
from simple_logger.logger import get_logger
from timeout_sampler import retry

from tests.model_registry.constants import DEFAULT_CUSTOM_MODEL_CATALOG
from tests.model_registry.mcp_servers.constants import (
MCP_CATALOG_API_PATH,
MCP_CATALOG_INVALID_SOURCE,
MCP_CATALOG_SOURCE,
MCP_CATALOG_SOURCE2,
MCP_SERVERS_YAML,
MCP_SERVERS_YAML2,
)
from tests.model_registry.utils import (
TransientUnauthorizedError,
execute_get_call,
execute_get_command,
wait_for_mcp_catalog_api,
wait_for_model_catalog_pod_ready_after_deletion,
)

LOGGER = get_logger(name=__name__)


@pytest.fixture(scope="class")
def mcp_catalog_rest_urls(model_registry_namespace: str, model_catalog_routes: list[Route]) -> list[str]:
"""Build MCP catalog REST URL from existing model catalog routes."""
assert model_catalog_routes, f"Model catalog routes do not exist in {model_registry_namespace}"
return [f"https://{route.instance.spec.host}:443{MCP_CATALOG_API_PATH}" for route in model_catalog_routes]


@retry(
wait_timeout=90,
sleep=5,
exceptions_dict={ResourceNotFoundError: [], TransientUnauthorizedError: []},
)
def wait_for_mcp_catalog_api(url: str, headers: dict[str, str]) -> requests.Response:
"""Wait for MCP catalog API to be ready and returning MCP server data."""
LOGGER.info(f"Waiting for MCP catalog API at {url}mcp_servers")
response = execute_get_call(url=f"{url}mcp_servers", headers=headers)
data = response.json()
if not data.get("items"):
raise ResourceNotFoundError("MCP catalog API returned empty items, catalog data not yet loaded")
return response


@pytest.fixture(scope="class")
def mcp_servers_response(
mcp_catalog_rest_urls: list[str],
Expand All @@ -64,50 +36,6 @@ def mcp_servers_response(
)


@pytest.fixture(scope="class")
def mcp_servers_configmap_patch(
admin_client: DynamicClient,
model_registry_namespace: str,
mcp_catalog_rest_urls: list[str],
model_registry_rest_headers: dict[str, str],
) -> Generator[None]:
"""
Class-scoped fixture that patches the model-catalog-sources ConfigMap

Sets two keys in the ConfigMap data:
- sources.yaml: catalog source definition pointing to the MCP servers YAML
- mcp-servers.yaml: the actual MCP server definitions
"""
catalog_config_map = ConfigMap(
name=DEFAULT_CUSTOM_MODEL_CATALOG,
client=admin_client,
namespace=model_registry_namespace,
)

current_data = yaml.safe_load(catalog_config_map.instance.data.get("sources.yaml", "{}") or "{}")
if "mcp_catalogs" not in current_data:
current_data["mcp_catalogs"] = []
current_data["mcp_catalogs"].append(MCP_CATALOG_SOURCE)

patches = {
"data": {
"sources.yaml": yaml.dump(current_data, default_flow_style=False),
"mcp-servers.yaml": MCP_SERVERS_YAML,
}
}

with ResourceEditor(patches={catalog_config_map: patches}):
wait_for_model_catalog_pod_ready_after_deletion(
client=admin_client, model_registry_namespace=model_registry_namespace
)
wait_for_mcp_catalog_api(url=mcp_catalog_rest_urls[0], headers=model_registry_rest_headers)
yield

wait_for_model_catalog_pod_ready_after_deletion(
client=admin_client, model_registry_namespace=model_registry_namespace
)


@pytest.fixture(scope="class")
def mcp_multi_source_configmap_patch(
admin_client: DynamicClient,
Expand Down
18 changes: 0 additions & 18 deletions tests/model_registry/mcp_servers/test_data_integrity.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from tests.model_registry.mcp_servers.constants import (
EXPECTED_MCP_SERVER_NAMES,
EXPECTED_MCP_SERVER_TIMESTAMPS,
EXPECTED_MCP_SERVER_TOOL_COUNTS,
EXPECTED_MCP_SERVER_TOOLS,
)
from tests.model_registry.utils import execute_get_command
Expand All @@ -30,9 +29,6 @@ def test_mcp_servers_loaded(
assert server["createTimeSinceEpoch"] == expected["createTimeSinceEpoch"]
assert server["lastUpdateTimeSinceEpoch"] == expected["lastUpdateTimeSinceEpoch"]

@pytest.mark.xfail(
reason="RHOAIENG-51765: Tool name returned as {server}@{version}:{tool} instead of YAML-defined name"
)
def test_mcp_server_tools_loaded(
self: Self,
mcp_catalog_rest_urls: list[str],
Expand All @@ -50,17 +46,3 @@ def test_mcp_server_tools_loaded(
assert server["toolCount"] == len(expected_tool_names)
actual_tool_names = [tool["name"] for tool in server["tools"]]
assert sorted(actual_tool_names) == sorted(expected_tool_names)

@pytest.mark.xfail(reason="RHOAIENG-51764: toolCount is 0 when not passing includeTools=true")
def test_mcp_server_tool_count_without_include(
self: Self,
mcp_servers_response: dict[str, Any],
):
"""Verify that toolCount reflects actual tools even when tools are not included."""
for server in mcp_servers_response.get("items", []):
name = server["name"]
expected_count = EXPECTED_MCP_SERVER_TOOL_COUNTS[name]
actual_count = server.get("toolCount", 0)
assert actual_count == expected_count, (
f"Server '{name}': expected toolCount {expected_count}, got {actual_count}"
)
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
from typing import Any, Self

import pytest
from simple_logger.logger import get_logger

from tests.model_registry.model_catalog.constants import REDHAT_AI_CATALOG_ID
from tests.model_registry.mcp_servers.constants import MCP_CATALOG_SOURCE_ID
from tests.model_registry.model_catalog.constants import REDHAT_AI_CATALOG_ID, VALIDATED_CATALOG_ID
from tests.model_registry.utils import execute_get_command

pytestmark = [pytest.mark.usefixtures("updated_dsc_component_state_scope_session", "model_registry_namespace")]
Expand Down Expand Up @@ -46,3 +49,39 @@ def test_sources_endpoint_returns_all_sources_regardless_of_enabled_field(
f"Sources endpoint returned {len(items)} total sources: "
f"{len(enabled_sources)} enabled, {len(disabled_sources)} disabled"
)


@pytest.mark.usefixtures("mcp_servers_configmap_patch")
class TestAssetTypeFilter:
"""RHOAIENG-51583: Tests for /sources endpoint assetType query parameter filtering."""

@pytest.mark.parametrize(
"asset_type,expected_ids",
[
(None, {REDHAT_AI_CATALOG_ID, VALIDATED_CATALOG_ID}),
("models", {REDHAT_AI_CATALOG_ID, VALIDATED_CATALOG_ID}),
("mcp_servers", {MCP_CATALOG_SOURCE_ID}),
("invalid_value", set()),
],
ids=["default-models", "explicit-models", "mcp-servers", "invalid-empty"],
)
def test_asset_type_filters_sources(
self: Self,
asset_type: str | None,
expected_ids: set[str],
model_catalog_rest_url: list[str],
model_registry_rest_headers: dict[str, str],
) -> None:
"""Test that the /sources endpoint filters by assetType, returning only catalog sources
for None/models, only MCP sources for mcp_servers, and an empty list for invalid values.
"""
sources_url = f"{model_catalog_rest_url[0]}sources"
params: dict[str, Any] = {}
if asset_type is not None:
params["assetType"] = asset_type

response = execute_get_command(url=sources_url, headers=model_registry_rest_headers, params=params or None)
source_ids = {item["id"] for item in response["items"]}

assert source_ids == expected_ids
LOGGER.info(f"assetType={asset_type} returned sources: {source_ids}")
15 changes: 15 additions & 0 deletions tests/model_registry/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -959,3 +959,18 @@ def wait_for_model_catalog_pod_created(client: DynamicClient, model_registry_nam
if pods:
return True
raise PodNotFound("Model catalog pod not found")


@retry(
wait_timeout=90,
sleep=5,
exceptions_dict={ResourceNotFoundError: [], TransientUnauthorizedError: []},
)
def wait_for_mcp_catalog_api(url: str, headers: dict[str, str]) -> requests.Response:
"""Wait for MCP catalog API to be ready and returning MCP server data."""
LOGGER.info(f"Waiting for MCP catalog API at {url}mcp_servers")
response = execute_get_call(url=f"{url}mcp_servers", headers=headers)
data = response.json()
if not data.get("items"):
raise ResourceNotFoundError("MCP catalog API returned empty items, catalog data not yet loaded")
Comment thread
fege marked this conversation as resolved.
return response