fix: update automation to reflect the changes around new source for other models (opendatahub-io#1232)

Signed-off-by: Debarati Basu-Nag <dbasunag@redhat.com>
Signed-off-by: Shehan Saleem <ssaleem@redhat.com>

Author: dbasunag

tests/model_registry/model_catalog/catalog_config/test_default_model_catalog.py

@@ -59,7 +59,7 @@ class TestModelCatalogGeneral:
             ),
             pytest.param(
                 {"configmap_name": DEFAULT_MODEL_CATALOG_CM},
-                2,
+                3,
                 True,
                 id="test_model_catalog_default_sources_configmap",
             ),

tests/model_registry/model_catalog/catalog_config/test_default_source_inclusion_exclusion_cleanup.py

@@ -208,6 +208,7 @@ def test_model_cleanup_on_exclusion_change(
                         model_registry_rest_headers=model_registry_rest_headers,
                         source_label=REDHAT_AI_CATALOG_NAME,
                         expected_models=granite_models,
+                        source_id=REDHAT_AI_CATALOG_ID,
                     )
                 except TimeoutExpiredError as e:
                     pytest.fail(f"Phase 1: Timeout waiting for granite models {granite_models}: {e}")
@@ -245,6 +246,7 @@ def test_model_cleanup_on_exclusion_change(
                         model_registry_rest_headers=model_registry_rest_headers,
                         source_label=REDHAT_AI_CATALOG_NAME,
                         expected_models=prometheus_models,
+                        source_id=REDHAT_AI_CATALOG_ID,
                     )
                 except TimeoutExpiredError as e:
                     pytest.fail(f"Phase 2: Timeout waiting for prometheus models {prometheus_models}: {e}")

tests/model_registry/model_catalog/catalog_config/utils.py

@@ -61,7 +61,7 @@ def validate_default_catalog(catalogs: list[dict[Any, Any]]) -> None:
 
 
 def get_validate_default_model_catalog_source(catalogs: list[dict[Any, Any]]) -> None:
-    assert len(catalogs) == 2, f"Expected no custom models to be present. Actual: {catalogs}"
+    assert len(catalogs) == 3, f"Expected no custom models to be present. Actual: {catalogs}"
     ids_actual = [entry["id"] for entry in catalogs]
     assert sorted(ids_actual) == sorted(DEFAULT_CATALOGS.keys()), (
         f"Actual default catalog entries: {ids_actual},Expected: {DEFAULT_CATALOGS.keys()}"
@@ -206,6 +206,7 @@ def validate_filter_test_result(
         model_registry_rest_headers=model_registry_rest_headers,
         source_label=REDHAT_AI_CATALOG_NAME,
         expected_models=expected_models,
+        source_id=REDHAT_AI_CATALOG_ID,
     )
 
     # Get database models
@@ -426,6 +427,7 @@ def wait_for_model_set_match(
     model_catalog_rest_url: list[str],
     model_registry_rest_headers: dict[str, str],
     source_label: str,
+    source_id: str,
     expected_models: set[str],
 ) -> set[str]:
     """
@@ -436,6 +438,7 @@ def wait_for_model_set_match(
         model_registry_rest_headers: API headers
         source_label: Source to query
         expected_models: Expected set of model names
+        source_id: Source to query
 
     Returns:
         Set of matched models
@@ -445,10 +448,13 @@ def wait_for_model_set_match(
         AssertionError: If models don't match (retried automatically)
         Exception: If API errors occur (retried automatically)
     """
-    current_models = get_api_models_by_source_label(
-        model_catalog_rest_url=model_catalog_rest_url,
-        model_registry_rest_headers=model_registry_rest_headers,
-        source_label=source_label,
+    current_models = models_with_source_id(
+        models=get_api_models_by_source_label(
+            model_catalog_rest_url=model_catalog_rest_url,
+            model_registry_rest_headers=model_registry_rest_headers,
+            source_label=source_label,
+        ),
+        source_id=source_id,
     )
     # Raise AssertionError if condition not met - this will be retried
     assert current_models == expected_models, f"Expected models {expected_models}, got {current_models}"
@@ -528,6 +534,11 @@ def wait_for_catalog_source_restore(
     return True
 
 
+def models_with_source_id(models: set[str], source_id: str) -> set[str]:
+    """Prefix each model name with the source ID to create unique identifiers across sources."""
+    return {f"{source_id}:{model}" for model in models}
+
+
 def validate_model_catalog_sources(
     model_catalog_sources_url: str, rest_headers: dict[str, str], expected_catalog_values: dict[str, str]
 ) -> None:

tests/model_registry/model_catalog/conftest.py

@@ -459,7 +459,7 @@ def baseline_redhat_ai_models(
         model_registry_rest_headers=model_registry_rest_headers,
         source_label="Red Hat AI",
     )
-    api_models = {model["name"] for model in api_response.get("items", [])}
+    api_models = {f"{REDHAT_AI_CATALOG_ID}:{model['name']}" for model in api_response.get("items", [])}
 
     db_models = get_models_from_database_by_source(
         admin_client=admin_client, source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace

tests/model_registry/model_catalog/constants.py

@@ -16,28 +16,38 @@
 REDHAT_AI_VALIDATED_UNESCAPED_CATALOG_NAME: str = "Red Hat AI Validated"
 REDHAT_AI_FILTER: str = "Red+Hat+AI"
 REDHAT_AI_VALIDATED_FILTER = "Red+Hat+AI+Validated"
-
+OTHER_MODELS_CATALOG_ID: str = "other_models"
 SAMPLE_MODEL_NAME3 = "mistralai/Ministral-8B-Instruct-2410"
 CATALOG_CONTAINER: str = "catalog"
+REDHAT_AI_CATALOG_ID: str = "redhat_ai_models"
+OTHER_MODELS: str = "Other Models"
+VALIDATED_CATALOG_ID: str = "redhat_ai_validated_models"
 DEFAULT_CATALOGS: dict[str, Any] = {
-    "redhat_ai_models": {
+    REDHAT_AI_CATALOG_ID: {
         "name": REDHAT_AI_CATALOG_NAME,
         "type": "yaml",
         "properties": {"yamlCatalogPath": "/shared-data/models-catalog.yaml"},
         "labels": [REDHAT_AI_CATALOG_NAME],
         "enabled": True,
     },
-    "redhat_ai_validated_models": {
+    VALIDATED_CATALOG_ID: {
         "name": REDHAT_AI_VALIDATED_CATALOG_NAME,
         "type": "yaml",
         "properties": {"yamlCatalogPath": "/shared-data/validated-models-catalog.yaml"},
         "labels": [REDHAT_AI_VALIDATED_CATALOG_NAME],
         "enabled": True,
     },
+    OTHER_MODELS_CATALOG_ID: {
+        "name": OTHER_MODELS,
+        "type": "yaml",
+        "properties": {"yamlCatalogPath": "/shared-data/other-models-catalog.yaml"},
+        "labels": None,
+        "enabled": True,
+    },
 }
-REDHAT_AI_CATALOG_ID: str = "redhat_ai_models"
+
 DEFAULT_CATALOG_FILE: str = DEFAULT_CATALOGS[REDHAT_AI_CATALOG_ID]["properties"]["yamlCatalogPath"]
-VALIDATED_CATALOG_ID: str = "redhat_ai_validated_models"
+
 VALIDATED_CATALOG_FILE: str = DEFAULT_CATALOGS[VALIDATED_CATALOG_ID]["properties"]["yamlCatalogPath"]
 
 MODEL_ARTIFACT_TYPE: str = "model-artifact"

tests/model_registry/model_catalog/metadata/test_sources_endpoint.py

@@ -4,7 +4,11 @@
 from simple_logger.logger import get_logger
 
 from tests.model_registry.mcp_servers.constants import MCP_CATALOG_SOURCE_ID
-from tests.model_registry.model_catalog.constants import REDHAT_AI_CATALOG_ID, VALIDATED_CATALOG_ID
+from tests.model_registry.model_catalog.constants import (
+    OTHER_MODELS_CATALOG_ID,
+    REDHAT_AI_CATALOG_ID,
+    VALIDATED_CATALOG_ID,
+)
 from tests.model_registry.utils import execute_get_command
 
 pytestmark = [pytest.mark.usefixtures("updated_dsc_component_state_scope_session", "model_registry_namespace")]
@@ -58,8 +62,8 @@ class TestAssetTypeFilter:
     @pytest.mark.parametrize(
         "asset_type,expected_ids",
         [
-            (None, {REDHAT_AI_CATALOG_ID, VALIDATED_CATALOG_ID}),
-            ("models", {REDHAT_AI_CATALOG_ID, VALIDATED_CATALOG_ID}),
+            (None, {REDHAT_AI_CATALOG_ID, VALIDATED_CATALOG_ID, OTHER_MODELS_CATALOG_ID}),
+            ("models", {REDHAT_AI_CATALOG_ID, VALIDATED_CATALOG_ID, OTHER_MODELS_CATALOG_ID}),
             ("mcp_servers", {MCP_CATALOG_SOURCE_ID}),
             ("invalid_value", set()),
         ],