Merge branch 'main' into gpufix

mwaykole · web-flow · commit 742e12a215c0 · 2026-02-26T08:28:31.000+05:30
diff --git a/tests/model_registry/model_catalog/catalog_config/conftest.py b/tests/model_registry/model_catalog/catalog_config/conftest.py
@@ -17,11 +17,11 @@
     wait_for_catalog_source_restore,
 )
 from tests.model_registry.model_catalog.constants import REDHAT_AI_CATALOG_ID, REDHAT_AI_CATALOG_NAME
-from tests.model_registry.utils import (
-    get_model_catalog_pod,
+from tests.model_registry.model_catalog.utils import (
     wait_for_model_catalog_api,
     wait_for_model_catalog_pod_ready_after_deletion,
 )
+from tests.model_registry.utils import get_model_catalog_pod
 
 LOGGER = get_logger(name=__name__)
 
diff --git a/tests/model_registry/model_catalog/catalog_config/test_catalog_source_merge.py b/tests/model_registry/model_catalog/catalog_config/test_catalog_source_merge.py
@@ -2,7 +2,7 @@
 from simple_logger.logger import get_logger
 
 from tests.model_registry.model_catalog.constants import REDHAT_AI_CATALOG_ID
-from tests.model_registry.utils import execute_get_command
+from tests.model_registry.model_catalog.utils import execute_get_command
 
 LOGGER = get_logger(name=__name__)
 
diff --git a/tests/model_registry/model_catalog/catalog_config/test_custom_model_catalog.py b/tests/model_registry/model_catalog/catalog_config/test_custom_model_catalog.py
@@ -6,6 +6,7 @@
 from simple_logger.logger import get_logger
 
 from tests.model_registry.constants import CUSTOM_CATALOG_ID1, SAMPLE_MODEL_NAME1
+from tests.model_registry.model_catalog.catalog_config.utils import validate_model_catalog_sources
 from tests.model_registry.model_catalog.constants import (
     CUSTOM_CATALOG_ID2,
     EXPECTED_CUSTOM_CATALOG_VALUES,
@@ -15,12 +16,11 @@
     SAMPLE_MODEL_NAME2,
     SAMPLE_MODEL_NAME3,
 )
-from tests.model_registry.model_catalog.utils import get_hf_catalog_str
-from tests.model_registry.utils import (
+from tests.model_registry.model_catalog.utils import (
     execute_get_command,
     get_catalog_str,
+    get_hf_catalog_str,
     get_sample_yaml_str,
-    validate_model_catalog_sources,
 )
 
 LOGGER = get_logger(name=__name__)
diff --git a/tests/model_registry/model_catalog/catalog_config/test_default_model_catalog.py b/tests/model_registry/model_catalog/catalog_config/test_default_model_catalog.py
@@ -23,7 +23,8 @@
     validate_model_catalog_resource,
 )
 from tests.model_registry.model_catalog.constants import CATALOG_CONTAINER, DEFAULT_CATALOGS, REDHAT_AI_CATALOG_ID
-from tests.model_registry.utils import execute_get_command, get_model_catalog_pod, get_rest_headers
+from tests.model_registry.model_catalog.utils import execute_get_command
+from tests.model_registry.utils import get_model_catalog_pod, get_rest_headers
 from utilities.user_utils import UserTestSession
 
 LOGGER = get_logger(name=__name__)
diff --git a/tests/model_registry/model_catalog/catalog_config/test_default_source_inclusion_exclusion_cleanup.py b/tests/model_registry/model_catalog/catalog_config/test_default_source_inclusion_exclusion_cleanup.py
@@ -18,7 +18,7 @@
     REDHAT_AI_CATALOG_ID,
     REDHAT_AI_CATALOG_NAME,
 )
-from tests.model_registry.utils import wait_for_model_catalog_api
+from tests.model_registry.model_catalog.utils import wait_for_model_catalog_api
 
 LOGGER = get_logger(name=__name__)
 
diff --git a/tests/model_registry/model_catalog/catalog_config/utils.py b/tests/model_registry/model_catalog/catalog_config/utils.py
@@ -19,6 +19,7 @@
 from tests.model_registry.model_catalog.db_constants import GET_MODELS_BY_SOURCE_ID_DB_QUERY
 from tests.model_registry.model_catalog.utils import (
     execute_database_query,
+    execute_get_command,
     get_models_from_catalog_api,
     parse_psql_output,
 )
@@ -526,3 +527,17 @@ def wait_for_catalog_source_restore(
 
     LOGGER.info("Found expected number of models: %s for source: %s", expected_count, source_label)
     return True
+
+
+def validate_model_catalog_sources(
+    model_catalog_sources_url: str, rest_headers: dict[str, str], expected_catalog_values: dict[str, str]
+) -> None:
+    results = execute_get_command(
+        url=model_catalog_sources_url,
+        headers=rest_headers,
+    )["items"]
+    LOGGER.info(f"Model catalog sources: {results}")
+    ids_from_query = [result_entry["id"] for result_entry in results]
+    ids_expected = [expected_entry["id"] for expected_entry in expected_catalog_values]
+    LOGGER.info(f"IDs expected: {ids_expected}, IDs found: {ids_from_query}")
+    assert set(ids_expected).issubset(set(ids_from_query)), f"Expected: {expected_catalog_values}. Actual: {results}"
diff --git a/tests/model_registry/model_catalog/conftest.py b/tests/model_registry/model_catalog/conftest.py
@@ -24,16 +24,18 @@
     REDHAT_AI_CATALOG_ID,
     SAMPLE_MODEL_NAME3,
 )
-from tests.model_registry.model_catalog.utils import get_models_from_catalog_api
-from tests.model_registry.utils import (
+from tests.model_registry.model_catalog.utils import (
     execute_get_command,
-    get_model_catalog_pod,
     get_model_str,
-    get_mr_user_token,
-    get_rest_headers,
+    get_models_from_catalog_api,
     wait_for_model_catalog_api,
     wait_for_model_catalog_pod_ready_after_deletion,
 )
+from tests.model_registry.utils import (
+    get_model_catalog_pod,
+    get_mr_user_token,
+    get_rest_headers,
+)
 from utilities.infra import create_inference_token, get_openshift_token, login_with_user_password
 
 LOGGER = get_logger(name=__name__)
diff --git a/tests/model_registry/model_catalog/huggingface/test_huggingface_source_error_validation.py b/tests/model_registry/model_catalog/huggingface/test_huggingface_source_error_validation.py
@@ -7,7 +7,7 @@
 from simple_logger.logger import get_logger
 
 from tests.model_registry.model_catalog.huggingface.utils import assert_accessible_models_via_catalog_api
-from tests.model_registry.utils import execute_get_command
+from tests.model_registry.model_catalog.utils import execute_get_command
 
 LOGGER = get_logger(name=__name__)
 INACCESSIBLE_MODELS: list[str] = [
diff --git a/tests/model_registry/model_catalog/huggingface/utils.py b/tests/model_registry/model_catalog/huggingface/utils.py
@@ -7,8 +7,8 @@
 from timeout_sampler import retry
 
 from tests.model_registry.model_catalog.constants import HF_SOURCE_ID
-from tests.model_registry.model_catalog.utils import get_models_from_catalog_api
-from tests.model_registry.utils import execute_get_command, get_model_catalog_pod
+from tests.model_registry.model_catalog.utils import execute_get_command, get_models_from_catalog_api
+from tests.model_registry.utils import get_model_catalog_pod
 
 LOGGER = get_logger(name=__name__)
 
diff --git a/tests/model_registry/model_catalog/metadata/test_custom_properties.py b/tests/model_registry/model_catalog/metadata/test_custom_properties.py
@@ -10,7 +10,8 @@
     get_metadata_from_catalog_pod,
     validate_custom_properties_match_metadata,
 )
-from tests.model_registry.utils import execute_get_command, get_model_catalog_pod
+from tests.model_registry.model_catalog.utils import execute_get_command
+from tests.model_registry.utils import get_model_catalog_pod
 
 LOGGER = get_logger(name=__name__)
 
diff --git a/tests/model_registry/model_catalog/metadata/test_filter_options_endpoint.py b/tests/model_registry/model_catalog/metadata/test_filter_options_endpoint.py
@@ -14,9 +14,10 @@
 )
 from tests.model_registry.model_catalog.utils import (
     execute_database_query,
+    execute_get_command,
     parse_psql_output,
 )
-from tests.model_registry.utils import execute_get_command, get_rest_headers
+from tests.model_registry.utils import get_rest_headers
 from utilities.user_utils import UserTestSession
 
 LOGGER = get_logger(name=__name__)
diff --git a/tests/model_registry/model_catalog/metadata/test_sources_endpoint.py b/tests/model_registry/model_catalog/metadata/test_sources_endpoint.py
@@ -2,7 +2,7 @@
 from simple_logger.logger import get_logger
 
 from tests.model_registry.model_catalog.constants import REDHAT_AI_CATALOG_ID
-from tests.model_registry.utils import execute_get_command
+from tests.model_registry.model_catalog.utils import execute_get_command
 
 pytestmark = [pytest.mark.usefixtures("updated_dsc_component_state_scope_session", "model_registry_namespace")]
 
diff --git a/tests/model_registry/model_catalog/metadata/utils.py b/tests/model_registry/model_catalog/metadata/utils.py
@@ -10,7 +10,8 @@
 from simple_logger.logger import get_logger
 
 from tests.model_registry.constants import DEFAULT_CUSTOM_MODEL_CATALOG, DEFAULT_MODEL_CATALOG_CM
-from tests.model_registry.utils import execute_get_command, get_rest_headers
+from tests.model_registry.model_catalog.utils import execute_get_command
+from tests.model_registry.utils import get_rest_headers
 
 LOGGER = get_logger(name=__name__)
 CATALOG_CONTAINER = "catalog"
diff --git a/tests/model_registry/model_catalog/search/utils.py b/tests/model_registry/model_catalog/search/utils.py
@@ -20,8 +20,7 @@
     SEARCH_MODELS_DB_QUERY,
     SEARCH_MODELS_WITH_SOURCE_ID_DB_QUERY,
 )
-from tests.model_registry.model_catalog.utils import execute_database_query, parse_psql_output
-from tests.model_registry.utils import execute_get_command
+from tests.model_registry.model_catalog.utils import execute_database_query, execute_get_command, parse_psql_output
 
 LOGGER = get_logger(name=__name__)
 
diff --git a/tests/model_registry/model_catalog/sorting/utils.py b/tests/model_registry/model_catalog/sorting/utils.py
@@ -9,10 +9,10 @@
 )
 from tests.model_registry.model_catalog.utils import (
     execute_database_query,
+    execute_get_command,
     get_models_from_catalog_api,
     parse_psql_output,
 )
-from tests.model_registry.utils import execute_get_command
 
 LOGGER = get_logger(name=__name__)
 
diff --git a/tests/model_registry/model_catalog/upgrade/test_model_catalog_upgrade.py b/tests/model_registry/model_catalog/upgrade/test_model_catalog_upgrade.py
@@ -9,7 +9,7 @@
 from simple_logger.logger import get_logger
 
 from tests.model_registry.constants import CUSTOM_CATALOG_ID1, SAMPLE_MODEL_NAME1
-from tests.model_registry.utils import (
+from tests.model_registry.model_catalog.utils import (
     get_catalog_str,
     get_sample_yaml_str,
     wait_for_model_catalog_api,
diff --git a/tests/model_registry/model_catalog/utils.py b/tests/model_registry/model_catalog/utils.py
@@ -1,11 +1,18 @@
+import json
+import time
 from typing import Any
 
+import requests
 from kubernetes.dynamic import DynamicClient
+from kubernetes.dynamic.exceptions import ResourceNotFoundError
 from ocp_resources.pod import Pod
 from simple_logger.logger import get_logger
+from timeout_sampler import retry
 
 from tests.model_registry.model_catalog.constants import HF_MODELS
-from tests.model_registry.utils import execute_get_command
+from tests.model_registry.utils import get_model_catalog_pod
+from utilities.constants import PodNotFound
+from utilities.general import wait_for_pods_running
 
 LOGGER = get_logger(name=__name__)
 
@@ -275,3 +282,122 @@ def assert_source_error_state_message(
     assert expected_error_message in matched_source[0]["error"], (
         f"Expected error: {expected_error_message} not found in {matched_source[0]['error']}"
     )
+
+
+class TransientUnauthorizedError(Exception):
+    """Exception for transient 401 Unauthorized errors that should be retried."""
+
+
+def execute_get_call(
+    url: str, headers: dict[str, str], verify: bool | str = False, params: dict[str, Any] | None = None
+) -> requests.Response:
+    LOGGER.info(f"Executing get call: {url}")
+    if params:
+        LOGGER.info(f"params: {params}")
+    resp = requests.get(url=url, headers=headers, verify=verify, timeout=60, params=params)
+    LOGGER.info(f"Encoded url from requests library: {resp.url}")
+    if resp.status_code not in [200, 201]:
+        # Raise custom exception for 401 errors that can be retried (OAuth/kube-rbac-proxy initialization)
+        if resp.status_code == 401:
+            raise TransientUnauthorizedError(f"Get call failed for resource: {url}, 401: {resp.text}")
+        # Raise regular exception for other errors (400, 403, 404, etc.) that should fail immediately
+        raise ResourceNotFoundError(f"Get call failed for resource: {url}, {resp.status_code}: {resp.text}")
+    return resp
+
+
+def execute_get_command(
+    url: str, headers: dict[str, str], verify: bool | str = False, params: dict[str, Any] | None = None
+) -> dict[Any, Any]:
+    resp = execute_get_call(url=url, headers=headers, verify=verify, params=params)
+    try:
+        return json.loads(resp.text)
+    except json.JSONDecodeError:
+        LOGGER.error(f"Unable to parse {resp.text}")
+        raise
+
+
+@retry(wait_timeout=90, sleep=5, exceptions_dict={ResourceNotFoundError: [], TransientUnauthorizedError: []})
+def wait_for_model_catalog_api(url: str, headers: dict[str, str], verify: bool | str = False) -> requests.Response:
+    """
+    Wait for model catalog API to be ready and fully initialized checks both /sources and /models endpoints
+    to ensure OAuth/kube-rbac-proxy is fully initialized.
+    """
+    LOGGER.info(f"Waiting for model catalog API at {url}sources")
+    execute_get_call(url=f"{url}sources", headers=headers, verify=verify)
+    LOGGER.info(f"Verifying model catalog API readiness at {url}models")
+
+    return execute_get_call(url=f"{url}models", headers=headers, verify=verify)
+
+
+def get_model_str(model: str) -> str:
+    current_time = int(time.time() * 1000)
+    return f"""
+- name: {model}
+  description: test description.
+  readme: |-
+    # test read me information {model}
+  provider: Mistral AI
+  logo: temp placeholder logo
+  license: apache-2.0
+  licenseLink: https://www.apache.org/licenses/LICENSE-2.0.txt
+  libraryName: transformers
+  artifacts:
+    - uri: https://huggingface.co/{model}/resolve/main/consolidated.safetensors
+  createTimeSinceEpoch: \"{current_time - 10000!s}\"
+  lastUpdateTimeSinceEpoch: \"{current_time!s}\"
+"""
+
+
+def get_sample_yaml_str(models: list[str]) -> str:
+    model_str: str = ""
+    for model in models:
+        model_str += f"""
+{get_model_str(model=model)}
+"""
+    return f"""source: Hugging Face
+models:
+{model_str}
+"""
+
+
+def get_catalog_str(ids: list[str]) -> str:
+    catalog_str: str = ""
+    for index, id in enumerate(ids):
+        catalog_str += f"""
+- name: Sample Catalog {index}
+  id: {id}
+  type: yaml
+  enabled: true
+  properties:
+    yamlCatalogPath: {id.replace("_", "-")}.yaml
+"""
+    return f"""catalogs:
+{catalog_str}
+"""
+
+
+def wait_for_model_catalog_pod_ready_after_deletion(
+    client: DynamicClient, model_registry_namespace: str, consecutive_try: int = 6
+) -> bool:
+    model_catalog_pods = get_model_catalog_pod(
+        client=client,
+        model_registry_namespace=model_registry_namespace,
+    )
+    # We can wait for the pods to reflect updated catalog, however, deleting them ensures the updated config is
+    # applied immediately.
+    for pod in model_catalog_pods:
+        pod.delete()
+    # After the deletion, we need to wait for the pod to be spinned up and get to ready state.
+    assert wait_for_model_catalog_pod_created(client=client, model_registry_namespace=model_registry_namespace)
+    wait_for_pods_running(
+        admin_client=client, namespace_name=model_registry_namespace, number_of_consecutive_checks=consecutive_try
+    )
+    return True
+
+
+@retry(wait_timeout=30, sleep=5, exceptions_dict={PodNotFound: []})
+def wait_for_model_catalog_pod_created(client: DynamicClient, model_registry_namespace: str) -> bool:
+    pods = get_model_catalog_pod(client=client, model_registry_namespace=model_registry_namespace)
+    if pods:
+        return True
+    raise PodNotFound("Model catalog pod not found")
diff --git a/tests/model_registry/utils.py b/tests/model_registry/utils.py
diff --git a/tests/model_serving/model_runtime/image_validation/__init__.py b/tests/model_serving/model_runtime/image_validation/__init__.py
diff --git a/tests/model_serving/model_runtime/image_validation/conftest.py b/tests/model_serving/model_runtime/image_validation/conftest.py
diff --git a/tests/model_serving/model_runtime/image_validation/constant.py b/tests/model_serving/model_runtime/image_validation/constant.py
diff --git a/tests/model_serving/model_runtime/image_validation/test_verify_serving_runtime_images.py b/tests/model_serving/model_runtime/image_validation/test_verify_serving_runtime_images.py
diff --git a/tests/workbenches/test_imagestream_health.py b/tests/workbenches/test_imagestream_health.py

Original file line number	Diff line number	Diff line change
`@@ -17,11 +17,11 @@`
`17`	`17`	`wait_for_catalog_source_restore,`
`18`	`18`	`)`
`19`	`19`	`from tests.model_registry.model_catalog.constants import REDHAT_AI_CATALOG_ID, REDHAT_AI_CATALOG_NAME`
`20`		`-from tests.model_registry.utils import (`
`21`		`- get_model_catalog_pod,`
	`20`	`+from tests.model_registry.model_catalog.utils import (`
`22`	`21`	`wait_for_model_catalog_api,`
`23`	`22`	`wait_for_model_catalog_pod_ready_after_deletion,`
`24`	`23`	`)`
	`24`	`+from tests.model_registry.utils import get_model_catalog_pod`
`25`	`25`
`26`	`26`	`LOGGER = get_logger(name=__name__)`
`27`	`27`
Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,8 @@`
`23`	`23`	`validate_model_catalog_resource,`
`24`	`24`	`)`
`25`	`25`	`from tests.model_registry.model_catalog.constants import CATALOG_CONTAINER, DEFAULT_CATALOGS, REDHAT_AI_CATALOG_ID`
`26`		`-from tests.model_registry.utils import execute_get_command, get_model_catalog_pod, get_rest_headers`
	`26`	`+from tests.model_registry.model_catalog.utils import execute_get_command`
	`27`	`+from tests.model_registry.utils import get_model_catalog_pod, get_rest_headers`
`27`	`28`	`from utilities.user_utils import UserTestSession`
`28`	`29`
`29`	`30`	`LOGGER = get_logger(name=__name__)`
Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@`
`18`	`18`	`REDHAT_AI_CATALOG_ID,`
`19`	`19`	`REDHAT_AI_CATALOG_NAME,`
`20`	`20`	`)`
`21`		`-from tests.model_registry.utils import wait_for_model_catalog_api`
	`21`	`+from tests.model_registry.model_catalog.utils import wait_for_model_catalog_api`
`22`	`22`
`23`	`23`	`LOGGER = get_logger(name=__name__)`
`24`	`24`
Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,8 @@`
`10`	`10`	`get_metadata_from_catalog_pod,`
`11`	`11`	`validate_custom_properties_match_metadata,`
`12`	`12`	`)`
`13`		`-from tests.model_registry.utils import execute_get_command, get_model_catalog_pod`
	`13`	`+from tests.model_registry.model_catalog.utils import execute_get_command`
	`14`	`+from tests.model_registry.utils import get_model_catalog_pod`
`14`	`15`
`15`	`16`	`LOGGER = get_logger(name=__name__)`
`16`	`17`