test: optimize API readiness check in model catalog conftest (#919)

* test: optimize API readiness check in model catalog conftest

* test: improve API token authorization handling

- Extract URL and header creation into reusable utility function
- Add explicit token authorization wait for service account users

* fix: typo in comment

Author: fege

tests/model_registry/model_catalog/conftest.py

@@ -9,15 +9,14 @@
 
 from ocp_resources.config_map import ConfigMap
 from ocp_resources.resource import ResourceEditor
-
 from ocp_resources.service_account import ServiceAccount
 from tests.model_registry.model_catalog.constants import (
     SAMPLE_MODEL_NAME3,
     DEFAULT_CATALOG_FILE,
     CATALOG_CONTAINER,
     REDHAT_AI_CATALOG_ID,
 )
-from tests.model_registry.model_catalog.utils import get_models_from_catalog_api
+from tests.model_registry.model_catalog.utils import get_models_from_catalog_api, get_catalog_url_and_headers
 from tests.model_registry.constants import (
     CUSTOM_CATALOG_ID1,
     DEFAULT_MODEL_CATALOG_CM,
@@ -42,6 +41,7 @@
 def enabled_model_catalog_config_map(
     admin_client: DynamicClient,
     model_registry_namespace: str,
+    current_client_token: str,
 ) -> ConfigMap:
     """
     Enable all catalogs in the default model catalog configmap
@@ -74,7 +74,17 @@ def enabled_model_catalog_config_map(
     patches = {"data": {"sources.yaml": enabled_sources_yaml}}
 
     with ResourceEditor(patches={user_sources_cm: patches}):
+        # Wait for the model catalog pod to be ready
         is_model_catalog_ready(client=admin_client, model_registry_namespace=model_registry_namespace)
+
+        # Get the model catalog URL and headers and wait for the API to be fully ready
+        catalog_url, headers = get_catalog_url_and_headers(
+            admin_client=admin_client,
+            model_registry_namespace=model_registry_namespace,
+            token=current_client_token,
+        )
+        wait_for_model_catalog_api(url=catalog_url, headers=headers)
+
         yield user_sources_cm
 
 
@@ -145,33 +155,44 @@ def user_token_for_api_calls(
     api_server_url: str,
     user_credentials_rbac: dict[str, str],
     service_account: ServiceAccount,
+    model_catalog_rest_url: list[str],
 ) -> Generator[str, None, None]:
     param = getattr(request, "param", {})
     user = param.get("user_type", "admin")
     LOGGER.info("User used: %s", user)
+
+    token = None
     if user == "admin":
         LOGGER.info("Logging in as admin user")
-        yield get_openshift_token()
+        token = get_openshift_token()
     elif user == "test":
         if not is_byoidc:
             login_with_user_password(
                 api_address=api_server_url,
                 user=user_credentials_rbac["username"],
                 password=user_credentials_rbac["password"],
             )
-            yield get_openshift_token()
-            LOGGER.info(f"Logging in as {original_user}")
-            login_with_user_password(
-                api_address=api_server_url,
-                user=original_user,
-            )
+            token = get_openshift_token()
         else:
-            yield get_mr_user_token(admin_client=admin_client, user_credentials_rbac=user_credentials_rbac)
+            token = get_mr_user_token(admin_client=admin_client, user_credentials_rbac=user_credentials_rbac)
     elif user == "sa_user":
-        yield create_inference_token(service_account)
+        token = create_inference_token(service_account)
+        # retries on 401 errors for OAuth/kube-rbac-proxy initialization
+        headers = get_rest_headers(token=token)
+        wait_for_model_catalog_api(url=model_catalog_rest_url[0], headers=headers)
     else:
         raise RuntimeError(f"Unknown user type: {user}")
 
+    yield token
+
+    # Cleanup: log back in as original user if needed
+    if user == "test" and not is_byoidc:
+        LOGGER.info(f"Logging in as {original_user}")
+        login_with_user_password(
+            api_address=api_server_url,
+            user=original_user,
+        )
+
 
 @pytest.fixture(scope="function")
 def randomly_picked_model_from_catalog_api_by_source(
@@ -203,7 +224,6 @@ def randomly_picked_model_from_catalog_api_by_source(
         headers = model_registry_rest_headers
     else:
         headers = get_rest_headers(token=user_token_for_api_calls)
-    wait_for_model_catalog_api(url=f"{model_catalog_rest_url[0]}", headers=headers)
 
     if not model_name:
         LOGGER.info(f"Picking random model from catalog: {catalog_id} with header_type: {header_type}")

tests/model_registry/model_catalog/utils.py

@@ -3,10 +3,11 @@
 
 from kubernetes.dynamic import DynamicClient
 from simple_logger.logger import get_logger
-
+from timeout_sampler import retry
 
 from ocp_resources.pod import Pod
 from ocp_resources.config_map import ConfigMap
+from ocp_resources.route import Route
 from tests.model_registry.model_catalog.constants import DEFAULT_CATALOGS
 from tests.model_registry.model_catalog.db_constants import (
     SEARCH_MODELS_DB_QUERY,
@@ -23,8 +24,7 @@
     PERFORMANCE_DATA_DIR,
 )
 from tests.model_registry.constants import DEFAULT_CUSTOM_MODEL_CATALOG, DEFAULT_MODEL_CATALOG_CM
-from tests.model_registry.utils import execute_get_command
-from tests.model_registry.utils import get_rest_headers
+from tests.model_registry.utils import execute_get_command, get_rest_headers
 
 LOGGER = get_logger(name=__name__)
 
@@ -33,6 +33,24 @@ class ResourceNotFoundError(Exception):
     pass
 
 
+@retry(wait_timeout=60, sleep=5, exceptions_dict={AssertionError: []})
+def get_catalog_url_and_headers(
+    admin_client: DynamicClient,
+    model_registry_namespace: str,
+    token: str,
+) -> tuple[str, dict[str, str]]:
+    """
+    Get model catalog URL and authentication headers from route.
+    """
+    model_catalog_routes = list(
+        Route.get(namespace=model_registry_namespace, label_selector="component=model-catalog", dyn_client=admin_client)
+    )
+    assert model_catalog_routes, f"Model catalog routes not found in namespace {model_registry_namespace}"
+
+    catalog_url = f"https://{model_catalog_routes[0].instance.spec.host}:443/api/model_catalog/v1alpha1/"
+    return catalog_url, get_rest_headers(token=token)
+
+
 def validate_model_catalog_enabled(pod: Pod) -> bool:
     for container in pod.instance.spec.containers:
         for env in container.env:
@@ -1024,9 +1042,6 @@ def verify_custom_properties_sorted(items: list[dict], property_field: str, sort
 
     Returns:
         True if sorted correctly, False otherwise
-
-    Raises:
-        ValueError: If there are not enough items to verify sorting
     """
     property_name, value_type = property_field.rsplit(".", 1)
     # Separate items into two groups

tests/model_registry/utils.py

@@ -715,7 +715,7 @@ def execute_get_call(
     return resp
 
 
-@retry(wait_timeout=60, sleep=5, exceptions_dict={ResourceNotFoundError: [], TransientUnauthorizedError: []})
+@retry(wait_timeout=90, sleep=5, exceptions_dict={ResourceNotFoundError: [], TransientUnauthorizedError: []})
 def wait_for_model_catalog_api(url: str, headers: dict[str, str], verify: bool | str = False) -> requests.Response:
     """
     Wait for model catalog API to be ready and fully initialized checks both /sources and /models endpoints