Merge branch 'main' into gaussian-credit-model-revert

Author: dbasunag

tests/fixtures/guardrails.py

@@ -52,11 +52,11 @@ def guardrails_orchestrator(
         metrics_endpoint = request.getfixturevalue(argname="otelcol_metrics_endpoint")
         traces_endpoint = request.getfixturevalue(argname="tempo_traces_endpoint")
         gorch_kwargs["otel_exporter"] = {
-            "metricsEndpoint": metrics_endpoint,
-            "metricsProtocol": "grpc",
-            "otlpExport": "metrics,traces",
-            "tracesEndpoint": traces_endpoint,
-            "tracesProtocol": "grpc",
+            "otlpProtocol": "grpc",
+            "otlpMetricsEndpoint": metrics_endpoint,
+            "otlpTracesEndpoint": traces_endpoint,
+            "enableMetrics": True,
+            "enableTracing": True,
         }
 
     with GuardrailsOrchestrator(**gorch_kwargs) as gorch:

tests/model_explainability/guardrails/conftest.py

@@ -158,19 +158,24 @@ def installed_tempo_operator(admin_client: DynamicClient, model_namespace: Names
     """
     Installs the Tempo operator and waits for its deployment.
     """
-    operator_ns = Namespace(name="openshift-operators", ensure_exists=True)
+
+    operator_ns_name = "openshift-tempo-operator"
+    operator_ns = Namespace(name=operator_ns_name)
+    if not operator_ns.exists:
+        operator_ns.create()
+
     package_name = "tempo-product"
 
     install_operator(
         admin_client=admin_client,
-        target_namespaces=["openshift-operators"],
+        target_namespaces=None,
         name=package_name,
         channel="stable",
         source="redhat-operators",
         operator_namespace=operator_ns.name,
         timeout=Timeout.TIMEOUT_15MIN,
         install_plan_approval="Automatic",
-        starting_csv="tempo-operator.v0.18.0-1",
+        starting_csv="tempo-operator.v0.18.0-2",
     )
 
     deployment = Deployment(
@@ -199,8 +204,7 @@ def tempo_stack(
     minio_secret_otel: Secret,
 ) -> Generator[Any, Any, None]:
     """
-    Create a TempoStack CR in the test namespace, configured to use the MinIO backend.
-    Mirrors the structure and management pattern of the mariadb_operator_cr fixture.
+    Create a TempoStack CR in the test namespace, configured to use MinIO backend.
     """
     csv_prefix = "tempo-operator"
     tempo_name = "my-tempo-stack"
@@ -209,20 +213,21 @@ def tempo_stack(
     tempo_csv: ClusterServiceVersion = get_cluster_service_version(
         client=admin_client,
         prefix=csv_prefix,
-        namespace="openshift-operators",
+        namespace="openshift-tempo-operator",
     )
 
-    # Retrieve example CRs (ALM examples)
+    # Retrieve ALM examples and pick TempoStack CR
     alm_examples: list[dict[str, Any]] = tempo_csv.get_alm_examples()
-
-    # Find the TempoStack kind example
-    tempo_stack_dict: dict[str, Any] = next(
-        example
-        for example in alm_examples
-        if example["kind"] == "TempoStack" and example["apiVersion"].startswith("tempo.grafana.com/")
+    tempo_stack_dict = next(
+        (
+            example
+            for example in alm_examples
+            if example["kind"] == "TempoStack" and example["apiVersion"].startswith("tempo.grafana.com/")
+        ),
+        None,
     )
     if not tempo_stack_dict:
-        raise ResourceNotFoundError(f"No TempoStack dict found in alm_examples for CSV {tempo_csv.name}")
+        raise ResourceNotFoundError(f"No TempoStack dict found in ALM examples for CSV {tempo_csv.name}")
 
     # Customize metadata
     tempo_stack_dict["metadata"]["namespace"] = model_namespace.name
@@ -267,7 +272,10 @@ def installed_opentelemetry_operator(admin_client: DynamicClient) -> Generator[N
     """
     Installs the Red Hat OpenTelemetry Operator and waits for its deployment.
     """
-    operator_ns = Namespace(name="openshift-operators", ensure_exists=True)
+    operator_namespace = "openshift-opentelemetry-operator"
+    operator_ns = Namespace(name=operator_namespace)
+    if not operator_ns.exists:
+        operator_ns.create()
 
     package_name = "opentelemetry-product"
 
@@ -317,7 +325,7 @@ def otel_collector(
     otel_csv: ClusterServiceVersion = get_cluster_service_version(
         client=admin_client,
         prefix="opentelemetry",
-        namespace="openshift-operators",
+        namespace="openshift-opentelemetry-operator",
     )
 
     # Extract OpenTelemetryCollector CR example from ALM examples
@@ -568,7 +576,7 @@ def tempo_traces_service_portforward(admin_client, model_namespace, tempo_stack)
       oc -n <ns> port-forward svc/tempo-my-tempo-stack-query-frontend 16686:16686
     """
     namespace = model_namespace.name
-    service_name = f"tempo-{tempo_stack.name}-query-frontend"  # tempo-my-tempo-stack-query-frontend"
+    service_name = f"tempo-{tempo_stack.name}-query-frontend"
     local_port = 16686
     local_url = f"http://localhost:{local_port}"
 

tests/model_explainability/guardrails/test_guardrails.py

@@ -1,5 +1,3 @@
-import http
-
 import pytest
 import requests
 import yaml
@@ -403,15 +401,18 @@ def test_guardrails_traces_in_tempo(
 
         @retry(wait_timeout=Timeout.TIMEOUT_1MIN, sleep=5)
         def check_traces():
-            response = requests.get(f"{tempo_traces_service_portforward}/api/traces?service=fms_guardrails_orchestr8")
-            if response.status_code == http.HTTPStatus.OK:
-                data = response.json()
-                if data.get("data"):  # non-empty list of traces
-                    return data
-            return False
-
-        traces = check_traces()
-        assert traces["data"], "No traces found in Tempo for Guardrails Orchestrator"
+            services = requests.get(f"{tempo_traces_service_portforward}/api/services").json().get("data", [])
+
+            guardrails_services = [s for s in services if "guardrails" in s]
+            if not guardrails_services:
+                return False
+
+            svc = guardrails_services[0]
+
+            traces = requests.get(f"{tempo_traces_service_portforward}/api/traces?service={svc}").json()
+
+            if traces.get("data"):
+                return traces
 
 
 @pytest.mark.parametrize(

tests/model_registry/model_catalog/test_default_model_catalog.py

@@ -7,6 +7,8 @@
 from ocp_resources.deployment import Deployment
 from simple_logger.logger import get_logger
 from typing import Self, Any
+from timeout_sampler import TimeoutSampler
+from kubernetes.dynamic.exceptions import ResourceNotFoundError
 
 from ocp_resources.pod import Pod
 from ocp_resources.config_map import ConfigMap
@@ -170,10 +172,20 @@ def test_model_catalog_default_catalog_sources(
         Validate specific user can access default model catalog source
         """
         LOGGER.info("Attempting client connection with token")
-        result = execute_get_command(
-            url=f"{model_catalog_rest_url[0]}sources",
-            headers=get_rest_headers(token=user_token_for_api_calls),
-        )["items"]
+
+        url = f"{model_catalog_rest_url[0]}sources"
+        headers = get_rest_headers(token=user_token_for_api_calls)
+
+        # Retry for up to 2 minutes to allow RBAC propagation
+        # Accept ResourceNotFoundError (401) as a transient error during RBAC propagation
+        sampler = TimeoutSampler(
+            wait_timeout=120,
+            sleep=5,
+            func=lambda: execute_get_command(url=url, headers=headers)["items"],
+            exceptions_dict={ResourceNotFoundError: []},
+        )
+
+        result = next(iter(sampler))
         assert result
         items_to_validate = []
         if pytestconfig.option.pre_upgrade or pytestconfig.option.post_upgrade:

tests/model_registry/rbac/conftest.py

@@ -122,16 +122,16 @@ def created_role_binding_user(
     mr_access_role: Role,
     user_credentials_rbac: dict[str, str],
 ) -> Generator[RoleBinding, None, None]:
-    if is_byoidc:
-        user_credentials_rbac["username"] = "mr-non-admin"
-    LOGGER.info(f"Using user {user_credentials_rbac['username']}")
+    # Determine the username to use without mutating the shared fixture
+    username = "mr-non-admin" if is_byoidc else user_credentials_rbac["username"]
+    LOGGER.info(f"Using user {username}")
     yield from create_role_binding(
         admin_client=admin_client,
         model_registry_namespace=model_registry_namespace,
         name="test-model-registry-access",
         mr_access_role=mr_access_role,
         subjects_kind="User",
-        subjects_name=user_credentials_rbac["username"],
+        subjects_name=username,
     )
 
 

tests/model_registry/rbac/test_mr_rbac.py

@@ -91,15 +91,17 @@ def test_user_added_to_group(
         1. After adding the user to the appropriate group, they gain access
         """
         # Wait for access to be granted
-        user_credentials_rbac["username"] = "mr-user1"
+        # Create a copy to avoid mutating the shared fixture
+        creds_copy = user_credentials_rbac.copy()
+        creds_copy["username"] = "mr-user1"
         sampler = TimeoutSampler(
             wait_timeout=240,
             sleep=5,
             func=assert_positive_mr_registry,
             model_registry_instance_rest_endpoint=model_registry_instance_rest_endpoint[0],
             token=get_openshift_token()
             if not is_byoidc
-            else get_mr_user_token(admin_client=admin_client, user_credentials_rbac=user_credentials_rbac),
+            else get_mr_user_token(admin_client=admin_client, user_credentials_rbac=creds_copy),
         )
         for _ in sampler:
             break  # Break after first successful iteration
@@ -145,13 +147,15 @@ def test_add_single_user_role_binding(
         2. The user can access the Model Registry after being granted access
         """
         if is_byoidc:
-            user_credentials_rbac["username"] = "mr-non-admin"
+            # Create a copy to avoid mutating the shared fixture
+            creds_copy = user_credentials_rbac.copy()
+            creds_copy["username"] = "mr-non-admin"
             sampler = TimeoutSampler(
                 wait_timeout=120,
                 sleep=5,
                 func=assert_positive_mr_registry,
                 model_registry_instance_rest_endpoint=model_registry_instance_rest_endpoint[0],
-                token=get_mr_user_token(admin_client=admin_client, user_credentials_rbac=user_credentials_rbac),
+                token=get_mr_user_token(admin_client=admin_client, user_credentials_rbac=creds_copy),
             )
             for _ in sampler:
                 break  # Break after first successful iteration

tests/model_registry/rbac/test_mr_rbac_sa.py

@@ -1,12 +1,13 @@
 # AI Disclaimer: Google Gemini 2.5 pro has been used to generate a majority of this code, with human review and editing.
 import pytest
-from typing import Self
+from typing import Any, Self
 from simple_logger.logger import get_logger
 from model_registry import ModelRegistry as ModelRegistryClient
 from tests.model_registry.rbac.utils import build_mr_client_args
 from utilities.infra import create_inference_token
-from mr_openapi.exceptions import ForbiddenException
+from mr_openapi.exceptions import ForbiddenException, UnauthorizedException
 from ocp_resources.service_account import ServiceAccount
+from timeout_sampler import TimeoutSampler, retry
 
 LOGGER = get_logger(name=__name__)
 
@@ -44,12 +45,11 @@ def test_service_account_access_denied(
         )
         LOGGER.debug(f"Attempting client connection with args: {client_args}")
 
-        # Expect an exception related to HTTP 403
-        with pytest.raises(ForbiddenException) as exc_info:
-            _ = ModelRegistryClient(**client_args)
+        # Retry for up to 2 minutes if we get UnauthorizedException (401) during kube-rbac-proxy initialization
+        # Expect ForbiddenException (403) once kube-rbac-proxy is fully initialized
+        http_error = _try_connection_expect_forbidden(client_args=client_args)
 
         # Verify the status code from the caught exception
-        http_error = exc_info.value
         assert http_error.body is not None, "HTTPError should have a response object"
         LOGGER.info(f"Received expected HTTP error: Status Code {http_error.status}")
         assert http_error.status == 403, f"Expected HTTP 403 Forbidden, but got {http_error.status}"
@@ -69,21 +69,44 @@ def test_service_account_access_granted(
         LOGGER.info(f"Targeting Model Registry REST endpoint: {model_registry_instance_rest_endpoint[0]}")
         LOGGER.info("Applied RBAC Role/Binding via fixtures. Expecting access GRANT.")
 
-        # Create a fresh token to bypass OAuth proxy cache from previous test
+        # Create a fresh token to bypass kube-rbac-proxy cache from previous test
         fresh_token = create_inference_token(model_service_account=service_account)
+        client_args = build_mr_client_args(
+            rest_endpoint=model_registry_instance_rest_endpoint[0], token=fresh_token, author="rbac-test-granted"
+        )
+        LOGGER.debug(f"Attempting client connection with args: {client_args}")
+
+        # Retry for up to 2 minutes to allow RBAC propagation
+        # Accept UnauthorizedException (401) as a transient error during RBAC propagation
+        sampler = TimeoutSampler(
+            wait_timeout=120,
+            sleep=5,
+            func=lambda: ModelRegistryClient(**client_args),
+            exceptions_dict={UnauthorizedException: []},
+        )
 
         try:
-            client_args = build_mr_client_args(
-                rest_endpoint=model_registry_instance_rest_endpoint[0], token=fresh_token, author="rbac-test-granted"
-            )
-            LOGGER.debug(f"Attempting client connection with args: {client_args}")
-            mr_client_success = ModelRegistryClient(**client_args)
+            # Get the first successful result
+            mr_client_success = next(iter(sampler))
             assert mr_client_success is not None, "Client initialization failed after granting permissions"
             LOGGER.info("Client instantiated successfully after granting permissions.")
-
         except Exception as e:
-            # If we get an exception here, it's unexpected, especially 403
-            LOGGER.error(f"Received unexpected general error after granting access: {e}", exc_info=True)
+            LOGGER.error(f"Failed to access Model Registry after granting permissions: {e}", exc_info=True)
             raise
 
         LOGGER.info("--- RBAC Test Completed Successfully ---")
+
+
+@retry(wait_timeout=120, sleep=5, exceptions_dict={UnauthorizedException: []})
+def _try_connection_expect_forbidden(client_args: dict[str, Any]) -> ForbiddenException:
+    """
+    Attempts to create a ModelRegistryClient and expects ForbiddenException.
+    Retries on UnauthorizedException (401) during kube-rbac-proxy initialization.
+    Returns the ForbiddenException when received.
+    """
+    try:
+        ModelRegistryClient(**client_args)
+        raise AssertionError("Expected ForbiddenException but client connection succeeded")
+    except ForbiddenException as e:
+        # This is what we want - 403 Forbidden
+        return e