Configure OpenTelemetry on existing tests

	modified:   tests/model_explainability/guardrails/conftest.py
	modified:   tests/model_explainability/guardrails/test_guardrails.py

	modified:   pyproject.toml
	modified:   tests/model_explainability/guardrails/conftest.py
	modified:   tests/model_explainability/guardrails/test_guardrails.py

Author: kpunwatk

pyproject.toml

@@ -62,7 +62,7 @@ dependencies = [
     "timeout-sampler>=1.0.6",
     "shortuuid>=1.0.13",
     "jira>=3.8.0",
-    "openshift-python-wrapper>=11.0.57",
+    "openshift-python-wrapper>=11.0.92",
     "semver>=3.0.4",
     "sqlalchemy>=2.0.40",
     "pytest-order>=1.3.0",

tests/model_explainability/guardrails/conftest.py

@@ -6,25 +6,32 @@
 import pytest
 from _pytest.fixtures import FixtureRequest
 from kubernetes.dynamic import DynamicClient
+from kubernetes.dynamic.exceptions import ResourceNotFoundError
+from ocp_resources.cluster_service_version import ClusterServiceVersion
 from ocp_resources.config_map import ConfigMap
 from ocp_resources.deployment import Deployment
+from ocp_resources.exceptions import MissingRequiredArgumentError
 from ocp_resources.guardrails_orchestrator import GuardrailsOrchestrator
 from ocp_resources.inference_service import InferenceService
 from ocp_resources.namespace import Namespace
 from ocp_resources.pod import Pod
-from ocp_resources.resource import ResourceEditor
+from ocp_resources.resource import ResourceEditor, NamespacedResource
 from ocp_resources.route import Route
 from ocp_resources.secret import Secret
 from ocp_resources.serving_runtime import ServingRuntime
+from ocp_resources.subscription import Subscription
+from ocp_utilities.operators import install_operator, uninstall_operator
 from pytest_testconfig import py_config
+from timeout_sampler import TimeoutSampler
 
 from utilities.certificates_utils import create_ca_bundle_file
 from utilities.constants import (
     KServeDeploymentType,
     Labels,
+    Timeout, OPENSHIFT_OPERATORS,
 )
 from utilities.inference_utils import create_isvc
-
+from utilities.operator_utils import get_cluster_service_version
 
 GUARDRAILS_ORCHESTRATOR_NAME = "guardrails-orchestrator"
 
@@ -351,3 +358,197 @@ def hap_detector_route(
         service=hap_detector_isvc.name,
         wait_for_resource=True,
     )
+
+@pytest.fixture(scope="class")
+def installed_opentelemetry_operator(admin_client: DynamicClient) -> Generator[None, Any, None]:
+    """
+    Installs the OpenTelemetry Operator and waits for its deployment.
+    """
+    operator_ns = Namespace(name="openshift-operators", ensure_exists=True)
+
+    package_name = "opentelemetry-operator"
+
+    install_operator(
+        admin_client=admin_client,
+        target_namespaces=[operator_ns.name],
+        name=package_name,
+        channel="stable",
+        source="redhat-operators",
+        operator_namespace=operator_ns.name,
+        timeout=Timeout.TIMEOUT_15MIN,
+        install_plan_approval="Automatic",
+        starting_csv="opentelemetry-operator.v0.127.0-2"
+    )
+
+    deployment = Deployment(
+        client=admin_client,
+        namespace=operator_ns.name,
+        name="opentelemetry-operator-controller-manager",
+        wait_for_resource=True,
+    )
+    deployment.wait_for_replicas()
+
+    yield
+
+    uninstall_operator(
+        admin_client=admin_client,
+        name=package_name,
+        operator_namespace=operator_ns.name,
+        clean_up_namespace=False,
+    )
+
+@pytest.fixture(scope="class")
+def otel_operator_cr(
+    admin_client: DynamicClient,
+    installed_opentelemetry_operator: None,
+    model_namespace: Namespace,
+) -> Generator[OpenTelemetryCollector, Any, Any]:
+    """Create an OpenTelemetryCollector CR in the test namespace."""
+    otel_csv: ClusterServiceVersion = get_cluster_service_version(
+        client=admin_client,
+        prefix="opentelemetry",
+        namespace=model_namespace.name,
+    )
+
+    alm_examples: list[dict[str, Any]] = otel_csv.get_alm_examples()
+    otel_cr_dict: dict[str, Any] = next(
+        example for example in alm_examples if example["kind"] == "OpenTelemetryCollector"
+    )
+
+    if not otel_cr_dict:
+        raise ResourceNotFoundError(
+            f"No OpenTelemetryCollector dict found in alm_examples for CSV {otel_csv.name}"
+        )
+
+    otel_cr_dict["metadata"]["namespace"] = model_namespace.name
+
+    with OpenTelemetryCollector(kind_dict=otel_cr_dict) as otel_cr:
+        otel_cr.wait_for_condition(
+            condition="Available",
+            status=OpenTelemetryCollector.Condition.Status.TRUE,
+            timeout=Timeout.TIMEOUT_10MIN,
+        )
+        yield otel_cr
+
+@pytest.fixture(scope="class")
+def installed_jaeger_operator(
+        admin_client: DynamicClient, model_namespace: Namespace
+) -> Generator[None, Any, None]:
+    """
+    Installs the Jaeger operator and waits for its deployment.
+    """
+    operator_ns = Namespace(name="openshift-operators", ensure_exists=True)
+    package_name = "jaeger-product"
+
+    install_operator(
+        admin_client=admin_client,
+        target_namespaces=[operator_ns.name],
+        name=package_name,
+        channel="stable",
+        source="redhat-operators",
+        operator_namespace=operator_ns.name,
+        timeout=Timeout.TIMEOUT_15MIN,
+        install_plan_approval="Automatic",
+        starting_csv="jaeger-operator.v1.65.0-4",
+    )
+
+    deployment = Deployment(
+        client=admin_client,
+        namespace=operator_ns.name,
+        name="jaeger-operator",
+        wait_for_resource=True,
+    )
+    deployment.wait_for_replicas()
+
+    yield
+
+    uninstall_operator(
+        admin_client=admin_client,
+        name=package_name,
+        operator_namespace=operator_ns.name,
+        clean_up_namespace=False,
+    )
+
+
+@pytest.fixture(scope="class")
+def jaeger_instance(
+        admin_client: DynamicClient,
+        installed_jaeger_operator: None,
+        model_namespace: Namespace,
+) -> Generator[Jaeger, Any, None]:
+    """Create a Jaeger instance in the test namespace."""
+
+    # The CSV name is jaeger-operator.v1.65.0-4, so the prefix should be `jaeger-operator`.
+    csv_prefix = "jaeger-operator"
+
+    jaeger_csv: ClusterServiceVersion = get_cluster_service_version(
+        client=admin_client,
+        prefix=csv_prefix,
+        namespace=OPENSHIFT_OPERATORS,
+    )
+
+    alm_examples: list[dict[str, Any]] = jaeger_csv.get_alm_examples()
+    jaeger_dict: dict[str, Any] = next(
+        example for example in alm_examples if example["kind"] == "Jaeger"
+    )
+
+    if not jaeger_dict:
+        raise ResourceNotFoundError(
+            f"No Jaeger dict found in alm_examples for CSV {jaeger_csv.name}"
+        )
+
+    jaeger_dict["metadata"]["namespace"] = model_namespace.name
+    jaeger_dict["metadata"]["name"] = "simplest"
+
+    with Jaeger(kind_dict=jaeger_dict) as jaeger:
+        wait_for_jaeger_pods(
+            client=admin_client,
+            jaeger_name=jaeger.name,
+            namespace=model_namespace.name,
+        )
+        yield jaeger
+
+def wait_for_jaeger_operator_deployments(namespace: str) -> None:
+    """
+    Wait for the Jaeger operator deployment to be ready.
+    """
+    operator_deployment_name = "jaeger-operator"
+
+    deployment = Deployment(name=operator_deployment_name, namespace=namespace)
+    deployment.wait_for_replicas()
+
+
+def wait_for_jaeger_pods(
+    client: DynamicClient,
+    jaeger_name: str,
+    namespace: str,
+    timeout: int = Timeout.TIMEOUT_15MIN,
+) -> None:
+    """
+    Wait for pods created by a Jaeger instance to be ready.
+    """
+
+    def _get_jaeger_pods() -> list[Pod]:
+        return [
+            _pod
+            for _pod in Pod.get(
+                dyn_client=client,
+                namespace=namespace,
+                label_selector=f"app.kubernetes.io/instance={jaeger_name}",
+            )
+        ]
+
+    sampler = TimeoutSampler(
+        wait_timeout=timeout, sleep=1, func=lambda: bool(_get_jaeger_pods())
+    )
+
+    for sample in sampler:
+        if sample:
+            break
+
+    pods = _get_jaeger_pods()
+    for pod in pods:
+        pod.wait_for_condition(
+            condition=Pod.Condition.READY,
+            status="True",
+        )

tests/model_explainability/guardrails/test_guardrails.py

@@ -1,13 +1,16 @@
 import http
+import time
 from typing import Dict, Any
 
 import pytest
 import requests
 import yaml
+from ocp_resources.pod import Pod
 from simple_logger.logger import get_logger
 from timeout_sampler import retry
 
 from tests.model_explainability.constants import MNT_MODELS
+from tests.model_explainability.guardrails.conftest import wait_for_jaeger_pods
 from tests.model_explainability.guardrails.constants import (
     QWEN_ISVC_NAME,
     CHAT_GENERATION_CONFIG,
@@ -197,6 +200,41 @@ def test_guardrails_builtin_detectors_unsuitable_output(
             response=response, detector_id="regex", detection_name="email_address", detection_type="pii"
         )
 
+    def test_guardrails_traces_in_jaeger(
+            self,
+            admin_client,
+            jaeger_instance,
+            otel_operator_cr,
+            model_namespace,
+            minio_pod,
+            minio_data_connection,
+            orchestrator_config,
+            guardrails_orchestrator,
+            guardrails_gateway_config,
+
+    ):
+        """
+        Ensure that OpenTelemetry traces from Guardrails Orchestrator are collected in Jaeger.
+        Equivalent to clicking 'Find Traces' in the Jaeger UI.
+        """
+        # Jaeger query service URL (in-cluster)
+        jaeger_query_service = f"http://{jaeger_instance.name}-query.{model_namespace.name}.svc:16686/api/traces"
+
+        # Wait a bit to allow traces to be generated
+        time.sleep(10)
+
+        @retry(wait_timeout=Timeout.TIMEOUT_1MIN, sleep=5)
+        def check_traces():
+            response = requests.get(f"{jaeger_query_service}?service=jaeger-all-in-one")
+            if response.status_code == http.HTTPStatus.OK:
+                data = response.json()
+                if data.get("data"):  # non-empty list of traces
+                    return data
+            return False
+
+        traces = check_traces()
+        assert traces["data"], "No traces found in Jaeger for service jaeger-all-in-one"
+
     @pytest.mark.parametrize(
         "message, url_path",
         [
@@ -481,3 +519,4 @@ def test_guardrails_several_detector_negative_detection(
         )
 
         verify_negative_detection_response(response=response)
+