Merge branch 'main' into bug_template

Author: dbasunag

tests/model_explainability/evalhub/test_evalhub_health.py

@@ -1,22 +1,22 @@
 import pytest
 from ocp_resources.route import Route
 
-from tests.model_explainability.evalhub.utils import validate_evalhub_health
+from tests.model_explainability.evalhub.utils import validate_evalhub_health, validate_evalhub_providers
 
 
 @pytest.mark.parametrize(
     "model_namespace",
     [
         pytest.param(
-            {"name": "test-evalhub-health"},
+            {"name": "test-evalhub-health-providers"},
         ),
     ],
     indirect=True,
 )
 @pytest.mark.smoke
 @pytest.mark.model_explainability
-class TestEvalHubHealth:
-    """Tests for basic EvalHub service health and availability."""
+class TestEvalHub:
+    """Tests for basic EvalHub service health and providers."""
 
     def test_evalhub_health_endpoint(
         self,
@@ -30,3 +30,19 @@ def test_evalhub_health_endpoint(
             token=current_client_token,
             ca_bundle_file=evalhub_ca_bundle_file,
         )
+
+    def test_evalhub_providers_list(
+        self,
+        current_client_token: str,
+        evalhub_ca_bundle_file: str,
+        evalhub_route: Route,
+        model_namespace,
+    ) -> None:
+        """Test that the evaluations providers endpoint returns a non-empty list."""
+
+        validate_evalhub_providers(
+            host=evalhub_route.host,
+            token=current_client_token,
+            ca_bundle_file=evalhub_ca_bundle_file,
+            tenant=model_namespace.name,
+        )

tests/model_explainability/evalhub/utils.py

@@ -3,12 +3,31 @@
 from tests.model_explainability.evalhub.constants import (
     EVALHUB_HEALTH_PATH,
     EVALHUB_HEALTH_STATUS_HEALTHY,
+    EVALHUB_PROVIDERS_PATH,
 )
 from utilities.guardrails import get_auth_headers
 from utilities.opendatahub_logger import get_logger
 
 LOGGER = get_logger(name=__name__)
 
+TENANT_HEADER: str = "X-Tenant"
+
+
+def _build_headers(token: str, tenant: str | None = None) -> dict[str, str]:
+    """Build request headers with auth and optional tenant.
+
+    Args:
+        token: Bearer token for authentication.
+        tenant: Namespace for the X-Tenant header. Omitted if None.
+
+    Returns:
+        Headers dict.
+    """
+    headers = get_auth_headers(token=token)
+    if tenant is not None:
+        headers[TENANT_HEADER] = tenant
+    return headers
+
 
 def validate_evalhub_health(
     host: str,
@@ -45,3 +64,26 @@ def validate_evalhub_health(
         f"Expected status '{EVALHUB_HEALTH_STATUS_HEALTHY}', got '{data['status']}'"
     )
     assert "timestamp" in data, "Health response missing 'timestamp' field"
+
+
+def validate_evalhub_providers(
+    host: str,
+    token: str,
+    ca_bundle_file: str,
+    tenant: str | None = None,
+) -> dict:
+    """Smoke test for the EvalHub providers endpoint."""
+    url = f"https://{host}{EVALHUB_PROVIDERS_PATH}"
+
+    response = requests.get(
+        url=url,
+        headers=_build_headers(token=token, tenant=tenant),
+        verify=ca_bundle_file,
+        timeout=10,
+    )
+    response.raise_for_status()
+
+    data = response.json()
+    assert data.get("items"), f"Smoke test failed: Providers list is empty for tenant {tenant}"
+
+    return data

tests/model_serving/model_server/kserve/authentication/conftest.py

@@ -21,20 +21,16 @@
     ModelName,
     Protocols,
     RuntimeTemplates,
+    Timeout,
 )
 from utilities.inference_utils import create_isvc
 from utilities.infra import (
     create_inference_token,
     create_isvc_view_role,
-    get_pods_by_isvc_label,
 )
-from utilities.jira import is_jira_open
 from utilities.logger import RedactedString
-from utilities.opendatahub_logger import get_logger
 from utilities.serving_runtime import ServingRuntimeFromTemplate
 
-LOGGER = get_logger(name=__name__)
-
 
 # HTTP/REST model serving
 @pytest.fixture(scope="class")
@@ -77,15 +73,9 @@ def http_raw_inference_token(model_service_account: ServiceAccount, http_raw_rol
 
 @pytest.fixture()
 def patched_remove_raw_authentication_isvc(
-    admin_client: DynamicClient,
     unprivileged_client: DynamicClient,
     http_s3_ovms_raw_inference_service: InferenceService,
 ) -> Generator[InferenceService, Any, Any]:
-    predictor_pod = get_pods_by_isvc_label(
-        client=unprivileged_client,
-        isvc=http_s3_ovms_raw_inference_service,
-    )[0]
-
     with ResourceEditor(
         patches={
             http_s3_ovms_raw_inference_service: {
@@ -95,12 +85,20 @@ def patched_remove_raw_authentication_isvc(
             }
         }
     ):
-        if is_jira_open(jira_id="RHOAIENG-52129", admin_client=admin_client):
-            LOGGER.info("RHOAIENG-52129 is open; waiting for predictor pod rollout after auth toggle")
-            predictor_pod.wait_deleted()
-
+        http_s3_ovms_raw_inference_service.wait_for_condition(
+            condition=http_s3_ovms_raw_inference_service.Condition.READY,
+            status=http_s3_ovms_raw_inference_service.Condition.Status.TRUE,
+            timeout=Timeout.TIMEOUT_2MIN,
+        )
         yield http_s3_ovms_raw_inference_service
 
+    # ResourceEditor restores auth on exit; wait for ISVC to reconcile before next test
+    http_s3_ovms_raw_inference_service.wait_for_condition(
+        condition=http_s3_ovms_raw_inference_service.Condition.READY,
+        status=http_s3_ovms_raw_inference_service.Condition.Status.TRUE,
+        timeout=Timeout.TIMEOUT_2MIN,
+    )
+
 
 @pytest.fixture(scope="class")
 def model_service_account_2(

tests/model_serving/model_server/kserve/authentication/test_kserve_token_authentication_raw.py

@@ -1,11 +1,8 @@
 import pytest
-from ocp_resources.resource import ResourceEditor
 
 from tests.model_serving.model_server.utils import verify_inference_response
-from utilities.constants import Annotations, Protocols
+from utilities.constants import Protocols
 from utilities.inference_utils import Inference
-from utilities.infra import check_pod_status_in_time, get_pods_by_isvc_label
-from utilities.jira import is_jira_issue_open
 from utilities.manifests.onnx import ONNX_INFERENCE_CONFIG
 
 pytestmark = pytest.mark.usefixtures("valid_aws_config")
@@ -24,6 +21,16 @@
     indirect=True,
 )
 class TestKserveTokenAuthenticationRawForRest:
+    """Validate KServe raw deployment token-based authentication for REST inference.
+
+    Steps:
+        1. Deploy an OVMS model with authentication enabled in a raw deployment namespace.
+        2. Query the model with a valid token and verify a successful REST inference response.
+        3. Disable authentication and verify the model is still queryable without a token.
+        4. Re-enable authentication and verify the model requires a valid token again.
+        5. Attempt cross-model authentication using another model's token and verify access is denied.
+    """
+
     @pytest.mark.smoke
     @pytest.mark.ocp_interop
     @pytest.mark.dependency(name="test_model_authentication_using_rest_raw")
@@ -49,40 +56,6 @@ def test_disabled_raw_model_authentication(self, patched_remove_raw_authenticati
             use_default_query=True,
         )
 
-    @pytest.mark.smoke
-    @pytest.mark.xfail(condition=is_jira_issue_open(jira_id="RHOAIENG-52129"), reason="RHOAIENG-52129", run=False)
-    def test_raw_disable_enable_authentication_no_pod_rollout(self, http_s3_ovms_raw_inference_service):
-        """Verify no pod rollout when disabling and enabling authentication"""
-        pod = get_pods_by_isvc_label(
-            client=http_s3_ovms_raw_inference_service.client,
-            isvc=http_s3_ovms_raw_inference_service,
-        )[0]
-
-        ResourceEditor(
-            patches={
-                http_s3_ovms_raw_inference_service: {
-                    "metadata": {
-                        "annotations": {Annotations.KserveAuth.SECURITY: "false"},
-                    }
-                }
-            }
-        ).update()
-
-        check_pod_status_in_time(pod=pod, status={pod.Status.RUNNING})
-
-        ResourceEditor(
-            patches={
-                http_s3_ovms_raw_inference_service: {
-                    "metadata": {
-                        "annotations": {Annotations.KserveAuth.SECURITY: "true"},
-                    }
-                }
-            }
-        ).update()
-
-        check_pod_status_in_time(pod=pod, status={pod.Status.RUNNING})
-
-    @pytest.mark.dependency(depends=["test_disabled_raw_model_authentication"])
     def test_re_enabled_raw_model_authentication(self, http_s3_ovms_raw_inference_service, http_raw_inference_token):
         """Verify model query after authentication is re-enabled"""
         verify_inference_response(

tests/model_serving/model_server/kserve/authentication/test_non_admin_users.py

@@ -21,6 +21,14 @@
 @pytest.mark.smoke
 @pytest.mark.rawdeployment
 class TestRawUnprivilegedUser:
+    """Validate that a non-admin user can deploy and query a KServe raw deployment model.
+
+    Steps:
+        1. Create a namespace with unprivileged user credentials.
+        2. Deploy an OVMS model as a raw deployment using the non-admin user.
+        3. Query the deployed model via REST and verify a successful inference response.
+    """
+
     def test_non_admin_deploy_raw_and_query_model(
         self,
         unprivileged_s3_ovms_raw_inference_service,

tests/model_serving/model_server/kserve/inference_graph/test_inference_graph_raw.py

@@ -19,12 +19,22 @@
     indirect=True,
 )
 class TestInferenceGraphRaw:
+    """Validate KServe InferenceGraph functionality in raw deployment mode.
+
+    Steps:
+        1. Deploy an OVMS serving runtime with ONNX support in the test namespace.
+        2. Create inference graphs with various configurations (public, private, auth-enabled).
+        3. Send inference requests through each graph and verify correct routing and responses.
+        4. Verify authentication enforcement by testing with privileged and unprivileged tokens.
+    """
+
     @pytest.mark.parametrize(
         "dog_breed_inference_graph",
         [{"name": "dog-breed-raw-pipeline", "deployment-mode": KServeDeploymentType.RAW_DEPLOYMENT}],
         indirect=True,
     )
     def test_inference_graph_raw_deployment(self, dog_breed_inference_graph):
+        """Verify inference through a public raw deployment inference graph."""
         verify_inference_response(
             inference_service=dog_breed_inference_graph,
             inference_config=ONNX_INFERENCE_CONFIG,
@@ -46,6 +56,7 @@ def test_inference_graph_raw_deployment(self, dog_breed_inference_graph):
         indirect=True,
     )
     def test_private_inference_graph_raw_deployment(self, dog_breed_inference_graph):
+        """Verify inference through a private (no external route) raw deployment inference graph."""
         verify_inference_response(
             inference_service=dog_breed_inference_graph,
             inference_config=ONNX_INFERENCE_CONFIG,
@@ -69,6 +80,7 @@ def test_private_inference_graph_raw_deployment(self, dog_breed_inference_graph)
         indirect=True,
     )
     def test_inference_graph_raw_authentication(self, dog_breed_inference_graph, inference_graph_sa_token_with_access):
+        """Verify inference through an auth-enabled inference graph using an authorized service account token."""
         verify_inference_response(
             inference_service=dog_breed_inference_graph,
             inference_config=ONNX_INFERENCE_CONFIG,
@@ -94,6 +106,7 @@ def test_inference_graph_raw_authentication(self, dog_breed_inference_graph, inf
     def test_private_inference_graph_raw_authentication(
         self, dog_breed_inference_graph, inference_graph_sa_token_with_access
     ):
+        """Verify inference through a private auth-enabled inference graph using an authorized token."""
         verify_inference_response(
             inference_service=dog_breed_inference_graph,
             inference_config=ONNX_INFERENCE_CONFIG,
@@ -119,6 +132,7 @@ def test_private_inference_graph_raw_authentication(
     def test_inference_graph_raw_authentication_without_privileges(
         self, dog_breed_inference_graph, inference_graph_unprivileged_sa_token
     ):
+        """Verify that an unprivileged token is denied access to an auth-enabled inference graph."""
         verify_inference_response(
             inference_service=dog_breed_inference_graph,
             inference_config=ONNX_INFERENCE_CONFIG,
@@ -145,6 +159,7 @@ def test_inference_graph_raw_authentication_without_privileges(
     def test_private_inference_graph_raw_authentication_without_privileges(
         self, dog_breed_inference_graph, inference_graph_unprivileged_sa_token
     ):
+        """Verify that an unprivileged token is denied access to a private auth-enabled inference graph."""
         verify_inference_response(
             inference_service=dog_breed_inference_graph,
             inference_config=ONNX_INFERENCE_CONFIG,

tests/model_serving/model_server/kserve/inference_service_lifecycle/test_isvc_env_vars_updates.py

@@ -43,6 +43,15 @@
     indirect=True,
 )
 class TestRawISVCEnvVarsUpdates:
+    """Validate adding and removing environment variables on a KServe raw deployment ISVC.
+
+    Steps:
+        1. Deploy an OVMS inference service with custom environment variables.
+        2. Verify the environment variables are present in the predictor pods.
+        3. Remove the environment variables from the inference service.
+        4. Verify the environment variables are no longer present in the predictor pods.
+    """
+
     def test_raw_with_isvc_env_vars(self, ovms_kserve_inference_service):
         """Test adding environment variables to the inference service"""
         verify_env_vars_in_isvc_pods(isvc=ovms_kserve_inference_service, env_vars=ISVC_ENV_VARS, vars_exist=True)

tests/model_serving/model_server/kserve/inference_service_lifecycle/test_isvc_pull_secret_updates.py

@@ -24,6 +24,15 @@
     indirect=True,
 )
 class TestISVCPullSecretUpdate:
+    """Validate pull secret lifecycle operations on a KServe model car inference service.
+
+    Steps:
+        1. Deploy a model car ISVC with an initial pull secret attached.
+        2. Verify the initial pull secret is correctly set in the predictor pod.
+        3. Update the pull secret to a new value and verify it is reflected in the new pod.
+        4. Remove the pull secret and verify it is no longer present in the pod.
+    """
+
     @pytest.mark.tier1
     def test_initial_pull_secret_set(self, model_car_raw_inference_service_with_pull_secret):
         """Ensure initial pull secret is correctly set in the pod"""
@@ -36,4 +45,5 @@ def test_update_pull_secret(self, updated_isvc_pull_secret):
         verify_pull_secret(isvc=updated_isvc_pull_secret, pull_secret=UPDATED_PULL_SECRET, secret_exists=True)
 
     def test_remove_pull_secret(self, updated_isvc_remove_pull_secret):
+        """Remove the pull secret and verify it is no longer present in the pod."""
         verify_pull_secret(isvc=updated_isvc_remove_pull_secret, pull_secret=UPDATED_PULL_SECRET, secret_exists=False)

tests/model_serving/model_server/kserve/inference_service_lifecycle/test_isvc_replicas_update.py

@@ -39,6 +39,16 @@
     indirect=True,
 )
 class TestRawISVCReplicasUpdates:
+    """Validate scaling replica count up and down on a KServe raw deployment ISVC.
+
+    Steps:
+        1. Deploy an OVMS inference service with 2 min-replicas and 4 max-replicas.
+        2. Verify that 2 predictor pods are running after deployment.
+        3. Run inference to confirm the model responds correctly with multiple replicas.
+        4. Patch the ISVC to scale down to 1 replica and verify only 1 pod remains.
+        5. Run inference again to confirm the model responds correctly after scale-down.
+    """
+
     @pytest.mark.dependency(name="test_raw_increase_isvc_replicas")
     def test_raw_increase_isvc_replicas(self, isvc_pods, ovms_kserve_inference_service):
         """Test replicas increase"""