test(evalhub): add multi-tenancy integration tests

  Add 24 integration tests for EvalHub covering deployment topology,
  health, multi-tenancy authorisation (providers, collections, jobs),
  job lifecycle with vLLM emulator, and operator RBAC provisioning.

Signed-off-by: Rui Vieira <ruidevieira@googlemail.com>

Author: ruivieira

renovate.json

@@ -59,8 +59,12 @@
       "schedule": [
         "before 6pm on wednesday"
       ],
-      "prPriority": 5,
-      "pinDigests": true
+      "prPriority": 5
+    },
+    {
+      "matchDatasources": ["docker"],
+      "matchPackageNames": ["quay.io/fedora/fedora"],
+      "enabled": false
     }
   ]
 }

tests/model_explainability/evalhub/multitenancy/conftest.py

@@ -255,7 +255,7 @@ def tenant_a_token(
 
 
 @pytest.fixture(scope="class")
-def vllm_emulator_deployment(
+def evalhub_vllm_emulator_deployment(
     admin_client: DynamicClient,
     tenant_a_namespace: Namespace,
     tenant_a_rbac_ready: None,
@@ -298,10 +298,10 @@ def vllm_emulator_deployment(
 
 
 @pytest.fixture(scope="class")
-def vllm_emulator_service(
+def evalhub_vllm_emulator_service(
     admin_client: DynamicClient,
     tenant_a_namespace: Namespace,
-    vllm_emulator_deployment: Deployment,
+    evalhub_vllm_emulator_deployment: Deployment,
 ) -> Generator[Service, Any, Any]:
     """Service fronting the vLLM emulator in tenant-a."""
     with Service(

tests/model_explainability/evalhub/multitenancy/test_evalhub_job_delete_mt.py

@@ -37,11 +37,11 @@ def test_job_delete_authorized_tenant(
         tenant_a_namespace: Namespace,
         evalhub_mt_ca_bundle_file: str,
         evalhub_mt_route: Route,
-        vllm_emulator_service: Service,
+        evalhub_vllm_emulator_service: Service,
     ) -> None:
         """User with evaluations-delete RBAC in tenant-a can delete a job."""
         payload = build_evalhub_job_payload(
-            model_service_name=vllm_emulator_service.name,
+            model_service_name=evalhub_vllm_emulator_service.name,
             tenant_namespace=tenant_a_namespace.name,
             job_name="evalhub-delete-test-job",
         )
@@ -70,12 +70,12 @@ def test_job_delete_cross_tenant_denied(
         tenant_b_namespace: Namespace,
         evalhub_mt_ca_bundle_file: str,
         evalhub_mt_route: Route,
-        vllm_emulator_service: Service,
+        evalhub_vllm_emulator_service: Service,
     ) -> None:
         """User with RBAC in tenant-a is denied job deletion for tenant-b."""
         # Submit in tenant-a first to get a valid job ID
         payload = build_evalhub_job_payload(
-            model_service_name=vllm_emulator_service.name,
+            model_service_name=evalhub_vllm_emulator_service.name,
             tenant_namespace=tenant_a_namespace.name,
             job_name="evalhub-delete-test-job",
         )
@@ -103,12 +103,12 @@ def test_job_delete_missing_tenant_rejected(
         tenant_a_namespace: Namespace,
         evalhub_mt_ca_bundle_file: str,
         evalhub_mt_route: Route,
-        vllm_emulator_service: Service,
+        evalhub_vllm_emulator_service: Service,
     ) -> None:
         """Job deletion without X-Tenant header is rejected with 400."""
         # Submit in tenant-a first to get a valid job ID
         payload = build_evalhub_job_payload(
-            model_service_name=vllm_emulator_service.name,
+            model_service_name=evalhub_vllm_emulator_service.name,
             tenant_namespace=tenant_a_namespace.name,
             job_name="evalhub-delete-test-job",
         )

tests/model_explainability/evalhub/multitenancy/test_evalhub_jobs_mt.py

@@ -43,11 +43,11 @@ def test_job_submit_authorized_tenant(
         tenant_a_namespace: Namespace,
         evalhub_mt_ca_bundle_file: str,
         evalhub_mt_route: Route,
-        vllm_emulator_service: Service,
+        evalhub_vllm_emulator_service: Service,
     ) -> None:
         """User with evaluations-create RBAC in tenant-a can submit a job."""
         payload = build_evalhub_job_payload(
-            model_service_name=vllm_emulator_service.name,
+            model_service_name=evalhub_vllm_emulator_service.name,
             tenant_namespace=tenant_a_namespace.name,
         )
         data = submit_evalhub_job(
@@ -69,11 +69,11 @@ def test_job_completion_and_results(
         tenant_a_namespace: Namespace,
         evalhub_mt_ca_bundle_file: str,
         evalhub_mt_route: Route,
-        vllm_emulator_service: Service,
+        evalhub_vllm_emulator_service: Service,
     ) -> None:
         """Submit a job and wait for it to complete with benchmark results."""
         payload = build_evalhub_job_payload(
-            model_service_name=vllm_emulator_service.name,
+            model_service_name=evalhub_vllm_emulator_service.name,
             tenant_namespace=tenant_a_namespace.name,
         )
         data = submit_evalhub_job(
@@ -102,11 +102,11 @@ def test_job_pod_reaches_succeeded(
         tenant_a_namespace: Namespace,
         evalhub_mt_ca_bundle_file: str,
         evalhub_mt_route: Route,
-        vllm_emulator_service: Service,
+        evalhub_vllm_emulator_service: Service,
     ) -> None:
         """After job completion, the K8s pod should be in Succeeded state."""
         payload = build_evalhub_job_payload(
-            model_service_name=vllm_emulator_service.name,
+            model_service_name=evalhub_vllm_emulator_service.name,
             tenant_namespace=tenant_a_namespace.name,
         )
         data = submit_evalhub_job(
@@ -150,11 +150,11 @@ def test_job_submit_cross_tenant_denied(
         tenant_b_namespace: Namespace,
         evalhub_mt_ca_bundle_file: str,
         evalhub_mt_route: Route,
-        vllm_emulator_service: Service,
+        evalhub_vllm_emulator_service: Service,
     ) -> None:
         """User with RBAC in tenant-a is denied job submission for tenant-b."""
         payload = build_evalhub_job_payload(
-            model_service_name=vllm_emulator_service.name,
+            model_service_name=evalhub_vllm_emulator_service.name,
             tenant_namespace=tenant_a_namespace.name,
         )
         validate_evalhub_post_denied(
@@ -172,11 +172,11 @@ def test_job_submit_missing_tenant_rejected(
         tenant_a_namespace: Namespace,
         evalhub_mt_ca_bundle_file: str,
         evalhub_mt_route: Route,
-        vllm_emulator_service: Service,
+        evalhub_vllm_emulator_service: Service,
     ) -> None:
         """Job submission without X-Tenant header is rejected with 400."""
         payload = build_evalhub_job_payload(
-            model_service_name=vllm_emulator_service.name,
+            model_service_name=evalhub_vllm_emulator_service.name,
             tenant_namespace=tenant_a_namespace.name,
         )
         validate_evalhub_post_no_tenant(
@@ -193,11 +193,11 @@ def test_job_list_authorized_tenant(
         tenant_a_namespace: Namespace,
         evalhub_mt_ca_bundle_file: str,
         evalhub_mt_route: Route,
-        vllm_emulator_service: Service,
+        evalhub_vllm_emulator_service: Service,
     ) -> None:
         """After submitting a job, listing jobs for tenant-a shows it."""
         payload = build_evalhub_job_payload(
-            model_service_name=vllm_emulator_service.name,
+            model_service_name=evalhub_vllm_emulator_service.name,
             tenant_namespace=tenant_a_namespace.name,
         )
         submitted = submit_evalhub_job(

tests/model_serving/maas_billing/maas_subscription/conftest.py

@@ -5,19 +5,24 @@
 import requests
 import structlog
 from kubernetes.dynamic import DynamicClient
+from ocp_resources.cron_job import CronJob
 from ocp_resources.llm_inference_service import LLMInferenceService
 from ocp_resources.maas_auth_policy import MaaSAuthPolicy
 from ocp_resources.maas_model_ref import MaaSModelRef
 from ocp_resources.maas_subscription import MaaSSubscription
 from ocp_resources.namespace import Namespace
+from ocp_resources.network_policy import NetworkPolicy
+from ocp_resources.pod import Pod
 from ocp_resources.resource import ResourceEditor
 from ocp_resources.service_account import ServiceAccount
 from pytest_testconfig import config as py_config
 
 from tests.model_serving.maas_billing.maas_subscription.utils import (
+    assert_api_key_created_ok,
     create_and_yield_api_key_id,
     create_api_key,
     create_maas_subscription,
+    get_maas_api_labels,
     patch_llmisvc_with_maas_router_and_tiers,
     resolve_api_key_username,
     revoke_api_key,
@@ -786,3 +791,87 @@ def short_expiration_api_key_id(
         key_name_prefix="e2e-exp-short",
         expires_in="1h",
     )
+
+
+@pytest.fixture()
+def maas_cleanup_cronjob(
+    admin_client: DynamicClient,
+) -> CronJob:
+    """Return the maas-api-key-cleanup CronJob, asserting it exists."""
+    applications_namespace = py_config["applications_namespace"]
+    cronjob = CronJob(
+        client=admin_client,
+        name="maas-api-key-cleanup",
+        namespace=applications_namespace,
+    )
+    assert cronjob.exists, f"CronJob maas-api-key-cleanup not found in {applications_namespace}"
+    return cronjob
+
+
+@pytest.fixture()
+def maas_cleanup_networkpolicy(
+    admin_client: DynamicClient,
+) -> NetworkPolicy:
+    """Return the maas-api-cleanup-restrict NetworkPolicy, asserting it exists."""
+    applications_namespace = py_config["applications_namespace"]
+    network_policy = NetworkPolicy(
+        client=admin_client,
+        name="maas-api-cleanup-restrict",
+        namespace=applications_namespace,
+    )
+    assert network_policy.exists, f"NetworkPolicy maas-api-cleanup-restrict not found in {applications_namespace}"
+    return network_policy
+
+
+@pytest.fixture()
+def maas_api_pod_name(
+    admin_client: DynamicClient,
+) -> str:
+    """Return the name of the single running maas-api pod (exactly one pod is expected)."""
+    applications_namespace = py_config["applications_namespace"]
+    label_selector = ",".join(f"{k}={v}" for k, v in get_maas_api_labels().items())
+    pods = list(
+        Pod.get(
+            client=admin_client,
+            namespace=applications_namespace,
+            label_selector=label_selector,
+        )
+    )
+    assert len(pods) == 1, f"Expected exactly 1 maas-api pod in {applications_namespace}, found {len(pods)}"
+    assert pods[0].instance.status.phase == "Running", (
+        f"maas-api pod '{pods[0].name}' is not Running (phase={pods[0].instance.status.phase})"
+    )
+    return pods[0].name
+
+
+@pytest.fixture()
+def ephemeral_api_key(
+    request_session_http: requests.Session,
+    base_url: str,
+    ocp_token_for_actor: str,
+) -> Generator[dict[str, Any]]:
+    """Create an ephemeral API key and revoke it on teardown."""
+    creation_response, api_key_data = create_api_key(
+        base_url=base_url,
+        ocp_user_token=ocp_token_for_actor,
+        request_session_http=request_session_http,
+        api_key_name=f"e2e-ephemeral-{generate_random_name()}",
+        expires_in="1h",
+        ephemeral=True,
+        raise_on_error=False,
+    )
+    assert_api_key_created_ok(resp=creation_response, body=api_key_data, required_fields=("key", "id"))
+    LOGGER.info(
+        f"[ephemeral] Created ephemeral key: id={api_key_data['id']}, expiresAt={api_key_data.get('expiresAt')}"
+    )
+    yield api_key_data
+    revoke_response, _ = revoke_api_key(
+        request_session_http=request_session_http,
+        base_url=base_url,
+        key_id=api_key_data["id"],
+        ocp_user_token=ocp_token_for_actor,
+    )
+    if revoke_response.status_code not in (200, 404):
+        raise AssertionError(
+            f"Unexpected teardown status for ephemeral key id={api_key_data['id']}: {revoke_response.status_code}"
+        )