tests(maas): add multiple auth policy and cascade deletion tests

tests/model_serving/maas_billing/maas_subscription/conftest.py

@@ -25,6 +25,7 @@
     MAAS_DB_NAMESPACE,
     MAAS_SUBSCRIPTION_NAMESPACE,
     create_api_key,
+    create_maas_subscription,
     get_maas_postgres_resources,
     patch_llmisvc_with_maas_router_and_tiers,
     wait_for_postgres_connection_log,
@@ -447,3 +448,111 @@ def maas_subscription_namespace(unprivileged_client, admin_client):
         admin_client=admin_client,
     ) as ns:
         yield ns
+
+
+@pytest.fixture(scope="function")
+def temporary_system_authenticated_subscription(
+    admin_client: DynamicClient,
+    maas_subscription_tinyllama_free: MaaSSubscription,
+    maas_model_tinyllama_free: MaaSModelRef,
+) -> Generator[MaaSSubscription, Any, Any]:
+    """
+    Creates a temporary subscription owned by system:authenticated.
+    Used for cascade deletion tests.
+    """
+
+    subscription_name = f"e2e-temp-sub-{generate_random_name()}"
+
+    with create_maas_subscription(
+        admin_client=admin_client,
+        subscription_namespace=maas_subscription_tinyllama_free.namespace,
+        subscription_name=subscription_name,
+        owner_group_name="system:authenticated",
+        model_name=maas_model_tinyllama_free.name,
+        model_namespace=maas_model_tinyllama_free.namespace,
+        tokens_per_minute=50,
+        window="1m",
+        priority=0,
+        teardown=False,
+        wait_for_resource=True,
+    ) as temporary_subscription:
+        temporary_subscription.wait_for_condition(
+            condition="Ready",
+            status="True",
+            timeout=300,
+        )
+
+        LOGGER.info(
+            f"Created temporary subscription {temporary_subscription.name} for model {maas_model_tinyllama_free.name}"
+        )
+
+        yield temporary_subscription
+
+        LOGGER.info(f"Fixture teardown: ensuring subscription {temporary_subscription.name} is removed")
+        temporary_subscription.clean_up(wait=True)
+
+
+@pytest.fixture(scope="function")
+def premium_system_authenticated_access(
+    admin_client: DynamicClient,
+    maas_model_tinyllama_premium: MaaSModelRef,
+    maas_subscription_tinyllama_premium: MaaSSubscription,
+) -> Generator[dict[str, Any], Any, Any]:
+    """
+    Creates an extra AuthPolicy and matching subscription for system:authenticated
+    on the premium model.
+    """
+
+    auth_policy_name = f"e2e-premium-system-auth-{generate_random_name()}"
+    subscription_name = f"e2e-premium-system-auth-sub-{generate_random_name()}"
+
+    with (
+        MaaSAuthPolicy(
+            client=admin_client,
+            name=auth_policy_name,
+            namespace=maas_subscription_tinyllama_premium.namespace,
+            model_refs=[
+                {
+                    "name": maas_model_tinyllama_premium.name,
+                    "namespace": maas_model_tinyllama_premium.namespace,
+                }
+            ],
+            subjects={"groups": [{"name": "system:authenticated"}]},
+            teardown=False,
+            wait_for_resource=True,
+        ) as extra_auth_policy,
+        create_maas_subscription(
+            admin_client=admin_client,
+            subscription_namespace=maas_subscription_tinyllama_premium.namespace,
+            subscription_name=subscription_name,
+            owner_group_name="system:authenticated",
+            model_name=maas_model_tinyllama_premium.name,
+            model_namespace=maas_model_tinyllama_premium.namespace,
+            tokens_per_minute=100,
+            window="1m",
+            priority=0,
+            teardown=True,
+            wait_for_resource=True,
+        ) as system_authenticated_subscription,
+    ):
+        extra_auth_policy.wait_for_condition(condition="Ready", status="True", timeout=300)
+        system_authenticated_subscription.wait_for_condition(
+            condition="Ready",
+            status="True",
+            timeout=300,
+        )
+
+        LOGGER.info(
+            f"Created extra AuthPolicy {extra_auth_policy.name} and subscription "
+            f"{system_authenticated_subscription.name} for premium model "
+            f"{maas_model_tinyllama_premium.name}"
+        )
+
+        yield {
+            "auth_policy": extra_auth_policy,
+            "subscription": system_authenticated_subscription,
+        }
+
+        if extra_auth_policy.exists:
+            LOGGER.info(f"Fixture teardown: ensuring AuthPolicy {extra_auth_policy.name} is removed")
+            extra_auth_policy.clean_up(wait=True)

tests/model_serving/maas_billing/maas_subscription/test_cascade_deletion.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import pytest
+import requests
+from kubernetes.dynamic import DynamicClient
+from simple_logger.logger import get_logger
+
+from tests.model_serving.maas_billing.maas_subscription.utils import (
+    chat_payload_for_url,
+    poll_expected_status,
+)
+from utilities.resources.maa_s_subscription import MaaSSubscription
+
+LOGGER = get_logger(name=__name__)
+
+
+@pytest.mark.usefixtures(
+    "maas_unprivileged_model_namespace",
+    "maas_subscription_controller_enabled_latest",
+    "maas_gateway_api",
+    "maas_api_gateway_reachable",
+    "maas_inference_service_tinyllama_free",
+    "maas_model_tinyllama_free",
+    "maas_auth_policy_tinyllama_free",
+    "maas_subscription_tinyllama_free",
+)
+class TestCascadeDeletion:
+    """
+    Tests that deleting MaaSSubscription CRs triggers proper cleanup/rebuild behavior.
+    """
+
+    @pytest.mark.tier1
+    @pytest.mark.parametrize("ocp_token_for_actor", [{"type": "free"}], indirect=True)
+    def test_delete_subscription_rebuilds_trlp(
+        self,
+        request_session_http: requests.Session,
+        model_url_tinyllama_free: str,
+        maas_headers_for_actor_api_key: dict[str, str],
+        temporary_system_authenticated_subscription: MaaSSubscription,
+    ) -> None:
+        """
+        Add a second subscription for the same model, then delete it.
+        Verify the original subscription still allows access (HTTP 200).
+        """
+
+        LOGGER.info(f"Deleting temporary subscription {temporary_system_authenticated_subscription.name}")
+
+        temporary_system_authenticated_subscription.clean_up(wait=True)
+
+        payload = chat_payload_for_url(model_url=model_url_tinyllama_free)
+
+        response = poll_expected_status(
+            request_session_http=request_session_http,
+            model_url=model_url_tinyllama_free,
+            headers=maas_headers_for_actor_api_key,
+            payload=payload,
+            expected_statuses={200},
+            wait_timeout=240,
+            sleep=5,
+            request_timeout=60,
+        )
+
+        assert response.status_code == 200, (
+            f"Expected HTTP 200 after deleting temporary subscription and rebuilding policies, "
+            f"but received {response.status_code}: {(response.text or '')[:200]}"
+        )
+
+    @pytest.mark.tier1
+    @pytest.mark.parametrize("ocp_token_for_actor", [{"type": "free"}], indirect=True)
+    def test_delete_last_subscription_denies_access(
+        self,
+        request_session_http: requests.Session,
+        admin_client: DynamicClient,
+        maas_free_group: str,
+        maas_model_tinyllama_free,
+        model_url_tinyllama_free: str,
+        maas_headers_for_actor_api_key: dict[str, str],
+        maas_subscription_tinyllama_free: MaaSSubscription,
+    ) -> None:
+        """
+        Delete the only/original subscription for the model.
+        Verify access is denied with 403 or 429.
+
+        Expected behavior:
+        - 403 if auth policy denies because no valid subscription exists
+        - 429 if default deny token rate limit policy applies first
+
+        Both mean: no subscription => no access.
+        """
+
+        original_name = maas_subscription_tinyllama_free.name
+        original_priority = getattr(maas_subscription_tinyllama_free, "priority", 0)
+
+        LOGGER.info("Deleting original subscription %s", original_name)
+        maas_subscription_tinyllama_free.clean_up(wait=True)
+
+        payload = chat_payload_for_url(model_url=model_url_tinyllama_free)
+
+        try:
+            response = poll_expected_status(
+                request_session_http=request_session_http,
+                model_url=model_url_tinyllama_free,
+                headers=maas_headers_for_actor_api_key,
+                payload=payload,
+                expected_statuses={403, 429},
+                wait_timeout=120,
+                sleep=5,
+                request_timeout=60,
+            )
+
+            LOGGER.info(
+                "No subscription present for model %s -> received %s as expected",
+                maas_model_tinyllama_free.name,
+                response.status_code,
+            )
+
+            assert response.status_code in {403, 429}, (
+                "Expected 403 or 429 after deleting the last subscription, "
+                f"got {response.status_code}: {(response.text or '')[:200]}"
+            )
+        finally:
+            with MaaSSubscription(
+                client=admin_client,
+                name=original_name,
+                namespace=maas_subscription_tinyllama_free.namespace,
+                owner={
+                    "groups": [{"name": maas_free_group}],
+                },
+                model_refs=[
+                    {
+                        "name": maas_model_tinyllama_free.name,
+                        "namespace": maas_model_tinyllama_free.namespace,
+                        "tokenRateLimits": [{"limit": 100, "window": "1m"}],
+                    }
+                ],
+                priority=original_priority,
+                teardown=False,
+                wait_for_resource=True,
+            ) as restored_subscription:
+                restored_subscription.wait_for_condition(
+                    condition="Ready",
+                    status="True",
+                    timeout=300,
+                )
+                LOGGER.info("Restored original subscription %s", restored_subscription.name)

tests/model_serving/maas_billing/maas_subscription/test_multiple_auth_policies_per_model.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import pytest
+import requests
+from simple_logger.logger import get_logger
+
+from tests.model_serving.maas_billing.maas_subscription.utils import (
+    chat_payload_for_url,
+    poll_expected_status,
+)
+
+LOGGER = get_logger(name=__name__)
+
+MAAS_SUBSCRIPTION_HEADER = "x-maas-subscription"
+
+
+@pytest.mark.usefixtures(
+    "maas_unprivileged_model_namespace",
+    "maas_subscription_controller_enabled_latest",
+    "maas_gateway_api",
+    "maas_api_gateway_reachable",
+    "maas_inference_service_tinyllama_premium",
+    "maas_model_tinyllama_premium",
+    "maas_auth_policy_tinyllama_premium",
+    "maas_subscription_tinyllama_premium",
+)
+class TestMultipleAuthPoliciesPerModel:
+    @pytest.mark.smoke
+    @pytest.mark.parametrize("ocp_token_for_actor", [{"type": "free"}], indirect=True)
+    def test_premium_model_denies_free_actor_by_default(
+        self,
+        request_session_http: requests.Session,
+        model_url_tinyllama_premium: str,
+        maas_subscription_tinyllama_premium,
+        maas_headers_for_actor_api_key: dict[str, str],
+    ) -> None:
+        """
+        Verify FREE actor is denied by default on the premium model.
+        """
+
+        baseline_headers = dict(maas_headers_for_actor_api_key)
+        baseline_headers[MAAS_SUBSCRIPTION_HEADER] = maas_subscription_tinyllama_premium.name
+        baseline_payload = chat_payload_for_url(model_url=model_url_tinyllama_premium)
+
+        baseline_response = poll_expected_status(
+            request_session_http=request_session_http,
+            model_url=model_url_tinyllama_premium,
+            headers=baseline_headers,
+            payload=baseline_payload,
+            expected_statuses={403},
+        )
+
+        assert baseline_response.status_code == 403, (
+            f"Expected baseline 403 for FREE actor on premium model, got "
+            f"{baseline_response.status_code}: {(baseline_response.text or '')[:200]}"
+        )
+
+    @pytest.mark.smoke
+    @pytest.mark.parametrize("ocp_token_for_actor", [{"type": "free"}], indirect=True)
+    def test_two_auth_policies_or_logic_allows_access(
+        self,
+        request_session_http: requests.Session,
+        model_url_tinyllama_premium: str,
+        maas_headers_for_actor_api_key: dict[str, str],
+        premium_system_authenticated_access,
+    ) -> None:
+        """
+        Verify FREE actor can access the premium model when an extra AuthPolicy
+        and matching subscription for system:authenticated exist.
+        """
+
+        payload = chat_payload_for_url(model_url=model_url_tinyllama_premium)
+        explicit_headers = dict(maas_headers_for_actor_api_key)
+        explicit_headers[MAAS_SUBSCRIPTION_HEADER] = premium_system_authenticated_access["subscription"].name
+
+        LOGGER.info(
+            f"Polling for 200 on premium model with OR auth policy: "
+            f"auth_policy={premium_system_authenticated_access['auth_policy'].name}, "
+            f"subscription={premium_system_authenticated_access['subscription'].name}"
+        )
+
+        response = poll_expected_status(
+            request_session_http=request_session_http,
+            model_url=model_url_tinyllama_premium,
+            headers=explicit_headers,
+            payload=payload,
+            expected_statuses={200},
+        )
+
+        assert response.status_code == 200, (
+            f"Expected 200 with second AuthPolicy (OR logic), got {response.status_code}: {(response.text or '')[:200]}"
+        )
+
+    @pytest.mark.tier1
+    @pytest.mark.parametrize("ocp_token_for_actor", [{"type": "free"}], indirect=True)
+    def test_delete_extra_auth_policy_denies_access_on_premium_model(
+        self,
+        request_session_http: requests.Session,
+        model_url_tinyllama_premium: str,
+        maas_headers_for_actor_api_key: dict[str, str],
+        premium_system_authenticated_access,
+    ) -> None:
+        """
+        Verify FREE actor loses access again after the extra AuthPolicy is deleted.
+        """
+
+        payload = chat_payload_for_url(model_url=model_url_tinyllama_premium)
+        explicit_headers = dict(maas_headers_for_actor_api_key)
+        explicit_headers[MAAS_SUBSCRIPTION_HEADER] = premium_system_authenticated_access["subscription"].name
+
+        poll_expected_status(
+            request_session_http=request_session_http,
+            model_url=model_url_tinyllama_premium,
+            headers=explicit_headers,
+            payload=payload,
+            expected_statuses={200},
+        )
+
+        premium_system_authenticated_access["auth_policy"].delete(wait=True)
+
+        response = poll_expected_status(
+            request_session_http=request_session_http,
+            model_url=model_url_tinyllama_premium,
+            headers=explicit_headers,
+            payload=payload,
+            expected_statuses={403},
+        )
+
+        assert response.status_code == 403, (
+            f"Expected 403 after deleting extra AuthPolicy, got {response.status_code}: {(response.text or '')[:200]}"
+        )