From 57ee0e2d1abe2bd721f9eb20b563aefd37522306 Mon Sep 17 00:00:00 2001 From: Swati Mukund Bagal Date: Thu, 26 Mar 2026 14:31:08 -0500 Subject: [PATCH] fix: move maas component_health under maas_billing (#1310) Signed-off-by: Swati Mukund Bagal Co-authored-by: Shelton Cyril --- .../maas_billing/component_health/__init__.py | 0 .../test_maas_api_health_check.py} | 0 .../test_maas_controller_health_check.py} | 0 tests/model_serving/maas_billing/conftest.py | 244 +++++++++++++++++- .../maas_subscription/conftest.py | 226 +--------------- 5 files changed, 243 insertions(+), 227 deletions(-) create mode 100644 tests/model_serving/maas_billing/component_health/__init__.py rename tests/model_serving/maas_billing/{maas_subscription/component_health/test_maas_api_health.py => component_health/test_maas_api_health_check.py} (100%) rename tests/model_serving/maas_billing/{maas_subscription/component_health/test_maas_controller_health.py => component_health/test_maas_controller_health_check.py} (100%) diff --git a/tests/model_serving/maas_billing/component_health/__init__.py b/tests/model_serving/maas_billing/component_health/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/model_serving/maas_billing/maas_subscription/component_health/test_maas_api_health.py b/tests/model_serving/maas_billing/component_health/test_maas_api_health_check.py similarity index 100% rename from tests/model_serving/maas_billing/maas_subscription/component_health/test_maas_api_health.py rename to tests/model_serving/maas_billing/component_health/test_maas_api_health_check.py diff --git a/tests/model_serving/maas_billing/maas_subscription/component_health/test_maas_controller_health.py b/tests/model_serving/maas_billing/component_health/test_maas_controller_health_check.py similarity index 100% rename from tests/model_serving/maas_billing/maas_subscription/component_health/test_maas_controller_health.py rename to tests/model_serving/maas_billing/component_health/test_maas_controller_health_check.py diff --git a/tests/model_serving/maas_billing/conftest.py b/tests/model_serving/maas_billing/conftest.py index c1ec8d1d9..f290f07b0 100644 --- a/tests/model_serving/maas_billing/conftest.py +++ b/tests/model_serving/maas_billing/conftest.py @@ -1,5 +1,8 @@ import base64 +import secrets +import string from collections.abc import Generator +from contextlib import ExitStack from typing import Any import pytest @@ -11,16 +14,28 @@ from ocp_resources.gateway_gateway_networking_k8s_io import Gateway from ocp_resources.infrastructure import Infrastructure from ocp_resources.llm_inference_service import LLMInferenceService +from ocp_resources.maas_auth_policy import MaaSAuthPolicy +from ocp_resources.maas_model_ref import MaaSModelRef +from ocp_resources.maas_subscription import MaaSSubscription from ocp_resources.namespace import Namespace from ocp_resources.oauth import OAuth from ocp_resources.resource import ResourceEditor from ocp_resources.secret import Secret +from ocp_resources.service import Service from ocp_resources.service_account import ServiceAccount from pytest import FixtureRequest from pytest_testconfig import config as py_config from simple_logger.logger import get_logger from timeout_sampler import TimeoutSampler +from tests.model_serving.maas_billing.maas_subscription.utils import ( + MAAS_DB_NAMESPACE, + MAAS_SUBSCRIPTION_NAMESPACE, + get_maas_postgres_resources, + patch_llmisvc_with_maas_router_and_tiers, + wait_for_postgres_connection_log, + wait_for_postgres_deployment_ready, +) from tests.model_serving.maas_billing.utils import ( build_maas_headers, create_maas_group, @@ -828,7 +843,10 @@ def maas_gateway_api( if gw.exists: LOGGER.info(f"Reusing existing gateway {MAAS_GATEWAY_NAMESPACE}/{MAAS_GATEWAY_NAME}") gw.wait_for_condition(condition="Programmed", status="True", timeout=300) - yield + with ResourceEditor( + patches={gw: {"metadata": {"annotations": {"security.opendatahub.io/authorino-tls-bootstrap": "true"}}}} + ): + yield else: LOGGER.info(f"Creating gateway {MAAS_GATEWAY_NAMESPACE}/{MAAS_GATEWAY_NAME}") with Gateway( @@ -837,7 +855,10 @@ def maas_gateway_api( namespace=MAAS_GATEWAY_NAMESPACE, gateway_class_name="openshift-default", listeners=maas_gateway_listeners(hostname=maas_gateway_api_hostname), - annotations={"opendatahub.io/managed": "false"}, + annotations={ + "opendatahub.io/managed": "false", + "security.opendatahub.io/authorino-tls-bootstrap": "true", + }, label={ "app.kubernetes.io/name": "maas", "app.kubernetes.io/instance": MAAS_GATEWAY_NAME, @@ -950,3 +971,222 @@ def revoke_maas_tokens_for_actor( ) LOGGER.info(f"[{actor_label}] revoke response: status={r_del.status_code} body={(r_del.text or '')[:200]}") + + +@pytest.fixture(scope="session") +def maas_postgres_credentials() -> dict[str, str]: + alphabet = string.ascii_letters + string.digits + + postgres_user = f"maas-{generate_random_name()}" + postgres_db = f"maas-{generate_random_name()}" + postgres_password = "".join(secrets.choice(alphabet) for _ in range(32)) + + LOGGER.info( + f"Generated PostgreSQL test credentials: " + f"user={postgres_user}, db={postgres_db}, password_length={len(postgres_password)}" + ) + + return { + "postgres_user": postgres_user, + "postgres_password": postgres_password, + "postgres_db": postgres_db, + } + + +@pytest.fixture(scope="session") +def maas_postgres_prereqs( + admin_client: DynamicClient, + maas_postgres_credentials: dict[str, str], +) -> Generator[dict[Any, Any], Any, Any]: + """ + Prepare PostgreSQL resources required by maas-api before MaaS API key tests run. + """ + resources = get_maas_postgres_resources( + client=admin_client, + namespace=MAAS_DB_NAMESPACE, + teardown_resources=True, + postgres_user=maas_postgres_credentials["postgres_user"], + postgres_password=maas_postgres_credentials["postgres_password"], + postgres_db=maas_postgres_credentials["postgres_db"], + ) + + resources_instances: dict[Any, Any] = {} + + with ExitStack() as stack: + for kind_name in [Secret, Service, Deployment]: + resources_instances[kind_name] = [] + + for resource_obj in resources[kind_name]: + resources_instances[kind_name].append(stack.enter_context(resource_obj)) + + for deployment in resources_instances[Deployment]: + deployment.wait_for_condition(condition="Available", status="True", timeout=180) + + wait_for_postgres_deployment_ready( + admin_client=admin_client, + namespace=MAAS_DB_NAMESPACE, + timeout=180, + ) + wait_for_postgres_connection_log( + admin_client=admin_client, + namespace=MAAS_DB_NAMESPACE, + timeout=180, + ) + + yield resources_instances + + +@pytest.fixture(scope="session") +def maas_subscription_namespace( + unprivileged_client: DynamicClient, admin_client: DynamicClient +) -> Generator[Namespace, Any, Any]: + existing_ns = Namespace(client=admin_client, name=MAAS_SUBSCRIPTION_NAMESPACE) + if existing_ns.exists: + LOGGER.info(f"Namespace {MAAS_SUBSCRIPTION_NAMESPACE} already exists, reusing it") + yield existing_ns + else: + with create_ns( + name=MAAS_SUBSCRIPTION_NAMESPACE, + unprivileged_client=unprivileged_client, + admin_client=admin_client, + ) as ns: + yield ns + + +@pytest.fixture(scope="session") +def maas_subscription_controller_enabled_latest( + dsc_resource: DataScienceCluster, + maas_postgres_prereqs: None, + maas_gateway_api: None, + maas_subscription_namespace: Namespace, +) -> Generator[DataScienceCluster, Any, Any]: + """ + ensures postgres prerequisites and subscription namespace exist before MaaS is switched to Managed. + """ + component_patch = { + DscComponents.KSERVE: {"modelsAsService": {"managementState": DscComponents.ManagementState.MANAGED}} + } + + with ResourceEditor(patches={dsc_resource: {"spec": {"components": component_patch}}}): + dsc_resource.wait_for_condition( + condition="ModelsAsServiceReady", + status="True", + timeout=900, + ) + dsc_resource.wait_for_condition( + condition="Ready", + status="True", + timeout=600, + ) + yield dsc_resource + + dsc_resource.wait_for_condition(condition="Ready", status="True", timeout=600) + + +@pytest.fixture(scope="class") +def maas_inference_service_tinyllama_free( + admin_client: DynamicClient, + maas_unprivileged_model_namespace: Namespace, + maas_model_service_account: ServiceAccount, + maas_gateway_api: None, +) -> Generator[LLMInferenceService, Any, Any]: + with ( + create_llmisvc( + client=admin_client, + name="llm-s3-tinyllama-free", + namespace=maas_unprivileged_model_namespace.name, + storage_uri=ModelStorage.TINYLLAMA_S3, + container_image=ContainerImages.VLLM_CPU, + container_resources={ + "limits": {"cpu": "2", "memory": "12Gi"}, + "requests": {"cpu": "1", "memory": "8Gi"}, + }, + service_account=maas_model_service_account.name, + wait=False, + timeout=900, + ) as llm_service, + patch_llmisvc_with_maas_router_and_tiers(llm_service=llm_service, tiers=[]), + ): + llm_service.wait_for_condition(condition="Ready", status="True", timeout=900) + yield llm_service + + +@pytest.fixture(scope="class") +def maas_model_tinyllama_free( + admin_client: DynamicClient, + maas_inference_service_tinyllama_free: LLMInferenceService, +) -> Generator[MaaSModelRef, Any, Any]: + + with MaaSModelRef( + client=admin_client, + name=maas_inference_service_tinyllama_free.name, + namespace=maas_inference_service_tinyllama_free.namespace, + model_ref={ + "name": maas_inference_service_tinyllama_free.name, + "namespace": maas_inference_service_tinyllama_free.namespace, + "kind": "LLMInferenceService", + }, + teardown=True, + wait_for_resource=True, + ) as maas_model: + yield maas_model + + +@pytest.fixture(scope="class") +def maas_auth_policy_tinyllama_free( + admin_client: DynamicClient, + maas_free_group: str, + maas_model_tinyllama_free: MaaSModelRef, + maas_subscription_namespace: Namespace, +) -> Generator[MaaSAuthPolicy, Any, Any]: + + with MaaSAuthPolicy( + client=admin_client, + name="tinyllama-free-access", + namespace=maas_subscription_namespace.name, + model_refs=[ + { + "name": maas_model_tinyllama_free.name, + "namespace": maas_model_tinyllama_free.namespace, + } + ], + subjects={ + "groups": [ + {"name": "system:authenticated"}, + {"name": maas_free_group}, + ], + }, + teardown=True, + wait_for_resource=True, + ) as maas_auth_policy_free: + yield maas_auth_policy_free + + +@pytest.fixture(scope="class") +def maas_subscription_tinyllama_free( + admin_client: DynamicClient, + maas_free_group: str, + maas_model_tinyllama_free: MaaSModelRef, + maas_subscription_namespace: Namespace, +) -> Generator[MaaSSubscription, Any, Any]: + + with MaaSSubscription( + client=admin_client, + name="tinyllama-free-subscription", + namespace=maas_subscription_namespace.name, + owner={ + "groups": [{"name": maas_free_group}], + }, + model_refs=[ + { + "name": maas_model_tinyllama_free.name, + "namespace": maas_model_tinyllama_free.namespace, + "tokenRateLimits": [{"limit": 100, "window": "1m"}], + } + ], + priority=0, + teardown=True, + wait_for_resource=True, + ) as maas_subscription_free: + maas_subscription_free.wait_for_condition(condition="Ready", status="True", timeout=300) + yield maas_subscription_free diff --git a/tests/model_serving/maas_billing/maas_subscription/conftest.py b/tests/model_serving/maas_billing/maas_subscription/conftest.py index 0d37ae993..a050ebfe7 100644 --- a/tests/model_serving/maas_billing/maas_subscription/conftest.py +++ b/tests/model_serving/maas_billing/maas_subscription/conftest.py @@ -1,44 +1,31 @@ -import secrets -import string from collections.abc import Generator -from contextlib import ExitStack from typing import Any import pytest import requests from kubernetes.dynamic import DynamicClient -from ocp_resources.data_science_cluster import DataScienceCluster -from ocp_resources.deployment import Deployment from ocp_resources.llm_inference_service import LLMInferenceService from ocp_resources.maas_auth_policy import MaaSAuthPolicy from ocp_resources.maas_model_ref import MaaSModelRef from ocp_resources.maas_subscription import MaaSSubscription from ocp_resources.namespace import Namespace from ocp_resources.resource import ResourceEditor -from ocp_resources.secret import Secret -from ocp_resources.service import Service from ocp_resources.service_account import ServiceAccount from pytest_testconfig import config as py_config from simple_logger.logger import get_logger from tests.model_serving.maas_billing.maas_subscription.utils import ( - MAAS_DB_NAMESPACE, - MAAS_SUBSCRIPTION_NAMESPACE, create_and_yield_api_key_id, create_api_key, create_maas_subscription, - get_maas_postgres_resources, patch_llmisvc_with_maas_router_and_tiers, resolve_api_key_username, revoke_api_key, wait_for_auth_ready, - wait_for_postgres_connection_log, - wait_for_postgres_deployment_ready, ) from tests.model_serving.maas_billing.utils import build_maas_headers -from utilities.constants import DscComponents from utilities.general import generate_random_name -from utilities.infra import create_inference_token, create_ns, get_openshift_token, login_with_user_password +from utilities.infra import create_inference_token, get_openshift_token, login_with_user_password from utilities.llmd_constants import ContainerImages, ModelStorage from utilities.llmd_utils import create_llmisvc from utilities.plugins.constant import OpenAIEnpoints @@ -49,63 +36,6 @@ CHAT_COMPLETIONS = OpenAIEnpoints.CHAT_COMPLETIONS -@pytest.fixture(scope="session") -def maas_subscription_controller_enabled_latest( - dsc_resource: DataScienceCluster, - maas_postgres_prereqs: None, - maas_gateway_api: None, -) -> Generator[DataScienceCluster, Any, Any]: - """ - ensures postgres prerequisites exist before MaaS is switched to Managed. - """ - component_patch = { - DscComponents.KSERVE: {"modelsAsService": {"managementState": DscComponents.ManagementState.MANAGED}} - } - - with ResourceEditor(patches={dsc_resource: {"spec": {"components": component_patch}}}): - dsc_resource.wait_for_condition( - condition="ModelsAsServiceReady", - status="True", - timeout=900, - ) - dsc_resource.wait_for_condition( - condition="Ready", - status="True", - timeout=600, - ) - yield dsc_resource - - dsc_resource.wait_for_condition(condition="Ready", status="True", timeout=600) - - -@pytest.fixture(scope="class") -def maas_inference_service_tinyllama_free( - admin_client: DynamicClient, - maas_unprivileged_model_namespace: Namespace, - maas_model_service_account: ServiceAccount, - maas_gateway_api: None, -) -> Generator[LLMInferenceService, Any, Any]: - with ( - create_llmisvc( - client=admin_client, - name="llm-s3-tinyllama-free", - namespace=maas_unprivileged_model_namespace.name, - storage_uri=ModelStorage.TINYLLAMA_S3, - container_image=ContainerImages.VLLM_CPU, - container_resources={ - "limits": {"cpu": "2", "memory": "12Gi"}, - "requests": {"cpu": "1", "memory": "8Gi"}, - }, - service_account=maas_model_service_account.name, - wait=False, - timeout=900, - ) as llm_service, - patch_llmisvc_with_maas_router_and_tiers(llm_service=llm_service, tiers=[]), - ): - llm_service.wait_for_condition(condition="Ready", status="True", timeout=900) - yield llm_service - - @pytest.fixture(scope="class") def maas_inference_service_tinyllama_premium( admin_client: DynamicClient, @@ -134,27 +64,6 @@ def maas_inference_service_tinyllama_premium( yield llm_service -@pytest.fixture(scope="class") -def maas_model_tinyllama_free( - admin_client: DynamicClient, - maas_inference_service_tinyllama_free: LLMInferenceService, -) -> Generator[MaaSModelRef]: - - with MaaSModelRef( - client=admin_client, - name=maas_inference_service_tinyllama_free.name, - namespace=maas_inference_service_tinyllama_free.namespace, - model_ref={ - "name": maas_inference_service_tinyllama_free.name, - "namespace": maas_inference_service_tinyllama_free.namespace, - "kind": "LLMInferenceService", - }, - teardown=True, - wait_for_resource=True, - ) as maas_model: - yield maas_model - - @pytest.fixture(scope="class") def maas_model_tinyllama_premium( admin_client: DynamicClient, @@ -176,36 +85,6 @@ def maas_model_tinyllama_premium( yield maas_model -@pytest.fixture(scope="class") -def maas_auth_policy_tinyllama_free( - admin_client: DynamicClient, - maas_free_group: str, - maas_model_tinyllama_free: MaaSModelRef, - maas_subscription_namespace: Namespace, -) -> Generator[MaaSAuthPolicy]: - - with MaaSAuthPolicy( - client=admin_client, - name="tinyllama-free-access", - namespace=maas_subscription_namespace.name, - model_refs=[ - { - "name": maas_model_tinyllama_free.name, - "namespace": maas_model_tinyllama_free.namespace, - } - ], - subjects={ - "groups": [ - {"name": "system:authenticated"}, - {"name": maas_free_group}, - ], - }, - teardown=True, - wait_for_resource=True, - ) as maas_auth_policy_free: - yield maas_auth_policy_free - - @pytest.fixture(scope="class") def maas_auth_policy_tinyllama_premium( admin_client: DynamicClient, @@ -233,36 +112,6 @@ def maas_auth_policy_tinyllama_premium( yield maas_auth_policy_premium -@pytest.fixture(scope="class") -def maas_subscription_tinyllama_free( - admin_client: DynamicClient, - maas_free_group: str, - maas_model_tinyllama_free: MaaSModelRef, - maas_subscription_namespace: Namespace, -) -> Generator[MaaSSubscription]: - - with MaaSSubscription( - client=admin_client, - name="tinyllama-free-subscription", - namespace=maas_subscription_namespace.name, - owner={ - "groups": [{"name": maas_free_group}], - }, - model_refs=[ - { - "name": maas_model_tinyllama_free.name, - "namespace": maas_model_tinyllama_free.namespace, - "tokenRateLimits": [{"limit": 100, "window": "1m"}], - } - ], - priority=0, - teardown=True, - wait_for_resource=True, - ) as maas_subscription_free: - maas_subscription_free.wait_for_condition(condition="Ready", status="True", timeout=300) - yield maas_subscription_free - - @pytest.fixture(scope="class") def maas_subscription_tinyllama_premium( admin_client: DynamicClient, @@ -317,69 +166,6 @@ def model_url_tinyllama_premium( return url -@pytest.fixture(scope="session") -def maas_postgres_credentials() -> dict[str, str]: - alphabet = string.ascii_letters + string.digits - - postgres_user = f"maas-{generate_random_name()}" - postgres_db = f"maas-{generate_random_name()}" - postgres_password = "".join(secrets.choice(alphabet) for _ in range(32)) - - LOGGER.info( - f"Generated PostgreSQL test credentials: " - f"user={postgres_user}, db={postgres_db}, password_length={len(postgres_password)}" - ) - - return { - "postgres_user": postgres_user, - "postgres_password": postgres_password, - "postgres_db": postgres_db, - } - - -@pytest.fixture(scope="session") -def maas_postgres_prereqs( - admin_client: DynamicClient, - maas_postgres_credentials: dict[str, str], -) -> Generator[dict[Any, Any], Any, Any]: - """ - Prepare PostgreSQL resources required by maas-api before MaaS API key tests run. - """ - resources = get_maas_postgres_resources( - client=admin_client, - namespace=MAAS_DB_NAMESPACE, - teardown_resources=True, - postgres_user=maas_postgres_credentials["postgres_user"], - postgres_password=maas_postgres_credentials["postgres_password"], - postgres_db=maas_postgres_credentials["postgres_db"], - ) - - resources_instances: dict[Any, Any] = {} - - with ExitStack() as stack: - for kind_name in [Secret, Service, Deployment]: - resources_instances[kind_name] = [] - - for resource_obj in resources[kind_name]: - resources_instances[kind_name].append(stack.enter_context(resource_obj)) - - for deployment in resources_instances[Deployment]: - deployment.wait_for_condition(condition="Available", status="True", timeout=180) - - wait_for_postgres_deployment_ready( - admin_client=admin_client, - namespace=MAAS_DB_NAMESPACE, - timeout=180, - ) - wait_for_postgres_connection_log( - admin_client=admin_client, - namespace=MAAS_DB_NAMESPACE, - timeout=180, - ) - - yield resources_instances - - @pytest.fixture(scope="class") def maas_api_key_for_actor( request_session_http: requests.Session, @@ -445,16 +231,6 @@ def maas_headers_for_wrong_group_sa(maas_wrong_group_service_account_token: str) return build_maas_headers(token=maas_wrong_group_service_account_token) -@pytest.fixture(scope="session") -def maas_subscription_namespace(unprivileged_client, admin_client): - with create_ns( - name=MAAS_SUBSCRIPTION_NAMESPACE, - unprivileged_client=unprivileged_client, - admin_client=admin_client, - ) as ns: - yield ns - - @pytest.fixture(scope="function") def temporary_system_authenticated_subscription( admin_client: DynamicClient,