Skip to content

Commit 4957b23

Browse files
SB159pre-commit-ci[bot]mwaykole
authored
test: add MaaS subscription resources and auth enforcement tests for TinyLlama (#1150)
* Add MaaS subscription resources and auth enforcement tests for TinyLlama * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix MaaS subscription auth tests and utils; cleanup headers * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix: add logging * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix: flake8 FCN001 get_logger keyword arg * Implemeneted Review Comments * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Milind Waykole <mwaykole@redhat.com>
1 parent 8a28c9b commit 4957b23

File tree

8 files changed

+633
-0
lines changed

8 files changed

+633
-0
lines changed

‎tests/model_serving/model_server/maas_billing/maas_subscription/__init__.py‎

Whitespace-only changes.

‎tests/model_serving/model_server/maas_billing/maas_subscription/conftest.py‎

Lines changed: 287 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,287 @@
1+
from collections.abc import Generator
2+
from typing import Any
3+
4+
import pytest
5+
from kubernetes.dynamic import DynamicClient
6+
from ocp_resources.llm_inference_service import LLMInferenceService
7+
from ocp_resources.namespace import Namespace
8+
from ocp_resources.service_account import ServiceAccount
9+
from pytest_testconfig import config as py_config
10+
from simple_logger.logger import get_logger
11+
12+
from tests.model_serving.model_server.maas_billing.maas_subscription.utils import (
13+
patch_llmisvc_with_maas_router_and_tiers,
14+
)
15+
from tests.model_serving.model_server.maas_billing.utils import build_maas_headers
16+
from utilities.infra import create_inference_token, login_with_user_password
17+
from utilities.llmd_constants import ContainerImages, ModelStorage
18+
from utilities.llmd_utils import create_llmisvc
19+
from utilities.plugins.constant import OpenAIEnpoints
20+
from utilities.resources.maa_s_auth_policy import MaaSAuthPolicy
21+
from utilities.resources.maa_s_model import MaaSModel
22+
from utilities.resources.maa_s_subscription import MaaSSubscription
23+
24+
LOGGER = get_logger(name=__name__)
25+
26+
CHAT_COMPLETIONS = OpenAIEnpoints.CHAT_COMPLETIONS
27+
28+
29+
@pytest.fixture(scope="class")
30+
def maas_inference_service_tinyllama_free(
31+
admin_client: DynamicClient,
32+
maas_unprivileged_model_namespace: Namespace,
33+
maas_model_service_account: ServiceAccount,
34+
maas_gateway_api: None,
35+
) -> Generator[LLMInferenceService, Any, Any]:
36+
with (
37+
create_llmisvc(
38+
client=admin_client,
39+
name="llm-s3-tinyllama-free",
40+
namespace=maas_unprivileged_model_namespace.name,
41+
storage_uri=ModelStorage.TINYLLAMA_S3,
42+
container_image=ContainerImages.VLLM_CPU,
43+
container_resources={
44+
"limits": {"cpu": "2", "memory": "12Gi"},
45+
"requests": {"cpu": "1", "memory": "8Gi"},
46+
},
47+
service_account=maas_model_service_account.name,
48+
wait=False,
49+
timeout=900,
50+
) as llm_service,
51+
patch_llmisvc_with_maas_router_and_tiers(llm_service=llm_service, tiers=[]),
52+
):
53+
llm_service.wait_for_condition(condition="Ready", status="True", timeout=900)
54+
yield llm_service
55+
56+
57+
@pytest.fixture(scope="class")
58+
def maas_inference_service_tinyllama_premium(
59+
admin_client: DynamicClient,
60+
maas_unprivileged_model_namespace: Namespace,
61+
maas_model_service_account: ServiceAccount,
62+
maas_gateway_api: None,
63+
) -> Generator[LLMInferenceService, Any, Any]:
64+
with (
65+
create_llmisvc(
66+
client=admin_client,
67+
name="llm-s3-tinyllama-premium",
68+
namespace=maas_unprivileged_model_namespace.name,
69+
storage_uri=ModelStorage.TINYLLAMA_S3,
70+
container_image=ContainerImages.VLLM_CPU,
71+
container_resources={
72+
"limits": {"cpu": "2", "memory": "12Gi"},
73+
"requests": {"cpu": "1", "memory": "8Gi"},
74+
},
75+
service_account=maas_model_service_account.name,
76+
wait=False,
77+
timeout=900,
78+
) as llm_service,
79+
patch_llmisvc_with_maas_router_and_tiers(llm_service=llm_service, tiers=["premium"]),
80+
):
81+
llm_service.wait_for_condition(condition="Ready", status="True", timeout=900)
82+
yield llm_service
83+
84+
85+
@pytest.fixture(scope="class")
86+
def maas_model_tinyllama_free(
87+
admin_client: DynamicClient,
88+
maas_inference_service_tinyllama_free: LLMInferenceService,
89+
) -> Generator[MaaSModel]:
90+
applications_namespace = py_config["applications_namespace"]
91+
92+
with MaaSModel(
93+
client=admin_client,
94+
name=maas_inference_service_tinyllama_free.name,
95+
namespace=applications_namespace,
96+
model_ref={
97+
"name": maas_inference_service_tinyllama_free.name,
98+
"namespace": maas_inference_service_tinyllama_free.namespace,
99+
"kind": "LLMInferenceService",
100+
"apiGroup": "serving.kserve.io",
101+
"group": "serving.kserve.io",
102+
},
103+
teardown=True,
104+
wait_for_resource=True,
105+
) as m:
106+
yield m
107+
108+
109+
@pytest.fixture(scope="class")
110+
def maas_model_tinyllama_premium(
111+
admin_client: DynamicClient,
112+
maas_inference_service_tinyllama_premium: LLMInferenceService,
113+
) -> Generator[MaaSModel]:
114+
applications_namespace = py_config["applications_namespace"]
115+
116+
with MaaSModel(
117+
client=admin_client,
118+
name=maas_inference_service_tinyllama_premium.name,
119+
namespace=applications_namespace,
120+
model_ref={
121+
"name": maas_inference_service_tinyllama_premium.name,
122+
"namespace": maas_inference_service_tinyllama_premium.namespace,
123+
"kind": "LLMInferenceService",
124+
"apiGroup": "serving.kserve.io",
125+
"group": "serving.kserve.io",
126+
},
127+
teardown=True,
128+
wait_for_resource=True,
129+
) as m:
130+
yield m
131+
132+
133+
@pytest.fixture(scope="class")
134+
def maas_auth_policy_tinyllama_free(
135+
admin_client: DynamicClient,
136+
maas_free_group: str,
137+
maas_model_tinyllama_free: MaaSModel,
138+
) -> Generator[MaaSAuthPolicy]:
139+
applications_namespace = py_config["applications_namespace"]
140+
141+
with MaaSAuthPolicy(
142+
client=admin_client,
143+
name="tinyllama-free-access",
144+
namespace=applications_namespace,
145+
model_refs=[maas_model_tinyllama_free.name],
146+
subjects={
147+
"groups": [
148+
{"name": "system:authenticated"},
149+
{"name": maas_free_group},
150+
],
151+
},
152+
teardown=True,
153+
wait_for_resource=True,
154+
) as maas_auth_policy_free:
155+
yield maas_auth_policy_free
156+
157+
158+
@pytest.fixture(scope="class")
159+
def maas_auth_policy_tinyllama_premium(
160+
admin_client: DynamicClient,
161+
maas_premium_group: str,
162+
maas_model_tinyllama_premium: MaaSModel,
163+
) -> Generator[MaaSAuthPolicy]:
164+
applications_namespace = py_config["applications_namespace"]
165+
166+
with MaaSAuthPolicy(
167+
client=admin_client,
168+
name="tinyllama-premium-access",
169+
namespace=applications_namespace,
170+
model_refs=[maas_model_tinyllama_premium.name],
171+
subjects={
172+
"groups": [{"name": maas_premium_group}],
173+
},
174+
teardown=True,
175+
wait_for_resource=True,
176+
) as maas_auth_policy_premium:
177+
yield maas_auth_policy_premium
178+
179+
180+
@pytest.fixture(scope="class")
181+
def maas_subscription_tinyllama_free(
182+
admin_client: DynamicClient,
183+
maas_free_group: str,
184+
maas_model_tinyllama_free: MaaSModel,
185+
) -> Generator[MaaSSubscription]:
186+
applications_namespace = py_config["applications_namespace"]
187+
188+
with MaaSSubscription(
189+
client=admin_client,
190+
name="tinyllama-free-subscription",
191+
namespace=applications_namespace,
192+
owner={
193+
"kind": "Group",
194+
"name": maas_free_group,
195+
},
196+
model_refs=[
197+
{
198+
"name": maas_model_tinyllama_free.name,
199+
"tokensPerMinute": 100,
200+
}
201+
],
202+
priority=0,
203+
teardown=True,
204+
wait_for_resource=True,
205+
) as maas_subscription_free:
206+
yield maas_subscription_free
207+
208+
209+
@pytest.fixture(scope="class")
210+
def maas_subscription_tinyllama_premium(
211+
admin_client: DynamicClient,
212+
maas_premium_group: str,
213+
maas_model_tinyllama_premium: MaaSModel,
214+
) -> Generator[MaaSSubscription]:
215+
applications_namespace = py_config["applications_namespace"]
216+
217+
with MaaSSubscription(
218+
client=admin_client,
219+
name="tinyllama-premium-subscription",
220+
namespace=applications_namespace,
221+
owner={
222+
"kind": "Group",
223+
"name": maas_premium_group,
224+
},
225+
model_refs=[
226+
{
227+
"name": maas_model_tinyllama_premium.name,
228+
"tokensPerMinute": 1000,
229+
}
230+
],
231+
priority=0,
232+
teardown=True,
233+
wait_for_resource=True,
234+
) as maas_subscription_premium:
235+
yield maas_subscription_premium
236+
237+
238+
@pytest.fixture(scope="class")
239+
def model_url_tinyllama_free(
240+
maas_scheme: str,
241+
maas_host: str,
242+
maas_inference_service_tinyllama_free: LLMInferenceService,
243+
) -> str:
244+
deployment_name = maas_inference_service_tinyllama_free.name
245+
url = f"{maas_scheme}://{maas_host}/llm/{deployment_name}{CHAT_COMPLETIONS}"
246+
LOGGER.info("MaaS: constructed model_url=%s (deployment=%s)", url, deployment_name)
247+
return url
248+
249+
250+
@pytest.fixture(scope="class")
251+
def model_url_tinyllama_premium(
252+
maas_scheme: str,
253+
maas_host: str,
254+
maas_inference_service_tinyllama_premium: LLMInferenceService,
255+
) -> str:
256+
deployment_name = maas_inference_service_tinyllama_premium.name
257+
url = f"{maas_scheme}://{maas_host}/llm/{deployment_name}{CHAT_COMPLETIONS}"
258+
LOGGER.info("MaaS: constructed model_url=%s (deployment=%s)", url, deployment_name)
259+
return url
260+
261+
262+
@pytest.fixture(scope="class")
263+
def maas_wrong_group_service_account_token(
264+
maas_api_server_url: str,
265+
original_user: str,
266+
admin_client: DynamicClient,
267+
) -> Generator[str]:
268+
applications_namespace = py_config["applications_namespace"]
269+
270+
with ServiceAccount(
271+
client=admin_client,
272+
namespace=applications_namespace,
273+
name="e2e-wrong-group-sa",
274+
teardown=True,
275+
) as sa:
276+
sa.wait(timeout=60)
277+
278+
ok = login_with_user_password(api_address=maas_api_server_url, user=original_user)
279+
assert ok, f"Failed to login as original_user={original_user}"
280+
281+
raw_token = create_inference_token(model_service_account=sa)
282+
yield raw_token
283+
284+
285+
@pytest.fixture(scope="class")
286+
def maas_headers_for_wrong_group_sa(maas_wrong_group_service_account_token: str) -> dict:
287+
return build_maas_headers(token=maas_wrong_group_service_account_token)

‎tests/model_serving/model_server/maas_billing/maas_subscription/test_maas_auth_enforcement.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
from __future__ import annotations
2+
3+
import pytest
4+
import requests
5+
from simple_logger.logger import get_logger
6+
7+
from tests.model_serving.model_server.maas_billing.maas_subscription.utils import chat_payload_for_url
8+
from tests.model_serving.model_server.maas_billing.utils import build_maas_headers
9+
from utilities.plugins.constant import RestHeader
10+
11+
LOGGER = get_logger(name=__name__)
12+
13+
14+
@pytest.mark.usefixtures(
15+
"maas_unprivileged_model_namespace",
16+
"maas_controller_enabled_latest",
17+
"maas_gateway_api",
18+
"maas_api_gateway_reachable",
19+
"maas_model_tinyllama_free",
20+
"maas_model_tinyllama_premium",
21+
"maas_auth_policy_tinyllama_free",
22+
"maas_auth_policy_tinyllama_premium",
23+
"maas_subscription_tinyllama_free",
24+
"maas_subscription_tinyllama_premium",
25+
)
26+
class TestMaaSAuthPolicyEnforcementTinyLlama:
27+
@pytest.mark.sanity
28+
@pytest.mark.parametrize("ocp_token_for_actor", [{"type": "free"}], indirect=True)
29+
def test_authorized_user_gets_200(
30+
self,
31+
request_session_http: requests.Session,
32+
model_url_tinyllama_free: str,
33+
ocp_token_for_actor: str,
34+
) -> None:
35+
headers = build_maas_headers(token=ocp_token_for_actor)
36+
payload = chat_payload_for_url(model_url=model_url_tinyllama_free)
37+
38+
resp = request_session_http.post(
39+
url=model_url_tinyllama_free,
40+
headers=headers,
41+
json=payload,
42+
timeout=60,
43+
)
44+
LOGGER.info(f"test_authorized_user_gets_200 -> POST {model_url_tinyllama_free} returned {resp.status_code}")
45+
assert resp.status_code == 200, f"Expected 200, got {resp.status_code}: {resp.text[:200]}"
46+
47+
@pytest.mark.sanity
48+
def test_no_auth_header_gets_401(
49+
self,
50+
request_session_http: requests.Session,
51+
model_url_tinyllama_free: str,
52+
) -> None:
53+
payload = chat_payload_for_url(model_url=model_url_tinyllama_free)
54+
55+
resp = request_session_http.post(
56+
url=model_url_tinyllama_free,
57+
headers=RestHeader.HEADERS,
58+
json=payload,
59+
timeout=60,
60+
)
61+
LOGGER.info(f"test_no_auth_header_gets_401 -> POST {model_url_tinyllama_free} returned {resp.status_code}")
62+
assert resp.status_code == 401, f"Expected 401, got {resp.status_code}: {resp.text[:200]}"
63+
64+
@pytest.mark.sanity
65+
def test_invalid_token_gets_401(
66+
self,
67+
request_session_http: requests.Session,
68+
model_url_tinyllama_free: str,
69+
) -> None:
70+
headers = build_maas_headers(token="totally-invalid-garbage-token")
71+
payload = chat_payload_for_url(model_url=model_url_tinyllama_free)
72+
73+
resp = request_session_http.post(
74+
url=model_url_tinyllama_free,
75+
headers=headers,
76+
json=payload,
77+
timeout=60,
78+
)
79+
LOGGER.info(f"test_invalid_token_gets_401 -> POST {model_url_tinyllama_free} returned {resp.status_code}")
80+
assert resp.status_code == 401, f"Expected 401, got {resp.status_code}: {resp.text[:200]}"
81+
82+
@pytest.mark.sanity
83+
def test_wrong_group_sa_denied_on_premium_model(
84+
self,
85+
request_session_http: requests.Session,
86+
model_url_tinyllama_premium: str,
87+
maas_headers_for_wrong_group_sa: dict,
88+
) -> None:
89+
payload = chat_payload_for_url(model_url=model_url_tinyllama_premium)
90+
91+
resp = request_session_http.post(
92+
url=model_url_tinyllama_premium,
93+
headers=maas_headers_for_wrong_group_sa,
94+
json=payload,
95+
timeout=60,
96+
)
97+
LOGGER.info(
98+
"test_wrong_group_sa_denied_on_premium_model -> "
99+
f"POST {model_url_tinyllama_premium} returned {resp.status_code}"
100+
)
101+
assert resp.status_code == 403, f"Expected 403, got {resp.status_code}: {resp.text[:200]}"

0 commit comments

Comments
 (0)