Skip to content

Commit bb20f21

Browse files
kpunwatkkpunwatk
committed
feat: Added multi-tenancy test cases for evalhub
 modified: tests/model_explainability/evalhub/conftest.py modified: tests/model_explainability/evalhub/constants.py new file: tests/model_explainability/evalhub/test_evalhub_multi_tenancy.py modified: tests/model_explainability/evalhub/utils.py
1 parent 8a67a44 commit bb20f21

File tree

4 files changed

+1026
-0
lines changed

4 files changed

+1026
-0
lines changed

‎tests/model_explainability/evalhub/conftest.py‎

Lines changed: 286 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,29 @@
11
from collections.abc import Generator
2+
from contextlib import ExitStack
23
from typing import Any
4+
import shlex
35

46
import pytest
57
import structlog
68
from kubernetes.dynamic import DynamicClient
79
from ocp_resources.deployment import Deployment
810
from ocp_resources.namespace import Namespace
11+
from ocp_resources.role import Role
12+
from ocp_resources.role_binding import RoleBinding
913
from ocp_resources.route import Route
14+
from ocp_resources.service_account import ServiceAccount
15+
from pyhelper_utils.shell import run_command
1016

17+
from tests.model_explainability.evalhub.constants import (
18+
EVALHUB_EVALUATOR_ROLE,
19+
EVALHUB_PROVIDERS_ACCESS_CLUSTER_ROLE,
20+
EVALHUB_TENANT_LABEL,
21+
TENANT_A_NAME,
22+
TENANT_A_SA_NAME,
23+
TENANT_B_NAME,
24+
TENANT_B_SA_NAME,
25+
TENANT_UNAUTHORISED_SA_NAME,
26+
)
1127
from utilities.certificates_utils import create_ca_bundle_file
1228
from utilities.constants import Timeout
1329
from utilities.resources.evalhub import EvalHub
@@ -67,3 +83,273 @@ def evalhub_ca_bundle_file(
6783
) -> str:
6884
"""Create a CA bundle file for verifying the EvalHub route TLS certificate."""
6985
return create_ca_bundle_file(client=admin_client)
86+
87+
88+
# Multi-Tenancy Fixtures
89+
90+
91+
@pytest.fixture(scope="class")
92+
def tenant_a_namespace(
93+
admin_client: DynamicClient,
94+
) -> Generator[Namespace, Any, Any]:
95+
"""Create tenant-a namespace with EvalHub tenant label."""
96+
with Namespace(
97+
client=admin_client,
98+
name=TENANT_A_NAME,
99+
label={EVALHUB_TENANT_LABEL: ""},
100+
) as ns:
101+
ns.wait_for_status(status=Namespace.Status.ACTIVE, timeout=Timeout.TIMEOUT_2MIN)
102+
yield ns
103+
104+
105+
@pytest.fixture(scope="class")
106+
def tenant_b_namespace(
107+
admin_client: DynamicClient,
108+
) -> Generator[Namespace, Any, Any]:
109+
"""Create tenant-b namespace with EvalHub tenant label."""
110+
with Namespace(
111+
client=admin_client,
112+
name=TENANT_B_NAME,
113+
label={EVALHUB_TENANT_LABEL: ""},
114+
) as ns:
115+
ns.wait_for_status(status=Namespace.Status.ACTIVE, timeout=Timeout.TIMEOUT_2MIN)
116+
yield ns
117+
118+
119+
@pytest.fixture(scope="class")
120+
def tenant_a_service_account(
121+
admin_client: DynamicClient,
122+
tenant_a_namespace: Namespace,
123+
) -> Generator[ServiceAccount, Any, Any]:
124+
"""Create service account for tenant-a."""
125+
with ServiceAccount(
126+
client=admin_client,
127+
name=TENANT_A_SA_NAME,
128+
namespace=tenant_a_namespace.name,
129+
) as sa:
130+
yield sa
131+
132+
133+
@pytest.fixture(scope="class")
134+
def tenant_b_service_account(
135+
admin_client: DynamicClient,
136+
tenant_b_namespace: Namespace,
137+
) -> Generator[ServiceAccount, Any, Any]:
138+
"""Create service account for tenant-b."""
139+
with ServiceAccount(
140+
client=admin_client,
141+
name=TENANT_B_SA_NAME,
142+
namespace=tenant_b_namespace.name,
143+
) as sa:
144+
yield sa
145+
146+
147+
@pytest.fixture(scope="class")
148+
def tenant_unauthorised_service_account(
149+
admin_client: DynamicClient,
150+
model_namespace: Namespace,
151+
) -> Generator[ServiceAccount, Any, Any]:
152+
"""Create service account without any EvalHub permissions."""
153+
with ServiceAccount(
154+
client=admin_client,
155+
name=TENANT_UNAUTHORISED_SA_NAME,
156+
namespace=model_namespace.name,
157+
) as sa:
158+
yield sa
159+
160+
161+
@pytest.fixture(scope="class")
162+
def tenant_a_evaluator_role(
163+
admin_client: DynamicClient,
164+
tenant_a_namespace: Namespace,
165+
) -> Generator[Role, Any, Any]:
166+
"""Create evaluator role for tenant-a with permissions to manage evaluations."""
167+
with Role(
168+
client=admin_client,
169+
name=EVALHUB_EVALUATOR_ROLE,
170+
namespace=tenant_a_namespace.name,
171+
rules=[
172+
{
173+
"apiGroups": ["trustyai.opendatahub.io"],
174+
"resources": ["evaluations", "collections", "providers"],
175+
"verbs": ["get", "list", "create", "update", "delete"],
176+
},
177+
{
178+
"apiGroups": ["mlflow.kubeflow.org"],
179+
"resources": ["experiments"],
180+
"verbs": ["create", "get"],
181+
},
182+
],
183+
) as role:
184+
yield role
185+
186+
187+
@pytest.fixture(scope="class")
188+
def tenant_b_evaluator_role(
189+
admin_client: DynamicClient,
190+
tenant_b_namespace: Namespace,
191+
) -> Generator[Role, Any, Any]:
192+
"""Create evaluator role for tenant-b with permissions to manage evaluations."""
193+
with Role(
194+
client=admin_client,
195+
name=EVALHUB_EVALUATOR_ROLE,
196+
namespace=tenant_b_namespace.name,
197+
rules=[
198+
{
199+
"apiGroups": ["trustyai.opendatahub.io"],
200+
"resources": ["evaluations", "collections", "providers"],
201+
"verbs": ["get", "list", "create", "update", "delete"],
202+
},
203+
{
204+
"apiGroups": ["mlflow.kubeflow.org"],
205+
"resources": ["experiments"],
206+
"verbs": ["create", "get"],
207+
},
208+
],
209+
) as role:
210+
yield role
211+
212+
213+
@pytest.fixture(scope="class")
214+
def tenant_a_evaluator_role_binding(
215+
admin_client: DynamicClient,
216+
tenant_a_namespace: Namespace,
217+
tenant_a_evaluator_role: Role,
218+
tenant_a_service_account: ServiceAccount,
219+
) -> Generator[RoleBinding, Any, Any]:
220+
"""Bind evaluator role to tenant-a service account."""
221+
with RoleBinding(
222+
client=admin_client,
223+
name=f"{EVALHUB_EVALUATOR_ROLE}-binding",
224+
namespace=tenant_a_namespace.name,
225+
subjects_kind="ServiceAccount",
226+
subjects_name=tenant_a_service_account.name,
227+
role_ref_kind="Role",
228+
role_ref_name=tenant_a_evaluator_role.name,
229+
) as rb:
230+
yield rb
231+
232+
233+
@pytest.fixture(scope="class")
234+
def tenant_b_evaluator_role_binding(
235+
admin_client: DynamicClient,
236+
tenant_b_namespace: Namespace,
237+
tenant_b_evaluator_role: Role,
238+
tenant_b_service_account: ServiceAccount,
239+
) -> Generator[RoleBinding, Any, Any]:
240+
"""Bind evaluator role to tenant-b service account."""
241+
with RoleBinding(
242+
client=admin_client,
243+
name=f"{EVALHUB_EVALUATOR_ROLE}-binding",
244+
namespace=tenant_b_namespace.name,
245+
subjects_kind="ServiceAccount",
246+
subjects_name=tenant_b_service_account.name,
247+
role_ref_kind="Role",
248+
role_ref_name=tenant_b_evaluator_role.name,
249+
) as rb:
250+
yield rb
251+
252+
253+
@pytest.fixture(scope="class")
254+
def evalhub_providers_role_binding_tenant_a(
255+
admin_client: DynamicClient,
256+
tenant_a_namespace: Namespace,
257+
tenant_a_service_account: ServiceAccount,
258+
) -> Generator[RoleBinding, Any, Any]:
259+
"""Bind providers access cluster role to tenant-a service account."""
260+
with RoleBinding(
261+
client=admin_client,
262+
name="evalhub-providers-access",
263+
namespace=tenant_a_namespace.name,
264+
subjects_kind="ServiceAccount",
265+
subjects_name=tenant_a_service_account.name,
266+
role_ref_kind="ClusterRole",
267+
role_ref_name=EVALHUB_PROVIDERS_ACCESS_CLUSTER_ROLE,
268+
) as rb:
269+
yield rb
270+
271+
272+
@pytest.fixture(scope="class")
273+
def evalhub_providers_role_binding_tenant_b(
274+
admin_client: DynamicClient,
275+
tenant_b_namespace: Namespace,
276+
tenant_b_service_account: ServiceAccount,
277+
) -> Generator[RoleBinding, Any, Any]:
278+
"""Bind providers access cluster role to tenant-b service account."""
279+
with RoleBinding(
280+
client=admin_client,
281+
name="evalhub-providers-access",
282+
namespace=tenant_b_namespace.name,
283+
subjects_kind="ServiceAccount",
284+
subjects_name=tenant_b_service_account.name,
285+
role_ref_kind="ClusterRole",
286+
role_ref_name=EVALHUB_PROVIDERS_ACCESS_CLUSTER_ROLE,
287+
) as rb:
288+
yield rb
289+
290+
291+
@pytest.fixture(scope="class")
292+
def tenant_a_token(
293+
tenant_a_service_account: ServiceAccount,
294+
tenant_a_evaluator_role_binding: RoleBinding,
295+
evalhub_providers_role_binding_tenant_a: RoleBinding,
296+
) -> str:
297+
"""Generate bearer token for tenant-a service account (30 min validity)."""
298+
return run_command(
299+
shlex.split(
300+
f"oc create token -n {tenant_a_service_account.namespace} "
301+
f"{tenant_a_service_account.name} --duration=30m"
302+
)
303+
)[1].strip()
304+
305+
306+
@pytest.fixture(scope="class")
307+
def tenant_b_token(
308+
tenant_b_service_account: ServiceAccount,
309+
tenant_b_evaluator_role_binding: RoleBinding,
310+
evalhub_providers_role_binding_tenant_b: RoleBinding,
311+
) -> str:
312+
"""Generate bearer token for tenant-b service account (30 min validity)."""
313+
return run_command(
314+
shlex.split(
315+
f"oc create token -n {tenant_b_service_account.namespace} "
316+
f"{tenant_b_service_account.name} --duration=30m"
317+
)
318+
)[1].strip()
319+
320+
321+
@pytest.fixture(scope="class")
322+
def tenant_unauthorised_token(
323+
tenant_unauthorised_service_account: ServiceAccount,
324+
) -> str:
325+
"""Generate bearer token for unauthorised service account (30 min validity)."""
326+
return run_command(
327+
shlex.split(
328+
f"oc create token -n {tenant_unauthorised_service_account.namespace} "
329+
f"{tenant_unauthorised_service_account.name} --duration=30m"
330+
)
331+
)[1].strip()
332+
333+
334+
@pytest.fixture(scope="class")
335+
def multi_tenant_setup(
336+
tenant_a_namespace: Namespace,
337+
tenant_b_namespace: Namespace,
338+
tenant_a_token: str,
339+
tenant_b_token: str,
340+
tenant_unauthorised_token: str,
341+
) -> dict[str, Any]:
342+
"""Composite fixture providing all multi-tenancy setup."""
343+
return {
344+
"tenant_a": {
345+
"namespace": tenant_a_namespace,
346+
"token": tenant_a_token,
347+
},
348+
"tenant_b": {
349+
"namespace": tenant_b_namespace,
350+
"token": tenant_b_token,
351+
},
352+
"unauthorised": {
353+
"token": tenant_unauthorised_token,
354+
},
355+
}

‎tests/model_explainability/evalhub/constants.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
EVALHUB_CONTAINER_PORT: int = 8080
44
EVALHUB_HEALTH_PATH: str = "/api/v1/health"
55
EVALHUB_PROVIDERS_PATH: str = "/api/v1/evaluations/providers"
6+
EVALHUB_JOBS_PATH: str = "/api/v1/evaluations/jobs"
7+
EVALHUB_BENCHMARKS_PATH: str = "/api/v1/evaluations/benchmarks"
8+
EVALHUB_COLLECTIONS_PATH: str = "/api/v1/evaluations/collections"
69
EVALHUB_HEALTH_STATUS_HEALTHY: str = "healthy"
710

811
EVALHUB_APP_LABEL: str = "eval-hub"
@@ -12,3 +15,15 @@
1215
EVALHUB_API_VERSION: str = "v1alpha1"
1316
EVALHUB_KIND: str = "EvalHub"
1417
EVALHUB_PLURAL: str = "evalhubs"
18+
19+
# RBAC
20+
EVALHUB_PROVIDERS_ACCESS_CLUSTER_ROLE: str = "trustyai-service-operator-evalhub-providers-access"
21+
EVALHUB_EVALUATOR_ROLE: str = "evalhub-evaluator"
22+
EVALHUB_TENANT_LABEL: str = "evalhub.trustyai.opendatahub.io/tenant"
23+
24+
# Multi-tenancy test constants
25+
TENANT_A_NAME: str = "evalhub-tenant-a"
26+
TENANT_B_NAME: str = "evalhub-tenant-b"
27+
TENANT_A_SA_NAME: str = "team-a-user"
28+
TENANT_B_SA_NAME: str = "team-b-user"
29+
TENANT_UNAUTHORISED_SA_NAME: str = "evalhub-no-access-user"

0 commit comments

Comments
 (0)