Skip to content

Commit 1cd5013

Browse files
christinaexyouchristinaexyou
committed
Create HF access token secret
1 parent 3fa64f7 commit 1cd5013

File tree

3 files changed

+128
-135
lines changed

3 files changed

+128
-135
lines changed

‎tests/conftest.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -573,4 +573,3 @@ def autouse_fixtures(
573573
) -> None:
574574
"""Fixture to control the order of execution of some of the fixtures"""
575575
return
576-

‎tests/model_explainability/lm_eval/conftest.py‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
import os
2-
from typing import Any, Generator
1+
from typing import Generator, Any
32

43
import pytest
4+
from ocp_resources.route import Route
5+
from ocp_resources.secret import Secret
6+
from ocp_resources.service import Service
7+
from pytest import FixtureRequest
58
from kubernetes.dynamic import DynamicClient
69
from ocp_resources.config_map import ConfigMap
710
from ocp_resources.deployment import Deployment
@@ -10,10 +13,6 @@
1013
from ocp_resources.persistent_volume_claim import PersistentVolumeClaim
1114
from ocp_resources.pod import Pod
1215
from ocp_resources.resource import ResourceEditor
13-
from ocp_resources.route import Route
14-
from ocp_resources.secret import Secret
15-
from ocp_resources.service import Service
16-
from pytest import FixtureRequest, Config
1716
from pytest_testconfig import py_config
1817

1918
from tests.model_explainability.lm_eval.utils import get_lmevaljob_pod
@@ -31,14 +30,8 @@ def lmevaljob_hf(
3130
admin_client: DynamicClient,
3231
model_namespace: Namespace,
3332
patched_trustyai_operator_configmap_allow_online: ConfigMap,
34-
pytestconfig: Config,
33+
lmeval_hf_access_token: Secret,
3534
) -> Generator[LMEvalJob, None, None]:
36-
hf_access_token=pytestconfig.option.hf_access_token
37-
if not hf_access_token:
38-
raise ValueError(
39-
"HF access token is not set. "
40-
"Either pass with `--hf-access-token` or set `HF_ACCESS_TOKEN` environment variable"
41-
)
4235
with LMEvalJob(
4336
client=admin_client,
4437
name=LMEVALJOB_NAME,
@@ -467,6 +460,7 @@ def lmeval_hf_access_token(
467460
string_data={
468461
"HF_ACCESS_TOKEN": hf_access_token,
469462
},
463+
470464
wait_for_resource=True,
471465
) as secret:
472466
yield secret

‎tests/model_explainability/lm_eval/test_lm_eval.py‎

Lines changed: 121 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -59,124 +59,124 @@ def test_lmeval_huggingface_model(admin_client, model_namespace, lmevaljob_hf_po
5959
lmevaljob_hf_pod.wait_for_status(status=lmevaljob_hf_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_40MIN)
6060

6161

62-
# @pytest.mark.parametrize(
63-
# "model_namespace, lmeval_data_downloader_pod, lmevaljob_local_offline",
64-
# [
65-
# pytest.param(
66-
# {"name": "test-lmeval-local-offline-builtin"},
67-
# {
68-
# "image": "quay.io/trustyai_testing/lmeval-assets-flan-arceasy"
69-
# "@sha256:11cc9c2f38ac9cc26c4fab1a01a8c02db81c8f4801b5d2b2b90f90f91b97ac98"
70-
# },
71-
# {"task_list": {"taskNames": ["arc_easy"]}},
72-
# )
73-
# ],
74-
# indirect=True,
75-
# )
76-
# @pytest.mark.smoke
77-
# def test_lmeval_local_offline_builtin_tasks_flan_arceasy(
78-
# admin_client,
79-
# model_namespace,
80-
# lmeval_data_downloader_pod,
81-
# lmevaljob_local_offline_pod,
82-
# ):
83-
# """Test that verifies that LMEval can run successfully in local, offline mode using builtin tasks"""
84-
# lmevaljob_local_offline_pod.wait_for_status(
85-
# status=lmevaljob_local_offline_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_20MIN
86-
# )
87-
88-
89-
# @pytest.mark.parametrize(
90-
# "model_namespace, lmeval_data_downloader_pod, lmevaljob_local_offline",
91-
# [
92-
# pytest.param(
93-
# {"name": "test-lmeval-local-offline-unitxt"},
94-
# {
95-
# "image": "quay.io/trustyai_testing/lmeval-assets-flan-20newsgroups"
96-
# "@sha256:3778c15079f11ef338a82ee35ae1aa43d6db52bac7bbfdeab343ccabe2608a0c"
97-
# },
98-
# {
99-
# "task_list": {
100-
# "taskRecipes": [
101-
# {
102-
# "card": {"name": "cards.20_newsgroups_short"},
103-
# "template": {"name": "templates.classification.multi_class.title"},
104-
# }
105-
# ]
106-
# }
107-
# },
108-
# )
109-
# ],
110-
# indirect=True,
111-
# )
112-
# def test_lmeval_local_offline_unitxt_tasks_flan_20newsgroups(
113-
# admin_client,
114-
# model_namespace,
115-
# lmeval_data_downloader_pod,
116-
# lmevaljob_local_offline_pod,
117-
# ):
118-
# """Test that verifies that LMEval can run successfully in local, offline mode using unitxt"""
119-
# lmevaljob_local_offline_pod.wait_for_status(
120-
# status=lmevaljob_local_offline_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_20MIN
121-
# )
122-
123-
124-
# @pytest.mark.parametrize(
125-
# "model_namespace",
126-
# [
127-
# pytest.param(
128-
# {"name": "test-lmeval-vllm"},
129-
# )
130-
# ],
131-
# indirect=True,
132-
# )
133-
# def test_lmeval_vllm_emulator(admin_client, model_namespace, lmevaljob_vllm_emulator_pod):
134-
# """Basic test that verifies LMEval works with vLLM using a vLLM emulator for more efficient evaluation"""
135-
# lmevaljob_vllm_emulator_pod.wait_for_status(
136-
# status=lmevaljob_vllm_emulator_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_20MIN
137-
# )
138-
139-
140-
# @pytest.mark.parametrize(
141-
# "model_namespace, minio_data_connection",
142-
# [
143-
# pytest.param(
144-
# {"name": "test-s3-lmeval"},
145-
# {"bucket": "models"},
146-
# )
147-
# ],
148-
# indirect=True,
149-
# )
150-
# def test_lmeval_s3_storage(
151-
# admin_client,
152-
# model_namespace,
153-
# lmevaljob_s3_offline_pod,
154-
# ):
155-
# """Test to verify that LMEval works with a model stored in a S3 bucket"""
156-
# lmevaljob_s3_offline_pod.wait_for_status(
157-
# status=lmevaljob_s3_offline_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_20MIN
158-
# )
159-
160-
161-
# @pytest.mark.parametrize(
162-
# "model_namespace, minio_data_connection",
163-
# [
164-
# pytest.param(
165-
# {"name": "test-lmeval-images"},
166-
# {"bucket": "models"},
167-
# )
168-
# ],
169-
# indirect=True,
170-
# )
171-
# @pytest.mark.smoke
172-
# def test_verify_lmeval_pod_images(lmevaljob_s3_offline_pod, trustyai_operator_configmap) -> None:
173-
# """Test to verify LMEval pod images.
174-
# Checks if the image tag from the ConfigMap is used within the Pod and if it's pinned using a sha256 digest.
175-
176-
# Verifies:
177-
# - lmeval driver image
178-
# - lmeval job runner image
179-
# """
180-
# validate_tai_component_images(
181-
# pod=lmevaljob_s3_offline_pod, tai_operator_configmap=trustyai_operator_configmap, include_init_containers=True
182-
# )
62+
@pytest.mark.parametrize(
63+
"model_namespace, lmeval_data_downloader_pod, lmevaljob_local_offline",
64+
[
65+
pytest.param(
66+
{"name": "test-lmeval-local-offline-builtin"},
67+
{
68+
"image": "quay.io/trustyai_testing/lmeval-assets-flan-arceasy"
69+
"@sha256:11cc9c2f38ac9cc26c4fab1a01a8c02db81c8f4801b5d2b2b90f90f91b97ac98"
70+
},
71+
{"task_list": {"taskNames": ["arc_easy"]}},
72+
)
73+
],
74+
indirect=True,
75+
)
76+
@pytest.mark.smoke
77+
def test_lmeval_local_offline_builtin_tasks_flan_arceasy(
78+
admin_client,
79+
model_namespace,
80+
lmeval_data_downloader_pod,
81+
lmevaljob_local_offline_pod,
82+
):
83+
"""Test that verifies that LMEval can run successfully in local, offline mode using builtin tasks"""
84+
lmevaljob_local_offline_pod.wait_for_status(
85+
status=lmevaljob_local_offline_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_20MIN
86+
)
87+
88+
89+
@pytest.mark.parametrize(
90+
"model_namespace, lmeval_data_downloader_pod, lmevaljob_local_offline",
91+
[
92+
pytest.param(
93+
{"name": "test-lmeval-local-offline-unitxt"},
94+
{
95+
"image": "quay.io/trustyai_testing/lmeval-assets-flan-20newsgroups"
96+
"@sha256:3778c15079f11ef338a82ee35ae1aa43d6db52bac7bbfdeab343ccabe2608a0c"
97+
},
98+
{
99+
"task_list": {
100+
"taskRecipes": [
101+
{
102+
"card": {"name": "cards.20_newsgroups_short"},
103+
"template": {"name": "templates.classification.multi_class.title"},
104+
}
105+
]
106+
}
107+
},
108+
)
109+
],
110+
indirect=True,
111+
)
112+
def test_lmeval_local_offline_unitxt_tasks_flan_20newsgroups(
113+
admin_client,
114+
model_namespace,
115+
lmeval_data_downloader_pod,
116+
lmevaljob_local_offline_pod,
117+
):
118+
"""Test that verifies that LMEval can run successfully in local, offline mode using unitxt"""
119+
lmevaljob_local_offline_pod.wait_for_status(
120+
status=lmevaljob_local_offline_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_20MIN
121+
)
122+
123+
124+
@pytest.mark.parametrize(
125+
"model_namespace",
126+
[
127+
pytest.param(
128+
{"name": "test-lmeval-vllm"},
129+
)
130+
],
131+
indirect=True,
132+
)
133+
def test_lmeval_vllm_emulator(admin_client, model_namespace, lmevaljob_vllm_emulator_pod):
134+
"""Basic test that verifies LMEval works with vLLM using a vLLM emulator for more efficient evaluation"""
135+
lmevaljob_vllm_emulator_pod.wait_for_status(
136+
status=lmevaljob_vllm_emulator_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_20MIN
137+
)
138+
139+
140+
@pytest.mark.parametrize(
141+
"model_namespace, minio_data_connection",
142+
[
143+
pytest.param(
144+
{"name": "test-s3-lmeval"},
145+
{"bucket": "models"},
146+
)
147+
],
148+
indirect=True,
149+
)
150+
def test_lmeval_s3_storage(
151+
admin_client,
152+
model_namespace,
153+
lmevaljob_s3_offline_pod,
154+
):
155+
"""Test to verify that LMEval works with a model stored in a S3 bucket"""
156+
lmevaljob_s3_offline_pod.wait_for_status(
157+
status=lmevaljob_s3_offline_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_20MIN
158+
)
159+
160+
161+
@pytest.mark.parametrize(
162+
"model_namespace, minio_data_connection",
163+
[
164+
pytest.param(
165+
{"name": "test-lmeval-images"},
166+
{"bucket": "models"},
167+
)
168+
],
169+
indirect=True,
170+
)
171+
@pytest.mark.smoke
172+
def test_verify_lmeval_pod_images(lmevaljob_s3_offline_pod, trustyai_operator_configmap) -> None:
173+
"""Test to verify LMEval pod images.
174+
Checks if the image tag from the ConfigMap is used within the Pod and if it's pinned using a sha256 digest.
175+
176+
Verifies:
177+
- lmeval driver image
178+
- lmeval job runner image
179+
"""
180+
validate_tai_component_images(
181+
pod=lmevaljob_s3_offline_pod, tai_operator_configmap=trustyai_operator_configmap, include_init_containers=True
182+
)

0 commit comments

Comments
 (0)