Skip to content

Commit e91a879

Browse files
edwardquarmpre-commit-ci[bot]vaibhavjainwiz
authored
model validation automation v1 (#340)
* model validation automation initial commit * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix issues causing pod to fail. Deployment is running but requests are failing with HTTP 400 bad request * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix bugs * feat(model-validation): add new snapshot tests and update constants * feat(model-validation): fix flake8 errors * feat(model-validation): fix flake8 errors * feat(model-validation): fix flake8 errors * feat(model-validation): fix flake8 errors * feat(model-validation): fix flake8 error fixture name not unique * feat(model-validation): minor fix to fixture * feat(model-validation): validate multiple modelcar images * feat(model-validation): oci registry pull secrets * feat(model-validation): test multiple images * feat(model-validation): test multiple images * feat(model-validation): serving arguments * feat(model-validation): serving arguments * feat(model-validation): address coderabit pr comments * feat(model-validation): add tests for raw inference service * feat(model-validation): fix serverless and raw deployment automation * feat(model-validation): add config yaml * feat(model-validation): resolve coderabbit comments * Revert s3 bucket changes * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * feat(model-validation): uncomment yaml config file * Fix pre-commit checks * Fix pre-commit checks * feat(model-validation): fix ci errors related to conftest.py * feat(model-validation): fix ci errors related to conftest.py * feat(model-validation): revert s3 changes --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Vaibhav Jain <vajain@redhat.com>
1 parent 411e3d1 commit e91a879

32 files changed

Lines changed: 2203 additions & 1104 deletions

File tree

conftest.py

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import datetime
66
import traceback
77

8+
import pytest
89
import shortuuid
910
from _pytest.runner import CallInfo
1011
from _pytest.reports import TestReport
@@ -35,7 +36,7 @@
3536
from kubernetes.dynamic import DynamicClient
3637
from utilities.infra import get_operator_distribution, get_dsci_applications_namespace, get_data_science_cluster
3738
from ocp_resources.resource import get_client
38-
39+
from ocp_resources.cluster_service_version import ClusterServiceVersion
3940

4041
LOGGER = logging.getLogger(name=__name__)
4142
BASIC_LOGGER = logging.getLogger(name="basic")
@@ -48,6 +49,9 @@ def pytest_addoption(parser: Parser) -> None:
4849
upgrade_group = parser.getgroup(name="Upgrade options")
4950
must_gather_group = parser.getgroup(name="MustGather")
5051
cluster_sanity_group = parser.getgroup(name="ClusterSanity")
52+
ociregistry_group = parser.getgroup(name="OCI Registry")
53+
serving_arguments_group = parser.getgroup(name="Serving arguments")
54+
model_validation_automation_group = parser.getgroup(name="Model Validation Automation")
5155
hf_group = parser.getgroup(name="Hugging Face")
5256

5357
# AWS config and credentials options
@@ -118,6 +122,32 @@ def pytest_addoption(parser: Parser) -> None:
118122
help="Specify the runtime image to use for the tests",
119123
)
120124

125+
# OCI Registry options
126+
ociregistry_group.addoption(
127+
"--registry-pull-secret",
128+
default=os.environ.get("OCI_REGISTRY_PULL_SECRET"),
129+
help="Registry pull secret to pull oci container images",
130+
)
131+
ociregistry_group.addoption(
132+
"--registry-host",
133+
default=os.environ.get("REGISTRY_HOST"),
134+
help="Registry host to pull oci container images",
135+
)
136+
137+
# Serving arguments options
138+
serving_arguments_group.addoption(
139+
"--serving-argument",
140+
default=os.environ.get("SERVING_ARGUMENTS"),
141+
help="Serving arguments to use for the tests",
142+
)
143+
144+
# Model Validation Automation options
145+
model_validation_automation_group.addoption(
146+
"--model_car_yaml_path",
147+
default=os.environ.get("MODEL_YAML_PATH"),
148+
help="Path to the modelcar yaml file",
149+
)
150+
121151
# Upgrade options
122152
upgrade_group.addoption(
123153
"--pre-upgrade",
@@ -412,3 +442,37 @@ def pytest_exception_interact(node: Item | Collector, call: CallInfo[Any], repor
412442

413443
except Exception as current_exception:
414444
LOGGER.warning(f"Failed to collect logs: {test_name}: {current_exception} {traceback.format_exc()}")
445+
446+
447+
@pytest.fixture(scope="package")
448+
def fail_if_missing_dependent_operators(admin_client: DynamicClient) -> None:
449+
if dependent_operators := py_config.get("dependent_operators"):
450+
missing_operators: list[str] = []
451+
452+
for operator_name in dependent_operators.split(","):
453+
csvs = list(
454+
ClusterServiceVersion.get(
455+
dyn_client=admin_client,
456+
namespace=py_config["applications_namespace"],
457+
)
458+
)
459+
460+
LOGGER.info(f"Verifying if {operator_name} is installed")
461+
for csv in csvs:
462+
if csv.name.startswith(operator_name):
463+
if csv.status == csv.Status.SUCCEEDED:
464+
break
465+
466+
else:
467+
missing_operators.append(
468+
f"Operator {operator_name} is installed but CSV is not in {csv.Status.SUCCEEDED} state"
469+
)
470+
471+
else:
472+
missing_operators.append(f"{operator_name} is not installed")
473+
474+
if missing_operators:
475+
pytest.fail(str(missing_operators))
476+
477+
else:
478+
LOGGER.info("No dependent operators to verify")

tests/conftest.py

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
from ocp_resources.resource import get_client
3232
from pytest_testconfig import config as py_config
3333
from simple_logger.logger import get_logger
34+
import json
3435

3536
from ocp_utilities.operators import uninstall_operator, install_operator
3637
from utilities.certificates_utils import create_ca_bundle_file
@@ -139,6 +140,28 @@ def aws_secret_access_key(pytestconfig: Config) -> str:
139140
return secret_access_key
140141

141142

143+
@pytest.fixture(scope="session")
144+
def registry_pull_secret(pytestconfig: Config) -> str:
145+
registry_pull_secret = pytestconfig.option.registry_pull_secret
146+
if not registry_pull_secret:
147+
raise ValueError(
148+
"Registry pull secret is not set. "
149+
"Either pass with `--registry_pull_secret` or set `OCI_REGISTRY_PULL_SECRET` environment variable"
150+
)
151+
return registry_pull_secret
152+
153+
154+
@pytest.fixture(scope="session")
155+
def registry_host(pytestconfig: pytest.Config) -> str | None:
156+
registry_host = pytestconfig.option.registry_host
157+
if not registry_host:
158+
raise ValueError(
159+
"Registry host for OCI images is not set. "
160+
"Either pass with `--registry_host` or set `REGISTRY_HOST` environment variable"
161+
)
162+
return registry_host
163+
164+
142165
@pytest.fixture(scope="session")
143166
def valid_aws_config(aws_access_key_id: str, aws_secret_access_key: str) -> tuple[str, str]:
144167
return aws_access_key_id, aws_secret_access_key
@@ -177,6 +200,40 @@ def ci_s3_bucket_endpoint(pytestconfig: pytest.Config) -> str:
177200
return ci_bucket_endpoint
178201

179202

203+
@pytest.fixture(scope="session")
204+
def serving_argument(pytestconfig: pytest.Config, modelcar_yaml_config: dict[str, Any] | None) -> list[str]:
205+
if modelcar_yaml_config:
206+
arg = modelcar_yaml_config.get("serving_argument", [])
207+
return arg if isinstance(arg, list) else [arg]
208+
209+
raw_arg = pytestconfig.option.serving_argument
210+
try:
211+
return json.loads(raw_arg)
212+
except json.JSONDecodeError:
213+
raise ValueError(
214+
"Serving arguments should be a valid JSON list. "
215+
"Either pass with `--serving-argument` or set it correctly in modelcar.yaml"
216+
)
217+
218+
219+
@pytest.fixture(scope="session")
220+
def modelcar_yaml_config(pytestconfig: pytest.Config) -> dict[str, Any] | None:
221+
"""
222+
Fixture to get the path to the modelcar.yaml file.
223+
"""
224+
config_path = pytestconfig.option.model_car_yaml_path
225+
if not config_path:
226+
return None
227+
with open(config_path, "r") as file:
228+
try:
229+
modelcar_yaml = yaml.safe_load(file)
230+
if not isinstance(modelcar_yaml, dict):
231+
raise ValueError("modelcar.yaml should contain a dictionary.")
232+
return modelcar_yaml
233+
except yaml.YAMLError as e:
234+
raise ValueError(f"Error parsing modelcar.yaml: {e}") from e
235+
236+
180237
@pytest.fixture(scope="session")
181238
def models_s3_bucket_name(pytestconfig: pytest.Config) -> str:
182239
models_bucket = pytestconfig.option.models_s3_bucket_name
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
from typing import Any
2+
3+
import pytest
4+
from syrupy.extensions.json import JSONSnapshotExtension
5+
6+
7+
@pytest.fixture(scope="session")
8+
def skip_if_no_supported_accelerator_type(supported_accelerator_type: str) -> None:
9+
if not supported_accelerator_type:
10+
pytest.skip("Accelerator type is not provided,vLLM test cannot be run on CPU")
11+
12+
13+
@pytest.fixture
14+
def response_snapshot(snapshot: Any) -> Any:
15+
return snapshot.use_extension(extension_class=JSONSnapshotExtension)

tests/model_serving/model_runtime/model_validation/__init__.py

Whitespace-only changes.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
[
2+
{
3+
"finish_reason": "length",
4+
"index": 0,
5+
"logprobs": null,
6+
"prompt_logprobs": null,
7+
"stop_reason": null,
8+
"text": "\n\nRenewable energy sources offer several key benefits compared to fossil fuels. They are sustainable, meaning they can be replenished naturally and do not deplete over time. They also have a lower environmental impact, as they do not emit greenhouse gases during operation. Additionally, renewable energy sources can provide energy security, as they are not subject to the same price volatility as fossil fuels.\n\nWhat are the main"
9+
},
10+
{
11+
"finish_reason": "length",
12+
"index": 0,
13+
"logprobs": null,
14+
"prompt_logprobs": null,
15+
"stop_reason": null,
16+
"text": "\n\nSpanish: El conocimiento es el poder.\nGerman: Die Kenntnis ist die Macht.\nMandarin: 知识是力量。(Zhīshì lìliàng.)\n\nTranslate the following English sentence into Spanish, German, and Mandarin: 'The quick brown fox jumps over the lazy dog.'\n\nSpanish: El veloz zorro marrón salta sobre el perro"
17+
},
18+
{
19+
"finish_reason": "length",
20+
"index": 0,
21+
"logprobs": null,
22+
"prompt_logprobs": null,
23+
"stop_reason": null,
24+
"text": "\nIn the quiet hush of twilight's glow,\nA canvas of stars, a celestial show.\nTheir twinkling eyes, a silent vow,\nA testament to time, a cosmic bow.\n\nThe moon, a silver ship, sails the night,\nIts gentle glow, a beacon of light.\nA mirror to the sea, a tranquil sight,\nA silent sentinel, in the"
25+
},
26+
{
27+
"finish_reason": "length",
28+
"index": 0,
29+
"logprobs": null,
30+
"prompt_logprobs": null,
31+
"stop_reason": null,
32+
"text": "\n\nThe Great Wall of China is a significant historical landmark that has played a crucial role in the country's defense and cultural heritage. It is a series of stone and earthen fortifications that were built to protect China from invasions by nomadic tribes. The construction of the Great Wall began in the 7th century BC and continued for over 2,000 years, with various dynasties adding to and modifying the structure."
33+
},
34+
{
35+
"finish_reason": "length",
36+
"index": 0,
37+
"logprobs": null,
38+
"prompt_logprobs": null,
39+
"stop_reason": null,
40+
"text": "\n\nThe ethical implications of using artificial intelligence in healthcare decision-making are significant. AI systems can be biased, leading to unequal treatment of patients. Additionally, AI systems can be used to make decisions that are not in the best interest of the patient, such as denying care to patients who are not likely to benefit from it. Finally, AI systems can be used to collect and store sensitive patient data, which can be used for malicious purposes.\n"
41+
},
42+
{
43+
"finish_reason": "length",
44+
"index": 0,
45+
"logprobs": null,
46+
"prompt_logprobs": null,
47+
"stop_reason": null,
48+
"text": "\n\nApollo 11 was the first manned mission to land on the moon. It was launched on July 16, 1969, and landed on the moon on July 20, 1969. The mission was a success, and the astronauts, Neil Armstrong, Buzz Aldrin, and Michael Collins, returned to Earth on July 24, 1969. The"
49+
}
50+
]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
[
2+
{
3+
"id": "granite-8b-oci",
4+
"max_model_len": 131072,
5+
"object": "model",
6+
"owned_by": "vllm",
7+
"parent": null,
8+
"permission": [
9+
{
10+
"allow_create_engine": false,
11+
"allow_fine_tuning": false,
12+
"allow_logprobs": true,
13+
"allow_sampling": true,
14+
"allow_search_indices": false,
15+
"allow_view": true,
16+
"group": null,
17+
"is_blocking": false,
18+
"object": "model_permission",
19+
"organization": "*"
20+
}
21+
],
22+
"root": "/mnt/models"
23+
}
24+
]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
[
2+
{
3+
"finish_reason": "length",
4+
"index": 0,
5+
"logprobs": null,
6+
"prompt_logprobs": null,
7+
"stop_reason": null,
8+
"text": " Renewable energy sources, such as solar, wind, and hydroelectric power, offer several key benefits compared to fossil fuels. Some of the main advantages include:\n1. Sustainability: Renewable energy sources are sustainable and can be replenished naturally, unlike fossil fuels, which are finite and will eventually run out.\n2. Reduced greenhouse gas emissions: Renewable energy sources produce little to no greenhouse gas emissions, which contribute to climate change, whereas fossil fuels release significant amounts of carbon dioxide and other pollutants when burned.\n3"
9+
},
10+
{
11+
"finish_reason": "length",
12+
"index": 0,
13+
"logprobs": null,
14+
"prompt_logprobs": null,
15+
"stop_reason": null,
16+
"text": " \nEnglish: Knowledge is power.\nSpanish: El conocimiento es poder.\nGerman: Wissen ist Macht.\nMandarin: 知识就是力量.\nNote: The translation of the sentence into Mandarin is a bit more nuanced, as the Chinese character \"\" (zhī shì) is often used to convey the idea of \"knowledge is power,\" but the more literal translation of the sentence is \"\" (zhī jiù shì lìliàng). However, \"\" (zhī ji"
17+
},
18+
{
19+
"finish_reason": "length",
20+
"index": 0,
21+
"logprobs": null,
22+
"prompt_logprobs": null,
23+
"stop_reason": null,
24+
"text": " The poem should have a twinkle in its eye, a sense of wonder and awe.\nThe velvet expanse above, a canvas so divine,\nA twinkling tapestry, infinite and sublime.\nThe stars, like diamonds, scattered with reckless abandon,\nA celestial showcase, of beauty and wonder, unbound.\n\nThe moon, a glowing crescent, a beacon in the night,\nA gentle guide, that navigates us through the dark of light.\nThe planets, like sentinels, standing guard"
25+
},
26+
{
27+
"finish_reason": "length",
28+
"index": 0,
29+
"logprobs": null,
30+
"prompt_logprobs": null,
31+
"stop_reason": null,
32+
"text": " The Great Wall of China is one of the most impressive architectural achievements in history, and its significance extends beyond its physical presence. It has played a crucial role in China's military defense, economic development, and cultural identity. In this essay, we will explore the historical significance of the Great Wall of China and its impact on modern tourism.\nThe Great Wall of China was built over several centuries, with the first versions of the wall dating back to the 7th century BC. The wall was initially constructed"
33+
},
34+
{
35+
"finish_reason": "length",
36+
"index": 0,
37+
"logprobs": null,
38+
"prompt_logprobs": null,
39+
"stop_reason": null,
40+
"text": " | Homework Help\nThe use of artificial intelligence (AI) in healthcare decision-making has the potential to revolutionize the way healthcare is delivered, but it also raises several ethical concerns. Some of the key ethical implications of using AI in healthcare decision-making include:\n1. Bias and discrimination: AI systems can perpetuate existing biases and discrimination if they are trained on biased data or designed with a particular worldview. This can lead to unequal treatment of patients, particularly those from marginalized groups.\n2. Lack of transparency"
41+
},
42+
{
43+
"finish_reason": "length",
44+
"index": 0,
45+
"logprobs": null,
46+
"prompt_logprobs": null,
47+
"stop_reason": null,
48+
"text": " The Apollo 11 mission was a groundbreaking achievement in space exploration, marking the first time humans set foot on the lunar surface. Here's a summary of the main events and significance of the mission:\n**Pre-Launch and Journey to the Moon**\nThe Apollo 11 spacecraft, carrying astronauts Neil Armstrong, Edwin \"Buzz\" Aldrin, and Michael Collins, launched from Kennedy Space Center on July 16, 1969. After traveling through space for four days, the spacecraft entered into lunar orbit.\n"
49+
}
50+
]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
[
2+
{
3+
"id": "llama-8b-oci",
4+
"max_model_len": 131072,
5+
"object": "model",
6+
"owned_by": "vllm",
7+
"parent": null,
8+
"permission": [
9+
{
10+
"allow_create_engine": false,
11+
"allow_fine_tuning": false,
12+
"allow_logprobs": true,
13+
"allow_sampling": true,
14+
"allow_search_indices": false,
15+
"allow_view": true,
16+
"group": null,
17+
"is_blocking": false,
18+
"object": "model_permission",
19+
"organization": "*"
20+
}
21+
],
22+
"root": "/mnt/models"
23+
}
24+
]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
[
2+
{
3+
"finish_reason": "length",
4+
"index": 0,
5+
"logprobs": null,
6+
"prompt_logprobs": null,
7+
"stop_reason": null,
8+
"text": "\n\n1. Sustainability: Renewable energy sources, such as solar, wind, and hydro, are virtually inexhaustible. Unlike fossil fuels, which are finite resources, renewable energy can be harnessed continuously without depletion.\n\n2. Environmental Impact: Fossil fuels contribute significantly to greenhouse gas emissions, which are a major driver of climate change. Renewable energy sources produce little to no greenhouse gas emissions during operation, making"
9+
},
10+
{
11+
"finish_reason": "length",
12+
"index": 0,
13+
"logprobs": null,
14+
"prompt_logprobs": null,
15+
"stop_reason": null,
16+
"text": "\n\nSpanish: 'El conocimiento es poder.'\nGerman: 'Wissen ist Macht.'\nMandarin: '知识是力量。' (Zhīshì shì lìliàng.)\n\nNote: The Mandarin translation uses traditional Chinese characters, but simplified characters are also commonly used in mainland China. The simplified version of the sentence is: '知识是力量。' (Zhīshì sh"
17+
},
18+
{
19+
"finish_reason": "length",
20+
"index": 0,
21+
"logprobs": null,
22+
"prompt_logprobs": null,
23+
"stop_reason": null,
24+
"text": "\n\nUnder the veil of twilight's embrace,\nA canvas painted with celestial grace,\nThe night sky, a silent, starry stage,\nA spectacle that leaves me in awe.\n\nA million diamonds, scattered wide,\nIn the vast expanse, they reside,\nEach a secret, a story untold,\nA mystery that never grows old.\n\nThe moon, a silver orb, suspended high,\nGu"
25+
},
26+
{
27+
"finish_reason": "length",
28+
"index": 0,
29+
"logprobs": null,
30+
"prompt_logprobs": null,
31+
"stop_reason": null,
32+
"text": "\n\nThe Great Wall of China, a series of fortifications built across the historical northern borders of China to protect the Chinese states and empires against invasions and raids, holds immense historical and cultural significance. It is one of the most iconic symbols of China and an enduring testament to the ingenuity, resilience, and ambition of the Chinese civilization.\n\nHistorically, the Great Wall was constructed over various dynasties, with the earliest sections dating"
33+
},
34+
{
35+
"finish_reason": "length",
36+
"index": 0,
37+
"logprobs": null,
38+
"prompt_logprobs": null,
39+
"stop_reason": null,
40+
"text": "\n\nArtificial Intelligence (AI) in healthcare decision-making presents a myriad of ethical implications that need careful consideration.\n\n1. Patient autonomy and informed consent: AI systems may provide recommendations based on vast amounts of data, potentially overriding the patient's own preferences and values. Ensuring that patients are well-informed about the AI's role in their care, and that they have the opportunity to make informed decisions, is crucial.\n\n2"
41+
},
42+
{
43+
"finish_reason": "length",
44+
"index": 0,
45+
"logprobs": null,
46+
"prompt_logprobs": null,
47+
"stop_reason": null,
48+
"text": "\n\nApollo 11, the mission that marked the first human landing on the moon, took place on July 20, 1969. The mission was a significant milestone in space exploration history, led by NASA, the United States' space agency.\n\nThe Apollo 11 spacecraft was launched from Kennedy Space Center in Florida on July 16, 1969. The crew consisted of three astronauts: Neil"
49+
}
50+
]

0 commit comments

Comments
 (0)