Skip to content

Commit f69676b

Browse files
dbasunagmwaykole
dbasunag
authored and
mwaykole
committed
Add negative test for allowedOrganization (opendatahub-io#1004)
* Add negative test for allowedOrganization * Move HF tests to separate location
1 parent 4010007 commit f69676b

File tree

4 files changed

+161
-94
lines changed

4 files changed

+161
-94
lines changed

‎tests/model_registry/model_catalog/catalog_config/test_model_catalog_negative.py‎

Lines changed: 8 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from tests.model_registry.constants import DEFAULT_MODEL_CATALOG_CM
88
from tests.model_registry.model_catalog.constants import DEFAULT_CATALOGS
99
from tests.model_registry.model_catalog.catalog_config.utils import validate_model_catalog_configmap_data
10-
from tests.model_registry.utils import execute_get_command
10+
from tests.model_registry.model_catalog.utils import assert_source_error_state_message
1111

1212
LOGGER = get_logger(name=__name__)
1313

@@ -75,105 +75,19 @@ def test_modify_default_catalog_configmap_reconciles(
7575
"non-existent-catalog.yaml: no such file or directory",
7676
id="test_source_error_invalid_path",
7777
),
78-
pytest.param(
79-
"""
80-
catalogs:
81-
- name: HuggingFace Hub
82-
id: error_catalog
83-
type: hf
84-
enabled: true
85-
""",
86-
"includedModels cannot be empty for Hugging Face catalog",
87-
id="test_hf_source_no_include_model",
88-
),
89-
pytest.param(
90-
"""
91-
catalogs:
92-
- name: HuggingFace Hub
93-
id: error_catalog
94-
type: hf
95-
enabled: true
96-
includedModels:
97-
- abc-random*
98-
""",
99-
'failed to expand model patterns: wildcard pattern "abc-random*" is not supported - '
100-
"Hugging Face requires a specific organization",
101-
id="test_hf_source_invalid_organization",
102-
),
103-
pytest.param(
104-
"""
105-
catalogs:
106-
- name: HuggingFace Hub
107-
id: error_catalog
108-
type: hf
109-
enabled: true
110-
includedModels:
111-
- '*'
112-
""",
113-
'failed to expand model patterns: wildcard pattern "*" is not supported - '
114-
"Hugging Face requires a specific organization",
115-
id="test_hf_source_global_wildcard",
116-
),
117-
pytest.param(
118-
"""
119-
catalogs:
120-
- name: HuggingFace Hub
121-
id: error_catalog
122-
type: hf
123-
enabled: true
124-
includedModels:
125-
- '*/*'
126-
""",
127-
'failed to expand model patterns: wildcard pattern "*/*" is not supported - '
128-
"Hugging Face requires a specific organization",
129-
id="test_hf_source_global_org_wildcard",
130-
),
131-
pytest.param(
132-
"""
133-
catalogs:
134-
- name: HuggingFace Hub
135-
id: error_catalog
136-
type: hf
137-
enabled: true
138-
includedModels:
139-
- 'RedHatAI/'
140-
""",
141-
'failed to expand model patterns: wildcard pattern "RedHatAI/" is not supported - '
142-
"Hugging Face requires a specific organization",
143-
id="test_hf_source_empty_model_name",
144-
),
145-
pytest.param(
146-
"""
147-
catalogs:
148-
- name: HuggingFace Hub
149-
id: error_catalog
150-
type: hf
151-
enabled: true
152-
includedModels:
153-
- '*RedHatAI*'
154-
""",
155-
'failed to expand model patterns: wildcard pattern "*RedHatAI*" is not supported - '
156-
"Hugging Face requires a specific organization",
157-
id="test_hf_source_multiple_wildcards",
158-
),
15978
],
16079
indirect=["updated_catalog_config_map_scope_function"],
16180
)
162-
def test_source_error_state(
81+
def test_default_source_error_state(
16382
self: Self,
16483
updated_catalog_config_map_scope_function: ConfigMap,
16584
model_catalog_rest_url: list[str],
16685
model_registry_rest_headers: dict[str, str],
167-
expected_error_message,
86+
expected_error_message: str,
16887
):
169-
results = execute_get_command(
170-
url=f"{model_catalog_rest_url[0]}sources",
171-
headers=model_registry_rest_headers,
172-
)["items"]
173-
# pick the relevant source first by id:
174-
matched_source = [result for result in results if result["id"] == "error_catalog"]
175-
assert matched_source, f"Matched expected source not found: {results}"
176-
assert matched_source[0]["status"] == "error"
177-
assert expected_error_message in matched_source[0]["error"], (
178-
f"Expected error: {expected_error_message} not found in {matched_source[0]['error']}"
88+
assert_source_error_state_message(
89+
model_catalog_rest_url=model_catalog_rest_url,
90+
model_registry_rest_headers=model_registry_rest_headers,
91+
expected_error_message=expected_error_message,
92+
source_id="error_catalog",
17993
)

‎tests/model_registry/model_catalog/conftest.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,7 @@ def updated_catalog_config_map_scope_function(
394394
wait_for_model_catalog_api(url=model_catalog_rest_url[0], headers=model_registry_rest_headers)
395395
yield catalog_config_map
396396
is_model_catalog_ready(client=admin_client, model_registry_namespace=model_registry_namespace)
397+
wait_for_model_catalog_api(url=model_catalog_rest_url[0], headers=model_registry_rest_headers)
397398

398399

399400
@pytest.fixture(scope="class")

‎tests/model_registry/model_catalog/huggingface/test_huggingface_negative.py‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import pytest
2+
from simple_logger.logger import get_logger
3+
from typing import Self
4+
5+
from ocp_resources.config_map import ConfigMap
6+
from tests.model_registry.model_catalog.utils import assert_source_error_state_message
7+
8+
LOGGER = get_logger(name=__name__)
9+
10+
pytestmark = [
11+
pytest.mark.usefixtures(
12+
"updated_dsc_component_state_scope_session",
13+
"model_registry_namespace",
14+
)
15+
]
16+
17+
18+
class TestHuggingFaceNegative:
19+
@pytest.mark.parametrize(
20+
"updated_catalog_config_map_scope_function, expected_error_message",
21+
[
22+
pytest.param(
23+
"""
24+
catalogs:
25+
- name: HuggingFace Hub
26+
id: error_catalog
27+
type: hf
28+
enabled: true
29+
""",
30+
"includedModels cannot be empty for Hugging Face catalog",
31+
id="test_hf_source_no_include_model",
32+
),
33+
pytest.param(
34+
"""
35+
catalogs:
36+
- name: HuggingFace Hub
37+
id: error_catalog
38+
type: hf
39+
enabled: true
40+
includedModels:
41+
- abc-random*
42+
""",
43+
'failed to expand model patterns: wildcard pattern "abc-random*" is not supported - '
44+
"Hugging Face requires a specific organization",
45+
id="test_hf_source_invalid_organization",
46+
),
47+
pytest.param(
48+
"""
49+
catalogs:
50+
- name: HuggingFace Hub
51+
id: error_catalog
52+
type: hf
53+
enabled: true
54+
includedModels:
55+
- '*'
56+
""",
57+
'failed to expand model patterns: wildcard pattern "*" is not supported - '
58+
"Hugging Face requires a specific organization",
59+
id="test_hf_source_global_wildcard",
60+
),
61+
pytest.param(
62+
"""
63+
catalogs:
64+
- name: HuggingFace Hub
65+
id: error_catalog
66+
type: hf
67+
enabled: true
68+
includedModels:
69+
- '*/*'
70+
""",
71+
'failed to expand model patterns: wildcard pattern "*/*" is not supported - '
72+
"Hugging Face requires a specific organization",
73+
id="test_hf_source_global_org_wildcard",
74+
),
75+
pytest.param(
76+
"""
77+
catalogs:
78+
- name: HuggingFace Hub
79+
id: error_catalog
80+
type: hf
81+
enabled: true
82+
includedModels:
83+
- 'RedHatAI/'
84+
""",
85+
'failed to expand model patterns: wildcard pattern "RedHatAI/" is not supported - '
86+
"Hugging Face requires a specific organization",
87+
id="test_hf_source_empty_model_name",
88+
),
89+
pytest.param(
90+
"""
91+
catalogs:
92+
- name: HuggingFace Hub
93+
id: error_catalog
94+
type: hf
95+
enabled: true
96+
includedModels:
97+
- '*RedHatAI*'
98+
""",
99+
'failed to expand model patterns: wildcard pattern "*RedHatAI*" is not supported - '
100+
"Hugging Face requires a specific organization",
101+
id="test_hf_source_multiple_wildcards",
102+
),
103+
pytest.param(
104+
"""
105+
catalogs:
106+
- name: HuggingFace Hub
107+
id: error_catalog
108+
type: hf
109+
enabled: true
110+
properties:
111+
allowedOrganization: "abc-random"
112+
includedModels:
113+
- '*'
114+
""",
115+
"failed to expand model patterns: no models found",
116+
id="test_hf_source_non_existent_allowed_organization",
117+
),
118+
],
119+
indirect=["updated_catalog_config_map_scope_function"],
120+
)
121+
def test_huggingface_source_error_state(
122+
self: Self,
123+
updated_catalog_config_map_scope_function: ConfigMap,
124+
model_catalog_rest_url: list[str],
125+
model_registry_rest_headers: dict[str, str],
126+
expected_error_message: str,
127+
):
128+
assert_source_error_state_message(
129+
model_catalog_rest_url=model_catalog_rest_url,
130+
model_registry_rest_headers=model_registry_rest_headers,
131+
expected_error_message=expected_error_message,
132+
source_id="error_catalog",
133+
)

‎tests/model_registry/model_catalog/utils.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,3 +251,22 @@ def get_excluded_model_str(models: list[str]) -> str:
251251
- {model_name}
252252
"""
253253
return excluded_models
254+
255+
256+
def assert_source_error_state_message(
257+
model_catalog_rest_url: list[str],
258+
model_registry_rest_headers: dict[str, str],
259+
expected_error_message: str,
260+
source_id: str,
261+
):
262+
results = execute_get_command(
263+
url=f"{model_catalog_rest_url[0]}sources",
264+
headers=model_registry_rest_headers,
265+
)["items"]
266+
# pick the relevant source first by id:
267+
matched_source = [result for result in results if result["id"] == source_id]
268+
assert matched_source, f"Matched expected source not found: {results}"
269+
assert matched_source[0]["status"] == "error"
270+
assert expected_error_message in matched_source[0]["error"], (
271+
f"Expected error: {expected_error_message} not found in {matched_source[0]['error']}"
272+
)

0 commit comments

Comments
 (0)