Skip to content

Commit a06a245

Browse files
fegefege
committed
test: reduce smoke time execution run
1 parent dc4aada commit a06a245

10 files changed

Lines changed: 190 additions & 18 deletions

File tree

‎tests/conftest.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -947,3 +947,62 @@ def oci_registry_route(admin_client: DynamicClient, oci_registry_service: Servic
947947
def oci_registry_host(oci_registry_route: Route) -> str:
948948
"""Get the OCI registry host from the route"""
949949
return oci_registry_route.host
950+
951+
952+
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
953+
"""
954+
Enforce test tier hierarchy by excluding tests with explicit higher-tier markers.
955+
956+
Test tier hierarchy (from lowest to highest):
957+
smoke < sanity < tier1 < tier2
958+
959+
When running tests with a specific tier marker (e.g., -m smoke), this hook
960+
automatically excludes tests that have explicit markers for higher tiers,
961+
preventing marker inheritance issues from class-level parametrization.
962+
963+
Example:
964+
- Running 'pytest -m smoke' excludes tests marked @pytest.mark.sanity/tier1/tier2
965+
- Running 'pytest -m sanity' excludes tests marked @pytest.mark.tier1/tier2
966+
- Running 'pytest -m tier1' excludes tests marked @pytest.mark.tier2
967+
"""
968+
# Define tier hierarchy (order matters: lower tiers first)
969+
tier_hierarchy = ["smoke", "sanity", "tier1", "tier2"]
970+
971+
# Get the marker expression from command line
972+
markexpr = config.getoption("-m", "") # noqa: FCN001
973+
if not markexpr:
974+
return
975+
976+
# Find which tier is being requested
977+
requested_tier = None
978+
for tier in tier_hierarchy:
979+
if tier in markexpr:
980+
requested_tier = tier
981+
break
982+
983+
if not requested_tier:
984+
return
985+
986+
# Get index of requested tier
987+
requested_tier_idx = tier_hierarchy.index(requested_tier)
988+
989+
# Get all higher tiers that should be excluded
990+
excluded_tiers = set(tier_hierarchy[requested_tier_idx + 1 :]) # noqa: E203
991+
992+
if not excluded_tiers:
993+
return # No higher tiers to exclude
994+
995+
# Skip tests that have explicit markers for higher tiers
996+
skip_marker = pytest.mark.skip(
997+
reason=f"Has explicit {'/'.join(excluded_tiers)} marker, excluded from {requested_tier}"
998+
)
999+
1000+
skipped_count = 0
1001+
for item in items:
1002+
marker_names = {mark.name for mark in item.iter_markers()}
1003+
if marker_names & excluded_tiers: # If any excluded tier marker is present
1004+
item.add_marker(skip_marker) # noqa: FCN001
1005+
skipped_count += 1
1006+
1007+
if skipped_count > 0:
1008+
print(f"\n[Tier Filter] Excluded {skipped_count} tests with {excluded_tiers} markers from {requested_tier} run")

‎tests/model_registry/image_validation/test_verify_rhoai_images.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
from pytest_testconfig import config as py_config
1919

2020
LOGGER = get_logger(name=__name__)
21-
pytestmark = [pytest.mark.downstream_only, pytest.mark.skip_must_gather, pytest.mark.smoke]
21+
pytestmark = [pytest.mark.downstream_only, pytest.mark.skip_must_gather]
2222

2323

2424
class TestAIHubResourcesImages:
@@ -35,7 +35,7 @@ class TestAIHubResourcesImages:
3535
"namespace": py_config["applications_namespace"],
3636
"label_selector": f"{Labels.OpenDataHubIo.NAME}={MR_OPERATOR_NAME}",
3737
},
38-
marks=pytest.mark.smoke,
38+
marks=pytest.mark.sanity,
3939
id="test_model_registry_operator_pods_images",
4040
),
4141
],
@@ -53,7 +53,7 @@ def test_verify_pod_images(
5353
@pytest.mark.parametrize(
5454
"model_registry_metadata_db_resources, model_registry_instance, model_registry_instance_pods_by_label",
5555
[
56-
pytest.param({}, {}, {"label_selectors": [f"app={MR_INSTANCE_NAME}"]}),
56+
pytest.param({}, {}, {"label_selectors": [f"app={MR_INSTANCE_NAME}"]}, marks=pytest.mark.smoke),
5757
pytest.param(
5858
{"db_name": "default"},
5959
{"db_name": "default"},
@@ -63,6 +63,7 @@ def test_verify_pod_images(
6363
f"app.kubernetes.io/name={MR_POSTGRES_DEPLOYMENT_NAME_STR}",
6464
]
6565
},
66+
marks=pytest.mark.tier1,
6667
),
6768
],
6869
indirect=True,

‎tests/model_registry/model_catalog/catalog_config/test_default_source_inclusion_exclusion_cleanup.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class TestModelInclusionFiltering:
3434
[
3535
pytest.param(
3636
{"filter_type": "inclusion", "pattern": "granite", "filter_value": "*granite*"},
37-
marks=pytest.mark.smoke,
37+
marks=pytest.mark.tier1,
3838
id="test_include_granite_models_only",
3939
),
4040
pytest.param(
@@ -82,7 +82,7 @@ class TestModelExclusionFiltering:
8282
[
8383
pytest.param(
8484
{"filter_type": "exclusion", "pattern": "granite", "filter_value": "*granite*"},
85-
marks=pytest.mark.smoke,
85+
marks=pytest.mark.tier1,
8686
id="test_exclude_granite_models",
8787
),
8888
pytest.param(
@@ -131,7 +131,7 @@ class TestCombinedIncludeExcludeFiltering:
131131
"exclude_pattern": "lab",
132132
"exclude_filter_value": "*lab*",
133133
},
134-
marks=pytest.mark.smoke,
134+
marks=pytest.mark.tier1,
135135
id="include_granite_exclude_lab",
136136
),
137137
pytest.param(
@@ -275,7 +275,7 @@ def test_model_cleanup_on_exclusion_change(
275275
class TestSourceLifecycleCleanup:
276276
"""Test source disabling cleanup scenarios (RHOAIENG-41846)"""
277277

278-
@pytest.mark.smoke
278+
@pytest.mark.tier1
279279
def test_source_disabling_removes_models(
280280
self,
281281
admin_client: DynamicClient,

‎tests/model_registry/model_catalog/conftest.py‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,99 @@
3838
LOGGER = get_logger(name=__name__)
3939

4040

41+
@pytest.fixture(scope="session")
42+
def catalog_pod_model_counts(
43+
admin_client: DynamicClient,
44+
recreated_model_catalog_configmap: ConfigMap,
45+
) -> dict[str, int]:
46+
"""
47+
Session-scoped fixture that extracts model counts from catalog pod logs.
48+
49+
Scrapes logs for earliest occurrences of:
50+
- "redhat_ai_validated_models: loaded x models"
51+
- "redhat_ai_models: loaded y models"
52+
53+
Returns:
54+
Dictionary with keys "redhat_ai_validated_models" and "redhat_ai_models"
55+
containing the extracted model counts
56+
"""
57+
# Get the model catalog pod
58+
namespace_name = py_config["model_registry_namespace"]
59+
catalog_pods = get_model_catalog_pod(client=admin_client, model_registry_namespace=namespace_name)
60+
assert len(catalog_pods) > 0, f"No model catalog pods found in namespace {namespace_name}"
61+
62+
catalog_pod = catalog_pods[0] # Use the first pod if multiple exist
63+
64+
# Get pod logs
65+
logs = catalog_pod.log(container="catalog")
66+
67+
# Define regex patterns for extraction
68+
patterns = {
69+
"redhat_ai_validated_models": r"redhat_ai_validated_models: loaded (\d+) models",
70+
"redhat_ai_models": r"redhat_ai_models: loaded (\d+) models",
71+
}
72+
73+
# Extract counts
74+
model_counts = {}
75+
for key, pattern in patterns.items():
76+
match = re.search(pattern, logs)
77+
if match:
78+
model_counts[key] = int(match.group(1))
79+
else:
80+
LOGGER.warning(f"Pattern '{pattern}' not found in catalog pod logs")
81+
model_counts[key] = 0 # Default to 0 if not found
82+
83+
LOGGER.info(f"Extracted model counts from catalog pod logs: {model_counts}")
84+
return model_counts
85+
86+
87+
@pytest.fixture(scope="session")
88+
def recreated_model_catalog_configmap(
89+
admin_client: DynamicClient,
90+
) -> ConfigMap:
91+
"""
92+
Session-scoped fixture that deletes the DEFAULT_CUSTOM_MODEL_CATALOG ConfigMap
93+
and waits for it to be automatically recreated, and cleans up catalog pod, to start with a fresh log
94+
95+
Returns:
96+
ConfigMap: The recreated ConfigMap instance
97+
"""
98+
namespace_name = py_config["model_registry_namespace"]
99+
# TODO: RHOAIENG-46741 would require changing this to look for configmaps based on label
100+
# Get the existing ConfigMap
101+
configmap = ConfigMap(
102+
name=DEFAULT_CUSTOM_MODEL_CATALOG, client=admin_client, namespace=namespace_name, ensure_exists=True
103+
)
104+
105+
LOGGER.info(f"Deleting ConfigMap {DEFAULT_CUSTOM_MODEL_CATALOG} to test recreation")
106+
107+
# Delete the ConfigMap
108+
configmap.delete()
109+
110+
LOGGER.info(f"ConfigMap {DEFAULT_CUSTOM_MODEL_CATALOG} deleted, waiting for recreation")
111+
112+
# Wait for it to be recreated using TimeoutSampler
113+
recreated_configmap = ConfigMap(
114+
name=DEFAULT_CUSTOM_MODEL_CATALOG,
115+
client=admin_client,
116+
namespace=namespace_name,
117+
)
118+
119+
# Use TimeoutSampler to wait for recreation (2 minutes timeout)
120+
for sample in TimeoutSampler(
121+
wait_timeout=120, # 2 minutes
122+
sleep=5,
123+
func=lambda: recreated_configmap.exists,
124+
exceptions_dict={NotFoundError: []},
125+
):
126+
if sample: # ConfigMap exists
127+
break
128+
129+
LOGGER.info(f"ConfigMap {DEFAULT_CUSTOM_MODEL_CATALOG} recreated successfully")
130+
wait_for_model_catalog_pod_ready_after_deletion(client=admin_client, model_registry_namespace=namespace_name)
131+
return recreated_configmap
132+
133+
41134
@pytest.fixture()
42135
def sparse_override_catalog_source(
43136
request: pytest.FixtureRequest,

‎tests/model_registry/model_catalog/metadata/test_sources_endpoint.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ class TestSourcesEndpoint:
1818
[{"id": REDHAT_AI_CATALOG_ID, "field_name": "enabled", "field_value": False}],
1919
indirect=True,
2020
)
21-
@pytest.mark.smoke
21+
@pytest.mark.sanity
2222
def test_sources_endpoint_returns_all_sources_regardless_of_enabled_field(
2323
self,
2424
sparse_override_catalog_source: dict,

‎tests/model_registry/model_catalog/rbac/test_catalog_rbac.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,16 @@
2626
class TestCatalogRBAC:
2727
"""Test suite for catalog ConfigMap RBAC"""
2828

29-
@pytest.mark.smoke
3029
@pytest.mark.pre_upgrade
3130
@pytest.mark.post_upgrade
3231
@pytest.mark.install
33-
@pytest.mark.parametrize("configmap_name", [DEFAULT_MODEL_CATALOG_CM, DEFAULT_CUSTOM_MODEL_CATALOG])
32+
@pytest.mark.parametrize(
33+
"configmap_name",
34+
[
35+
pytest.param(DEFAULT_MODEL_CATALOG_CM, marks=pytest.mark.smoke),
36+
pytest.param(DEFAULT_CUSTOM_MODEL_CATALOG, marks=pytest.mark.sanity),
37+
],
38+
)
3439
def test_admin_can_read_catalog_configmaps(
3540
self,
3641
admin_client: DynamicClient,
@@ -64,8 +69,13 @@ def test_admin_can_read_catalog_configmaps(
6469

6570
LOGGER.info(f"Admin successfully read ConfigMap '{configmap_name}'")
6671

67-
@pytest.mark.smoke
68-
@pytest.mark.parametrize("configmap_name", [DEFAULT_MODEL_CATALOG_CM, DEFAULT_CUSTOM_MODEL_CATALOG])
72+
@pytest.mark.parametrize(
73+
"configmap_name",
74+
[
75+
pytest.param(DEFAULT_MODEL_CATALOG_CM, marks=pytest.mark.smoke),
76+
pytest.param(DEFAULT_CUSTOM_MODEL_CATALOG, marks=pytest.mark.sanity),
77+
],
78+
)
6979
def test_non_admin_cannot_access_catalog_configmaps(
7080
self,
7181
is_byoidc: bool,

‎tests/model_registry/model_registry/rbac/test_mr_rbac_sa.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ def test_service_account_access_denied(
5555
assert http_error.status == 403, f"Expected HTTP 403 Forbidden, but got {http_error.status}"
5656
LOGGER.info("Successfully received expected HTTP 403 status code.")
5757

58-
@pytest.mark.smoke
58+
@pytest.mark.sanity
5959
@pytest.mark.usefixtures("sa_namespace", "service_account", "mr_access_role", "mr_access_role_binding")
6060
def test_service_account_access_granted(
6161
self: Self,

‎tests/model_registry/model_registry/rest_api/test_model_registry_rest_api.py‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,15 +44,19 @@
4444
{"db_name": "postgres"},
4545
{"db_name": "postgres"},
4646
MODEL_REGISTER_DATA,
47-
marks=(pytest.mark.smoke),
47+
marks=(pytest.mark.sanity),
48+
),
49+
pytest.param(
50+
{"db_name": "default"},
51+
{"db_name": "default"},
52+
MODEL_REGISTER_DATA,
53+
marks=(pytest.mark.tier1),
4854
),
4955
pytest.param(
5056
{"db_name": "mariadb"},
5157
{"db_name": "mariadb"},
5258
MODEL_REGISTER_DATA,
53-
marks=(pytest.mark.sanity),
5459
),
55-
pytest.param({"db_name": "default"}, {"db_name": "default"}, MODEL_REGISTER_DATA, marks=(pytest.mark.smoke)),
5660
],
5761
indirect=True,
5862
)
@@ -102,6 +106,7 @@ def test_validate_model_registry_resource(
102106
resource_name=data_key,
103107
)
104108

109+
@pytest.mark.sanity
105110
@pytest.mark.parametrize(
106111
"kind, resource_name",
107112
[
@@ -150,6 +155,7 @@ def test_default_postgres_db_resource_exists(
150155
for field in ["controller", "blockOwnerDeletion"]:
151156
assert owner_reference[0][field] is True
152157

158+
@pytest.mark.sanity
153159
def test_default_postgres_db_pod_log(
154160
self: Self,
155161
skip_if_not_default_db: None,
@@ -169,6 +175,7 @@ def test_default_postgres_db_pod_log(
169175
postgres_pod_log = pods[0].log(container="postgres")
170176
assert CONNECTION_STRING in postgres_pod_log
171177

178+
@pytest.mark.tier1
172179
def test_model_registry_validate_api_version(
173180
self: Self,
174181
admin_client: DynamicClient,
@@ -184,6 +191,7 @@ def test_model_registry_validate_api_version(
184191
expected_version = f"{ModelRegistry.ApiGroup.MODELREGISTRY_OPENDATAHUB_IO}/{ModelRegistry.ApiVersion.V1BETA1}"
185192
assert api_version == expected_version
186193

194+
@pytest.mark.tier1
187195
def test_model_registry_validate_kuberbacproxy_enabled(
188196
self: Self,
189197
model_registry_instance: list[ModelRegistry],
@@ -226,6 +234,7 @@ def test_model_registry_validate_kuberbacproxy_enabled(
226234
],
227235
indirect=["updated_model_registry_resource"],
228236
)
237+
@pytest.mark.tier1
229238
def test_create_update_model_artifact(
230239
self,
231240
updated_model_registry_resource: dict[str, Any],
@@ -270,6 +279,7 @@ def test_create_update_model_artifact(
270279
],
271280
indirect=["updated_model_registry_resource"],
272281
)
282+
@pytest.mark.tier1
273283
def test_updated_model_version(
274284
self,
275285
updated_model_registry_resource: dict[str, Any],
@@ -315,6 +325,7 @@ def test_updated_model_version(
315325
],
316326
indirect=["updated_model_registry_resource"],
317327
)
328+
@pytest.mark.tier1
318329
def test_updated_registered_model(
319330
self,
320331
updated_model_registry_resource: dict[str, Any],

‎tests/model_registry/model_registry/rest_api/test_model_registry_secure_db.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,6 @@ def test_register_model_with_invalid_ca(
9999
"ca_configmap_for_test",
100100
"patch_external_deployment_with_ssl_ca",
101101
)
102-
@pytest.mark.smoke
103102
def test_register_model_with_valid_ca(
104103
self: Self,
105104
admin_client: DynamicClient,

‎tests/model_registry/model_registry/rest_api/test_multiple_mr.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
"model_registry_instance",
4141
)
4242
class TestModelRegistryMultipleInstances:
43-
@pytest.mark.smoke
4443
def test_validate_multiple_model_registry(
4544
self: Self,
4645
admin_client: DynamicClient,

0 commit comments

Comments
 (0)