Reduce smoke for model_registry component (#1122)

fege · dbasunag · web-flow · commit ec4297d76aef · 2026-03-09T17:04:59.000Z
* test: reduce smoke time execution run

* fix: merge changes from remote

* test: reduce model registry smoke tests

* fix: Rename sanity-&gt;tier1, cascade tier1-&gt;tier2, add tier3 for negatives

* fix: move tests from tier1 to tier2

* chore: tier3 marker

* fix: reduce tier1 run time

---------

Co-authored-by: Debarati Basu-Nag &lt;dbasunag@redhat.com&gt;
diff --git a/pytest.ini b/pytest.ini
@@ -10,10 +10,11 @@ markers =
     parallel: marks tests that can run in parallel along with pytest-xdist
 
     # CI
-    smoke: Mark tests as smoke tests; covers core functionality of the product. Aims to ensure that the build is stable enough for further testing.
+    smoke: Mark tests as smoke tests; very high critical priority tests. Covers core functionality of the product. Aims to ensure that the build is stable enough for further testing.
     sanity: Mark tests as sanity tests. Aims to verify that specific functionality is working as expected.
-    tier1: Mark tests as tier1. Aims to cover frequently used functionality of the product and basic user flows.
-    tier2: Mark tests as tier2. Aims to cover more advanced functionality of the product.
+    tier1: Mark tests as tier1. High-priority tests.
+    tier2: Mark tests as tier2. Medium/low-priority positive tests.
+    tier3: Mark tests as tier3. Negative and destructive tests.
     slow: Mark tests which take more than 10 minutes as slow tests.
     pre_upgrade: Mark tests which should be run before upgrading the product.
     post_upgrade: Mark tests which should be run after upgrading the product.
diff --git a/tests/model_registry/image_validation/test_verify_rhoai_images.py b/tests/model_registry/image_validation/test_verify_rhoai_images.py
@@ -19,7 +19,7 @@
 from utilities.constants import Labels
 
 LOGGER = get_logger(name=__name__)
-pytestmark = [pytest.mark.downstream_only, pytest.mark.skip_must_gather, pytest.mark.smoke]
+pytestmark = [pytest.mark.downstream_only, pytest.mark.skip_must_gather]
 
 
 class TestAIHubResourcesImages:
@@ -28,15 +28,15 @@ class TestAIHubResourcesImages:
         [
             pytest.param(
                 {"namespace": py_config["model_registry_namespace"], "label_selector": "component=model-catalog"},
-                marks=pytest.mark.smoke,
+                marks=pytest.mark.tier1,
                 id="test_model_catalog_pods_images",
             ),
             pytest.param(
                 {
                     "namespace": py_config["applications_namespace"],
                     "label_selector": f"{Labels.OpenDataHubIo.NAME}={MR_OPERATOR_NAME}",
                 },
-                marks=pytest.mark.smoke,
+                marks=pytest.mark.tier1,
                 id="test_model_registry_operator_pods_images",
             ),
         ],
@@ -54,7 +54,7 @@ def test_verify_pod_images(
 @pytest.mark.parametrize(
     "model_registry_metadata_db_resources, model_registry_instance, model_registry_instance_pods_by_label",
     [
-        pytest.param({}, {}, {"label_selectors": [f"app={MR_INSTANCE_NAME}"]}),
+        pytest.param({}, {}, {"label_selectors": [f"app={MR_INSTANCE_NAME}"]}, marks=pytest.mark.smoke),
         pytest.param(
             {"db_name": "default"},
             {"db_name": "default"},
@@ -64,6 +64,7 @@ def test_verify_pod_images(
                     f"app.kubernetes.io/name={MR_POSTGRES_DEPLOYMENT_NAME_STR}",
                 ]
             },
+            marks=pytest.mark.tier2,
         ),
     ],
     indirect=True,
diff --git a/tests/model_registry/model_catalog/catalog_config/test_default_source_inclusion_exclusion_cleanup.py b/tests/model_registry/model_catalog/catalog_config/test_default_source_inclusion_exclusion_cleanup.py
@@ -35,22 +35,22 @@ class TestModelInclusionFiltering:
         [
             pytest.param(
                 {"filter_type": "inclusion", "pattern": "granite", "filter_value": "*granite*"},
-                marks=pytest.mark.smoke,
+                marks=pytest.mark.tier2,
                 id="test_include_granite_models_only",
             ),
             pytest.param(
                 {"filter_type": "inclusion", "pattern": "prometheus", "filter_value": "*prometheus*"},
-                marks=pytest.mark.sanity,
+                marks=pytest.mark.tier2,
                 id="test_include_prometheus_models_only",
             ),
             pytest.param(
                 {"filter_type": "inclusion", "pattern": "-8b-", "filter_value": "*-8b-*"},
-                marks=pytest.mark.sanity,
+                marks=pytest.mark.tier2,
                 id="test_include_eight_b_models_only",
             ),
             pytest.param(
                 {"filter_type": "inclusion", "pattern": "code", "filter_value": "*code*"},
-                marks=pytest.mark.sanity,
+                marks=pytest.mark.tier2,
                 id="test_include_code_models_only",
             ),
         ],
@@ -83,17 +83,17 @@ class TestModelExclusionFiltering:
         [
             pytest.param(
                 {"filter_type": "exclusion", "pattern": "granite", "filter_value": "*granite*"},
-                marks=pytest.mark.smoke,
+                marks=pytest.mark.tier2,
                 id="test_exclude_granite_models",
             ),
             pytest.param(
                 {"filter_type": "exclusion", "pattern": "prometheus", "filter_value": "*prometheus*"},
-                marks=pytest.mark.sanity,
+                marks=pytest.mark.tier1,
                 id="test_exclude_prometheus_models",
             ),
             pytest.param(
                 {"filter_type": "exclusion", "pattern": "lab", "filter_value": "*lab*"},
-                marks=pytest.mark.sanity,
+                marks=pytest.mark.tier2,
                 id="test_exclude_lab_models",
             ),
         ],
@@ -132,7 +132,7 @@ class TestCombinedIncludeExcludeFiltering:
                     "exclude_pattern": "lab",
                     "exclude_filter_value": "*lab*",
                 },
-                marks=pytest.mark.smoke,
+                marks=pytest.mark.tier2,
                 id="include_granite_exclude_lab",
             ),
             pytest.param(
@@ -143,7 +143,7 @@ class TestCombinedIncludeExcludeFiltering:
                     "exclude_pattern": "code",
                     "exclude_filter_value": "*code*",
                 },
-                marks=pytest.mark.sanity,
+                marks=pytest.mark.tier2,
                 id="include_eight_b_exclude_code",
             ),
         ],
@@ -171,7 +171,7 @@ def test_combined_include_exclude_filtering(
 class TestModelCleanupLifecycle:
     """Test automatic model cleanup during lifecycle changes (RHOAIENG-41846)"""
 
-    @pytest.mark.sanity
+    @pytest.mark.tier2
     def test_model_cleanup_on_exclusion_change(
         self,
         admin_client: DynamicClient,
@@ -276,7 +276,7 @@ def test_model_cleanup_on_exclusion_change(
 class TestSourceLifecycleCleanup:
     """Test source disabling cleanup scenarios (RHOAIENG-41846)"""
 
-    @pytest.mark.smoke
+    @pytest.mark.tier2
     def test_source_disabling_removes_models(
         self,
         admin_client: DynamicClient,
@@ -294,7 +294,7 @@ def test_source_disabling_removes_models(
             model_registry_namespace=model_registry_namespace,
         )
 
-    @pytest.mark.sanity
+    @pytest.mark.tier2
     def test_source_disabling_logging(
         self,
         admin_client: DynamicClient,
@@ -323,7 +323,7 @@ class TestLoggingValidation:
         [
             pytest.param(
                 {"filter_type": "exclusion", "pattern": "granite", "filter_value": "*granite*", "log_cleanup": True},
-                marks=pytest.mark.sanity,
+                marks=pytest.mark.tier2,
                 id="test_exclude_granite_models_for_logging",
             )
         ],
diff --git a/tests/model_registry/model_catalog/catalog_config/test_model_catalog_negative.py b/tests/model_registry/model_catalog/catalog_config/test_model_catalog_negative.py
@@ -21,6 +21,7 @@
 
 
 @pytest.mark.skip_must_gather
+@pytest.mark.tier3
 class TestDefaultCatalogNegative:
     """Negative tests for default catalog configuration"""
 
diff --git a/tests/model_registry/model_catalog/huggingface/test_huggingface_model_deployment.py b/tests/model_registry/model_catalog/huggingface/test_huggingface_model_deployment.py
@@ -33,7 +33,7 @@ class TestHuggingFaceModelDeployment:
     Tests the complete deployment workflow from ServingRuntime to InferenceService.
     """
 
-    @pytest.mark.sanity
+    @pytest.mark.tier2
     def test_huggingface_model_deployment_end_to_end(
         self,
         admin_client: DynamicClient,
diff --git a/tests/model_registry/model_catalog/huggingface/test_huggingface_negative.py b/tests/model_registry/model_catalog/huggingface/test_huggingface_negative.py
@@ -17,6 +17,7 @@
 ]
 
 
+@pytest.mark.tier3
 class TestHuggingFaceNegative:
     @pytest.mark.parametrize(
         "updated_catalog_config_map_scope_function, expected_error_message",
diff --git a/tests/model_registry/model_catalog/metadata/test_catalog_preview.py b/tests/model_registry/model_catalog/metadata/test_catalog_preview.py
@@ -200,6 +200,7 @@ def test_catalog_preview_filter_status(
             LOGGER.info(f"filterStatus={filter_status} validation passed: {len(items)} items")
 
 
+@pytest.mark.tier3
 class TestCatalogPreviewErrorHandling:
     """
     Test class for validating the catalog preview API error handling
diff --git a/tests/model_registry/model_catalog/metadata/test_labels_endpoint.py b/tests/model_registry/model_catalog/metadata/test_labels_endpoint.py
@@ -41,7 +41,7 @@ def test_labels_endpoint_default_data(
         # Verify they match
         verify_labels_match(expected_labels=expected_labels, api_labels=api_labels)
 
-    @pytest.mark.sanity
+    @pytest.mark.tier1
     def test_labels_endpoint_configmap_updates(
         self,
         admin_client: DynamicClient,
diff --git a/tests/model_registry/model_catalog/metadata/test_sources_endpoint.py b/tests/model_registry/model_catalog/metadata/test_sources_endpoint.py
@@ -20,7 +20,7 @@ class TestSourcesEndpoint:
         [{"id": REDHAT_AI_CATALOG_ID, "field_name": "enabled", "field_value": False}],
         indirect=True,
     )
-    @pytest.mark.smoke
+    @pytest.mark.tier1
     def test_sources_endpoint_returns_all_sources_regardless_of_enabled_field(
         self,
         sparse_override_catalog_source: dict,
diff --git a/tests/model_registry/model_catalog/rbac/test_catalog_rbac.py b/tests/model_registry/model_catalog/rbac/test_catalog_rbac.py
@@ -25,11 +25,16 @@
 class TestCatalogRBAC:
     """Test suite for catalog ConfigMap RBAC"""
 
-    @pytest.mark.smoke
     @pytest.mark.pre_upgrade
     @pytest.mark.post_upgrade
     @pytest.mark.install
-    @pytest.mark.parametrize("configmap_name", [DEFAULT_MODEL_CATALOG_CM, DEFAULT_CUSTOM_MODEL_CATALOG])
+    @pytest.mark.parametrize(
+        "configmap_name",
+        [
+            pytest.param(DEFAULT_MODEL_CATALOG_CM, marks=pytest.mark.smoke),
+            pytest.param(DEFAULT_CUSTOM_MODEL_CATALOG, marks=pytest.mark.tier1),
+        ],
+    )
     def test_admin_can_read_catalog_configmaps(
         self,
         admin_client: DynamicClient,
@@ -63,8 +68,13 @@ def test_admin_can_read_catalog_configmaps(
 
         LOGGER.info(f"Admin successfully read ConfigMap '{configmap_name}'")
 
-    @pytest.mark.smoke
-    @pytest.mark.parametrize("configmap_name", [DEFAULT_MODEL_CATALOG_CM, DEFAULT_CUSTOM_MODEL_CATALOG])
+    @pytest.mark.parametrize(
+        "configmap_name",
+        [
+            pytest.param(DEFAULT_MODEL_CATALOG_CM, marks=pytest.mark.tier1),
+            pytest.param(DEFAULT_CUSTOM_MODEL_CATALOG, marks=pytest.mark.tier1),
+        ],
+    )
     def test_non_admin_cannot_access_catalog_configmaps(
         self,
         is_byoidc: bool,
diff --git a/tests/model_registry/model_catalog/search/test_model_search.py b/tests/model_registry/model_catalog/search/test_model_search.py
@@ -62,6 +62,7 @@ def test_search_model_catalog_source_label(
         assert no_filtered_models_size == both_filtered_models_size
         assert redhat_ai_filter_moldels_size + redhat_ai_validated_filter_models_size == both_filtered_models_size
 
+    @pytest.mark.tier3
     def test_search_model_catalog_invalid_source_label(
         self: Self,
         model_catalog_rest_url: list[str],
@@ -230,7 +231,7 @@ def test_q_parameter_with_source_label_filter(
 
 
 class TestSearchModelsByFilterQuery:
-    @pytest.mark.sanity
+    @pytest.mark.tier1
     def test_search_models_by_filter_query(
         self: Self,
         admin_client: DynamicClient,
@@ -278,6 +279,7 @@ def test_search_models_by_filter_query(
 
         LOGGER.info("All models match the filter query and database validation passed")
 
+    @pytest.mark.tier3
     def test_search_models_by_invalid_filter_query(
         self: Self,
         admin_client: DynamicClient,
diff --git a/tests/model_registry/model_catalog/sorting/test_model_sorting.py b/tests/model_registry/model_catalog/sorting/test_model_sorting.py
@@ -31,7 +31,7 @@ class TestAccuracySorting:
             "DESC",
         ],
     )
-    @pytest.mark.sanity
+    @pytest.mark.tier1
     def test_accuracy_sorting_works_correctly(
         self: Self,
         admin_client: DynamicClient,
@@ -74,7 +74,7 @@ def test_accuracy_sorting_works_correctly(
         "use_case",
         [
             "code_fixing",
-            pytest.param("chatbot", marks=pytest.mark.sanity),  # Dashboard default use case
+            pytest.param("chatbot", marks=pytest.mark.tier1),  # Dashboard default use case
             "long_rag",
             "rag",
         ],
diff --git a/tests/model_registry/model_catalog/sorting/test_sorting_functionality.py b/tests/model_registry/model_catalog/sorting/test_sorting_functionality.py
@@ -46,6 +46,7 @@ def test_sources_sorting_works_correctly(
 
         assert validate_items_sorted_correctly(response["items"], order_by, sort_order)
 
+    @pytest.mark.tier3
     @pytest.mark.parametrize("unsupported_field", ["CREATE_TIME", "LAST_UPDATE_TIME"])
     def test_sources_rejects_unsupported_fields(
         self: Self,
diff --git a/tests/model_registry/model_registry/negative_tests/test_db_migration.py b/tests/model_registry/model_registry/negative_tests/test_db_migration.py
@@ -17,7 +17,7 @@
     "updated_dsc_component_state_scope_session", "model_registry_metadata_db_resources", "model_registry_instance"
 )
 class TestDBMigration:
-    @pytest.mark.sanity
+    @pytest.mark.tier3
     def test_db_migration_negative(
         self: Self,
         admin_client: DynamicClient,
diff --git a/tests/model_registry/model_registry/negative_tests/test_model_registry_creation_negative.py b/tests/model_registry/model_registry/negative_tests/test_model_registry_creation_negative.py
@@ -29,7 +29,7 @@
 )
 @pytest.mark.custom_namespace
 class TestModelRegistryCreationNegative:
-    @pytest.mark.sanity
+    @pytest.mark.tier3
     def test_registering_model_negative(
         self: Self,
         admin_client: DynamicClient,
diff --git a/tests/model_registry/model_registry/python_client/test_model_registry_creation.py b/tests/model_registry/model_registry/python_client/test_model_registry_creation.py
@@ -65,7 +65,7 @@ def test_registering_model(
         if errors:
             pytest.fail("errors found in model registry response validation:\n{}".format("\n".join(errors)))
 
-    @pytest.mark.sanity
+    @pytest.mark.tier1
     def test_model_registry_operator_env(
         self,
         model_registry_namespace: str,
@@ -80,7 +80,7 @@ def test_model_registry_operator_env(
         if not namespace_env:
             pytest.fail("Missing environment variable REGISTRIES_NAMESPACE")
 
-    @pytest.mark.sanity
+    @pytest.mark.tier1
     def test_model_registry_grpc_container_removal(self, model_registry_deployment_containers: list[dict[str, Any]]):
         """
         RHOAIENG-26239: Test to ensure removal of grpc container from model registry deployment
@@ -91,7 +91,7 @@ def test_model_registry_grpc_container_removal(self, model_registry_deployment_c
         """
         validate_no_grpc_container(deployment_containers=model_registry_deployment_containers)
 
-    @pytest.mark.sanity
+    @pytest.mark.tier1
     def test_model_registry_pod_log_mlmd_removal(
         self, model_registry_deployment_containers: list[dict[str, Any]], model_registry_pod: Pod
     ):
@@ -117,7 +117,7 @@ def test_model_registry_pod_log_mlmd_removal(
             ),
         ],
     )
-    @pytest.mark.sanity
+    @pytest.mark.tier1
     def test_model_registry_endpoint_response(
         self, model_registry_rest_url: list[str], model_registry_rest_headers: dict[str, str], endpoint: str
     ):
diff --git a/tests/model_registry/model_registry/rbac/test_mr_rbac.py b/tests/model_registry/model_registry/rbac/test_mr_rbac.py
@@ -52,7 +52,7 @@
 )
 @pytest.mark.custom_namespace
 class TestUserPermission:
-    @pytest.mark.sanity
+    @pytest.mark.tier1
     def test_user_permission_non_admin_user(
         self: Self,
         is_byoidc: bool,
@@ -76,7 +76,7 @@ def test_user_permission_non_admin_user(
         assert exc_info.value.status == 403, f"Expected HTTP 403 ForbiddenException, but got {exc_info.value.status}"
         LOGGER.info("Successfully received expected HTTP 403 status code")
 
-    @pytest.mark.sanity
+    @pytest.mark.tier1
     def test_user_added_to_group(
         self: Self,
         is_byoidc: bool,
@@ -108,7 +108,7 @@ def test_user_added_to_group(
             break  # Break after first successful iteration
         LOGGER.info("Successfully accessed Model Registry")
 
-    @pytest.mark.sanity
+    @pytest.mark.tier1
     def test_create_group(
         self: Self,
         skip_test_on_byoidc: None,
@@ -129,7 +129,7 @@ def test_create_group(
             model_registry_instance_rest_endpoint=model_registry_instance_rest_endpoint[0],
         )
 
-    @pytest.mark.sanity
+    @pytest.mark.tier1
     def test_add_single_user_role_binding(
         self: Self,
         is_byoidc: bool,
@@ -181,7 +181,7 @@ class TestUserMultiProjectPermission:
         MR_MULTIPROJECT_TEST_SCENARIO_PARAMS,
         indirect=True,
     )
-    @pytest.mark.sanity
+    @pytest.mark.tier2
     def test_user_permission_multi_project_parametrized(
         self: Self,
         is_byoidc: bool,
diff --git a/tests/model_registry/model_registry/rbac/test_mr_rbac_sa.py b/tests/model_registry/model_registry/rbac/test_mr_rbac_sa.py
diff --git a/tests/model_registry/model_registry/rest_api/test_model_registry_rest_api.py b/tests/model_registry/model_registry/rest_api/test_model_registry_rest_api.py
diff --git a/tests/model_registry/model_registry/rest_api/test_model_registry_secure_db.py b/tests/model_registry/model_registry/rest_api/test_model_registry_secure_db.py
diff --git a/tests/model_registry/model_registry/rest_api/test_multiple_mr.py b/tests/model_registry/model_registry/rest_api/test_multiple_mr.py
diff --git a/tests/model_registry/scc/test_model_catalog_scc.py b/tests/model_registry/scc/test_model_catalog_scc.py
diff --git a/tests/model_registry/scc/test_model_registry_scc.py b/tests/model_registry/scc/test_model_registry_scc.py

Original file line number	Diff line number	Diff line change
`@@ -17,6 +17,7 @@`
`17`	`17`	`]`
`18`	`18`
`19`	`19`
	`20`	`+@pytest.mark.tier3`
`20`	`21`	`class TestHuggingFaceNegative:`
`21`	`22`	`@pytest.mark.parametrize(`
`22`	`23`	`"updated_catalog_config_map_scope_function, expected_error_message",`
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,7 @@ class TestSourcesEndpoint:`
`20`	`20`	`[{"id": REDHAT_AI_CATALOG_ID, "field_name": "enabled", "field_value": False}],`
`21`	`21`	`indirect=True,`
`22`	`22`	`)`
`23`		`- @pytest.mark.smoke`
	`23`	`+ @pytest.mark.tier1`
`24`	`24`	`def test_sources_endpoint_returns_all_sources_regardless_of_enabled_field(`
`25`	`25`	`self,`
`26`	`26`	`sparse_override_catalog_source: dict,`