Add QG Markers for AI Safety and marker deprecation warning (#1193)

* feat: add smoke and t1 markers

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Debarati Basu-Nag <dbasunag@redhat.com>

Author: 3 people

pytest.ini

@@ -11,7 +11,7 @@ markers =
 
     # CI
     smoke: Mark tests as smoke tests; very high critical priority tests. Covers core functionality of the product. Aims to ensure that the build is stable enough for further testing.
-    sanity: Mark tests as sanity tests. Aims to verify that specific functionality is working as expected.
+    sanity: <<DEPRECATION WARNING: to be superseded by tier1>> Mark tests as sanity tests. Aims to verify that specific functionality is working as expected.
     tier1: Mark tests as tier1. High-priority tests.
     tier2: Mark tests as tier2. Medium/low-priority positive tests.
     tier3: Mark tests as tier3. Negative and destructive tests.

tests/llama_stack/safety/test_trustyai_fms_provider.py

@@ -9,6 +9,7 @@
 SECURE_SHIELD_ID: str = "secure_shield"
 
 
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace, minio_pod, minio_data_connection, "
     "orchestrator_config, guardrails_orchestrator, llama_stack_server_config",

tests/model_explainability/guardrails/test_guardrails.py

@@ -38,6 +38,7 @@
 LOGGER = get_logger(name=__name__)
 
 
+@pytest.mark.smoke
 @pytest.mark.parametrize(
     "model_namespace, orchestrator_config, guardrails_orchestrator",
     [
@@ -56,7 +57,6 @@
     ],
     indirect=True,
 )
-@pytest.mark.smoke
 def test_validate_guardrails_orchestrator_images(
     model_namespace,
     orchestrator_config,
@@ -70,6 +70,7 @@ def test_validate_guardrails_orchestrator_images(
     validate_tai_component_images(pod=guardrails_orchestrator_pod, tai_operator_configmap=trustyai_operator_configmap)
 
 
+@pytest.mark.smoke
 @pytest.mark.parametrize(
     "model_namespace, orchestrator_config, guardrails_gateway_config, guardrails_orchestrator",
     [
@@ -115,7 +116,6 @@ def test_validate_guardrails_orchestrator_images(
     ],
     indirect=True,
 )
-@pytest.mark.smoke
 @pytest.mark.rawdeployment
 @pytest.mark.usefixtures("patched_dsc_kserve_headed", "guardrails_gateway_config")
 class TestGuardrailsOrchestratorWithBuiltInDetectors:
@@ -212,6 +212,7 @@ def test_guardrails_builtin_detectors_negative_detection(
         )
 
 
+@pytest.mark.smoke
 @pytest.mark.parametrize(
     "model_namespace, orchestrator_config, guardrails_gateway_config,guardrails_orchestrator",
     [
@@ -421,6 +422,7 @@ def check_traces():
     indirect=True,
 )
 @pytest.mark.usefixtures("patched_dsc_kserve_headed")
+@pytest.mark.tier1
 @pytest.mark.rawdeployment
 class TestGuardrailsOrchestratorAutoConfig:
     """
@@ -495,6 +497,7 @@ def test_guardrails_autoconfig_negative_detection(
     indirect=True,
 )
 @pytest.mark.usefixtures("patched_dsc_kserve_headed")
+@pytest.mark.tier2
 @pytest.mark.rawdeployment
 class TestGuardrailsOrchestratorAutoConfigWithGateway:
     """

tests/model_explainability/lm_eval/test_lm_eval.py

@@ -28,17 +28,14 @@
 
 
 @pytest.mark.skip_on_disconnected
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace, lmevaljob_hf",
     [
         pytest.param(
             {"name": "test-lmeval-hf-tier1"},
             {"task_list": {"taskNames": TIER1_LMEVAL_TASKS}},
         ),
-        pytest.param(
-            {"name": "test-lmeval-hf-tier2"},
-            {"task_list": {"taskNames": TIER2_LMEVAL_TASKS}},
-        ),
         pytest.param(
             {"name": "test-lmeval-hf-custom-task"},
             CUSTOM_UNITXT_TASK_DATA,
@@ -58,6 +55,24 @@ def test_lmeval_huggingface_model(admin_client, model_namespace, lmevaljob_hf_po
     validate_lmeval_job_pod_and_logs(lmevaljob_pod=lmevaljob_hf_pod)
 
 
+@pytest.mark.skip_on_disconnected
+@pytest.mark.tier2
+@pytest.mark.parametrize(
+    "model_namespace, lmevaljob_hf",
+    [
+        pytest.param(
+            {"name": "test-lmeval-hf-tier2"},
+            {"task_list": {"taskNames": TIER2_LMEVAL_TASKS}},
+        ),
+    ],
+    indirect=True,
+)
+def test_lmeval_huggingface_model_tier2(admin_client, model_namespace, lmevaljob_hf_pod):
+    """Tests that verify running common evaluations (and a custom one) on a model pulled directly from HuggingFace.
+    On each test we run a different evaluation task, limiting it to 0.5% of the questions on each eval."""
+    validate_lmeval_job_pod_and_logs(lmevaljob_pod=lmevaljob_hf_pod)
+
+
 @pytest.mark.parametrize(
     "model_namespace, lmeval_data_downloader_pod, lmevaljob_local_offline",
     [
@@ -80,6 +95,7 @@ def test_lmeval_local_offline_builtin_tasks_flan_arceasy(
     validate_lmeval_job_pod_and_logs(lmevaljob_pod=lmevaljob_local_offline_pod)
 
 
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace",
     [
@@ -95,6 +111,7 @@ def test_lmeval_vllm_emulator(admin_client, model_namespace, lmevaljob_vllm_emul
     validate_lmeval_job_pod_and_logs(lmevaljob_pod=lmevaljob_vllm_emulator_pod)
 
 
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace, minio_data_connection",
     [
@@ -138,6 +155,7 @@ def test_verify_lmeval_pod_images(lmevaljob_s3_offline_pod, trustyai_operator_co
     )
 
 
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace, oci_registry_pod_with_minio, lmeval_data_downloader_pod, lmevaljob_local_offline_oci",
     [

tests/model_explainability/trustyai_service/service/test_trustyai_service.py

@@ -22,6 +22,7 @@
 from utilities.constants import MinIo
 
 
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace",
     [
@@ -45,6 +46,7 @@ def test_trustyai_service_with_invalid_db_cert(
     )
 
 
+@pytest.mark.smoke
 @pytest.mark.parametrize(
     "model_namespace, trustyai_service",
     [
@@ -55,7 +57,6 @@ def test_trustyai_service_with_invalid_db_cert(
     ],
     indirect=True,
 )
-@pytest.mark.smoke
 def test_validate_trustyai_service_image(
     admin_client,
     model_namespace: Namespace,
@@ -72,6 +73,7 @@ def test_validate_trustyai_service_image(
     )
 
 
+@pytest.mark.tier1
 @pytest.mark.parametrize(
     "model_namespace, minio_pod, minio_data_connection, trustyai_service",
     [