feat: Added drift tests using DB storage (opendatahub-io#400)

* faet: drift test case
	modified:   tests/model_explainability/trustyai_service/drift/test_drift.py
	modified:   tests/model_explainability/trustyai_service/fairness/test_fairness.py

	modified:   tests/model_explainability/trustyai_service/drift/test_drift.py
	modified:   tests/model_explainability/trustyai_service/fairness/test_fairness.py

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: kpunwatk

tests/model_explainability/trustyai_service/drift/test_drift.py

@@ -151,3 +151,134 @@ def test_drift_metric_delete(
             token=current_client_token,
             metric_name=metric_name,
         )
+
+
+@pytest.mark.parametrize(
+    "model_namespace, minio_pod, minio_data_connection",
+    [
+        pytest.param(
+            {"name": "test-drift"},
+            MinIo.PodConfig.MODEL_MESH_MINIO_CONFIG,
+            {"bucket": MinIo.Buckets.MODELMESH_EXAMPLE_MODELS},
+        )
+    ],
+    indirect=True,
+)
+@pytest.mark.usefixtures("minio_pod")
+class TestDriftMetricsWithDBStorage:
+    """
+    Verifies all the basic operations with a drift metric (meanshift, kstest, approxkstest and fouriermmd)
+     available in TrustyAI, using MariaDB storage.
+
+    1. Send data to the model and verify that TrustyAI registers the observations.
+    2. Apply name mappings
+    3. Send metric request (meanshift, kstest, approxkstest and fouriermmd) and verify the response.
+    4. Send metric scheduling request and verify the response.
+    5. Send metric deletion request and verify that the scheduled metric has been deleted.
+    """
+
+    def test_drift_send_inference_and_verify_trustyai_service_with_db_storage(
+        self,
+        admin_client,
+        current_client_token,
+        model_namespace,
+        trustyai_service_with_db_storage,
+        gaussian_credit_model,
+        isvc_getter_token,
+    ) -> None:
+        send_inferences_and_verify_trustyai_service_registered(
+            client=admin_client,
+            token=current_client_token,
+            data_path=f"{DRIFT_BASE_DATA_PATH}/data_batches",
+            trustyai_service=trustyai_service_with_db_storage,
+            inference_service=gaussian_credit_model,
+            inference_config=OPENVINO_KSERVE_INFERENCE_CONFIG,
+            inference_token=isvc_getter_token,
+        )
+
+    def test_upload_data_to_trustyai_service_with_db_storage(
+        self,
+        admin_client,
+        minio_data_connection,
+        current_client_token,
+        trustyai_service_with_db_storage,
+    ) -> None:
+        verify_upload_data_to_trustyai_service(
+            client=admin_client,
+            trustyai_service=trustyai_service_with_db_storage,
+            token=current_client_token,
+            data_path=f"{DRIFT_BASE_DATA_PATH}/training_data.json",
+        )
+
+    @pytest.mark.parametrize("metric_name", DRIFT_METRICS)
+    def test_drift_metric_request_with_db_storage(
+        self,
+        admin_client,
+        current_client_token,
+        trustyai_service_with_db_storage,
+        gaussian_credit_model,
+        metric_name,
+    ):
+        verify_trustyai_service_metric_request(
+            client=admin_client,
+            trustyai_service=trustyai_service_with_db_storage,
+            token=current_client_token,
+            metric_name=metric_name,
+            json_data={
+                "modelId": gaussian_credit_model.name,
+                "referenceTag": "TRAINING",
+            },
+        )
+
+    @pytest.mark.parametrize("metric_name", DRIFT_METRICS)
+    def test_drift_metric_schedule_with_db_storage(
+        self,
+        admin_client,
+        current_client_token,
+        trustyai_service_with_db_storage,
+        gaussian_credit_model,
+        metric_name,
+    ):
+        verify_trustyai_service_metric_scheduling_request(
+            client=admin_client,
+            trustyai_service=trustyai_service_with_db_storage,
+            token=current_client_token,
+            metric_name=metric_name,
+            json_data={
+                "modelId": gaussian_credit_model.name,
+                "referenceTag": "TRAINING",
+            },
+        )
+
+    @pytest.mark.parametrize("metric_name", DRIFT_METRICS)
+    def test_drift_metric_prometheus_with_db_storage(
+        self,
+        admin_client,
+        model_namespace,
+        trustyai_service_with_db_storage,
+        gaussian_credit_model,
+        prometheus,
+        metric_name,
+    ):
+        validate_metrics_field(
+            prometheus=prometheus,
+            metrics_query=f'trustyai_{metric_name}{{namespace="{model_namespace.name}"}}',
+            expected_value=metric_name.upper(),
+            field_getter=partial(get_metric_label, label_name="metricName"),
+        )
+
+    @pytest.mark.parametrize("metric_name", DRIFT_METRICS)
+    def test_drift_metric_delete_with_db_storage(
+        self,
+        admin_client,
+        minio_data_connection,
+        current_client_token,
+        trustyai_service_with_db_storage,
+        metric_name,
+    ):
+        verify_trustyai_service_metric_delete_request(
+            client=admin_client,
+            trustyai_service=trustyai_service_with_db_storage,
+            token=current_client_token,
+            metric_name=metric_name,
+        )

tests/model_explainability/trustyai_service/fairness/test_fairness.py

@@ -64,11 +64,11 @@ def get_fairness_request_json_data(isvc: InferenceService) -> dict[str, Any]:
 @pytest.mark.usefixtures("minio_pod")
 class TestFairnessMetricsWithPVCStorage:
     """
-    Verifies all the basic operations with a fairness metric (spd) available in TrustyAI, using PVC storage.
+    Verifies all the basic operations with a fairness metric (spd and dir) available in TrustyAI, using PVC storage.
 
     1. Send data to the model and verify that TrustyAI registers the observations.
     2. Apply name mappings
-    3. Send metric request (spd) and verify the response.
+    3. Send metric request (spd and dir) and verify the response.
     4. Send metric scheduling request and verify the response.
     5. Send metric deletion request and verify that the scheduled metric has been deleted.
     """
@@ -165,11 +165,11 @@ def test_fairness_metric_delete_with_pvc_storage(
 @pytest.mark.usefixtures("minio_pod")
 class TestFairnessMetricsWithDBStorage:
     """
-    Verifies all the basic operations with a fairness metric (spd) available in TrustyAI, using MariaDB storage.
+    Verifies all the basic operations with a fairness metric (spd and dir) available in TrustyAI, using MariaDB storage.
 
     1. Send data to the model and verify that TrustyAI registers the observations.
     2. Apply name mappings
-    3. Send metric request (spd) and verify the response.
+    3. Send metric request (spd and dir) and verify the response.
     4. Send metric scheduling request and verify the response.
     5. Send metric deletion request and verify that the scheduled metric has been deleted.
     """