Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 131 additions & 0 deletions tests/model_explainability/trustyai_service/drift/test_drift.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,3 +151,134 @@ def test_drift_metric_delete(
token=current_client_token,
metric_name=metric_name,
)


@pytest.mark.parametrize(
"model_namespace, minio_pod, minio_data_connection",
[
pytest.param(
{"name": "test-drift"},
MinIo.PodConfig.MODEL_MESH_MINIO_CONFIG,
{"bucket": MinIo.Buckets.MODELMESH_EXAMPLE_MODELS},
)
],
indirect=True,
)
@pytest.mark.usefixtures("minio_pod")
class TestDriftMetricsWithDBStorage:
"""
Verifies all the basic operations with a drift metric (meanshift, kstest, approxkstest and fouriermmd)
available in TrustyAI, using MariaDB storage.

1. Send data to the model and verify that TrustyAI registers the observations.
2. Apply name mappings
3. Send metric request (meanshift, kstest, approxkstest and fouriermmd) and verify the response.
4. Send metric scheduling request and verify the response.
5. Send metric deletion request and verify that the scheduled metric has been deleted.
"""

def test_drift_send_inference_and_verify_trustyai_service_with_db_storage(
self,
admin_client,
current_client_token,
model_namespace,
trustyai_service_with_db_storage,
gaussian_credit_model,
isvc_getter_token,
) -> None:
send_inferences_and_verify_trustyai_service_registered(
client=admin_client,
token=current_client_token,
data_path=f"{DRIFT_BASE_DATA_PATH}/data_batches",
trustyai_service=trustyai_service_with_db_storage,
inference_service=gaussian_credit_model,
inference_config=OPENVINO_KSERVE_INFERENCE_CONFIG,
inference_token=isvc_getter_token,
)

def test_upload_data_to_trustyai_service_with_db_storage(
self,
admin_client,
minio_data_connection,
current_client_token,
trustyai_service_with_db_storage,
) -> None:
verify_upload_data_to_trustyai_service(
client=admin_client,
trustyai_service=trustyai_service_with_db_storage,
token=current_client_token,
data_path=f"{DRIFT_BASE_DATA_PATH}/training_data.json",
)

@pytest.mark.parametrize("metric_name", DRIFT_METRICS)
def test_drift_metric_request_with_db_storage(
self,
admin_client,
current_client_token,
trustyai_service_with_db_storage,
gaussian_credit_model,
metric_name,
):
verify_trustyai_service_metric_request(
client=admin_client,
trustyai_service=trustyai_service_with_db_storage,
token=current_client_token,
metric_name=metric_name,
json_data={
"modelId": gaussian_credit_model.name,
"referenceTag": "TRAINING",
},
)

@pytest.mark.parametrize("metric_name", DRIFT_METRICS)
def test_drift_metric_schedule_with_db_storage(
self,
admin_client,
current_client_token,
trustyai_service_with_db_storage,
gaussian_credit_model,
metric_name,
):
verify_trustyai_service_metric_scheduling_request(
client=admin_client,
trustyai_service=trustyai_service_with_db_storage,
token=current_client_token,
metric_name=metric_name,
json_data={
"modelId": gaussian_credit_model.name,
"referenceTag": "TRAINING",
},
)

@pytest.mark.parametrize("metric_name", DRIFT_METRICS)
def test_drift_metric_prometheus_with_db_storage(
self,
admin_client,
model_namespace,
trustyai_service_with_db_storage,
gaussian_credit_model,
prometheus,
metric_name,
):
validate_metrics_field(
prometheus=prometheus,
metrics_query=f'trustyai_{metric_name}{{namespace="{model_namespace.name}"}}',
expected_value=metric_name.upper(),
field_getter=partial(get_metric_label, label_name="metricName"),
)

@pytest.mark.parametrize("metric_name", DRIFT_METRICS)
def test_drift_metric_delete_with_db_storage(
self,
admin_client,
minio_data_connection,
current_client_token,
trustyai_service_with_db_storage,
metric_name,
):
verify_trustyai_service_metric_delete_request(
client=admin_client,
trustyai_service=trustyai_service_with_db_storage,
token=current_client_token,
metric_name=metric_name,
)
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,11 @@ def get_fairness_request_json_data(isvc: InferenceService) -> dict[str, Any]:
@pytest.mark.usefixtures("minio_pod")
class TestFairnessMetricsWithPVCStorage:
"""
Verifies all the basic operations with a fairness metric (spd) available in TrustyAI, using PVC storage.
Verifies all the basic operations with a fairness metric (spd and dir) available in TrustyAI, using PVC storage.

1. Send data to the model and verify that TrustyAI registers the observations.
2. Apply name mappings
3. Send metric request (spd) and verify the response.
3. Send metric request (spd and dir) and verify the response.
4. Send metric scheduling request and verify the response.
5. Send metric deletion request and verify that the scheduled metric has been deleted.
"""
Expand Down Expand Up @@ -165,11 +165,11 @@ def test_fairness_metric_delete_with_pvc_storage(
@pytest.mark.usefixtures("minio_pod")
class TestFairnessMetricsWithDBStorage:
"""
Verifies all the basic operations with a fairness metric (spd) available in TrustyAI, using MariaDB storage.
Verifies all the basic operations with a fairness metric (spd and dir) available in TrustyAI, using MariaDB storage.

1. Send data to the model and verify that TrustyAI registers the observations.
2. Apply name mappings
3. Send metric request (spd) and verify the response.
3. Send metric request (spd and dir) and verify the response.
4. Send metric scheduling request and verify the response.
5. Send metric deletion request and verify that the scheduled metric has been deleted.
"""
Expand Down
Loading