fix: server metrics delete pvc

Author: rnetser

pytest.ini

@@ -22,6 +22,7 @@ markers =
     rawdeployment: Mark tests which are raw deployment tests
     minio: Mark tests which are using MinIO storage
     tls: Mark tests which are testing TLS
+    metrics: Mark tests which are testing metrics
 
 addopts =
     -s

tests/model_serving/model_server/authentication/test_non_admin_users.py

@@ -64,7 +64,8 @@ def test_non_admin_deploy_raw_and_query_model(
             use_default_query=True,
         )
 
-    def test_raw_metrics(
+    @pytest.mark.metrics
+    def test_non_admin_raw_metrics(
         self,
         unprivileged_s3_caikit_raw_inference_service,
         prometheus,
@@ -84,5 +85,5 @@ def test_raw_metrics(
         validate_metrics_value(
             prometheus=prometheus,
             metrics_query="tgi_request_count",
-            expected_value=str(total_runs + 1),
+            expected_value=str(total_runs),
         )

tests/model_serving/model_server/conftest.py

@@ -444,6 +444,8 @@ def prometheus(admin_client: DynamicClient) -> Prometheus:
 def user_workload_monitoring_config_map(
     admin_client: DynamicClient, cluster_monitoring_config: ConfigMap
 ) -> Generator[ConfigMap, None, None]:
+    uwm_namespace = "openshift-user-workload-monitoring"
+
     data = {
         "config.yaml": yaml.dump({
             "prometheus": {
@@ -457,11 +459,15 @@ def user_workload_monitoring_config_map(
     with update_configmap_data(
         client=admin_client,
         name="user-workload-monitoring-config",
-        namespace="openshift-user-workload-monitoring",
+        namespace=uwm_namespace,
         data=data,
     ) as cm:
         yield cm
 
+    # UWM PVCs are not deleted once the configmap is deleted; forcefully deleting the PVCs to avoid having left-overs
+    for pvc in PersistentVolumeClaim.get(dyn_client=admin_client, namespace=uwm_namespace):
+        pvc.clean_up()
+
 
 @pytest.fixture(scope="class")
 def http_s3_ovms_external_route_model_mesh_serving_runtime(

tests/model_serving/model_server/metrics/conftest.py

@@ -19,6 +19,7 @@
 pytestmark = [
     pytest.mark.serverless,
     pytest.mark.usefixtures("valid_aws_config", "user_workload_monitoring_config_map"),
+    pytest.mark.metrics,
 ]
 
 
@@ -46,8 +47,7 @@
 class TestModelMetrics:
     @pytest.mark.smoke
     @pytest.mark.polarion("ODS-2555")
-    @pytest.mark.dependency(name="test_model_metrics_num_success_requests")
-    def test_model_metrics_num_success_requests(self, s3_models_inference_service, deleted_metrics, prometheus):
+    def test_model_metrics_num_success_requests(self, s3_models_inference_service, prometheus):
         """Verify number of successful model requests in OpenShift monitoring system (UserWorkloadMonitoring) metrics"""
         verify_inference_response(
             inference_service=s3_models_inference_service,
@@ -65,10 +65,6 @@ def test_model_metrics_num_success_requests(self, s3_models_inference_service, d
 
     @pytest.mark.smoke
     @pytest.mark.polarion("ODS-2555")
-    @pytest.mark.dependency(
-        name="test_model_metrics_num_total_requests",
-        depends=["test_model_metrics_num_success_requests"],
-    )
     def test_model_metrics_num_total_requests(self, s3_models_inference_service, prometheus):
         """Verify number of total model requests in OpenShift monitoring system (UserWorkloadMonitoring) metrics"""
         total_runs = 5
@@ -90,7 +86,6 @@ def test_model_metrics_num_total_requests(self, s3_models_inference_service, pro
 
     @pytest.mark.smoke
     @pytest.mark.polarion("ODS-2555")
-    @pytest.mark.dependency(depends=["test_model_metrics_num_total_requests"])
     def test_model_metrics_cpu_utilization(self, s3_models_inference_service, prometheus):
         """Verify CPU utilization data in OpenShift monitoring system (UserWorkloadMonitoring) metrics"""
         assert get_metrics_value(