Add fix and extra step for metrices test

Signed-off-by: Milind Waykole <mwaykole@redhat.com>

Author: mwaykole

tests/model_serving/model_server/metrics/test_model_metrics.py

@@ -1,44 +1,43 @@
 import pytest
 
+from tests.model_serving.model_server.metrics.utils import validate_metrics_configuration
 from tests.model_serving.model_server.utils import (
     run_inference_multiple_times,
     verify_inference_response,
 )
 from utilities.constants import (
     KServeDeploymentType,
-    ModelFormat,
+    ModelAndFormat,
     ModelInferenceRuntime,
     ModelStoragePath,
+    ModelVersion,
     Protocols,
-    RuntimeTemplates,
 )
+from timeout_sampler import TimeoutSampler
 from utilities.inference_utils import Inference
-from utilities.manifests.caikit_tgis import CAIKIT_TGIS_INFERENCE_CONFIG
+from utilities.manifests.openvino import OPENVINO_KSERVE_INFERENCE_CONFIG
 from utilities.monitoring import get_metrics_value, validate_metrics_field
 
 pytestmark = [
-    pytest.mark.serverless,
     pytest.mark.usefixtures("valid_aws_config", "user_workload_monitoring_config_map"),
     pytest.mark.metrics,
 ]
 
 
-@pytest.mark.serverless
 @pytest.mark.parametrize(
-    "unprivileged_model_namespace, serving_runtime_from_template, s3_models_inference_service",
+    "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service",
     [
         pytest.param(
-            {"name": "test-kserve-tgis-metrics"},
+            {"name": "test-ovms-metrics"},
             {
-                "name": f"{Protocols.HTTP}-{ModelInferenceRuntime.CAIKIT_TGIS_RUNTIME}",
-                "template-name": RuntimeTemplates.CAIKIT_TGIS_SERVING,
-                "multi-model": False,
-                "enable-http": True,
+                "runtime-name": ModelInferenceRuntime.OPENVINO_KSERVE_RUNTIME,
+                "model-format": {ModelAndFormat.OPENVINO_IR: ModelVersion.OPSET1},
             },
             {
-                "name": f"{Protocols.HTTP}-{ModelFormat.CAIKIT}",
-                "deployment-mode": KServeDeploymentType.SERVERLESS,
-                "model-dir": ModelStoragePath.FLAN_T5_SMALL_CAIKIT,
+                "name": "ovms-metrics",
+                "model-dir": ModelStoragePath.KSERVE_OPENVINO_EXAMPLE_MODEL,
+                "model-version": ModelVersion.OPSET1,
+                "deployment-mode": KServeDeploymentType.RAW_DEPLOYMENT,
             },
         )
     ],
@@ -47,48 +46,72 @@
 class TestModelMetrics:
     @pytest.mark.smoke
     @pytest.mark.polarion("ODS-2555")
-    def test_model_metrics_num_success_requests(self, s3_models_inference_service, prometheus):
+    def test_model_metrics_num_success_requests(self, ovms_kserve_inference_service, prometheus):
         """Verify number of successful model requests in OpenShift monitoring system (UserWorkloadMonitoring) metrics"""
+        validate_metrics_configuration(inference_service=ovms_kserve_inference_service)
+
         verify_inference_response(
-            inference_service=s3_models_inference_service,
-            inference_config=CAIKIT_TGIS_INFERENCE_CONFIG,
-            inference_type=Inference.ALL_TOKENS,
+            inference_service=ovms_kserve_inference_service,
+            inference_config=OPENVINO_KSERVE_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
             protocol=Protocols.HTTPS,
-            model_name=ModelFormat.CAIKIT,
             use_default_query=True,
         )
+
+        metrics_query = (
+            f'ovms_requests_success{{namespace="{ovms_kserve_inference_service.namespace}", '
+            f'name="{ovms_kserve_inference_service.name}"}}'
+        )
+
         validate_metrics_field(
             prometheus=prometheus,
-            metrics_query="tgi_request_success",
+            metrics_query=metrics_query,
             expected_value="1",
         )
 
     @pytest.mark.smoke
     @pytest.mark.polarion("ODS-2555")
-    def test_model_metrics_num_total_requests(self, s3_models_inference_service, prometheus):
+    def test_model_metrics_num_total_requests(self, ovms_kserve_inference_service, prometheus):
         """Verify number of total model requests in OpenShift monitoring system (UserWorkloadMonitoring) metrics"""
+        validate_metrics_configuration(inference_service=ovms_kserve_inference_service)
+
         total_runs = 5
 
         run_inference_multiple_times(
-            isvc=s3_models_inference_service,
-            inference_config=CAIKIT_TGIS_INFERENCE_CONFIG,
-            inference_type=Inference.ALL_TOKENS,
+            isvc=ovms_kserve_inference_service,
+            inference_config=OPENVINO_KSERVE_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
             protocol=Protocols.HTTPS,
-            model_name=ModelFormat.CAIKIT,
             iterations=total_runs,
             run_in_parallel=True,
         )
+
+        metrics_query = (
+            f'ovms_requests_success{{namespace="{ovms_kserve_inference_service.namespace}", '
+            f'name="{ovms_kserve_inference_service.name}"}}'
+        )
+
         validate_metrics_field(
             prometheus=prometheus,
-            metrics_query="tgi_request_count",
-            expected_value=str(total_runs + 1),
+            metrics_query=metrics_query,
+            expected_value=str(total_runs),
+            greater_than=True,
         )
 
     @pytest.mark.smoke
     @pytest.mark.polarion("ODS-2555")
-    def test_model_metrics_cpu_utilization(self, s3_models_inference_service, prometheus):
+    def test_model_metrics_cpu_utilization(self, ovms_kserve_inference_service, prometheus):
         """Verify CPU utilization data in OpenShift monitoring system (UserWorkloadMonitoring) metrics"""
-        assert get_metrics_value(
+        validate_metrics_configuration(inference_service=ovms_kserve_inference_service)
+
+        metrics_query = f"pod:container_cpu_usage:sum{{namespace='{ovms_kserve_inference_service.namespace}'}}"
+
+        for cpu_value in TimeoutSampler(
+            wait_timeout=120,
+            sleep=10,
+            func=get_metrics_value,
             prometheus=prometheus,
-            metrics_query=f"pod:container_cpu_usage:sum{{namespace='{s3_models_inference_service.namespace}'}}",
-        )
+            metrics_query=metrics_query,
+        ):
+            if cpu_value is not None:
+                break

tests/model_serving/model_server/metrics/test_non_admin_users.py

@@ -1,47 +1,37 @@
 import pytest
 
+from tests.model_serving.model_server.metrics.utils import validate_metrics_configuration
 from tests.model_serving.model_server.utils import (
     run_inference_multiple_times,
-    verify_inference_response,
 )
-from utilities.constants import ModelFormat, ModelStoragePath, Protocols
+from utilities.constants import (
+    KServeDeploymentType,
+    ModelAndFormat,
+    ModelInferenceRuntime,
+    ModelStoragePath,
+    ModelVersion,
+    Protocols,
+)
 from utilities.inference_utils import Inference
-from utilities.manifests.caikit_tgis import CAIKIT_TGIS_INFERENCE_CONFIG
+from utilities.manifests.openvino import OPENVINO_KSERVE_INFERENCE_CONFIG
 from utilities.monitoring import validate_metrics_field
 
 
 @pytest.mark.parametrize(
-    "unprivileged_model_namespace, unprivileged_s3_caikit_serverless_inference_service",
-    [
-        pytest.param(
-            {"name": "test-non-admin-serverless"},
-            {"model-dir": ModelStoragePath.FLAN_T5_SMALL_CAIKIT},
-        )
-    ],
-    indirect=True,
-)
-@pytest.mark.smoke
-@pytest.mark.serverless
-class TestServerlessUnprivilegedUser:
-    @pytest.mark.polarion("ODS-2552")
-    def test_non_admin_deploy_serverless_and_query_metrics(self, unprivileged_s3_caikit_serverless_inference_service):
-        """Verify non admin can deploy a model and query using REST"""
-        verify_inference_response(
-            inference_service=unprivileged_s3_caikit_serverless_inference_service,
-            inference_config=CAIKIT_TGIS_INFERENCE_CONFIG,
-            inference_type=Inference.ALL_TOKENS,
-            protocol=Protocols.HTTPS,
-            model_name=ModelFormat.CAIKIT,
-            use_default_query=True,
-        )
-
-
-@pytest.mark.parametrize(
-    "unprivileged_model_namespace, unprivileged_s3_caikit_raw_inference_service",
+    "unprivileged_model_namespace, ovms_kserve_serving_runtime, ovms_kserve_inference_service",
     [
         pytest.param(
             {"name": "test-non-admin-metrics"},
-            {"model-dir": ModelStoragePath.FLAN_T5_SMALL_HF},
+            {
+                "runtime-name": ModelInferenceRuntime.OPENVINO_KSERVE_RUNTIME,
+                "model-format": {ModelAndFormat.OPENVINO_IR: ModelVersion.OPSET1},
+            },
+            {
+                "name": "ovms-non-admin",
+                "model-dir": ModelStoragePath.KSERVE_OPENVINO_EXAMPLE_MODEL,
+                "model-version": ModelVersion.OPSET1,
+                "deployment-mode": KServeDeploymentType.RAW_DEPLOYMENT,
+            },
         )
     ],
     indirect=True,
@@ -52,23 +42,31 @@ class TestRawUnprivilegedUserMetrics:
     @pytest.mark.metrics
     def test_non_admin_raw_metrics(
         self,
-        unprivileged_s3_caikit_raw_inference_service,
+        ovms_kserve_inference_service,
         prometheus,
         user_workload_monitoring_config_map,
     ):
         """Verify number of total model requests in OpenShift monitoring system (UserWorkloadMonitoring) metrics"""
+        validate_metrics_configuration(inference_service=ovms_kserve_inference_service)
+
         total_runs = 5
 
         run_inference_multiple_times(
-            isvc=unprivileged_s3_caikit_raw_inference_service,
-            inference_config=CAIKIT_TGIS_INFERENCE_CONFIG,
-            inference_type=Inference.ALL_TOKENS,
-            protocol=Protocols.HTTP,
-            model_name=ModelFormat.CAIKIT,
+            isvc=ovms_kserve_inference_service,
+            inference_config=OPENVINO_KSERVE_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
+            protocol=Protocols.HTTPS,
             iterations=total_runs,
         )
+
+        metrics_query = (
+            f'ovms_requests_success{{namespace="{ovms_kserve_inference_service.namespace}", '
+            f'name="{ovms_kserve_inference_service.name}"}}'
+        )
+
         validate_metrics_field(
             prometheus=prometheus,
-            metrics_query="tgi_request_count",
+            metrics_query=metrics_query,
             expected_value=str(total_runs),
+            greater_than=True,
         )

tests/model_serving/model_server/metrics/utils.py

@@ -0,0 +1,35 @@
+from ocp_resources.config_map import ConfigMap
+from ocp_resources.inference_service import InferenceService
+
+
+def validate_metrics_configuration(inference_service: InferenceService) -> None:
+    """
+    Validate that the InferenceService has proper metrics configuration.
+
+    Checks:
+    - Metrics dashboard ConfigMap has supported=true
+
+    Args:
+        inference_service: InferenceService object
+
+    Raises:
+        AssertionError: If validation fails
+    """
+    metrics_cm_name = f"{inference_service.name}-metrics-dashboard"
+    metrics_cm = ConfigMap(
+        client=inference_service.client,
+        name=metrics_cm_name,
+        namespace=inference_service.namespace,
+    )
+
+    assert metrics_cm.exists, (
+        f"Metrics dashboard ConfigMap '{metrics_cm_name}' not found in namespace "
+        f"'{inference_service.namespace}'"
+    )
+
+    supported_value = metrics_cm.instance.data.get("supported")
+
+    assert supported_value == "true", (
+        f"Metrics dashboard ConfigMap '{metrics_cm_name}' has 'supported: {supported_value}'. "
+        f"Expected 'supported: true' for metrics to be available. "
+    )