ci: merge main branch

Author: rnetser

pytest.ini

@@ -22,6 +22,7 @@ markers =
     rawdeployment: Mark tests which are raw deployment tests
     minio: Mark tests which are using MinIO storage
     tls: Mark tests which are testing TLS
+    metrics: Mark tests which are testing metrics
 
 addopts =
     -s

tests/model_serving/model_server/authentication/conftest.py

@@ -17,13 +17,11 @@
 from utilities.infra import (
     create_isvc_view_role,
     get_pods_by_isvc_label,
-    s3_endpoint_secret,
     create_inference_token,
 )
 from utilities.constants import (
     KServeDeploymentType,
     ModelFormat,
-    ModelStoragePath,
     Protocols,
     ModelInferenceRuntime,
     RuntimeTemplates,
@@ -327,72 +325,6 @@ def http_s3_caikit_raw_inference_service_2(
         yield isvc
 
 
-# Unprivileged user tests
-
-
-@pytest.fixture(scope="class")
-def unprivileged_s3_caikit_serving_runtime(
-    admin_client: DynamicClient,
-    unprivileged_client: DynamicClient,
-    unprivileged_model_namespace: Namespace,
-) -> Generator[ServingRuntime, Any, Any]:
-    with ServingRuntimeFromTemplate(
-        client=admin_client,
-        unprivileged_client=unprivileged_client,
-        name=f"{Protocols.HTTP}-{ModelInferenceRuntime.CAIKIT_TGIS_RUNTIME}",
-        namespace=unprivileged_model_namespace.name,
-        template_name=RuntimeTemplates.CAIKIT_TGIS_SERVING,
-        multi_model=False,
-        enable_http=True,
-        enable_grpc=False,
-    ) as model_runtime:
-        yield model_runtime
-
-
-@pytest.fixture(scope="class")
-def unprivileged_models_endpoint_s3_secret(
-    unprivileged_client: DynamicClient,
-    unprivileged_model_namespace: Namespace,
-    aws_access_key_id: str,
-    aws_secret_access_key: str,
-    models_s3_bucket_name: str,
-    models_s3_bucket_region: str,
-    models_s3_bucket_endpoint: str,
-) -> Generator[Secret, Any, Any]:
-    with s3_endpoint_secret(
-        admin_client=unprivileged_client,
-        name="models-bucket-secret",
-        namespace=unprivileged_model_namespace.name,
-        aws_access_key=aws_access_key_id,
-        aws_secret_access_key=aws_secret_access_key,
-        aws_s3_region=models_s3_bucket_region,
-        aws_s3_bucket=models_s3_bucket_name,
-        aws_s3_endpoint=models_s3_bucket_endpoint,
-    ) as secret:
-        yield secret
-
-
-@pytest.fixture(scope="class")
-def unprivileged_s3_caikit_serverless_inference_service(
-    request: FixtureRequest,
-    unprivileged_client: DynamicClient,
-    unprivileged_model_namespace: Namespace,
-    unprivileged_s3_caikit_serving_runtime: ServingRuntime,
-    unprivileged_models_endpoint_s3_secret: Secret,
-) -> Generator[InferenceService, Any, Any]:
-    with create_isvc(
-        client=unprivileged_client,
-        name=f"{Protocols.HTTP}-{ModelFormat.CAIKIT}",
-        namespace=unprivileged_model_namespace.name,
-        runtime=unprivileged_s3_caikit_serving_runtime.name,
-        model_format=unprivileged_s3_caikit_serving_runtime.instance.spec.supportedModelFormats[0].name,
-        deployment_mode=KServeDeploymentType.SERVERLESS,
-        storage_key=unprivileged_models_endpoint_s3_secret.name,
-        storage_path=request.param["model-dir"],
-    ) as isvc:
-        yield isvc
-
-
 @pytest.fixture(scope="class")
 def http_s3_caikit_tgis_serving_runtime(
     request: FixtureRequest,
@@ -411,27 +343,6 @@ def http_s3_caikit_tgis_serving_runtime(
         yield model_runtime
 
 
-@pytest.fixture(scope="class")
-def unprivileged_s3_caikit_raw_inference_service(
-    request: FixtureRequest,
-    unprivileged_client: DynamicClient,
-    unprivileged_model_namespace: Namespace,
-    unprivileged_s3_caikit_serving_runtime: ServingRuntime,
-    unprivileged_models_endpoint_s3_secret: Secret,
-) -> Generator[InferenceService, Any, Any]:
-    with create_isvc(
-        client=unprivileged_client,
-        name=f"{Protocols.HTTP}-{ModelFormat.CAIKIT}-raw",
-        namespace=unprivileged_model_namespace.name,
-        runtime=unprivileged_s3_caikit_serving_runtime.name,
-        model_format=unprivileged_s3_caikit_serving_runtime.instance.spec.supportedModelFormats[0].name,
-        deployment_mode=KServeDeploymentType.RAW_DEPLOYMENT,
-        storage_key=unprivileged_models_endpoint_s3_secret.name,
-        storage_path=ModelStoragePath.FLAN_T5_SMALL_CAIKIT,
-    ) as isvc:
-        yield isvc
-
-
 @pytest.fixture()
 def patched_remove_authentication_model_mesh_runtime(
     admin_client: DynamicClient,

tests/model_serving/model_server/authentication/test_non_admin_users.py

@@ -1,13 +1,11 @@
 import pytest
 
 from tests.model_serving.model_server.utils import (
-    run_inference_multiple_times,
     verify_inference_response,
 )
 from utilities.constants import ModelFormat, ModelStoragePath, Protocols
 from utilities.inference_utils import Inference
 from utilities.manifests.caikit_tgis import CAIKIT_TGIS_INFERENCE_CONFIG
-from utilities.monitoring import validate_metrics_value
 
 
 @pytest.mark.parametrize(
@@ -63,26 +61,3 @@ def test_non_admin_deploy_raw_and_query_model(
             model_name=ModelFormat.CAIKIT,
             use_default_query=True,
         )
-
-    def test_raw_metrics(
-        self,
-        unprivileged_s3_caikit_raw_inference_service,
-        prometheus,
-        user_workload_monitoring_config_map,
-    ):
-        """Verify number of total model requests in OpenShift monitoring system (UserWorkloadMonitoring) metrics"""
-        total_runs = 5
-
-        run_inference_multiple_times(
-            isvc=unprivileged_s3_caikit_raw_inference_service,
-            inference_config=CAIKIT_TGIS_INFERENCE_CONFIG,
-            inference_type=Inference.ALL_TOKENS,
-            protocol=Protocols.HTTP,
-            model_name=ModelFormat.CAIKIT,
-            iterations=total_runs,
-        )
-        validate_metrics_value(
-            prometheus=prometheus,
-            metrics_query="tgi_request_count",
-            expected_value=str(total_runs + 1),
-        )

tests/model_serving/model_server/conftest.py

@@ -18,13 +18,14 @@
 from ocp_utilities.monitoring import Prometheus
 from pytest_testconfig import config as py_config
 
-from utilities.constants import StorageClassName
 from utilities.constants import (
     KServeDeploymentType,
     ModelFormat,
     ModelInferenceRuntime,
+    ModelStoragePath,
     Protocols,
     RuntimeTemplates,
+    StorageClassName,
 )
 from utilities.constants import (
     ModelAndFormat,
@@ -444,6 +445,8 @@ def prometheus(admin_client: DynamicClient) -> Prometheus:
 def user_workload_monitoring_config_map(
     admin_client: DynamicClient, cluster_monitoring_config: ConfigMap
 ) -> Generator[ConfigMap, None, None]:
+    uwm_namespace = "openshift-user-workload-monitoring"
+
     data = {
         "config.yaml": yaml.dump({
             "prometheus": {
@@ -457,11 +460,15 @@ def user_workload_monitoring_config_map(
     with update_configmap_data(
         client=admin_client,
         name="user-workload-monitoring-config",
-        namespace="openshift-user-workload-monitoring",
+        namespace=uwm_namespace,
         data=data,
     ) as cm:
         yield cm
 
+    # UWM PVCs are not deleted once the configmap is deleted; forcefully deleting the PVCs to avoid having left-overs
+    for pvc in PersistentVolumeClaim.get(dyn_client=admin_client, namespace=uwm_namespace):
+        pvc.clean_up()
+
 
 @pytest.fixture(scope="class")
 def http_s3_ovms_external_route_model_mesh_serving_runtime(
@@ -515,3 +522,87 @@ def http_s3_openvino_second_model_mesh_inference_service(
         model_version=request.param["model-version"],
     ) as isvc:
         yield isvc
+
+
+@pytest.fixture(scope="class")
+def unprivileged_s3_caikit_raw_inference_service(
+    request: FixtureRequest,
+    unprivileged_client: DynamicClient,
+    unprivileged_model_namespace: Namespace,
+    unprivileged_s3_caikit_serving_runtime: ServingRuntime,
+    unprivileged_models_endpoint_s3_secret: Secret,
+) -> Generator[InferenceService, Any, Any]:
+    with create_isvc(
+        client=unprivileged_client,
+        name=f"{Protocols.HTTP}-{ModelFormat.CAIKIT}-raw",
+        namespace=unprivileged_model_namespace.name,
+        runtime=unprivileged_s3_caikit_serving_runtime.name,
+        model_format=unprivileged_s3_caikit_serving_runtime.instance.spec.supportedModelFormats[0].name,
+        deployment_mode=KServeDeploymentType.RAW_DEPLOYMENT,
+        storage_key=unprivileged_models_endpoint_s3_secret.name,
+        storage_path=ModelStoragePath.FLAN_T5_SMALL_CAIKIT,
+    ) as isvc:
+        yield isvc
+
+
+@pytest.fixture(scope="class")
+def unprivileged_s3_caikit_serving_runtime(
+    admin_client: DynamicClient,
+    unprivileged_client: DynamicClient,
+    unprivileged_model_namespace: Namespace,
+) -> Generator[ServingRuntime, Any, Any]:
+    with ServingRuntimeFromTemplate(
+        client=admin_client,
+        unprivileged_client=unprivileged_client,
+        name=f"{Protocols.HTTP}-{ModelInferenceRuntime.CAIKIT_TGIS_RUNTIME}",
+        namespace=unprivileged_model_namespace.name,
+        template_name=RuntimeTemplates.CAIKIT_TGIS_SERVING,
+        multi_model=False,
+        enable_http=True,
+        enable_grpc=False,
+    ) as model_runtime:
+        yield model_runtime
+
+
+@pytest.fixture(scope="class")
+def unprivileged_models_endpoint_s3_secret(
+    unprivileged_client: DynamicClient,
+    unprivileged_model_namespace: Namespace,
+    aws_access_key_id: str,
+    aws_secret_access_key: str,
+    models_s3_bucket_name: str,
+    models_s3_bucket_region: str,
+    models_s3_bucket_endpoint: str,
+) -> Generator[Secret, Any, Any]:
+    with s3_endpoint_secret(
+        admin_client=unprivileged_client,
+        name="models-bucket-secret",
+        namespace=unprivileged_model_namespace.name,
+        aws_access_key=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+        aws_s3_region=models_s3_bucket_region,
+        aws_s3_bucket=models_s3_bucket_name,
+        aws_s3_endpoint=models_s3_bucket_endpoint,
+    ) as secret:
+        yield secret
+
+
+@pytest.fixture(scope="class")
+def unprivileged_s3_caikit_serverless_inference_service(
+    request: FixtureRequest,
+    unprivileged_client: DynamicClient,
+    unprivileged_model_namespace: Namespace,
+    unprivileged_s3_caikit_serving_runtime: ServingRuntime,
+    unprivileged_models_endpoint_s3_secret: Secret,
+) -> Generator[InferenceService, Any, Any]:
+    with create_isvc(
+        client=unprivileged_client,
+        name=f"{Protocols.HTTP}-{ModelFormat.CAIKIT}",
+        namespace=unprivileged_model_namespace.name,
+        runtime=unprivileged_s3_caikit_serving_runtime.name,
+        model_format=unprivileged_s3_caikit_serving_runtime.instance.spec.supportedModelFormats[0].name,
+        deployment_mode=KServeDeploymentType.SERVERLESS,
+        storage_key=unprivileged_models_endpoint_s3_secret.name,
+        storage_path=request.param["model-dir"],
+    ) as isvc:
+        yield isvc

tests/model_serving/model_server/metrics/conftest.py

@@ -19,6 +19,7 @@
 pytestmark = [
     pytest.mark.serverless,
     pytest.mark.usefixtures("valid_aws_config", "user_workload_monitoring_config_map"),
+    pytest.mark.metrics,
 ]
 
 
@@ -46,8 +47,7 @@
 class TestModelMetrics:
     @pytest.mark.smoke
     @pytest.mark.polarion("ODS-2555")
-    @pytest.mark.dependency(name="test_model_metrics_num_success_requests")
-    def test_model_metrics_num_success_requests(self, s3_models_inference_service, deleted_metrics, prometheus):
+    def test_model_metrics_num_success_requests(self, s3_models_inference_service, prometheus):
         """Verify number of successful model requests in OpenShift monitoring system (UserWorkloadMonitoring) metrics"""
         verify_inference_response(
             inference_service=s3_models_inference_service,
@@ -65,10 +65,6 @@ def test_model_metrics_num_success_requests(self, s3_models_inference_service, d
 
     @pytest.mark.smoke
     @pytest.mark.polarion("ODS-2555")
-    @pytest.mark.dependency(
-        name="test_model_metrics_num_total_requests",
-        depends=["test_model_metrics_num_success_requests"],
-    )
     def test_model_metrics_num_total_requests(self, s3_models_inference_service, prometheus):
         """Verify number of total model requests in OpenShift monitoring system (UserWorkloadMonitoring) metrics"""
         total_runs = 5
@@ -90,7 +86,6 @@ def test_model_metrics_num_total_requests(self, s3_models_inference_service, pro
 
     @pytest.mark.smoke
     @pytest.mark.polarion("ODS-2555")
-    @pytest.mark.dependency(depends=["test_model_metrics_num_total_requests"])
     def test_model_metrics_cpu_utilization(self, s3_models_inference_service, prometheus):
         """Verify CPU utilization data in OpenShift monitoring system (UserWorkloadMonitoring) metrics"""
         assert get_metrics_value(