wip: migrate trustyaiservice tests to kserveraw

Author: adolfo-ab

tests/model_explainability/trustyai_service/conftest.py

@@ -1,3 +1,4 @@
+import json
 from typing import Generator, Any
 
 import pytest
@@ -14,11 +15,13 @@
 from ocp_resources.pod import Pod
 from ocp_resources.role import Role
 from ocp_resources.role_binding import RoleBinding
+from ocp_resources.route import Route
 from ocp_resources.secret import Secret
 from ocp_resources.service import Service
 from ocp_resources.service_account import ServiceAccount
 from ocp_resources.serving_runtime import ServingRuntime
 from ocp_resources.trustyai_service import TrustyAIService
+from pytest_testconfig import py_config
 
 from tests.model_explainability.trustyai_service.constants import (
     TAI_DATA_CONFIG,
@@ -50,7 +53,8 @@
 from utilities.operator_utils import get_cluster_service_version
 from utilities.constants import KServeDeploymentType, Labels, OPENSHIFT_OPERATORS, MARIADB, TRUSTYAI_SERVICE_NAME
 from utilities.inference_utils import create_isvc
-from utilities.infra import update_configmap_data, create_inference_token
+from ocp_resources.resource import ResourceEditor
+from utilities.infra import create_inference_token, get_kserve_storage_initialize_image
 
 DB_CREDENTIALS_SECRET_NAME: str = "db-credentials"
 DB_NAME: str = "trustyai_db"
@@ -96,15 +100,74 @@ def trustyai_service(
             yield trustyai_service
 
 
+@pytest.fixture(scope="session")
+def kserve_raw_config(admin_client: DynamicClient) -> Generator[ConfigMap, Any, Any]:
+    """Configure KServe for KServeRaw support by adding logger configuration."""
+
+    storage_initializer_image = get_kserve_storage_initialize_image(client=admin_client)
+    logger_config = {
+        "image": storage_initializer_image,
+        "memoryRequest": "100Mi",
+        "memoryLimit": "1Gi",
+        "cpuRequest": "100m",
+        "cpuLimit": "1",
+        "defaultUrl": "http://default-broker",
+        "caBundle": "kserve-logger-ca-bundle",
+        "caCertFile": "service-ca.crt",
+        "tlsSkipVerify": False,
+    }
+
+    data = {"logger": json.dumps(obj=logger_config)}
+
+    cm = ConfigMap(
+        client=admin_client,
+        name="inferenceservice-config",
+        namespace=py_config["applications_namespace"],
+        ensure_exists=True,
+    )
+
+    with ResourceEditor(
+        patches={
+            cm: {
+                "metadata": {"annotations": {"opendatahub.io/managed": "false"}},
+                "data": data,
+            }
+        }
+    ):
+        yield cm
+
+
+@pytest.fixture(scope="class")
+def kserve_logger_ca_bundle(admin_client: DynamicClient, model_namespace: Namespace) -> Generator[ConfigMap, Any, Any]:
+    """Create CA certificate ConfigMap required for KServeRaw logger."""
+    with ConfigMap(
+        client=admin_client,
+        name="kserve-logger-ca-bundle",
+        namespace=model_namespace.name,
+        annotations={"service.beta.openshift.io/inject-cabundle": "true"},
+        data={},
+    ) as ca_bundle:
+        yield ca_bundle
+
+
 @pytest.fixture(scope="session")
 def user_workload_monitoring_config(admin_client: DynamicClient) -> Generator[ConfigMap, Any, Any]:
-    data = {"config.yaml": yaml.dump({"prometheus": {"logLevel": "debug", "retention": "15d"}})}
-    with update_configmap_data(
+    data = {"config.yaml": yaml.dump(data={"prometheus": {"logLevel": "debug", "retention": "15d"}})}
+
+    cm = ConfigMap(
         client=admin_client,
         name="user-workload-monitoring-config",
         namespace="openshift-user-workload-monitoring",
-        data=data,
-    ) as cm:
+        ensure_exists=True,
+    )
+
+    with ResourceEditor(
+        patches={
+            cm: {
+                "data": data,
+            }
+        }
+    ):
         yield cm
 
 
@@ -218,6 +281,8 @@ def gaussian_credit_model(
     minio_service: Service,
     minio_data_connection: Secret,
     mlserver_runtime: ServingRuntime,
+    kserve_raw_config: ConfigMap,
+    kserve_logger_ca_bundle: ConfigMap,
     teardown_resources: bool,
 ) -> Generator[InferenceService, Any, Any]:
     gaussian_credit_model_kwargs = {
@@ -232,13 +297,15 @@ def gaussian_credit_model(
         isvc.clean_up()
     else:
         with create_isvc(
-            deployment_mode=KServeDeploymentType.SERVERLESS,
+            deployment_mode=KServeDeploymentType.RAW_DEPLOYMENT,
             model_format=XGBOOST,
             runtime=mlserver_runtime.name,
             storage_key=minio_data_connection.name,
             storage_path=GAUSSIAN_CREDIT_MODEL_STORAGE_PATH,
             enable_auth=True,
+            external_route=True,
             wait_for_predictor_pods=False,
+            wait=False,
             resources=GAUSSIAN_CREDIT_MODEL_RESOURCES,
             teardown=teardown_resources,
             **gaussian_credit_model_kwargs,
@@ -251,6 +318,22 @@ def gaussian_credit_model(
             yield isvc
 
 
+@pytest.fixture(scope="class")
+def gaussian_credit_model_route(
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    gaussian_credit_model: InferenceService,
+) -> Generator[Route, Any, Any]:
+    """Create Route for gaussian credit model InferenceService."""
+    with Route(
+        client=admin_client,
+        name=f"{gaussian_credit_model.name}-route",
+        namespace=model_namespace.name,
+        service=gaussian_credit_model.name,
+    ) as route:
+        yield route
+
+
 @pytest.fixture(scope="class")
 def isvc_getter_service_account(
     admin_client: DynamicClient, model_namespace: Namespace

tests/model_explainability/trustyai_service/drift/test_drift.py

@@ -33,18 +33,11 @@
             {"storage": "pvc"},
             id="pvc-storage",
         ),
-        pytest.param(
-            {"name": "test-drift-db"},
-            MinIo.PodConfig.MODEL_MESH_MINIO_CONFIG,
-            {"bucket": MinIo.Buckets.MODELMESH_EXAMPLE_MODELS},
-            {"storage": "db"},
-            id="db-storage",
-        ),
     ],
     indirect=True,
 )
 @pytest.mark.usefixtures("minio_pod")
-@pytest.mark.serverless
+@pytest.mark.rawdeployment
 @pytest.mark.smoke
 class TestDriftMetrics:
     """
@@ -67,6 +60,7 @@ def test_drift_send_inference(
         model_namespace,
         trustyai_service,
         gaussian_credit_model,
+        # gaussian_credit_model_route,
         isvc_getter_token,
     ) -> None:
         send_inferences_and_verify_trustyai_service_registered(

tests/model_explainability/trustyai_service/fairness/conftest.py

@@ -2,9 +2,11 @@
 
 import pytest
 from kubernetes.dynamic import DynamicClient
+from ocp_resources.config_map import ConfigMap
 from ocp_resources.inference_service import InferenceService
 from ocp_resources.namespace import Namespace
 from ocp_resources.pod import Pod
+from ocp_resources.route import Route
 from ocp_resources.secret import Secret
 from ocp_resources.service import Service
 from ocp_resources.serving_runtime import ServingRuntime
@@ -44,19 +46,22 @@ def onnx_loan_model(
     minio_service: Service,
     minio_data_connection: Secret,
     ovms_runtime: ServingRuntime,
+    kserve_raw_config: ConfigMap,
+    kserve_logger_ca_bundle: ConfigMap,
 ) -> Generator[InferenceService, Any, Any]:
     with create_isvc(
         client=admin_client,
         name="demo-loan-nn-onnx-alpha",
         namespace=model_namespace.name,
-        deployment_mode=KServeDeploymentType.SERVERLESS,
+        deployment_mode=KServeDeploymentType.RAW_DEPLOYMENT,
         model_format=ModelFormat.ONNX,
         runtime=ovms_runtime.name,
         storage_key=minio_data_connection.name,
         storage_path="ovms/loan_model_alpha",
         min_replicas=1,
         resources={"limits": {"cpu": "2", "memory": "8Gi"}, "requests": {"cpu": "1", "memory": "4Gi"}},
         enable_auth=True,
+        external_route=True,
         model_version="1",
         wait=True,
         wait_for_predictor_pods=False,
@@ -67,3 +72,19 @@ def onnx_loan_model(
             runtime_name=ovms_runtime.name,
         )
         yield isvc
+
+
+@pytest.fixture(scope="class")
+def onnx_loan_model_route(
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    onnx_loan_model: InferenceService,
+) -> Generator[Route, Any, Any]:
+    """Create Route for ONNX loan model InferenceService."""
+    with Route(
+        client=admin_client,
+        name=f"{onnx_loan_model.name}-route",
+        namespace=model_namespace.name,
+        service=onnx_loan_model.name,
+    ) as route:
+        yield route

tests/model_explainability/trustyai_service/fairness/test_fairness.py

@@ -71,7 +71,7 @@ def get_fairness_request_json_data(isvc: InferenceService) -> dict[str, Any]:
     indirect=True,
 )
 @pytest.mark.usefixtures("minio_pod")
-@pytest.mark.serverless
+@pytest.mark.rawdeployment
 @pytest.mark.smoke
 class TestFairnessMetrics:
     """
@@ -95,6 +95,7 @@ def test_fairness_send_inference(
         model_namespace,
         trustyai_service,
         onnx_loan_model,
+        onnx_loan_model_route,
         isvc_getter_token,
     ):
         send_inferences_and_verify_trustyai_service_registered(

tests/model_explainability/trustyai_service/service/multi_ns/conftest.py

@@ -3,7 +3,10 @@
 from typing import Generator, Any, List
 import pytest
 from ocp_resources.namespace import Namespace
+from ocp_resources.pod import Pod
+from ocp_resources.route import Route
 from ocp_resources.secret import Secret
+from ocp_resources.service import Service
 from ocp_resources.serving_runtime import ServingRuntime
 from ocp_resources.inference_service import InferenceService
 from ocp_resources.trustyai_service import TrustyAIService
@@ -12,6 +15,7 @@
 from ocp_resources.role_binding import RoleBinding
 from utilities.constants import OPENSHIFT_OPERATORS, MARIADB, TRUSTYAI_SERVICE_NAME
 from ocp_resources.maria_db import MariaDB
+from ocp_resources.config_map import ConfigMap
 from tests.model_explainability.trustyai_service.constants import (
     TAI_METRICS_CONFIG,
     TAI_DATA_CONFIG,
@@ -104,6 +108,28 @@ def trustyai_service_with_pvc_storage_multi_ns(
         yield services
 
 
+@pytest.fixture(scope="class")
+def kserve_logger_ca_bundle_multi_ns(
+    admin_client: DynamicClient, model_namespaces: List[Namespace]
+) -> Generator[List[ConfigMap], Any, None]:
+    """Create CA certificate ConfigMaps required for KServeRaw logger in each namespace."""
+    with ExitStack() as stack:
+        ca_bundles = [
+            stack.enter_context(
+                ConfigMap(
+                    client=admin_client,
+                    name="kserve-logger-ca-bundle",
+                    namespace=ns.name,
+                    annotations={"service.beta.openshift.io/inject-cabundle": "true"},
+                    data={},
+                    teardown=False,
+                )
+            )
+            for ns in model_namespaces
+        ]
+        yield ca_bundles
+
+
 @pytest.fixture(scope="class")
 def mlserver_runtime_multi_ns(admin_client, model_namespaces) -> Generator[List[ServingRuntime], Any, None]:
     with ExitStack() as stack:
@@ -128,7 +154,14 @@ def mlserver_runtime_multi_ns(admin_client, model_namespaces) -> Generator[List[
 
 @pytest.fixture(scope="class")
 def gaussian_credit_model_multi_ns(
-    admin_client, model_namespaces, minio_pod, minio_service, minio_data_connection_multi_ns, mlserver_runtime_multi_ns
+    admin_client: DynamicClient,
+    model_namespaces: List[Namespace],
+    minio_pod: Pod,
+    minio_service: Service,
+    minio_data_connection_multi_ns: List[Secret],
+    mlserver_runtime_multi_ns: List[ServingRuntime],
+    kserve_raw_config: ConfigMap,
+    kserve_logger_ca_bundle_multi_ns: List[ConfigMap],
 ) -> Generator[List[InferenceService], Any, None]:
     with ExitStack() as stack:
         models = []
@@ -137,12 +170,13 @@ def gaussian_credit_model_multi_ns(
                 client=admin_client,
                 namespace=ns.name,
                 name=GAUSSIAN_CREDIT_MODEL,
-                deployment_mode=KServeDeploymentType.SERVERLESS,
+                deployment_mode=KServeDeploymentType.RAW_DEPLOYMENT,
                 model_format=XGBOOST,
                 runtime=runtime.name,
                 storage_key=secret.name,
                 storage_path=GAUSSIAN_CREDIT_MODEL_STORAGE_PATH,
                 enable_auth=True,
+                external_route=True,
                 wait_for_predictor_pods=False,
                 resources=GAUSSIAN_CREDIT_MODEL_RESOURCES,
             )
@@ -159,6 +193,28 @@ def gaussian_credit_model_multi_ns(
         yield models
 
 
+@pytest.fixture(scope="class")
+def gaussian_credit_model_routes_multi_ns(
+    admin_client: DynamicClient,
+    model_namespaces: List[Namespace],
+    gaussian_credit_model_multi_ns: List[InferenceService],
+) -> Generator[List[Route], Any, None]:
+    """Create Routes for multi-namespace gaussian credit model InferenceServices."""
+    with ExitStack() as stack:
+        routes = []
+        for ns, isvc in zip(model_namespaces, gaussian_credit_model_multi_ns):
+            route = stack.enter_context(
+                Route(
+                    client=admin_client,
+                    name=f"{isvc.name}-route",
+                    namespace=ns.name,
+                    service=isvc.name,
+                )
+            )
+            routes.append(route)
+        yield routes
+
+
 @pytest.fixture(scope="class")
 def isvc_getter_service_account_multi_ns(admin_client, model_namespaces) -> Generator[List[ServiceAccount], None, None]:
     with ExitStack() as stack:

tests/model_explainability/trustyai_service/service/multi_ns/test_trustyai_service_multi_ns.py

@@ -29,7 +29,7 @@
     ],
     indirect=True,
 )
-@pytest.mark.serverless
+@pytest.mark.rawdeployment
 class TestTrustyAIServiceMultipleNS:
     """Verifies TrustyAIService operations across multiple namespaces,
     i.e. registering inference requests, uploading data, scheduling and deleting metrics,
@@ -41,6 +41,7 @@ def test_drift_send_inference_and_verify_trustyai_service_multiple_ns(
         current_client_token,
         trustyai_service_with_pvc_storage_multi_ns,
         gaussian_credit_model_multi_ns,
+        gaussian_credit_model_routes_multi_ns,
         isvc_getter_token_multi_ns,
     ):
         for tai, model, token in zip(
@@ -136,6 +137,7 @@ def test_drift_send_inference_and_verify_trustyai_service_with_db_storage(
         model_namespaces,
         trustyai_service_with_db_storage_multi_ns,
         gaussian_credit_model_multi_ns,
+        gaussian_credit_model_routes_multi_ns,
         isvc_getter_token_multi_ns,
     ):
         for tai, inference_model, inference_token in zip(

tests/model_explainability/trustyai_service/service/test_trustyai_service.py

@@ -85,7 +85,7 @@ def test_validate_trustyai_service_image(
     indirect=True,
 )
 @pytest.mark.usefixtures("minio_pod")
-@pytest.mark.serverless
+@pytest.mark.rawdeployment
 def test_trustyai_service_db_migration(
     admin_client,
     current_client_token,