Merge branch 'main' into custom_ns

Author: dbasunag

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
       - id: check-toml
 
   - repo: https://github.com/PyCQA/flake8
-    rev: "7.1.2"
+    rev: "7.2.0"
     hooks:
       - id: flake8
         args: [--config=.flake8, '--ignore=FCN001,W503']

README.md

@@ -10,3 +10,7 @@ Please follow the [Getting Started Guide](docs/GETTING_STARTED.md) on how to run
 
 ## Contribute to opendatahub-tests
 Please follow the [Contributing Guide](docs/CONTRIBUTING.md) and the [Developer guide](docs/DEVELOPER_GUIDE.md)
+
+
+## GitHub workflows
+Please follow the [GitHub workflows Guide](docs/GITHUB_WORKFLOWS.md) for more information.

.github/README.md docs/GITHUB_WORKFLOWS.md.github/README.md renamed to docs/GITHUB_WORKFLOWS.md

@@ -17,17 +17,6 @@
 )
 from utilities.constants import ModelAndFormat
 from utilities.inference_utils import create_isvc
-from utilities.jira import is_jira_open
-
-
-@pytest.fixture(scope="class")
-def skip_if_serverless_annotation_bug_present(
-    admin_client: DynamicClient,
-) -> None:
-    jira_id = "RHOAIENG-19654"
-
-    if is_jira_open(jira_id=jira_id, admin_client=admin_client):
-        pytest.skip(reason=f"Bug {jira_id} is not fixed")
 
 
 @pytest.fixture(scope="class")

tests/model_serving/model_server/components/kserve_dsc_deployment_mode/conftest.py

@@ -45,7 +45,6 @@
 class TestKServeDSCServerlessDefaultDeploymentMode:
     def test_isvc_contains_serverless_deployment_mode(
         self,
-        skip_if_serverless_annotation_bug_present,
         default_deployment_mode_in_dsc,
         ovms_inference_service,
     ):
@@ -75,7 +74,6 @@ def test_kserve_dsc_serverless_default_deployment_mode(
     )
     def test_isvc_on_dsc_default_deployment_mode_change_to_raw(
         self,
-        skip_if_serverless_annotation_bug_present,
         patched_default_deployment_mode_in_dsc,
         ovms_inference_service,
     ):
@@ -93,7 +91,6 @@ def test_isvc_on_dsc_default_deployment_mode_change_to_raw(
     )
     def test_restarted_pod_is_serverless(
         self,
-        skip_if_serverless_annotation_bug_present,
         patched_default_deployment_mode_in_dsc,
         restarted_inference_pod,
     ):

tests/model_serving/model_server/components/kserve_dsc_deployment_mode/test_kserve_dsc_default_deployment_mode.py

@@ -358,10 +358,39 @@ def ovms_kserve_inference_service(
     if max_replicas := request.param.get("max-replicas"):
         isvc_kwargs["max_replicas"] = max_replicas
 
+    if scale_metric := request.param.get("scale-metric"):
+        isvc_kwargs["scale_metric"] = scale_metric
+
+    if scale_target := request.param.get("scale-target"):
+        isvc_kwargs["scale_target"] = scale_target
+
     with create_isvc(**isvc_kwargs) as isvc:
         yield isvc
 
 
+@pytest.fixture(scope="class")
+def ovms_raw_inference_service(
+    request: FixtureRequest,
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    openvino_kserve_serving_runtime: ServingRuntime,
+    ci_endpoint_s3_secret: Secret,
+) -> Generator[InferenceService, Any, Any]:
+    with create_isvc(
+        client=admin_client,
+        name=f"{request.param['name']}-raw",
+        namespace=model_namespace.name,
+        external_route=True,
+        runtime=openvino_kserve_serving_runtime.name,
+        storage_path=request.param["model-dir"],
+        storage_key=ci_endpoint_s3_secret.name,
+        model_format=ModelAndFormat.OPENVINO_IR,
+        deployment_mode=KServeDeploymentType.RAW_DEPLOYMENT,
+        model_version=request.param["model-version"],
+    ) as isvc:
+        yield isvc
+
+
 @pytest.fixture(scope="class")
 def http_s3_tensorflow_model_mesh_inference_service(
     request: FixtureRequest,

tests/model_serving/model_server/conftest.py

@@ -12,15 +12,15 @@
 
 
 @pytest.fixture(scope="class")
-def model_car_tgis_inference_service(
+def model_car_serverless_inference_service(
     request: FixtureRequest,
     admin_client: DynamicClient,
     model_namespace: Namespace,
     serving_runtime_from_template: ServingRuntime,
 ) -> Generator[InferenceService, Any, Any]:
     with create_isvc(
         client=admin_client,
-        name="tgis-model-car",
+        name="serverless-model-car",
         namespace=model_namespace.name,
         runtime=serving_runtime_from_template.name,
         storage_uri=request.param["storage-uri"],

tests/model_serving/model_server/model_car/conftest.py

@@ -2,25 +2,26 @@
 
 from tests.model_serving.model_server.utils import verify_inference_response
 from utilities.infra import get_pods_by_isvc_label
-from utilities.constants import ModelName, Protocols
+from utilities.constants import ModelFormat, ModelName, Protocols, RuntimeTemplates
 from utilities.inference_utils import Inference
-from utilities.manifests.tgis_grpc import TGIS_INFERENCE_CONFIG
+from utilities.manifests.onnx import ONNX_INFERENCE_CONFIG
 
 pytestmark = pytest.mark.serverless
 
 
 @pytest.mark.parametrize(
-    "model_namespace, serving_runtime_from_template, model_car_tgis_inference_service",
+    "model_namespace, serving_runtime_from_template, model_car_serverless_inference_service",
     [
         pytest.param(
-            {"name": "tgsi-model-car"},
+            {"name": f"{ModelFormat.OPENVINO}-model-car"},
             {
-                "name": "tgis-runtime",
-                "template-name": "tgis-grpc-serving-template",
+                "name": f"{ModelName.MNIST}-runtime",
+                "template-name": RuntimeTemplates.OVMS_KSERVE,
                 "multi-model": False,
             },
             {
-                "storage-uri": "oci://quay.io/mwaykole/test@sha256:c526a1a3697253eb09adc65da6efaf7f36150205c3a51ab8d13b92b6a3af9c1c"  # noqa: E501
+                # Using mnist-8-1 model from OCI image
+                "storage-uri": "oci://quay.io/mwaykole/test@sha256:8a3217bcfa2cc5fa3d07496cff8b234acdf2c9725dd307dc0a80401f55e1a11c"  # noqa: E501
             },
         )
     ],
@@ -29,11 +30,11 @@
 class TestKserveModelCar:
     @pytest.mark.smoke
     @pytest.mark.jira("RHOAIENG-13465")
-    def test_model_car_no_restarts(self, model_car_tgis_inference_service):
+    def test_model_car_no_restarts(self, model_car_serverless_inference_service):
         """Verify that model pod doesn't restart"""
         pod = get_pods_by_isvc_label(
-            client=model_car_tgis_inference_service.client,
-            isvc=model_car_tgis_inference_service,
+            client=model_car_serverless_inference_service.client,
+            isvc=model_car_serverless_inference_service,
         )[0]
         restarted_containers = [
             container.name for container in pod.instance.status.containerStatuses if container.restartCount > 1
@@ -42,13 +43,12 @@ def test_model_car_no_restarts(self, model_car_tgis_inference_service):
 
     @pytest.mark.smoke
     @pytest.mark.jira("RHOAIENG-12306")
-    def test_model_car_using_rest(self, model_car_tgis_inference_service):
+    def test_model_car_using_rest(self, model_car_serverless_inference_service):
         """Verify model query with token using REST"""
         verify_inference_response(
-            inference_service=model_car_tgis_inference_service,
-            inference_config=TGIS_INFERENCE_CONFIG,
-            inference_type=Inference.ALL_TOKENS,
-            protocol=Protocols.GRPC,
-            model_name=ModelName.FLAN_T5_SMALL_HF,
+            inference_service=model_car_serverless_inference_service,
+            inference_config=ONNX_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
+            protocol=Protocols.HTTPS,
             use_default_query=True,
         )

tests/model_serving/model_server/model_car/test_oci_image.py

@@ -0,0 +1,54 @@
+import pytest
+
+from tests.model_serving.model_server.utils import verify_inference_response
+from tests.model_serving.model_server.raw_deployment.utils import assert_ingress_status_changed
+from utilities.constants import ModelFormat, ModelVersion, Protocols, RunTimeConfigs
+from utilities.inference_utils import Inference
+from utilities.manifests.onnx import ONNX_INFERENCE_CONFIG
+
+
+pytestmark = [pytest.mark.rawdeployment, pytest.mark.usefixtures("valid_aws_config")]
+
+
+@pytest.mark.parametrize(
+    "model_namespace, openvino_kserve_serving_runtime, ovms_raw_inference_service",
+    [
+        pytest.param(
+            {"name": "kserve-raw-route-reconciliation"},
+            RunTimeConfigs.ONNX_OPSET13_RUNTIME_CONFIG,
+            {"name": ModelFormat.ONNX, "model-version": ModelVersion.OPSET13, "model-dir": "test-dir"},
+        )
+    ],
+    indirect=True,
+)
+class TestONNXRawRouteReconciliation:
+    """Test suite for  Validating reconciliation"""
+
+    @pytest.mark.smoke
+    def test_raw_onnx_rout_reconciliation(self, admin_client, ovms_raw_inference_service):
+        """
+        Verify that the KServe Raw ONNX model can be queried using REST
+        and ensure that the model rout reconciliation works correctly .
+        """
+        # Initial inference validation
+        verify_inference_response(
+            inference_service=ovms_raw_inference_service,
+            inference_config=ONNX_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
+            protocol=Protocols.HTTPS,
+            use_default_query=True,
+        )
+
+    def test_route_value_before_and_after_deletion(self, admin_client, ovms_raw_inference_service):
+        # Validate ingress status before and after route deletion
+        assert_ingress_status_changed(admin_client=admin_client, inference_service=ovms_raw_inference_service)
+
+    def test_model_works_after_route_is_recreated(self, ovms_raw_inference_service):
+        # Final inference validation after route update
+        verify_inference_response(
+            inference_service=ovms_raw_inference_service,
+            inference_config=ONNX_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
+            protocol=Protocols.HTTPS,
+            use_default_query=True,
+        )

tests/model_serving/model_server/raw_deployment/test_kserve_raw_routes_reconciliation.py

@@ -0,0 +1,60 @@
+from kubernetes.dynamic import DynamicClient
+from kubernetes.dynamic.exceptions import ResourceNotFoundError
+from ocp_resources.inference_service import InferenceService
+from utilities.constants import Timeout
+from utilities.infra import get_model_route
+
+
+def assert_ingress_status_changed(admin_client: DynamicClient, inference_service: InferenceService) -> None:
+    """
+    Validates that the ingress status changes correctly after route deletion.
+
+    Args:
+        admin_client (DynamicClient): The administrative client used to manage the model route.
+        inference_service (InferenceService): The inference service whose route status is being checked.
+
+    Raises:
+        ResourceNotFoundError: If the route does not exist before or after deletion.
+        AssertionError: If any of the validation checks fail.
+
+    Returns:
+        None
+    """
+    route = get_model_route(admin_client, inference_service)
+    if not route.exists:
+        raise ResourceNotFoundError("Route before deletion not found: No active route is currently available.")
+
+    initial_status = route.instance.status["ingress"][0]["conditions"][0]
+    initial_host = route.host
+    initial_transition_time = initial_status["lastTransitionTime"]
+    initial_status_value = initial_status["status"]
+
+    route.delete(wait=True, timeout=Timeout.TIMEOUT_1MIN)
+
+    if not route.exists:
+        raise ResourceNotFoundError("Route after deletion not found: No active route is currently available.")
+
+    updated_status = route.instance.status["ingress"][0]["conditions"][0]
+    updated_host = route.host
+    updated_transition_time = updated_status["lastTransitionTime"]
+    updated_status_value = updated_status["status"]
+
+    # Collect failures instead of stopping at the first failed assertion
+    failures = []
+
+    if updated_host != initial_host:
+        failures.append(f"Host mismatch: before={initial_host}, after={updated_host}")
+
+    if updated_transition_time == initial_transition_time:
+        failures.append(
+            f"Transition time did not change: before={initial_transition_time}, after={updated_transition_time}"
+        )
+
+    if updated_status_value != "True":
+        failures.append(f"Updated ingress status incorrect: expected=True, actual={updated_status_value}")
+
+    if initial_status_value != "True":
+        failures.append(f"Initial ingress status incorrect: expected=True, actual={initial_status_value}")
+
+    # Assert all failures at once
+    assert not failures, "Ingress status validation failed:\n" + "\n".join(failures)