[model server] add scale to zero tests (#172)

* Create size-labeler.yml

* Delete .github/workflows/size-labeler.yml

* model mesh - add auth tests

* xx

* feat: add scale to zero tests

* feat: add scale to zero tests

Author: rnetser

tests/model_serving/model_server/serverless/__init__.py

@@ -0,0 +1,21 @@
+import pytest
+from _pytest.fixtures import FixtureRequest
+from ocp_resources.inference_service import InferenceService
+from ocp_resources.resource import ResourceEditor
+
+
+@pytest.fixture(scope="class")
+def inference_service_patched_replicas(
+    request: FixtureRequest, ovms_serverless_inference_service: InferenceService
+) -> InferenceService:
+    ResourceEditor(
+        patches={
+            ovms_serverless_inference_service: {
+                "spec": {
+                    "predictor": {"minReplicas": request.param["min-replicas"]},
+                }
+            }
+        }
+    ).update()
+
+    return ovms_serverless_inference_service

tests/model_serving/model_server/serverless/conftest.py

@@ -0,0 +1,96 @@
+import pytest
+from ocp_resources.deployment import Deployment
+
+from tests.model_serving.model_server.serverless.utils import verify_no_inference_pods
+from tests.model_serving.model_server.utils import verify_inference_response
+from utilities.constants import (
+    ModelFormat,
+    ModelInferenceRuntime,
+    ModelVersion,
+    Protocols,
+)
+from utilities.exceptions import DeploymentValidationError
+from utilities.inference_utils import Inference
+from utilities.manifests.onnx import ONNX_INFERENCE_CONFIG
+
+pytestmark = [
+    pytest.mark.serverless,
+    pytest.mark.sanity,
+    pytest.mark.usefixtures("valid_aws_config"),
+]
+
+
+@pytest.mark.serverless
+@pytest.mark.parametrize(
+    "model_namespace, openvino_kserve_serving_runtime, ovms_serverless_inference_service",
+    [
+        pytest.param(
+            {"name": "serverless-scale-zero"},
+            {
+                "runtime-name": ModelInferenceRuntime.ONNX_RUNTIME,
+                "model-format": {ModelFormat.ONNX: ModelVersion.OPSET13},
+            },
+            {
+                "name": ModelFormat.ONNX,
+                "model-version": ModelVersion.OPSET13,
+                "model-dir": "test-dir",
+            },
+        )
+    ],
+    indirect=True,
+)
+class TestServerlessScaleToZero:
+    def test_serverless_before_scale_to_zero(self, ovms_serverless_inference_service):
+        """Verify model can be queried before scaling to zero"""
+        verify_inference_response(
+            inference_service=ovms_serverless_inference_service,
+            inference_config=ONNX_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
+            protocol=Protocols.HTTPS,
+            use_default_query=True,
+        )
+
+    @pytest.mark.parametrize(
+        "inference_service_patched_replicas",
+        [pytest.param({"min-replicas": 0})],
+        indirect=True,
+    )
+    @pytest.mark.dependency(name="test_no_serverless_pods_after_scale_to_zero")
+    def test_no_serverless_pods_after_scale_to_zero(self, admin_client, inference_service_patched_replicas):
+        """Verify pods are scaled to zero"""
+        verify_no_inference_pods(client=admin_client, isvc=inference_service_patched_replicas)
+
+    @pytest.mark.dependency(depends=["test_no_serverless_pods_after_scale_to_zero"])
+    def test_serverless_inference_after_scale_to_zero(self, ovms_serverless_inference_service):
+        """Verify model can be queried after scaling to zero"""
+        verify_inference_response(
+            inference_service=ovms_serverless_inference_service,
+            inference_config=ONNX_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
+            protocol=Protocols.HTTPS,
+            use_default_query=True,
+        )
+
+    @pytest.mark.dependency(depends=["test_no_serverless_pods_after_scale_to_zero"])
+    def test_no_serverless_pods_when_no_traffic(self, admin_client, ovms_serverless_inference_service):
+        """Verify pods are scaled to zero when no traffic is sent"""
+        verify_no_inference_pods(client=admin_client, isvc=ovms_serverless_inference_service)
+
+    @pytest.mark.parametrize(
+        "inference_service_patched_replicas",
+        [pytest.param({"min-replicas": 1})],
+        indirect=True,
+    )
+    def test_serverless_pods_after_scale_to_one_replica(self, admin_client, inference_service_patched_replicas):
+        """Verify pod is running after scaling to 1 replica"""
+        for deployment in Deployment.get(
+            client=admin_client,
+            namespace=inference_service_patched_replicas.namespace,
+        ):
+            if deployment.labels["serving.knative.dev/configurationGeneration"] == "3":
+                deployment.wait_for_replicas()
+                return
+
+        raise DeploymentValidationError(
+            f"Inference Service {inference_service_patched_replicas.name} new deployment not found"
+        )

tests/model_serving/model_server/serverless/test_scale_to_zero.py

@@ -0,0 +1,40 @@
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.inference_service import InferenceService
+from simple_logger.logger import get_logger
+from timeout_sampler import TimeoutSampler
+
+from utilities.constants import Timeout
+from utilities.infra import get_pods_by_isvc_label
+
+
+LOGGER = get_logger(name=__name__)
+
+
+def verify_no_inference_pods(client: DynamicClient, isvc: InferenceService) -> None:
+    """
+    Verify that no inference pods are running for the given InferenceService.
+
+    Args:
+        client (DynamicClient): DynamicClient object
+        isvc (InferenceService): InferenceService object
+
+    Raises:
+        TimeoutError: If pods are exist after the timeout.
+
+    """
+    pods = []
+
+    try:
+        pods = TimeoutSampler(
+            wait_timeout=Timeout.TIMEOUT_4MIN,
+            sleep=5,
+            func=get_pods_by_isvc_label,
+            client=client,
+            isvc=isvc,
+        )
+        if not pods:
+            return
+
+    except TimeoutError:
+        LOGGER.error(f"{[pod.name for pod in pods]} were not deleted")
+        raise

tests/model_serving/model_server/serverless/utils.py

@@ -68,3 +68,7 @@ def __init__(self, type: str):
 
     def __str__(self) -> str:
         return f"The {self.type} is not supported"
+
+
+class DeploymentValidationError(Exception):
+    pass

utilities/exceptions.py

@@ -91,7 +91,7 @@ def create_ns(
 def wait_for_inference_deployment_replicas(
     client: DynamicClient,
     isvc: InferenceService,
-    runtime_name: str | None,
+    runtime_name: str | None = None,
     expected_num_deployments: int = 1,
     timeout: int = Timeout.TIMEOUT_5MIN,
 ) -> list[Deployment]: