ci: resolve conflicts

Author: rnetser

conftest.py

@@ -6,7 +6,15 @@
 import shutil
 
 import shortuuid
-from pytest import Parser, Session, FixtureRequest, FixtureDef, Item, Config, CollectReport
+from pytest import (
+    Parser,
+    Session,
+    FixtureRequest,
+    FixtureDef,
+    Item,
+    Config,
+    CollectReport,
+)
 from _pytest.terminal import TerminalReporter
 from typing import Optional, Any
 from pytest_testconfig import config as py_config
@@ -39,18 +47,26 @@ def pytest_addoption(parser: Parser) -> None:
 
     # Buckets options
     buckets_group.addoption(
-        "--ci-s3-bucket-name", default=os.environ.get("CI_S3_BUCKET_NAME"), help="Ci S3 bucket name"
+        "--ci-s3-bucket-name",
+        default=os.environ.get("CI_S3_BUCKET_NAME"),
+        help="Ci S3 bucket name",
     )
     buckets_group.addoption(
-        "--ci-s3-bucket-region", default=os.environ.get("CI_S3_BUCKET_REGION"), help="Ci S3 bucket region"
+        "--ci-s3-bucket-region",
+        default=os.environ.get("CI_S3_BUCKET_REGION"),
+        help="Ci S3 bucket region",
     )
 
     buckets_group.addoption(
-        "--ci-s3-bucket-endpoint", default=os.environ.get("CI_S3_BUCKET_ENDPOINT"), help="Ci S3 bucket endpoint"
+        "--ci-s3-bucket-endpoint",
+        default=os.environ.get("CI_S3_BUCKET_ENDPOINT"),
+        help="Ci S3 bucket endpoint",
     )
 
     buckets_group.addoption(
-        "--models-s3-bucket-name", default=os.environ.get("MODELS_S3_BUCKET_NAME"), help="Models S3 bucket name"
+        "--models-s3-bucket-name",
+        default=os.environ.get("MODELS_S3_BUCKET_NAME"),
+        help="Models S3 bucket name",
     )
     buckets_group.addoption(
         "--models-s3-bucket-region",
@@ -91,6 +107,11 @@ def pytest_addoption(parser: Parser) -> None:
         action="store_true",
         help="Delete pre-upgrade resources; useful when debugging pre-upgrade tests",
     )
+    upgrade_group.addoption(
+        "--upgrade-deployment-modes",
+        help="Coma-separated str; specify inference service deployment modes tests to run in upgrade tests. "
+        "If not set, all will be tested.",
+    )
 
 
 def pytest_cmdline_main(config: Any) -> None:
@@ -102,19 +123,45 @@ def pytest_collection_modifyitems(session: Session, config: Config, items: list[
     Pytest fixture to filter or re-order the items in-place.
 
     Filters upgrade tests based on '--pre-upgrade' / '--post-upgrade' option and marker.
+    If `--upgrade-deployment-modes` option is set, only tests with the specified deployment modes will be added.
     """
+
+    def _add_upgrade_test(_item: Item, _upgrade_deployment_modes: list[str]) -> bool:
+        """
+        Add upgrade test to the list of tests to run.
+
+        Args:
+            _item (Item): The test item.
+            _upgrade_deployment_modes (list[str]): The deployment modes to test.
+
+        Returns:
+            True if the test should be added, False otherwise.
+
+        """
+        if not _upgrade_deployment_modes:
+            return True
+
+        return any([keyword for keyword in _item.keywords if keyword in _upgrade_deployment_modes])
+
     pre_upgrade_tests: list[Item] = []
     post_upgrade_tests: list[Item] = []
     non_upgrade_tests: list[Item] = []
+    upgrade_deployment_modes: list[str] = []
 
     run_pre_upgrade_tests: str | None = config.getoption(name="pre_upgrade")
     run_post_upgrade_tests: str | None = config.getoption(name="post_upgrade")
+    if config_upgrade_deployment_modes := config.getoption(name="upgrade_deployment_modes"):
+        upgrade_deployment_modes = config_upgrade_deployment_modes.split(",")
 
     for item in items:
-        if "pre_upgrade" in item.keywords:
+        if "pre_upgrade" in item.keywords and _add_upgrade_test(
+            _item=item, _upgrade_deployment_modes=upgrade_deployment_modes
+        ):
             pre_upgrade_tests.append(item)
 
-        elif "post_upgrade" in item.keywords:
+        elif "post_upgrade" in item.keywords and _add_upgrade_test(
+            _item=item, _upgrade_deployment_modes=upgrade_deployment_modes
+        ):
             post_upgrade_tests.append(item)
 
         else:

tests/model_serving/model_server/serverless/conftest.py

@@ -12,6 +12,23 @@
 from utilities.manifests.caikit_tgis import CAIKIT_TGIS_INFERENCE_CONFIG
 
 
+@pytest.fixture(scope="class")
+def inference_service_patched_replicas(
+    request: FixtureRequest, ovms_serverless_inference_service: InferenceService
+) -> InferenceService:
+    ResourceEditor(
+        patches={
+            ovms_serverless_inference_service: {
+                "spec": {
+                    "predictor": {"minReplicas": request.param["min-replicas"]},
+                }
+            }
+        }
+    ).update()
+
+    return ovms_serverless_inference_service
+
+
 @pytest.fixture
 def inference_service_updated_canary_config(
     request: FixtureRequest, s3_models_inference_service: InferenceService

tests/model_serving/model_server/serverless/test_scale_to_zero.py

@@ -0,0 +1,96 @@
+import pytest
+from ocp_resources.deployment import Deployment
+
+from tests.model_serving.model_server.serverless.utils import verify_no_inference_pods
+from tests.model_serving.model_server.utils import verify_inference_response
+from utilities.constants import (
+    ModelFormat,
+    ModelInferenceRuntime,
+    ModelVersion,
+    Protocols,
+)
+from utilities.exceptions import DeploymentValidationError
+from utilities.inference_utils import Inference
+from utilities.manifests.onnx import ONNX_INFERENCE_CONFIG
+
+pytestmark = [
+    pytest.mark.serverless,
+    pytest.mark.sanity,
+    pytest.mark.usefixtures("valid_aws_config"),
+]
+
+
+@pytest.mark.serverless
+@pytest.mark.parametrize(
+    "model_namespace, openvino_kserve_serving_runtime, ovms_serverless_inference_service",
+    [
+        pytest.param(
+            {"name": "serverless-scale-zero"},
+            {
+                "runtime-name": ModelInferenceRuntime.ONNX_RUNTIME,
+                "model-format": {ModelFormat.ONNX: ModelVersion.OPSET13},
+            },
+            {
+                "name": ModelFormat.ONNX,
+                "model-version": ModelVersion.OPSET13,
+                "model-dir": "test-dir",
+            },
+        )
+    ],
+    indirect=True,
+)
+class TestServerlessScaleToZero:
+    def test_serverless_before_scale_to_zero(self, ovms_serverless_inference_service):
+        """Verify model can be queried before scaling to zero"""
+        verify_inference_response(
+            inference_service=ovms_serverless_inference_service,
+            inference_config=ONNX_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
+            protocol=Protocols.HTTPS,
+            use_default_query=True,
+        )
+
+    @pytest.mark.parametrize(
+        "inference_service_patched_replicas",
+        [pytest.param({"min-replicas": 0})],
+        indirect=True,
+    )
+    @pytest.mark.dependency(name="test_no_serverless_pods_after_scale_to_zero")
+    def test_no_serverless_pods_after_scale_to_zero(self, admin_client, inference_service_patched_replicas):
+        """Verify pods are scaled to zero"""
+        verify_no_inference_pods(client=admin_client, isvc=inference_service_patched_replicas)
+
+    @pytest.mark.dependency(depends=["test_no_serverless_pods_after_scale_to_zero"])
+    def test_serverless_inference_after_scale_to_zero(self, ovms_serverless_inference_service):
+        """Verify model can be queried after scaling to zero"""
+        verify_inference_response(
+            inference_service=ovms_serverless_inference_service,
+            inference_config=ONNX_INFERENCE_CONFIG,
+            inference_type=Inference.INFER,
+            protocol=Protocols.HTTPS,
+            use_default_query=True,
+        )
+
+    @pytest.mark.dependency(depends=["test_no_serverless_pods_after_scale_to_zero"])
+    def test_no_serverless_pods_when_no_traffic(self, admin_client, ovms_serverless_inference_service):
+        """Verify pods are scaled to zero when no traffic is sent"""
+        verify_no_inference_pods(client=admin_client, isvc=ovms_serverless_inference_service)
+
+    @pytest.mark.parametrize(
+        "inference_service_patched_replicas",
+        [pytest.param({"min-replicas": 1})],
+        indirect=True,
+    )
+    def test_serverless_pods_after_scale_to_one_replica(self, admin_client, inference_service_patched_replicas):
+        """Verify pod is running after scaling to 1 replica"""
+        for deployment in Deployment.get(
+            client=admin_client,
+            namespace=inference_service_patched_replicas.namespace,
+        ):
+            if deployment.labels["serving.knative.dev/configurationGeneration"] == "3":
+                deployment.wait_for_replicas()
+                return
+
+        raise DeploymentValidationError(
+            f"Inference Service {inference_service_patched_replicas.name} new deployment not found"
+        )

tests/model_serving/model_server/serverless/utils.py

@@ -5,16 +5,48 @@
 from kubernetes.dynamic import DynamicClient
 from ocp_resources.inference_service import InferenceService
 from simple_logger.logger import get_logger
-from timeout_sampler import TimeoutExpiredError, TimeoutSampler
+from timeout_sampler import TimeoutSampler
+from timeout_sampler import TimeoutExpiredError
 
 from tests.model_serving.model_server.utils import verify_inference_response
 from utilities.constants import Timeout
 from utilities.exceptions import InferenceCanaryTrafficError
 from utilities.infra import get_pods_by_isvc_label
 
+
 LOGGER = get_logger(name=__name__)
 
 
+def verify_no_inference_pods(client: DynamicClient, isvc: InferenceService) -> None:
+    """
+    Verify that no inference pods are running for the given InferenceService.
+
+    Args:
+        client (DynamicClient): DynamicClient object
+        isvc (InferenceService): InferenceService object
+
+    Raises:
+        TimeoutError: If pods are exist after the timeout.
+
+    """
+    pods = []
+
+    try:
+        pods = TimeoutSampler(
+            wait_timeout=Timeout.TIMEOUT_4MIN,
+            sleep=5,
+            func=get_pods_by_isvc_label,
+            client=client,
+            isvc=isvc,
+        )
+        if not pods:
+            return
+
+    except TimeoutError:
+        LOGGER.error(f"{[pod.name for pod in pods]} were not deleted")
+        raise
+
+
 def wait_for_canary_rollout(isvc: InferenceService, percentage: int, timeout: int = Timeout.TIMEOUT_5MIN) -> None:
     """
     Wait for inference service to be updated with canary rollout.

tests/model_serving/model_server/upgrade/README.md

@@ -30,3 +30,13 @@ uv run pytest --post-upgrade
 ```bash
 uv run pytest --pre-upgrade --post-upgrade
 ```
+
+## To run only specific deployment tests, pass --upgrade-deployment-modes with requested mode(s), for example:
+
+```bash
+uv run pytest --pre-upgrade --post-upgrade --upgrade-deployment-modes=servelerss
+```
+
+```bash
+uv run pytest --pre-upgrade --post-upgrade --upgrade-deployment-modes=servelerss,rawdeployment
+```

tests/model_serving/model_server/upgrade/test_upgrade.py

@@ -7,15 +7,14 @@
 from utilities.manifests.onnx import ONNX_INFERENCE_CONFIG
 from utilities.manifests.openvino import OPENVINO_INFERENCE_CONFIG
 
-pytestmark = [pytest.mark.serverless, pytest.mark.rawdeployment, pytest.mark.modelmesh]
-
 
 # TODO: add auth, external route and grpc tests
 
 
 @pytest.mark.usefixtures("valid_aws_config", "skipped_teardown_resources")
 class TestPreUpgradeModelServer:
     @pytest.mark.pre_upgrade
+    @pytest.mark.serverless
     def test_serverless_onnx_pre_upgrade_inference(self, ovms_serverless_inference_service_scope_session):
         """Verify that kserve Serverless ONNX model can be queried using REST before upgrade"""
         verify_inference_response(
@@ -27,6 +26,7 @@ def test_serverless_onnx_pre_upgrade_inference(self, ovms_serverless_inference_s
         )
 
     @pytest.mark.pre_upgrade
+    @pytest.mark.rawdeployment
     def test_raw_caikit_bge_pre_upgrade_inference(self, caikit_raw_inference_service_scope_session):
         """Test Caikit bge-large-en embedding model inference using internal route before upgrade"""
         verify_inference_response(
@@ -39,6 +39,7 @@ def test_raw_caikit_bge_pre_upgrade_inference(self, caikit_raw_inference_service
         )
 
     @pytest.mark.pre_upgrade
+    @pytest.mark.modelmesh
     def test_model_mesh_openvino_pre_upgrade_inference(self, openvino_model_mesh_inference_service_scope_session):
         """Test OpenVINO ModelMesh inference with internal route before upgrade"""
         verify_inference_response(
@@ -53,6 +54,7 @@ def test_model_mesh_openvino_pre_upgrade_inference(self, openvino_model_mesh_inf
 @pytest.mark.usefixtures("reused_resources")
 class TestPostUpgradeModelServer:
     @pytest.mark.post_upgrade
+    @pytest.mark.serverless
     @pytest.mark.dependency(name="test_serverless_onnx_post_upgrade_inference_service_exists")
     def test_serverless_onnx_post_upgrade_inference_service_exists(
         self, ovms_serverless_inference_service_scope_session
@@ -61,6 +63,7 @@ def test_serverless_onnx_post_upgrade_inference_service_exists(
         assert ovms_serverless_inference_service_scope_session.exists
 
     @pytest.mark.post_upgrade
+    @pytest.mark.serverless
     @pytest.mark.dependency(depends=["test_serverless_onnx_post_upgrade_inference_service_exists"])
     def test_serverless_onnx_post_upgrade_inference(self, ovms_serverless_inference_service_scope_session):
         """Verify that kserve Serverless ONNX model can be queried using REST after upgrade"""
@@ -73,12 +76,14 @@ def test_serverless_onnx_post_upgrade_inference(self, ovms_serverless_inference_
         )
 
     @pytest.mark.post_upgrade
+    @pytest.mark.rawdeployment
     @pytest.mark.dependency(name="test_raw_caikit_bge_post_upgrade_inference_exists")
     def test_raw_caikit_bge_post_upgrade_inference_exists(self, caikit_raw_inference_service_scope_session):
         """Test that raw deployment inference service exists after upgrade"""
         assert caikit_raw_inference_service_scope_session.exists
 
     @pytest.mark.post_upgrade
+    @pytest.mark.rawdeployment
     @pytest.mark.dependency(depends=["test_raw_caikit_bge_post_upgrade_inference_exists"])
     def test_raw_caikit_bge_post_upgrade_inference(self, caikit_raw_inference_service_scope_session):
         """Test Caikit bge-large-en embedding model inference using internal route after upgrade"""
@@ -92,6 +97,7 @@ def test_raw_caikit_bge_post_upgrade_inference(self, caikit_raw_inference_servic
         )
 
     @pytest.mark.post_upgrade
+    @pytest.mark.modelmesh
     @pytest.mark.dependency(name="test_model_mesh_openvino_post_upgrade_inference_exists")
     def test_model_mesh_openvino_post_upgrade_inference_exists(
         self, openvino_model_mesh_inference_service_scope_session
@@ -100,6 +106,7 @@ def test_model_mesh_openvino_post_upgrade_inference_exists(
         assert openvino_model_mesh_inference_service_scope_session.exists
 
     @pytest.mark.post_upgrade
+    @pytest.mark.modelmesh
     @pytest.mark.dependency(depends=["test_model_mesh_openvino_post_upgrade_inference_exists"])
     def test_model_mesh_openvino_post_upgrade_inference(self, openvino_model_mesh_inference_service_scope_session):
         """Test OpenVINO ModelMesh inference with internal route after upgrade"""

utilities/exceptions.py

@@ -70,5 +70,9 @@ def __str__(self) -> str:
         return f"The {self.type} is not supported"
 
 
+class DeploymentValidationError(Exception):
+    pass
+
+
 class InferenceCanaryTrafficError(Exception):
     pass

utilities/infra.py

@@ -91,7 +91,7 @@ def create_ns(
 def wait_for_inference_deployment_replicas(
     client: DynamicClient,
     isvc: InferenceService,
-    runtime_name: str | None,
+    runtime_name: str | None = None,
     expected_num_deployments: int = 1,
     timeout: int = Timeout.TIMEOUT_5MIN,
 ) -> list[Deployment]: