ci: resolve conflicts

Author: rnetser

conftest.py

@@ -6,7 +6,15 @@
 import shutil
 
 import shortuuid
-from pytest import Parser, Session, FixtureRequest, FixtureDef, Item, Config, CollectReport
+from pytest import (
+    Parser,
+    Session,
+    FixtureRequest,
+    FixtureDef,
+    Item,
+    Config,
+    CollectReport,
+)
 from _pytest.terminal import TerminalReporter
 from typing import Optional, Any
 from pytest_testconfig import config as py_config
@@ -39,18 +47,26 @@ def pytest_addoption(parser: Parser) -> None:
 
     # Buckets options
     buckets_group.addoption(
-        "--ci-s3-bucket-name", default=os.environ.get("CI_S3_BUCKET_NAME"), help="Ci S3 bucket name"
+        "--ci-s3-bucket-name",
+        default=os.environ.get("CI_S3_BUCKET_NAME"),
+        help="Ci S3 bucket name",
     )
     buckets_group.addoption(
-        "--ci-s3-bucket-region", default=os.environ.get("CI_S3_BUCKET_REGION"), help="Ci S3 bucket region"
+        "--ci-s3-bucket-region",
+        default=os.environ.get("CI_S3_BUCKET_REGION"),
+        help="Ci S3 bucket region",
     )
 
     buckets_group.addoption(
-        "--ci-s3-bucket-endpoint", default=os.environ.get("CI_S3_BUCKET_ENDPOINT"), help="Ci S3 bucket endpoint"
+        "--ci-s3-bucket-endpoint",
+        default=os.environ.get("CI_S3_BUCKET_ENDPOINT"),
+        help="Ci S3 bucket endpoint",
     )
 
     buckets_group.addoption(
-        "--models-s3-bucket-name", default=os.environ.get("MODELS_S3_BUCKET_NAME"), help="Models S3 bucket name"
+        "--models-s3-bucket-name",
+        default=os.environ.get("MODELS_S3_BUCKET_NAME"),
+        help="Models S3 bucket name",
     )
     buckets_group.addoption(
         "--models-s3-bucket-region",
@@ -91,6 +107,11 @@ def pytest_addoption(parser: Parser) -> None:
         action="store_true",
         help="Delete pre-upgrade resources; useful when debugging pre-upgrade tests",
     )
+    upgrade_group.addoption(
+        "--upgrade-deployment-modes",
+        help="Coma-separated str; specify inference service deployment modes tests to run in upgrade tests. "
+        "If not set, all will be tested.",
+    )
 
 
 def pytest_cmdline_main(config: Any) -> None:
@@ -102,19 +123,45 @@ def pytest_collection_modifyitems(session: Session, config: Config, items: list[
     Pytest fixture to filter or re-order the items in-place.
 
     Filters upgrade tests based on '--pre-upgrade' / '--post-upgrade' option and marker.
+    If `--upgrade-deployment-modes` option is set, only tests with the specified deployment modes will be added.
     """
+
+    def _add_upgrade_test(_item: Item, _upgrade_deployment_modes: list[str]) -> bool:
+        """
+        Add upgrade test to the list of tests to run.
+
+        Args:
+            _item (Item): The test item.
+            _upgrade_deployment_modes (list[str]): The deployment modes to test.
+
+        Returns:
+            True if the test should be added, False otherwise.
+
+        """
+        if not _upgrade_deployment_modes:
+            return True
+
+        return any([keyword for keyword in _item.keywords if keyword in _upgrade_deployment_modes])
+
     pre_upgrade_tests: list[Item] = []
     post_upgrade_tests: list[Item] = []
     non_upgrade_tests: list[Item] = []
+    upgrade_deployment_modes: list[str] = []
 
     run_pre_upgrade_tests: str | None = config.getoption(name="pre_upgrade")
     run_post_upgrade_tests: str | None = config.getoption(name="post_upgrade")
+    if config_upgrade_deployment_modes := config.getoption(name="upgrade_deployment_modes"):
+        upgrade_deployment_modes = config_upgrade_deployment_modes.split(",")
 
     for item in items:
-        if "pre_upgrade" in item.keywords:
+        if "pre_upgrade" in item.keywords and _add_upgrade_test(
+            _item=item, _upgrade_deployment_modes=upgrade_deployment_modes
+        ):
             pre_upgrade_tests.append(item)
 
-        elif "post_upgrade" in item.keywords:
+        elif "post_upgrade" in item.keywords and _add_upgrade_test(
+            _item=item, _upgrade_deployment_modes=upgrade_deployment_modes
+        ):
             post_upgrade_tests.append(item)
 
         else:

tests/model_serving/model_server/conftest.py

@@ -148,6 +148,12 @@ def s3_models_inference_service(
     if (enable_auth := request.param.get("enable-auth")) is not None:
         isvc_kwargs["enable_auth"] = enable_auth
 
+    if (scale_metric := request.param.get("scale-metric")) is not None:
+        isvc_kwargs["scale_metric"] = scale_metric
+
+    if (scale_target := request.param.get("scale-target")) is not None:
+        isvc_kwargs["scale_target"] = scale_target
+
     with create_isvc(**isvc_kwargs) as isvc:
         yield isvc
 

tests/model_serving/model_server/serverless/conftest.py

@@ -4,14 +4,69 @@
 from _pytest.fixtures import FixtureRequest
 from kubernetes.dynamic import DynamicClient
 from ocp_resources.inference_service import InferenceService
+from ocp_resources.resource import ResourceEditor
 from ocp_resources.namespace import Namespace
 from ocp_resources.secret import Secret
 from ocp_resources.serving_runtime import ServingRuntime
 
+from tests.model_serving.model_server.serverless.utils import wait_for_canary_rollout
+from tests.model_serving.model_server.utils import run_inference_multiple_times
+from utilities.constants import ModelFormat, Protocols
+from utilities.inference_utils import Inference
+from utilities.manifests.caikit_tgis import CAIKIT_TGIS_INFERENCE_CONFIG
 from utilities.constants import KServeDeploymentType, ModelName, ModelStoragePath
 from utilities.inference_utils import create_isvc
 
 
+@pytest.fixture(scope="class")
+def inference_service_patched_replicas(
+    request: FixtureRequest, ovms_serverless_inference_service: InferenceService
+) -> InferenceService:
+    ResourceEditor(
+        patches={
+            ovms_serverless_inference_service: {
+                "spec": {
+                    "predictor": {"minReplicas": request.param["min-replicas"]},
+                }
+            }
+        }
+    ).update()
+
+    return ovms_serverless_inference_service
+
+
+@pytest.fixture
+def inference_service_updated_canary_config(
+    request: FixtureRequest, s3_models_inference_service: InferenceService
+) -> Generator[InferenceService, Any, Any]:
+    canary_percent = request.param["canary-traffic-percent"]
+    predictor_config = {
+        "spec": {
+            "predictor": {"canaryTrafficPercent": canary_percent},
+        }
+    }
+
+    if model_path := request.param.get("model-path"):
+        predictor_config["spec"]["predictor"]["model"] = {"storage": {"path": model_path}}
+
+    with ResourceEditor(patches={s3_models_inference_service: predictor_config}):
+        wait_for_canary_rollout(isvc=s3_models_inference_service, percentage=canary_percent)
+        yield s3_models_inference_service
+
+
+@pytest.fixture
+def multiple_tgis_inference_requests(s3_models_inference_service: InferenceService) -> None:
+    run_inference_multiple_times(
+        isvc=s3_models_inference_service,
+        inference_config=CAIKIT_TGIS_INFERENCE_CONFIG,
+        inference_type=Inference.ALL_TOKENS,
+        protocol=Protocols.HTTPS,
+        model_name=ModelFormat.CAIKIT,
+        iterations=50,
+        run_in_parallel=True,
+    )
+
+
 @pytest.fixture(scope="class")
 def s3_flan_small_hf_caikit_serverless_inference_service(
     request: FixtureRequest,

tests/model_serving/model_server/serverless/test_canary_rollout.py

@@ -0,0 +1,99 @@
+import pytest
+
+from tests.model_serving.model_server.serverless.utils import verify_canary_traffic
+from tests.model_serving.model_server.utils import verify_inference_response
+from utilities.constants import (
+    KServeDeploymentType,
+    ModelAndFormat,
+    ModelName,
+    ModelStoragePath,
+    Protocols,
+    RuntimeTemplates,
+)
+from utilities.inference_utils import Inference
+from utilities.manifests.pytorch import PYTORCH_TGIS_INFERENCE_CONFIG
+from utilities.manifests.tgis_grpc import TGIS_INFERENCE_CONFIG
+
+pytestmark = [pytest.mark.serverless, pytest.mark.sanity]
+
+
+@pytest.mark.polarion("ODS-2371")
+@pytest.mark.parametrize(
+    "model_namespace, serving_runtime_from_template, s3_models_inference_service",
+    [
+        pytest.param(
+            {"name": "serverless-canary-rollout"},
+            {
+                "name": "tgis-runtime",
+                "template-name": RuntimeTemplates.TGIS_GRPC_SERVING,
+                "multi-model": False,
+                "enable-http": False,
+                "enable-grpc": True,
+            },
+            {
+                "name": f"{ModelName.BLOOM_560M}-model",
+                "deployment-mode": KServeDeploymentType.SERVERLESS,
+                "model-dir": f"{ModelStoragePath.BLOOM_560M_CAIKIT}/artifacts",
+                "external-route": True,
+            },
+        )
+    ],
+    indirect=True,
+)
+class TestServerlessCanaryRollout:
+    def test_serverless_before_model_update(
+        self,
+        s3_models_inference_service,
+    ):
+        """Test inference with Bloom before model is updated."""
+        verify_inference_response(
+            inference_service=s3_models_inference_service,
+            inference_config=PYTORCH_TGIS_INFERENCE_CONFIG,
+            inference_type=Inference.ALL_TOKENS,
+            protocol=Protocols.GRPC,
+            model_name=ModelAndFormat.BLOOM_560M_CAIKIT,
+            use_default_query=True,
+        )
+
+    @pytest.mark.parametrize(
+        "inference_service_updated_canary_config",
+        [
+            pytest.param(
+                {"canary-traffic-percent": 30, "model-path": ModelStoragePath.FLAN_T5_SMALL_HF},
+            )
+        ],
+        indirect=True,
+    )
+    def test_serverless_during_canary_rollout(self, inference_service_updated_canary_config):
+        """Test inference during canary rollout"""
+        verify_canary_traffic(
+            isvc=inference_service_updated_canary_config,
+            inference_config=TGIS_INFERENCE_CONFIG,
+            model_name=ModelAndFormat.FLAN_T5_SMALL_CAIKIT,
+            inference_type=Inference.ALL_TOKENS,
+            protocol=Protocols.GRPC,
+            iterations=20,
+            expected_percentage=30,
+            tolerance=10,
+        )
+
+    @pytest.mark.parametrize(
+        "inference_service_updated_canary_config",
+        [
+            pytest.param(
+                {"canary-traffic-percent": 100},
+            )
+        ],
+        indirect=True,
+    )
+    def test_serverless_after_canary_rollout(self, inference_service_updated_canary_config):
+        """Test inference after canary rollout"""
+        verify_canary_traffic(
+            isvc=inference_service_updated_canary_config,
+            inference_config=TGIS_INFERENCE_CONFIG,
+            model_name=ModelAndFormat.FLAN_T5_SMALL_CAIKIT,
+            inference_type=Inference.ALL_TOKENS,
+            protocol=Protocols.GRPC,
+            iterations=5,
+            expected_percentage=100,
+        )

tests/model_serving/model_server/serverless/test_concurrency_auto_scale.py

@@ -0,0 +1,71 @@
+import pytest
+
+from tests.model_serving.model_server.serverless.utils import (
+    inference_service_pods_sampler,
+)
+from utilities.constants import (
+    KServeDeploymentType,
+    ModelFormat,
+    ModelInferenceRuntime,
+    ModelStoragePath,
+    RuntimeTemplates,
+    Timeout,
+)
+
+pytestmark = [
+    pytest.mark.serverless,
+    pytest.mark.sanity,
+    pytest.mark.usefixtures("valid_aws_config"),
+]
+
+
+@pytest.mark.parametrize(
+    "model_namespace, serving_runtime_from_template, s3_models_inference_service",
+    [
+        pytest.param(
+            {"name": "serverless-auto-scale"},
+            {
+                "name": f"{ModelInferenceRuntime.CAIKIT_TGIS_RUNTIME}",
+                "template-name": RuntimeTemplates.CAIKIT_TGIS_SERVING,
+                "multi-model": False,
+                "enable-http": True,
+            },
+            {
+                "name": f"{ModelFormat.CAIKIT}-auto-scale",
+                "deployment-mode": KServeDeploymentType.SERVERLESS,
+                "model-dir": ModelStoragePath.FLAN_T5_SMALL_CAIKIT,
+                "scale-metric": "concurrency",
+                "scale-target": 1,
+            },
+        )
+    ],
+    indirect=True,
+)
+class TestConcurrencyAutoScale:
+    @pytest.mark.dependency(name="test_auto_scale_using_concurrency")
+    def test_auto_scale_using_concurrency(
+        self,
+        admin_client,
+        s3_models_inference_service,
+        multiple_tgis_inference_requests,
+    ):
+        """Verify model is successfully scaled up based on concurrency metrics (KPA)"""
+        for pods in inference_service_pods_sampler(
+            client=admin_client,
+            isvc=s3_models_inference_service,
+            timeout=Timeout.TIMEOUT_1MIN,
+        ):
+            if pods:
+                if len(pods) > 1 and all([pod.status == pod.Status.RUNNING for pod in pods]):
+                    return
+
+    @pytest.mark.dependency(requires=["test_auto_scale_using_concurrency"])
+    def test_pods_scaled_down_when_no_requests(self, admin_client, s3_models_inference_service):
+        """Verify auto-scaled pods are deleted when there are no inference requests"""
+        for pods in inference_service_pods_sampler(
+            client=admin_client,
+            isvc=s3_models_inference_service,
+            timeout=Timeout.TIMEOUT_4MIN,
+        ):
+            if pods and len(pods) == 1:
+                return