[model server] add deployment mode markers for upgrade tests (#173)

conftest.py

@@ -6,7 +6,15 @@
 import shutil
 
 import shortuuid
-from pytest import Parser, Session, FixtureRequest, FixtureDef, Item, Config, CollectReport
+from pytest import (
+    Parser,
+    Session,
+    FixtureRequest,
+    FixtureDef,
+    Item,
+    Config,
+    CollectReport,
+)
 from _pytest.terminal import TerminalReporter
 from typing import Optional, Any
 from pytest_testconfig import config as py_config
@@ -39,18 +47,26 @@ def pytest_addoption(parser: Parser) -> None:
 
     # Buckets options
     buckets_group.addoption(
-        "--ci-s3-bucket-name", default=os.environ.get("CI_S3_BUCKET_NAME"), help="Ci S3 bucket name"
+        "--ci-s3-bucket-name",
+        default=os.environ.get("CI_S3_BUCKET_NAME"),
+        help="Ci S3 bucket name",
     )
     buckets_group.addoption(
-        "--ci-s3-bucket-region", default=os.environ.get("CI_S3_BUCKET_REGION"), help="Ci S3 bucket region"
+        "--ci-s3-bucket-region",
+        default=os.environ.get("CI_S3_BUCKET_REGION"),
+        help="Ci S3 bucket region",
     )
 
     buckets_group.addoption(
-        "--ci-s3-bucket-endpoint", default=os.environ.get("CI_S3_BUCKET_ENDPOINT"), help="Ci S3 bucket endpoint"
+        "--ci-s3-bucket-endpoint",
+        default=os.environ.get("CI_S3_BUCKET_ENDPOINT"),
+        help="Ci S3 bucket endpoint",
     )
 
     buckets_group.addoption(
-        "--models-s3-bucket-name", default=os.environ.get("MODELS_S3_BUCKET_NAME"), help="Models S3 bucket name"
+        "--models-s3-bucket-name",
+        default=os.environ.get("MODELS_S3_BUCKET_NAME"),
+        help="Models S3 bucket name",
     )
     buckets_group.addoption(
         "--models-s3-bucket-region",
@@ -91,6 +107,11 @@ def pytest_addoption(parser: Parser) -> None:
         action="store_true",
         help="Delete pre-upgrade resources; useful when debugging pre-upgrade tests",
     )
+    upgrade_group.addoption(
+        "--upgrade-deployment-modes",
+        help="Coma-separated str; specify inference service deployment modes tests to run in upgrade tests. "
+        "If not set, all will be tested.",
+    )
 
 
 def pytest_cmdline_main(config: Any) -> None:
@@ -102,19 +123,45 @@ def pytest_collection_modifyitems(session: Session, config: Config, items: list[
     Pytest fixture to filter or re-order the items in-place.
 
     Filters upgrade tests based on '--pre-upgrade' / '--post-upgrade' option and marker.
+    If `--upgrade-deployment-modes` option is set, only tests with the specified deployment modes will be added.
     """
+
+    def _add_upgrade_test(_item: Item, _upgrade_deployment_modes: list[str]) -> bool:
+        """
+        Add upgrade test to the list of tests to run.
+
+        Args:
+            _item (Item): The test item.
+            _upgrade_deployment_modes (list[str]): The deployment modes to test.
+
+        Returns:
+            True if the test should be added, False otherwise.
+
+        """
+        if not _upgrade_deployment_modes:
+            return True
+
+        return any([keyword for keyword in _item.keywords if keyword in _upgrade_deployment_modes])
+
     pre_upgrade_tests: list[Item] = []
     post_upgrade_tests: list[Item] = []
     non_upgrade_tests: list[Item] = []
+    upgrade_deployment_modes: list[str] = []
 
     run_pre_upgrade_tests: str | None = config.getoption(name="pre_upgrade")
     run_post_upgrade_tests: str | None = config.getoption(name="post_upgrade")
+    if config_upgrade_deployment_modes := config.getoption(name="upgrade_deployment_modes"):
+        upgrade_deployment_modes = config_upgrade_deployment_modes.split(",")
 
     for item in items:
-        if "pre_upgrade" in item.keywords:
+        if "pre_upgrade" in item.keywords and _add_upgrade_test(
+            _item=item, _upgrade_deployment_modes=upgrade_deployment_modes
+        ):
             pre_upgrade_tests.append(item)
 
-        elif "post_upgrade" in item.keywords:
+        elif "post_upgrade" in item.keywords and _add_upgrade_test(
+            _item=item, _upgrade_deployment_modes=upgrade_deployment_modes
+        ):
             post_upgrade_tests.append(item)
 
         else:

tests/model_serving/model_server/upgrade/README.md

@@ -30,3 +30,13 @@ uv run pytest --post-upgrade
 ```bash
 uv run pytest --pre-upgrade --post-upgrade
 ```
+
+## To run only specific deployment tests, pass --upgrade-deployment-modes with requested mode(s), for example:
+
+```bash
+uv run pytest --pre-upgrade --post-upgrade --upgrade-deployment-modes=servelerss
+```
+
+```bash
+uv run pytest --pre-upgrade --post-upgrade --upgrade-deployment-modes=servelerss,rawdeployment
+```

tests/model_serving/model_server/upgrade/test_upgrade.py

@@ -7,15 +7,14 @@
 from utilities.manifests.onnx import ONNX_INFERENCE_CONFIG
 from utilities.manifests.openvino import OPENVINO_INFERENCE_CONFIG
 
-pytestmark = [pytest.mark.serverless, pytest.mark.rawdeployment, pytest.mark.modelmesh]
-
 
 # TODO: add auth, external route and grpc tests
 
 
 @pytest.mark.usefixtures("valid_aws_config", "skipped_teardown_resources")
 class TestPreUpgradeModelServer:
     @pytest.mark.pre_upgrade
+    @pytest.mark.serverless
     def test_serverless_onnx_pre_upgrade_inference(self, ovms_serverless_inference_service_scope_session):
         """Verify that kserve Serverless ONNX model can be queried using REST before upgrade"""
         verify_inference_response(
@@ -27,6 +26,7 @@ def test_serverless_onnx_pre_upgrade_inference(self, ovms_serverless_inference_s
         )
 
     @pytest.mark.pre_upgrade
+    @pytest.mark.rawdeployment
     def test_raw_caikit_bge_pre_upgrade_inference(self, caikit_raw_inference_service_scope_session):
         """Test Caikit bge-large-en embedding model inference using internal route before upgrade"""
         verify_inference_response(
@@ -39,6 +39,7 @@ def test_raw_caikit_bge_pre_upgrade_inference(self, caikit_raw_inference_service
         )
 
     @pytest.mark.pre_upgrade
+    @pytest.mark.modelmesh
     def test_model_mesh_openvino_pre_upgrade_inference(self, openvino_model_mesh_inference_service_scope_session):
         """Test OpenVINO ModelMesh inference with internal route before upgrade"""
         verify_inference_response(
@@ -53,6 +54,7 @@ def test_model_mesh_openvino_pre_upgrade_inference(self, openvino_model_mesh_inf
 @pytest.mark.usefixtures("reused_resources")
 class TestPostUpgradeModelServer:
     @pytest.mark.post_upgrade
+    @pytest.mark.serverless
     @pytest.mark.dependency(name="test_serverless_onnx_post_upgrade_inference_service_exists")
     def test_serverless_onnx_post_upgrade_inference_service_exists(
         self, ovms_serverless_inference_service_scope_session
@@ -61,6 +63,7 @@ def test_serverless_onnx_post_upgrade_inference_service_exists(
         assert ovms_serverless_inference_service_scope_session.exists
 
     @pytest.mark.post_upgrade
+    @pytest.mark.serverless
     @pytest.mark.dependency(depends=["test_serverless_onnx_post_upgrade_inference_service_exists"])
     def test_serverless_onnx_post_upgrade_inference(self, ovms_serverless_inference_service_scope_session):
         """Verify that kserve Serverless ONNX model can be queried using REST after upgrade"""
@@ -73,12 +76,14 @@ def test_serverless_onnx_post_upgrade_inference(self, ovms_serverless_inference_
         )
 
     @pytest.mark.post_upgrade
+    @pytest.mark.rawdeployment
     @pytest.mark.dependency(name="test_raw_caikit_bge_post_upgrade_inference_exists")
     def test_raw_caikit_bge_post_upgrade_inference_exists(self, caikit_raw_inference_service_scope_session):
         """Test that raw deployment inference service exists after upgrade"""
         assert caikit_raw_inference_service_scope_session.exists
 
     @pytest.mark.post_upgrade
+    @pytest.mark.rawdeployment
     @pytest.mark.dependency(depends=["test_raw_caikit_bge_post_upgrade_inference_exists"])
     def test_raw_caikit_bge_post_upgrade_inference(self, caikit_raw_inference_service_scope_session):
         """Test Caikit bge-large-en embedding model inference using internal route after upgrade"""
@@ -92,6 +97,7 @@ def test_raw_caikit_bge_post_upgrade_inference(self, caikit_raw_inference_servic
         )
 
     @pytest.mark.post_upgrade
+    @pytest.mark.modelmesh
     @pytest.mark.dependency(name="test_model_mesh_openvino_post_upgrade_inference_exists")
     def test_model_mesh_openvino_post_upgrade_inference_exists(
         self, openvino_model_mesh_inference_service_scope_session
@@ -100,6 +106,7 @@ def test_model_mesh_openvino_post_upgrade_inference_exists(
         assert openvino_model_mesh_inference_service_scope_session.exists
 
     @pytest.mark.post_upgrade
+    @pytest.mark.modelmesh
     @pytest.mark.dependency(depends=["test_model_mesh_openvino_post_upgrade_inference_exists"])
     def test_model_mesh_openvino_post_upgrade_inference(self, openvino_model_mesh_inference_service_scope_session):
         """Test OpenVINO ModelMesh inference with internal route after upgrade"""