feat: move to marking test branches

conftest.py

@@ -3,9 +3,11 @@
 import logging
 import os
 import pathlib
+import re
 import shutil
 
 import shortuuid
+from ocp_resources.resource import get_client
 from pytest import (
     Parser,
     Session,
@@ -18,8 +20,10 @@
 from _pytest.terminal import TerminalReporter
 from typing import Optional, Any
 from pytest_testconfig import config as py_config
+from semver import Version
 
 from utilities.constants import KServeDeploymentType
+from utilities.infra import get_product_version
 from utilities.logger import separator, setup_logging
 
 
@@ -154,13 +158,22 @@ def _add_upgrade_test(_item: Item, _upgrade_deployment_modes: list[str]) -> bool
     post_upgrade_tests: list[Item] = []
     non_upgrade_tests: list[Item] = []
     upgrade_deployment_modes: list[str] = []
+    formatted_product_version: Version | None = None
 
     run_pre_upgrade_tests: str | None = config.getoption(name="pre_upgrade")
     run_post_upgrade_tests: str | None = config.getoption(name="post_upgrade")
     if config_upgrade_deployment_modes := config.getoption(name="upgrade_deployment_modes"):
         upgrade_deployment_modes = config_upgrade_deployment_modes.split(",")
 
+    if not session.config.option.setupplan and not session.config.option.collectonly:
+        if product_version := get_product_version(admin_client=get_client(), raise_on_missing_csv=False):
+            formatted_product_version = Version.parse(
+                f"{product_version.major}.{product_version.minor}",
+                optional_minor_and_patch=True,
+            )
+
     for item in items:
+        # Upgrade tests filtering
         if "pre_upgrade" in item.keywords and _add_upgrade_test(
             _item=item, _upgrade_deployment_modes=upgrade_deployment_modes
         ):
@@ -174,6 +187,24 @@ def _add_upgrade_test(_item: Item, _upgrade_deployment_modes: list[str]) -> bool
         else:
             non_upgrade_tests.append(item)
 
+        try:
+            # Product version filtering - if version marker < product version, remove test from list
+            if formatted_product_version and (
+                product_marker := next(marker for marker in item.keywords if re.match(r"rhoai_\d+_\d+", marker))
+            ):
+                marker_major, marker_minor = product_marker.split("_")[1:]
+                test_version = Version.parse(f"{marker_major}.{marker_minor}", optional_minor_and_patch=True)
+                if test_version > formatted_product_version:
+                    LOGGER.info(
+                        f"Test {item.name} is not compatible with RHOAI version {formatted_product_version}; "
+                        "it will be skipped"
+                    )
+                    non_upgrade_tests.remove(item)
+
+        # TODO: Raise once all teams added a marker; need to make sure that we do not have tests without product release
+        except StopIteration:
+            LOGGER.error(f"Test {item.name} is missing RHOAI version marker")
+
     upgrade_tests = pre_upgrade_tests + post_upgrade_tests
 
     if run_pre_upgrade_tests and run_post_upgrade_tests:

docs/DEVELOPER_GUIDE.md

@@ -41,7 +41,7 @@ To contribute code to the project:
 
 ## Branching strategy
 The project follows RHOAI [release lifecyle strategy](https://access.redhat.com/support/policy/updates/rhoai-sm/lifecycle).  
-If needed, once your PR is merged to `main`, cherry-pick your PR to the relevant branch(es).
+Tests should be marked with the release they're tested against.
 
 
 ## Python
@@ -143,6 +143,13 @@ Check [pytest.ini](../pytest.ini) for available markers; additional markers can
 You should NOT group unrelated tests in one class (because it is misleading the reader).
 
 
+## Markers
+Do not forget to add the relevant marker(s) to your test(s)
+Check [pytest.ini](../pytest.ini) for available markers; additional markers can always be added when needed.
+For features which are only supported in upstream, add the `upstream` marker.
+For features which are only supported in ODH (`midstream`, add the `odh` marker.
+For features which are only supported from a specific RHOAI release, add the `rhoai_x_y` marker.  For example: `rhoai_2_19`
+
 ## Check the code
 ### pre-commit
 

pytest.ini

@@ -7,6 +7,14 @@ markers =
     polarion: Store polarion test ID
     jira: Store jira bug ID
 
+    # Distributions and supported downstream markers
+    upstream: Mark tests as ones suitable to run only upstream.
+    odh: Mark tests as ones suitable to run in ODH deployment (AKA `midstream`.
+    rhoai_2_16: Mark tests supported in RHOAI 2.16 (AKA `downstream`).
+    rhoai_2_17: Mark tests supported in RHOAI 2.16 (AKA `downstream`).
+    rhoai_2_18: Mark tests supported in RHOAI 2.16 (AKA `downstream`).
+    rhoai_2_19: Mark tests supported in RHOAI 2.19 (AKA `downstream`).
+
     # CI
     smoke: Mark tests as smoke tests; covers core functionality of the product. Aims to ensure that the build is stable enough for further testing.
     sanity: Mark tests as sanity tests. Aims to verify that specific functionality is working as expected.

tests/model_serving/model_server/authentication/test_kserve_token_authentication_raw.py

@@ -25,6 +25,7 @@
 )
 class TestKserveTokenAuthenticationRawForRest:
     @pytest.mark.smoke
+    @pytest.mark.rhoai_2_16
     @pytest.mark.dependency(name="test_model_authentication_using_rest_raw")
     def test_model_authentication_using_rest_raw(self, http_s3_caikit_raw_inference_service, http_raw_inference_token):
         """Verify RAW Kserve model query with token using REST"""
@@ -38,6 +39,7 @@ def test_model_authentication_using_rest_raw(self, http_s3_caikit_raw_inference_
             token=http_raw_inference_token,
         )
 
+    @pytest.mark.rhoai_2_16
     @pytest.mark.dependency(name="test_disabled_raw_model_authentication")
     def test_disabled_raw_model_authentication(self, patched_remove_raw_authentication_isvc):
         """Verify model query after authentication is disabled"""
@@ -51,6 +53,7 @@ def test_disabled_raw_model_authentication(self, patched_remove_raw_authenticati
         )
 
     @pytest.mark.sanity
+    @pytest.mark.rhoai_2_17
     @pytest.mark.jira("RHOAIENG-19275", run=False)
     def test_raw_disable_enable_authentication_no_pod_rollout(self, http_s3_caikit_raw_inference_service):
         """Verify no pod rollout when disabling and enabling authentication"""
@@ -83,6 +86,7 @@ def test_raw_disable_enable_authentication_no_pod_rollout(self, http_s3_caikit_r
 
         check_pod_status_in_time(pod=pod, status={pod.Status.RUNNING})
 
+    @pytest.mark.rhoai_2_17
     @pytest.mark.dependency(depends=["test_disabled_raw_model_authentication"])
     def test_re_enabled_raw_model_authentication(self, http_s3_caikit_raw_inference_service, http_raw_inference_token):
         """Verify model query after authentication is re-enabled"""
@@ -96,6 +100,7 @@ def test_re_enabled_raw_model_authentication(self, http_s3_caikit_raw_inference_
             token=http_raw_inference_token,
         )
 
+    @pytest.mark.rhoai_2_17
     @pytest.mark.dependency(name="test_cross_model_authentication_raw")
     def test_cross_model_authentication_raw(
         self, http_s3_caikit_raw_inference_service_2, http_raw_inference_token, admin_client

tests/model_serving/model_server/authentication/test_kserve_token_authentication_serverless.py

@@ -22,6 +22,7 @@
 )
 class TestKserveServerlessTokenAuthentication:
     @pytest.mark.smoke
+    @pytest.mark.rhoai_2_16
     @pytest.mark.dependency(name="test_model_authentication_using_rest")
     def test_model_authentication_using_rest(self, http_s3_caikit_serverless_inference_service, http_inference_token):
         """Verify model query with token using REST"""
@@ -36,6 +37,7 @@ def test_model_authentication_using_rest(self, http_s3_caikit_serverless_inferen
         )
 
     @pytest.mark.smoke
+    @pytest.mark.rhoai_2_16
     def test_model_authentication_using_grpc(self, grpc_s3_inference_service, grpc_inference_token):
         """Verify model query with token using GRPC"""
         verify_inference_response(
@@ -48,6 +50,7 @@ def test_model_authentication_using_grpc(self, grpc_s3_inference_service, grpc_i
             token=grpc_inference_token,
         )
 
+    @pytest.mark.rhoai_2_16
     @pytest.mark.dependency(name="test_disabled_model_authentication")
     def test_disabled_model_authentication(self, patched_remove_authentication_isvc):
         """Verify model query after authentication is disabled"""
@@ -60,6 +63,7 @@ def test_disabled_model_authentication(self, patched_remove_authentication_isvc)
             use_default_query=True,
         )
 
+    @pytest.mark.rhoai_2_16
     @pytest.mark.dependency(depends=["test_disabled_model_authentication"])
     def test_re_enabled_model_authentication(self, http_s3_caikit_serverless_inference_service, http_inference_token):
         """Verify model query after authentication is re-enabled"""
@@ -73,6 +77,7 @@ def test_re_enabled_model_authentication(self, http_s3_caikit_serverless_inferen
             token=http_inference_token,
         )
 
+    @pytest.mark.rhoai_2_16
     def test_model_authentication_using_invalid_token(self, http_s3_caikit_serverless_inference_service):
         """Verify model query with an invalid token"""
         verify_inference_response(
@@ -86,6 +91,7 @@ def test_model_authentication_using_invalid_token(self, http_s3_caikit_serverles
             authorized_user=False,
         )
 
+    @pytest.mark.rhoai_2_16
     def test_model_authentication_without_token(self, http_s3_caikit_serverless_inference_service):
         """Verify model query without providing a token"""
         verify_inference_response(
@@ -99,6 +105,7 @@ def test_model_authentication_without_token(self, http_s3_caikit_serverless_infe
         )
 
     @pytest.mark.sanity
+    @pytest.mark.rhoai_2_16
     def test_block_cross_model_authentication(self, http_s3_caikit_serverless_inference_service, grpc_inference_token):
         """Verify model query with a second model's token is blocked"""
         verify_inference_response(
@@ -113,6 +120,7 @@ def test_block_cross_model_authentication(self, http_s3_caikit_serverless_infere
         )
 
     @pytest.mark.sanity
+    @pytest.mark.rhoai_2_17
     def test_serverless_disable_enable_authentication_no_pod_rollout(self, http_s3_caikit_serverless_inference_service):
         """Verify no pod rollout when disabling and enabling authentication"""
         pod = get_pods_by_isvc_label(

tests/model_serving/model_server/authentication/test_model_mesh_authentication.py

@@ -8,7 +8,7 @@
 from utilities.inference_utils import Inference
 from utilities.manifests.openvino import OPENVINO_INFERENCE_CONFIG
 
-pytestmark = [pytest.mark.modelmesh, pytest.mark.sanity]
+pytestmark = [pytest.mark.modelmesh, pytest.mark.sanity, pytest.mark.rhoai_2_16]
 
 
 @pytest.mark.parametrize(

tests/model_serving/model_server/authentication/test_non_admin_users.py

@@ -21,6 +21,7 @@
     indirect=True,
 )
 @pytest.mark.smoke
+@pytest.mark.rhoai_2_16
 @pytest.mark.serverless
 class TestServerlessUnprivilegedUser:
     @pytest.mark.polarion("ODS-2552")
@@ -46,6 +47,7 @@ def test_non_admin_deploy_serverless_and_query_model(self, unprivileged_s3_caiki
     ],
     indirect=True,
 )
+@pytest.mark.rhoai_2_16
 @pytest.mark.sanity
 @pytest.mark.rawdeployment
 class TestRawUnprivilegedUser:

tests/model_serving/model_server/components/kserve_dsc_deployment_mode/test_kserve_dsc_default_deployment_mode.py

@@ -14,7 +14,7 @@
 from utilities.inference_utils import Inference
 from utilities.manifests.openvino import OPENVINO_KSERVE_INFERENCE_CONFIG
 
-pytestmark = [pytest.mark.sanity]
+pytestmark = [pytest.mark.sanity, pytest.mark.rhoai_2_17]
 
 RUNTIME_PARAMS = {
     "runtime-name": ModelInferenceRuntime.OPENVINO_KSERVE_RUNTIME,

tests/model_serving/model_server/components/model_mesh_kserve_co_exist/test_model_mesh_kserve_components_co_exist.py

@@ -7,7 +7,7 @@
 TIMEOUT = 6 * 60
 
 
-pytestmark = pytest.mark.smoke
+pytestmark = [pytest.mark.smoke, pytest.mark.rhoai_2_16]
 
 
 @pytest.mark.usefixtures("managed_modelmesh_kserve_in_dsc")

tests/model_serving/model_server/components/model_mesh_kserve_co_exist/test_model_mesh_kserve_inference_co_exist.py

@@ -31,6 +31,7 @@
     indirect=True,
 )
 class TestOpenVINOServerlessModelMesh:
+    @pytest.mark.rhoai_2_16
     def test_serverless_openvino_created_before_model_mesh_ns_rest_inference(
         self, ovms_kserve_inference_service, http_s3_openvino_model_mesh_inference_service
     ):
@@ -43,6 +44,7 @@ def test_serverless_openvino_created_before_model_mesh_ns_rest_inference(
             use_default_query=True,
         )
 
+    @pytest.mark.rhoai_2_16
     def test_model_mesh_openvino_created_after_serverless_in_namespace_rest_inference(
         self, ovms_kserve_inference_service, http_s3_openvino_model_mesh_inference_service
     ):
@@ -70,6 +72,7 @@ def test_model_mesh_openvino_created_after_serverless_in_namespace_rest_inferenc
     indirect=True,
 )
 class TestOpenVINOModelMeshServerless:
+    @pytest.mark.rhoai_2_16
     def test_model_mesh_openvino_created_before_serverless_in_namespace_rest_inference(
         self, http_s3_openvino_model_mesh_inference_service, ovms_kserve_inference_service
     ):
@@ -82,6 +85,7 @@ def test_model_mesh_openvino_created_before_serverless_in_namespace_rest_inferen
             use_default_query=True,
         )
 
+    @pytest.mark.rhoai_2_16
     def test_serverless_openvino_created_after_model_mesh_ns_rest_inference(
         self, http_s3_openvino_model_mesh_inference_service, ovms_kserve_inference_service
     ):

tests/model_serving/model_server/components/raw_deployment_serverless_co_exist/test_raw_deployment_serverless_inference_co_exist.py

@@ -47,6 +47,7 @@
     indirect=True,
 )
 class TestServerlessRawInternalDeploymentInferenceCoExist:
+    @pytest.mark.rhoai_2_16
     def test_serverless_openvino_created_before_raw_internal_deployment_caikit_inference(
         self, ovms_kserve_inference_service, s3_models_inference_service
     ):
@@ -59,6 +60,7 @@ def test_serverless_openvino_created_before_raw_internal_deployment_caikit_infer
             use_default_query=True,
         )
 
+    @pytest.mark.rhoai_2_19
     def test_raw_internal_deployment_caikit_created_after_serverless_in_namespace_rest_inference(
         self,
         ovms_kserve_inference_service,
@@ -90,6 +92,7 @@ def test_raw_internal_deployment_caikit_created_after_serverless_in_namespace_re
     indirect=True,
 )
 class TestServerlessRawExternalDeploymentInferenceCoExist:
+    @pytest.mark.rhoai_2_16
     def test_serverless_openvino_created_before_raw_external_deployment_caikit_inference(
         self, ovms_kserve_inference_service, s3_models_inference_service
     ):
@@ -102,6 +105,7 @@ def test_serverless_openvino_created_before_raw_external_deployment_caikit_infer
             use_default_query=True,
         )
 
+    @pytest.mark.rhoai_2_16
     def test_raw_external_deployment_caikit_created_after_serverless_in_namespace_rest_inference(
         self,
         ovms_kserve_inference_service,
@@ -133,6 +137,7 @@ def test_raw_external_deployment_caikit_created_after_serverless_in_namespace_re
     indirect=True,
 )
 class TestRawInternalDeploymentServerlessInferenceCoExist:
+    @pytest.mark.rhoai_2_16
     def test_raw_internal_deployment_caikit_created_before_serverless_openvino_in_namespace_rest_inference(
         self,
         s3_models_inference_service,
@@ -148,6 +153,7 @@ def test_raw_internal_deployment_caikit_created_before_serverless_openvino_in_na
             use_default_query=True,
         )
 
+    @pytest.mark.rhoai_2_19
     def test_serverless_openvino_created_after_raw_internal_deployment_caikit_ns_rest_inference(
         self,
         s3_models_inference_service,
@@ -178,6 +184,7 @@ def test_serverless_openvino_created_after_raw_internal_deployment_caikit_ns_res
     indirect=True,
 )
 class TestRawExternalDeploymentServerlessInferenceCoExist:
+    @pytest.mark.rhoai_2_19
     def test_raw_external_deployment_caikit_created_before_serverless_openvino_in_namespace_rest_inference(
         self,
         s3_models_inference_service,
@@ -193,6 +200,7 @@ def test_raw_external_deployment_caikit_created_before_serverless_openvino_in_na
             use_default_query=True,
         )
 
+    @pytest.mark.rhoai_2_16
     def test_serverless_openvino_created_after_raw_external_deployment_caikit_ns_rest_inference(
         self, s3_models_inference_service, ovms_kserve_inference_service
     ):

tests/model_serving/model_server/components/test_custom_resources.py

@@ -13,7 +13,13 @@
 
 LOGGER = get_logger(name=__name__)
 
-pytestmark = [pytest.mark.serverless, pytest.mark.tier2, pytest.mark.slow, pytest.mark.usefixtures("valid_aws_config")]
+pytestmark = [
+    pytest.mark.serverless,
+    pytest.mark.tier2,
+    pytest.mark.slow,
+    pytest.mark.usefixtures("valid_aws_config"),
+    pytest.mark.rhoai_2_16,
+]
 
 
 def wait_for_isvc_model_status(isvc: InferenceService, target_model_state: str, transition_status: str) -> None:

tests/model_serving/model_server/components/test_model_serving_components.py

@@ -41,6 +41,7 @@ def component_deployment(
 
 
 @pytest.mark.smoke
+@pytest.mark.rhoai_2_16
 @pytest.mark.parametrize(
     "component_deployment",
     [

tests/model_serving/model_server/inference_service_configuration/test_isvc_env_vars_updates.py

@@ -9,7 +9,7 @@
 )
 from utilities.constants import KServeDeploymentType, RunTimeConfigs
 
-pytestmark = [pytest.mark.sanity, pytest.mark.usefixtures("valid_aws_config")]
+pytestmark = [pytest.mark.sanity, pytest.mark.usefixtures("valid_aws_config"), pytest.mark.rhoai_2_19]
 
 
 ISVC_ENV_VARS_CONFIG = {

tests/model_serving/model_server/inference_service_configuration/test_isvc_replicas_update.py

@@ -19,7 +19,7 @@
 
 LOGGER = get_logger(name=__name__)
 
-pytestmark = [pytest.mark.sanity, pytest.mark.usefixtures("valid_aws_config")]
+pytestmark = [pytest.mark.sanity, pytest.mark.usefixtures("valid_aws_config"), pytest.mark.rhoai_2_19]
 
 
 @pytest.mark.rawdeployment