feat: add upstream and odh support

Author: rnetser

conftest.py

@@ -19,7 +19,7 @@
 from typing import Optional, Any
 from pytest_testconfig import config as py_config
 
-from utilities.constants import KServeDeploymentType
+from utilities.constants import Distributions, KServeDeploymentType
 from utilities.logger import separator, setup_logging
 
 
@@ -160,7 +160,32 @@ def _add_upgrade_test(_item: Item, _upgrade_deployment_modes: list[str]) -> bool
     if config_upgrade_deployment_modes := config.getoption(name="upgrade_deployment_modes"):
         upgrade_deployment_modes = config_upgrade_deployment_modes.split(",")
 
+    # Check if odh or upstream marker was passed from invocation command
+    skip_upstream = True
+    skip_odh = True
+
+    if user_markers_index := next(
+        (index for index, elem in enumerate(config.invocation_params.args) if elem == "-m"),
+        None,
+    ):
+        user_markers = config.invocation_params.args[user_markers_index + 1]
+        if Distributions.ODH in user_markers:
+            skip_odh = False
+
+        if Distributions.UPSTREAM in user_markers:
+            skip_upstream = False
+
     for item in items:
+        # Filter out `upstream` and `odh` tests if were not explicitly requested
+        if Distributions.UPSTREAM in item.keywords and skip_upstream:
+            items.remove(item)
+            continue
+
+        if Distributions.ODH in item.keywords and skip_odh:
+            items.remove(item)
+            continue
+
+        # Upgrade tests filtering
         if "pre_upgrade" in item.keywords and _add_upgrade_test(
             _item=item, _upgrade_deployment_modes=upgrade_deployment_modes
         ):

docs/DEVELOPER_GUIDE.md

@@ -143,6 +143,13 @@ Check [pytest.ini](../pytest.ini) for available markers; additional markers can
 You should NOT group unrelated tests in one class (because it is misleading the reader).
 
 
+## Markers
+Do not forget to add the relevant marker(s) to your test(s)
+Check [pytest.ini](../pytest.ini) for available markers; additional markers can always be added when needed.
+For features which are only supported in upstream, add the `upstream` marker.  Tests that are marked as `upstream` will be skipped by default.
+For features which are only supported in ODH (`midstream`, add the `odh` marker.  Tests that are marked as `odh` will be skipped by default.
+
+
 ## Check the code
 ### pre-commit
 

pytest.ini

@@ -7,6 +7,10 @@ markers =
     polarion: Store polarion test ID
     jira: Store jira bug ID
 
+    # Distributions
+    upstream: Mark tests as ones suitable to run only upstream.
+    odh: Mark tests as ones suitable to run in ODH deployment (AKA `midstream`).
+
     # CI
     smoke: Mark tests as smoke tests; covers core functionality of the product. Aims to ensure that the build is stable enough for further testing.
     sanity: Mark tests as sanity tests. Aims to verify that specific functionality is working as expected.

utilities/constants.py

@@ -253,6 +253,11 @@ class RunTimeConfig:
         IMAGE = "quay.io/opendatahub/openvino_model_server@sha256:564664371d3a21b9e732a5c1b4b40bacad714a5144c0a9aaf675baec4a04b148"  # noqa: E501
 
 
+class Distributions:
+    UPSTREAM: str = "upstream"
+    ODH: str = "odh"
+
+
 MODEL_REGISTRY: str = "model-registry"
 MODELMESH_SERVING: str = "modelmesh-serving"
 ISTIO_CA_BUNDLE_FILENAME: str = "istio_knative.crt"