Cherry-pick serverles ns deletion and distribution auto-detection

conftest.py

@@ -6,7 +6,15 @@
 import shutil
 
 import shortuuid
-from pytest import Parser, Session, FixtureRequest, FixtureDef, Item, Config, CollectReport
+from pytest import (
+    Parser,
+    Session,
+    FixtureRequest,
+    FixtureDef,
+    Item,
+    Config,
+    CollectReport,
+)
 from _pytest.terminal import TerminalReporter
 from typing import Optional, Any
 from pytest_testconfig import config as py_config
@@ -24,6 +32,7 @@ def pytest_addoption(parser: Parser) -> None:
     buckets_group = parser.getgroup(name="Buckets")
     runtime_group = parser.getgroup(name="Runtime details")
     upgrade_group = parser.getgroup(name="Upgrade options")
+    platform_group = parser.getgroup(name="Platform")
 
     # AWS config and credentials options
     aws_group.addoption(
@@ -91,6 +100,17 @@ def pytest_addoption(parser: Parser) -> None:
         action="store_true",
         help="Delete pre-upgrade resources; useful when debugging pre-upgrade tests",
     )
+    upgrade_group.addoption(
+        "--upgrade-deployment-modes",
+        help="Coma-separated str; specify inference service deployment modes tests to run in upgrade tests. "
+        "If not set, all will be tested.",
+    )
+
+    # Platform options
+    platform_group.addoption(
+        "--applications-namespace",
+        help="RHOAI/ODH applications namespace",
+    )
 
 
 def pytest_cmdline_main(config: Any) -> None:
@@ -102,19 +122,45 @@ def pytest_collection_modifyitems(session: Session, config: Config, items: list[
     Pytest fixture to filter or re-order the items in-place.
 
     Filters upgrade tests based on '--pre-upgrade' / '--post-upgrade' option and marker.
+    If `--upgrade-deployment-modes` option is set, only tests with the specified deployment modes will be added.
     """
+
+    def _add_upgrade_test(_item: Item, _upgrade_deployment_modes: list[str]) -> bool:
+        """
+        Add upgrade test to the list of tests to run.
+
+        Args:
+            _item (Item): The test item.
+            _upgrade_deployment_modes (list[str]): The deployment modes to test.
+
+        Returns:
+            True if the test should be added, False otherwise.
+
+        """
+        if not _upgrade_deployment_modes:
+            return True
+
+        return any([keyword for keyword in _item.keywords if keyword in _upgrade_deployment_modes])
+
     pre_upgrade_tests: list[Item] = []
     post_upgrade_tests: list[Item] = []
     non_upgrade_tests: list[Item] = []
+    upgrade_deployment_modes: list[str] = []
 
     run_pre_upgrade_tests: str | None = config.getoption(name="pre_upgrade")
     run_post_upgrade_tests: str | None = config.getoption(name="post_upgrade")
+    if config_upgrade_deployment_modes := config.getoption(name="upgrade_deployment_modes"):
+        upgrade_deployment_modes = config_upgrade_deployment_modes.split(",")
 
     for item in items:
-        if "pre_upgrade" in item.keywords:
+        if "pre_upgrade" in item.keywords and _add_upgrade_test(
+            _item=item, _upgrade_deployment_modes=upgrade_deployment_modes
+        ):
             pre_upgrade_tests.append(item)
 
-        elif "post_upgrade" in item.keywords:
+        elif "post_upgrade" in item.keywords and _add_upgrade_test(
+            _item=item, _upgrade_deployment_modes=upgrade_deployment_modes
+        ):
             post_upgrade_tests.append(item)
 
         else:
@@ -149,9 +195,6 @@ def pytest_sessionstart(session: Session) -> None:
         log_level=session.config.getoption("log_cli_level") or logging.INFO,
     )
 
-    if py_config.get("distribution") == "upstream":
-        py_config["applications_namespace"] = "opendatahub"
-
 
 def pytest_fixture_setup(fixturedef: FixtureDef[Any], request: FixtureRequest) -> None:
     LOGGER.info(f"Executing {fixturedef.scope} fixture: {fixturedef.argname}")
@@ -160,8 +203,9 @@ def pytest_fixture_setup(fixturedef: FixtureDef[Any], request: FixtureRequest) -
 def pytest_runtest_setup(item: Item) -> None:
     """
     Performs the following actions:
-    1. Adds skip fixture for kserve if serverless or authorino operators are not installed.
-    2. Adds skip fixture for serverless if authorino/serverless/service mesh are not deployed.
+    1. Updates global config (`updated_global_config`)
+    2. Adds skip fixture for kserve if serverless or authorino operators are not installed.
+    3. Adds skip fixture for serverless if authorino/serverless/service mesh are not deployed.
     """
 
     BASIC_LOGGER.info(f"\n{separator(symbol_='-', val=item.name)}")
@@ -179,6 +223,9 @@ def pytest_runtest_setup(item: Item) -> None:
     elif KServeDeploymentType.MODEL_MESH.lower() in item.keywords:
         item.fixturenames.insert(0, "enabled_modelmesh_in_dsc")
 
+    # The above fixtures require the global config to be updated before being called
+    item.fixturenames.insert(0, "updated_global_config")
+
 
 def pytest_runtest_call(item: Item) -> None:
     BASIC_LOGGER.info(f"{separator(symbol_='-', val='CALL')}")

tests/conftest.py

@@ -21,7 +21,13 @@
 from simple_logger.logger import get_logger
 
 from utilities.data_science_cluster_utils import update_components_in_dsc
-from utilities.infra import create_ns, login_with_user_password, get_openshift_token
+from utilities.infra import (
+    create_ns,
+    get_dsci_applications_namespace,
+    get_operator_distribution,
+    login_with_user_password,
+    get_openshift_token,
+)
 from utilities.constants import AcceleratorType, DscComponents
 from utilities.infra import update_configmap_data
 
@@ -45,20 +51,32 @@ def tests_tmp_dir(request: FixtureRequest, tmp_path_factory: TempPathFactory) ->
     shutil.rmtree(path=str(tests_tmp_path), ignore_errors=True)
 
 
+@pytest.fixture(scope="session")
+def updated_global_config(request: FixtureRequest, admin_client: DynamicClient) -> None:
+    if get_operator_distribution(client=admin_client) == "Open Data Hub":
+        py_config["distribution"] = "upstream"
+
+    else:
+        py_config["distribution"] = "downstream"
+
+    if applications_namespace := request.config.getoption("applications_namespace"):
+        py_config["applications_namespace"] = applications_namespace
+
+    else:
+        py_config["applications_namespace"] = get_dsci_applications_namespace(client=admin_client)
+
+
 @pytest.fixture(scope="session")
 def current_client_token(admin_client: DynamicClient) -> str:
     return get_openshift_token()
 
 
 @pytest.fixture(scope="class")
 def model_namespace(request: FixtureRequest, admin_client: DynamicClient) -> Generator[Namespace, Any, Any]:
-    ns_kwargs = {"admin_client": admin_client, "name": request.param["name"]}
-
     if request.param.get("modelmesh-enabled"):
         request.getfixturevalue(argname="enabled_modelmesh_in_dsc")
-        ns_kwargs["labels"] = {"modelmesh-enabled": "true"}
 
-    with create_ns(**ns_kwargs) as ns:
+    with create_ns(admin_client=admin_client, pytest_request=request) as ns:
         yield ns
 
 
@@ -275,6 +293,17 @@ def enabled_modelmesh_in_dsc(dsc_resource: DataScienceCluster) -> Generator[Data
         yield dsc
 
 
+@pytest.fixture(scope="package")
+def enabled_kserve_in_dsc(
+    dsc_resource: DataScienceCluster,
+) -> Generator[DataScienceCluster, Any, Any]:
+    with update_components_in_dsc(
+        dsc=dsc_resource,
+        components={DscComponents.KSERVE: DscComponents.ManagementState.MANAGED},
+    ) as dsc:
+        yield dsc
+
+
 @pytest.fixture(scope="session")
 def cluster_monitoring_config(admin_client: DynamicClient) -> Generator[ConfigMap, Any, Any]:
     data = {"config.yaml": yaml.dump({"enableUserWorkload": True})}

tests/model_serving/model_server/conftest.py

@@ -7,7 +7,6 @@
 from ocp_resources.authorino import Authorino
 from ocp_resources.cluster_service_version import ClusterServiceVersion
 from ocp_resources.config_map import ConfigMap
-from ocp_resources.data_science_cluster import DataScienceCluster
 from ocp_resources.inference_service import InferenceService
 from ocp_resources.namespace import Namespace
 from ocp_resources.persistent_volume_claim import PersistentVolumeClaim
@@ -19,7 +18,7 @@
 from ocp_utilities.monitoring import Prometheus
 from pytest_testconfig import config as py_config
 
-from utilities.constants import DscComponents, StorageClassName
+from utilities.constants import StorageClassName
 from utilities.constants import (
     KServeDeploymentType,
     ModelFormat,
@@ -37,7 +36,6 @@
     s3_endpoint_secret,
     update_configmap_data,
 )
-from utilities.data_science_cluster_utils import update_components_in_dsc
 from utilities.serving_runtime import ServingRuntimeFromTemplate
 
 
@@ -197,17 +195,6 @@ def skip_if_no_deployed_redhat_authorino_operator(admin_client: DynamicClient) -
         pytest.skip(f"Authorino {name} CR is missing from {namespace} namespace")
 
 
-@pytest.fixture(scope="package")
-def enabled_kserve_in_dsc(
-    dsc_resource: DataScienceCluster,
-) -> Generator[DataScienceCluster, Any, Any]:
-    with update_components_in_dsc(
-        dsc=dsc_resource,
-        components={DscComponents.KSERVE: DscComponents.ManagementState.MANAGED},
-    ) as dsc:
-        yield dsc
-
-
 @pytest.fixture(scope="package")
 def skip_if_no_deployed_openshift_service_mesh(admin_client: DynamicClient) -> None:
     smcp = ServiceMeshControlPlane(client=admin_client, name="data-science-smcp", namespace="istio-system")