feat: fail on missing operators (opendatahub-io#257)

* fix: update tests

* fix: update tests

* feat: fail on missing operators

* fix: rename to dependent

* fix: address comment

* fix: add log on failure

* fix: type in raise

* fix: remove MR check

* fix: remove MR check

* fix: use package scope

Author: rnetser

conftest.py

@@ -245,8 +245,8 @@ def pytest_runtest_setup(item: Item) -> None:
     """
     Performs the following actions:
     1. Updates global config (`updated_global_config`)
-    2. Adds skip fixture for kserve if serverless or authorino operators are not installed.
-    3. Adds skip fixture for serverless if authorino/serverless/service mesh are not deployed.
+    2. Adds `fail_if_missing_dependent_operators` fixture for Serverless tests.
+    3. Adds fixtures to enable KServe/model mesh in DSC for model server tests.
     """
     BASIC_LOGGER.info(f"\n{separator(symbol_='-', val=item.name)}")
     BASIC_LOGGER.info(f"{separator(symbol_='-', val='SETUP')}")
@@ -268,9 +268,9 @@ def pytest_runtest_setup(item: Item) -> None:
             LOGGER.error(f"Database error: {db_exception}. Must-gather collection may not be accurate")
 
     if KServeDeploymentType.SERVERLESS.lower() in item.keywords:
-        item.fixturenames.insert(0, "skip_if_no_deployed_redhat_authorino_operator")
-        item.fixturenames.insert(0, "skip_if_no_deployed_openshift_serverless")
-        item.fixturenames.insert(0, "skip_if_no_deployed_openshift_service_mesh")
+        item.fixturenames.insert(0, "fail_if_missing_dependent_operators")
+
+    if KServeDeploymentType.SERVERLESS.lower() in item.keywords:
         item.fixturenames.insert(0, "enabled_kserve_in_dsc")
 
     elif KServeDeploymentType.RAW_DEPLOYMENT.lower() in item.keywords:

docs/GETTING_STARTED.md

@@ -67,6 +67,13 @@ By default, cluster sanity checks are run to make cluster ready for tests.
 To skip cluster sanity checks, pass `--cluster-sanity-skip-check` to skip all tests.
 To skip RHOAI/ODH-related tests (for example when running in upstream), pass `--cluster-sanity-skip-rhoai-check`.
 
+### Check dependent operators
+By default, `Serveless` (serverless-operator), `Authorino` (authorino-operator) and `Service Mesh` (servicemeshoperator) operators are checked to be installed when  
+running model server Serverless tests.
+To check only specific operator, pass `--tc=dependent_operators:<operator_name>` to pytest.
+For example, to check only `Serveless` and `Service Mesh` operators, pass `--tc=dependent_operators:serverless-operator,servicemeshoperator`.
+
+
 ### jira integration
 To skip running tests which have open bugs, [pytest_jira](https://github.com/rhevm-qe-automation/pytest_jira) plugin is used.
 To run tests with jira integration, you need to set `PYTEST_JIRA_URL` and `PYTEST_JIRA_TOKEN` environment variables.

tests/model_serving/model_server/conftest.py

@@ -4,19 +4,18 @@
 import yaml
 from _pytest.fixtures import FixtureRequest
 from kubernetes.dynamic import DynamicClient
-from ocp_resources.authorino import Authorino
 from ocp_resources.cluster_service_version import ClusterServiceVersion
 from ocp_resources.config_map import ConfigMap
 from ocp_resources.inference_service import InferenceService
 from ocp_resources.namespace import Namespace
 from ocp_resources.persistent_volume_claim import PersistentVolumeClaim
 from ocp_resources.secret import Secret
 from ocp_resources.service_account import ServiceAccount
-from ocp_resources.service_mesh_control_plane import ServiceMeshControlPlane
 from ocp_resources.serving_runtime import ServingRuntime
 from ocp_resources.storage_class import StorageClass
 from ocp_utilities.monitoring import Prometheus
 from pytest_testconfig import config as py_config
+from simple_logger.logger import get_logger
 
 from utilities.constants import (
     KServeDeploymentType,
@@ -40,23 +39,7 @@
 from utilities.serving_runtime import ServingRuntimeFromTemplate
 
 
-@pytest.fixture(scope="package")
-def skip_if_no_deployed_openshift_serverless(admin_client: DynamicClient) -> None:
-    name = "openshift-serverless"
-    csvs = list(
-        ClusterServiceVersion.get(
-            client=admin_client,
-            namespace=name,
-            label_selector=f"operators.coreos.com/serverless-operator.{name}",
-        )
-    )
-    if not csvs:
-        pytest.skip("OpenShift Serverless is not deployed")
-
-    csv = csvs[0]
-
-    if not (csv.exists and csv.status == csv.Status.SUCCEEDED):
-        pytest.skip("OpenShift Serverless is not deployed")
+LOGGER = get_logger(name=__name__)
 
 
 @pytest.fixture(scope="class")
@@ -189,28 +172,6 @@ def skip_if_no_nfs_storage_class(admin_client: DynamicClient) -> None:
         pytest.skip(f"StorageClass {StorageClassName.NFS} is missing from the cluster")
 
 
-@pytest.fixture(scope="package")
-def skip_if_no_deployed_redhat_authorino_operator(admin_client: DynamicClient) -> None:
-    name = "authorino"
-    namespace = f"{py_config['applications_namespace']}-auth-provider"
-
-    if not Authorino(
-        client=admin_client,
-        name=name,
-        namespace=namespace,
-    ).exists:
-        pytest.skip(f"Authorino {name} CR is missing from {namespace} namespace")
-
-
-@pytest.fixture(scope="package")
-def skip_if_no_deployed_openshift_service_mesh(admin_client: DynamicClient) -> None:
-    smcp = ServiceMeshControlPlane(client=admin_client, name="data-science-smcp", namespace="istio-system")
-    if not smcp or not smcp.exists:
-        pytest.skip("OpenShift service mesh operator is not deployed")
-
-    smcp.wait_for_condition(condition=smcp.Condition.READY, status="True")
-
-
 @pytest.fixture(scope="class")
 def http_s3_openvino_model_mesh_inference_service(
     request: FixtureRequest,
@@ -608,3 +569,32 @@ def unprivileged_s3_caikit_serverless_inference_service(
         storage_path=request.param["model-dir"],
     ) as isvc:
         yield isvc
+
+
+@pytest.fixture(scope="package")
+def fail_if_missing_dependent_operators(admin_client: DynamicClient) -> None:
+    missing_operators: list[str] = []
+    csvs = list(
+        ClusterServiceVersion.get(
+            dyn_client=admin_client,
+            namespace=py_config["applications_namespace"],
+        )
+    )
+
+    for operator_name in py_config.get("dependent_operators", []).split(","):
+        LOGGER.info(f"Verifying if {operator_name} is installed")
+        for csv in csvs:
+            if csv.name.startswith(operator_name):
+                if csv.status == csv.Status.SUCCEEDED:
+                    break
+
+                else:
+                    missing_operators.append(
+                        f"Operator {operator_name} is installed but CSV is not in {csv.Status.SUCCEEDED} state"
+                    )
+
+        else:
+            missing_operators.append(f"{operator_name} is not installed")
+
+    if missing_operators:
+        pytest.fail(str(missing_operators))