Add Cluster sanity checks before test execution

conftest.py

@@ -31,6 +31,7 @@ def pytest_addoption(parser: Parser) -> None:
     runtime_group = parser.getgroup(name="Runtime details")
     upgrade_group = parser.getgroup(name="Upgrade options")
     platform_group = parser.getgroup(name="Platform")
+    cluster_sanity_group = parser.getgroup(name="ClusterSanity")
 
     # AWS config and credentials options
     aws_group.addoption(
@@ -118,6 +119,18 @@ def pytest_addoption(parser: Parser) -> None:
         help="RHOAI/ODH applications namespace",
     )
 
+    # Cluster sanity options
+    cluster_sanity_group.addoption(
+        "--cluster-sanity-skip-check",
+        help="Skip cluster_sanity check",
+        action="store_true",
+    )
+    cluster_sanity_group.addoption(
+        "--cluster-sanity-skip-rhoai-check",
+        help="Skip RHOAI/ODH-related resources (DSCI and DSC) checks",
+        action="store_true",
+    )
+
 
 def pytest_cmdline_main(config: Any) -> None:
     config.option.basetemp = py_config["tmp_base_dir"] = f"{config.option.basetemp}-{shortuuid.uuid()}"

docs/GETTING_STARTED.md

@@ -62,6 +62,11 @@ uv run pytest -k test_name
 Bt default, RHOAI distribution is set.  
 To run on ODH, pass `--tc=distribution:upstream` to pytest.
 
+### Skip cluster sanity checks
+By default, cluster sanity checks are run to make cluster ready for tests.
+To skip cluster sanity checks, pass `--cluster-sanity-skip-check` to skip all tests.
+To skip RHOAI/ODH-related tests (for example when running in upstream), pass `--cluster-sanity-skip-rhoai-check`.
+
 ### jira integration
 To skip running tests which have open bugs, [pytest_jira](https://github.com/rhevm-qe-automation/pytest_jira) plugin is used.
 To run tests with jira integration, you need to set `PYTEST_JIRA_URL` and `PYTEST_JIRA_TOKEN` environment variables.

pyproject.toml

@@ -65,6 +65,7 @@ dependencies = [
     "jira>=3.8.0",
     "openshift-python-wrapper>=11.0.38",
     "semver>=3.0.4",
+    "pytest-order>=1.3.0",
 ]
 
 [project.urls]

tests/conftest.py

@@ -1,20 +1,21 @@
 import base64
 import os
 import shutil
-from typing import Any, Generator
+from typing import Any, Callable, Generator
 
 import pytest
 import shortuuid
 import yaml
 from _pytest.tmpdir import TempPathFactory
 from ocp_resources.config_map import ConfigMap
+from ocp_resources.dsc_initialization import DSCInitialization
+from ocp_resources.node import Node
 from ocp_resources.pod import Pod
 from ocp_resources.secret import Secret
 from ocp_resources.service import Service
 from pyhelper_utils.shell import run_command
 from pytest import FixtureRequest, Config
 from kubernetes.dynamic import DynamicClient
-from kubernetes.dynamic.exceptions import ResourceNotFoundError
 from ocp_resources.data_science_cluster import DataScienceCluster
 from ocp_resources.namespace import Namespace
 from ocp_resources.resource import get_client
@@ -25,6 +26,7 @@
 from utilities.exceptions import ClusterLoginError
 from utilities.general import get_s3_secret_dict
 from utilities.infra import (
+    verify_cluster_sanity,
     create_ns,
     get_dsci_applications_namespace,
     get_operator_distribution,
@@ -277,12 +279,14 @@ def unprivileged_client(
             raise ClusterLoginError(user=non_admin_user_name)
 
 
+@pytest.fixture(scope="session")
+def dsci_resource(admin_client: DynamicClient) -> DSCInitialization:
+    return DSCInitialization(client=admin_client, name=py_config["dsci_name"], ensure_exists=True)
+
+
 @pytest.fixture(scope="session")
 def dsc_resource(admin_client: DynamicClient) -> DataScienceCluster:
-    name = py_config["dsc_name"]
-    for dsc in DataScienceCluster.get(dyn_client=admin_client, name=name):
-        return dsc
-    raise ResourceNotFoundError(f"DSC resource {name} not found")
+    return DataScienceCluster(client=admin_client, name=py_config["dsc_name"], ensure_exists=True)
 
 
 @pytest.fixture(scope="module")
@@ -444,3 +448,33 @@ def minio_data_connection(
         },
     ) as minio_secret:
         yield minio_secret
+
+
+@pytest.fixture(scope="session")
+def nodes(admin_client: DynamicClient) -> Generator[list[Node], Any, Any]:
+    yield list(Node.get(dyn_client=admin_client))
+
+
+@pytest.fixture(scope="session")
+def junitxml_plugin(
+    request: FixtureRequest, record_testsuite_property: Callable[[str, object], None]
+) -> Callable[[str, object], None] | None:
+    return record_testsuite_property if request.config.pluginmanager.has_plugin("junitxml") else None
+
+
+@pytest.fixture(scope="session", autouse=True)
+@pytest.mark.early(order=0)
+def cluster_sanity_scope_session(
+    request: FixtureRequest,
+    nodes: list[Node],
+    dsci_resource: DSCInitialization,
+    dsc_resource: DataScienceCluster,
+    junitxml_plugin: Callable[[str, object], None],
+) -> None:
+    verify_cluster_sanity(
+        request=request,
+        nodes=nodes,
+        dsc_resource=dsc_resource,
+        dsci_resource=dsci_resource,
+        junitxml_property=junitxml_plugin,
+    )

tests/global_config.py

@@ -3,6 +3,7 @@
 distribution: str = "downstream"
 applications_namespace: str = "redhat-ods-applications"  # overwritten in conftest.py if distribution is upstream
 dsc_name: str = "default-dsc"
+dsci_name: str = "default-dsci"
 
 for _dir in dir():
     val = locals()[_dir]

tests/model_serving/model_server/multi_node/conftest.py

@@ -28,11 +28,6 @@
 from utilities.serving_runtime import ServingRuntimeFromTemplate
 
 
-@pytest.fixture(scope="session")
-def nodes(admin_client: DynamicClient) -> list[Node]:
-    return list(Node.get(dyn_client=admin_client))
-
-
 @pytest.fixture(scope="session")
 def nvidia_gpu_nodes(nodes: list[Node]) -> list[Node]:
     return [node for node in nodes if "nvidia.com/gpu.present" in node.labels.keys()]

utilities/exceptions.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from typing import Optional
 
 from ocp_resources.service import Service
@@ -92,3 +94,7 @@ def __init__(self, user: str):
 
     def __str__(self) -> str:
         return f"Failed to log in as user {self.user}."
+
+
+class ResourceNotReadyError(Exception):
+    pass

utilities/infra.py

@@ -3,9 +3,10 @@
 import shlex
 from contextlib import contextmanager
 from functools import cache
-from typing import Any, Generator, Optional, Set
+from typing import Any, Callable, Generator, Optional, Set
 
 import kubernetes
+import pytest
 from _pytest.fixtures import FixtureRequest
 from kubernetes.dynamic import DynamicClient
 from kubernetes.dynamic.exceptions import ResourceNotFoundError, ResourceNotUniqueError
@@ -19,6 +20,7 @@
 from ocp_resources.inference_service import InferenceService
 from ocp_resources.infrastructure import Infrastructure
 from ocp_resources.namespace import Namespace
+from ocp_resources.node_config_openshift_io import Node
 from ocp_resources.pod import Pod
 from ocp_resources.project_project_openshift_io import Project
 from ocp_resources.project_request import ProjectRequest
@@ -29,6 +31,11 @@
 from ocp_resources.service import Service
 from ocp_resources.service_account import ServiceAccount
 from ocp_resources.serving_runtime import ServingRuntime
+from ocp_utilities.exceptions import NodeNotReadyError, NodeUnschedulableError
+from ocp_utilities.infra import (
+    assert_nodes_in_healthy_condition,
+    assert_nodes_schedulable,
+)
 from pyhelper_utils.shell import run_command
 from pytest_testconfig import config as py_config
 from semver import Version
@@ -37,7 +44,11 @@
 from utilities.constants import ApiGroups, Labels, Timeout
 from utilities.constants import KServeDeploymentType
 from utilities.constants import Annotations
-from utilities.exceptions import ClusterLoginError, FailedPodsError
+from utilities.exceptions import (
+    ClusterLoginError,
+    FailedPodsError,
+    ResourceNotReadyError,
+)
 from timeout_sampler import TimeoutExpiredError, TimeoutSampler, retry
 import utilities.general
 
@@ -719,13 +730,12 @@ def get_product_version(admin_client: DynamicClient) -> Version:
     return Version.parse(operator_version)
 
 
-def get_dsci_applications_namespace(client: DynamicClient, dsci_name: str = "default-dsci") -> str:
+def get_dsci_applications_namespace(client: DynamicClient) -> str:
     """
     Get the namespace where DSCI applications are deployed.
 
     Args:
         client (DynamicClient): DynamicClient object
-        dsci_name (str): DSCI name
 
     Returns:
         str: Namespace where DSCI applications are deployed.
@@ -735,6 +745,7 @@ def get_dsci_applications_namespace(client: DynamicClient, dsci_name: str = "def
             MissingResourceError: If DSCI not found
 
     """
+    dsci_name = py_config["dsci_name"]
     dsci = DSCInitialization(client=client, name=dsci_name)
 
     if dsci.exists:
@@ -798,7 +809,11 @@ def wait_for_serverless_pods_deletion(resource: Project | Namespace, admin_clien
             pod.wait_deleted(timeout=Timeout.TIMEOUT_1MIN)
 
 
-@retry(wait_timeout=Timeout.TIMEOUT_30SEC, sleep=1, exceptions_dict={ResourceNotFoundError: []})
+@retry(
+    wait_timeout=Timeout.TIMEOUT_30SEC,
+    sleep=1,
+    exceptions_dict={ResourceNotFoundError: []},
+)
 def wait_for_isvc_pods(client: DynamicClient, isvc: InferenceService, runtime_name: str | None = None) -> list[Pod]:
     """
     Wait for ISVC pods.
@@ -816,3 +831,67 @@ def wait_for_isvc_pods(client: DynamicClient, isvc: InferenceService, runtime_na
     """
     LOGGER.info("Waiting for pods to be created")
     return get_pods_by_isvc_label(client=client, isvc=isvc, runtime_name=runtime_name)
+
+
+def verify_dsci_status_ready(dsci_resource: DSCInitialization) -> None:
+    LOGGER.info(f"Verify DSCI {dsci_resource.name} are {dsci_resource.Status.READY}.")
+    if dsci_resource.status != dsci_resource.Status.READY:
+        raise ResourceNotReadyError(f"DSCI {dsci_resource.name} is not ready.\nStatus: {dsci_resource.instance.status}")
+
+
+def verify_dsc_status_ready(dsc_resource: DataScienceCluster) -> None:
+    LOGGER.info(f"Verify DSC {dsc_resource.name} are {dsc_resource.Status.READY}.")
+    if dsc_resource.status != dsc_resource.Status.READY:
+        raise ResourceNotReadyError(f"DSC {dsc_resource.name} is not ready.\nStatus: {dsc_resource.instance.status}")
+
+
+def verify_cluster_sanity(
+    request: FixtureRequest,
+    nodes: list[Node],
+    dsci_resource: DSCInitialization,
+    dsc_resource: DataScienceCluster,
+    junitxml_property: Callable[[str, object], None] | None = None,
+) -> None:
+    """
+    Check that cluster resources (Nodes, DSCI, DSC) are healthy and exists pytest execution on failure.
+
+    Args:
+        request (FixtureRequest): pytest request
+        nodes (list[Node]): list of nodes
+        dsci_resource (DSCInitialization): dsci resource
+        dsc_resource (DataScienceCluster): dsc resource
+        junitxml_property (property): Junitxml property
+
+    """
+    skip_cluster_sanity_check = "--cluster-sanity-skip-check"
+    skip_rhoai_check = "--cluster-sanity-skip-rhoai-check"
+
+    if request.session.config.getoption(skip_cluster_sanity_check):
+        LOGGER.warning(f"Skipping cluster sanity check, got {skip_cluster_sanity_check}")
+        return
+
+    try:
+        LOGGER.info("Check cluster sanity.")
+
+        assert_nodes_in_healthy_condition(nodes=nodes, healthy_node_condition_type={"KubeletReady": "True"})
+        assert_nodes_schedulable(nodes=nodes)
+
+        if request.session.config.getoption(skip_rhoai_check):
+            LOGGER.warning(f"Skipping RHOAI resource checks, got {skip_rhoai_check}")
+
+        else:
+            verify_dsci_status_ready(dsci_resource=dsci_resource)
+            verify_dsc_status_ready(dsc_resource=dsc_resource)
+
+    except (ResourceNotReadyError, NodeUnschedulableError, NodeNotReadyError) as ex:
+        error_msg = f"Cluster sanity check failed: {str(ex)}"
+        # return_code set to 99 to not collide with https://docs.pytest.org/en/stable/reference/exit-codes.html
+        return_code = 99
+
+        LOGGER.error(error_msg)
+
+        if junitxml_property:
+            junitxml_property(name="exit_code", value=return_code)  # type: ignore[call-arg]
+
+        # TODO: Write to file to easily report the failure in jenkins
+        pytest.exit(reason=error_msg, returncode=return_code)