Add Cluster sanity checks before test execution (opendatahub-io#235)

* Create size-labeler.yml

* Delete .github/workflows/size-labeler.yml

* model mesh - add auth tests

* xx

* feat: cluster sanity

* feat: cluster sanity

* feat: cluster sanity

* feat: cluster sanity add readme

* fix: tix str typo

* fix: address comments

* fix: address review comments

* fix: address comment

* fix: use dsci from global config

* fix: remove duplicate fixture

Author: rnetser

conftest.py

@@ -136,6 +136,18 @@ def pytest_addoption(parser: Parser) -> None:
         action="store_false",
     )
 
+    # Cluster sanity options
+    cluster_sanity_group.addoption(
+        "--cluster-sanity-skip-check",
+        help="Skip cluster_sanity check",
+        action="store_true",
+    )
+    cluster_sanity_group.addoption(
+        "--cluster-sanity-skip-rhoai-check",
+        help="Skip RHOAI/ODH-related resources (DSCI and DSC) checks",
+        action="store_true",
+    )
+
 
 def pytest_cmdline_main(config: Any) -> None:
     config.option.basetemp = py_config["tmp_base_dir"] = f"{config.option.basetemp}-{shortuuid.uuid()}"

docs/GETTING_STARTED.md

@@ -62,6 +62,11 @@ uv run pytest -k test_name
 Bt default, RHOAI distribution is set.  
 To run on ODH, pass `--tc=distribution:upstream` to pytest.
 
+### Skip cluster sanity checks
+By default, cluster sanity checks are run to make cluster ready for tests.
+To skip cluster sanity checks, pass `--cluster-sanity-skip-check` to skip all tests.
+To skip RHOAI/ODH-related tests (for example when running in upstream), pass `--cluster-sanity-skip-rhoai-check`.
+
 ### jira integration
 To skip running tests which have open bugs, [pytest_jira](https://github.com/rhevm-qe-automation/pytest_jira) plugin is used.
 To run tests with jira integration, you need to set `PYTEST_JIRA_URL` and `PYTEST_JIRA_TOKEN` environment variables.

pyproject.toml

@@ -66,6 +66,7 @@ dependencies = [
     "openshift-python-wrapper>=11.0.38",
     "semver>=3.0.4",
     "sqlalchemy>=2.0.40",
+    "pytest-order>=1.3.0",
 ]
 
 [project.urls]

tests/conftest.py

@@ -1,20 +1,21 @@
 import base64
 import os
 import shutil
-from typing import Any, Generator
+from typing import Any, Callable, Generator
 
 import pytest
 import shortuuid
 import yaml
 from _pytest.tmpdir import TempPathFactory
 from ocp_resources.config_map import ConfigMap
+from ocp_resources.dsc_initialization import DSCInitialization
+from ocp_resources.node import Node
 from ocp_resources.pod import Pod
 from ocp_resources.secret import Secret
 from ocp_resources.service import Service
 from pyhelper_utils.shell import run_command
 from pytest import FixtureRequest, Config
 from kubernetes.dynamic import DynamicClient
-from kubernetes.dynamic.exceptions import ResourceNotFoundError
 from ocp_resources.data_science_cluster import DataScienceCluster
 from ocp_resources.namespace import Namespace
 from ocp_resources.resource import get_client
@@ -25,6 +26,7 @@
 from utilities.exceptions import ClusterLoginError
 from utilities.general import get_s3_secret_dict
 from utilities.infra import (
+    verify_cluster_sanity,
     create_ns,
     get_dsci_applications_namespace,
     get_operator_distribution,
@@ -277,12 +279,14 @@ def unprivileged_client(
             raise ClusterLoginError(user=non_admin_user_name)
 
 
+@pytest.fixture(scope="session")
+def dsci_resource(admin_client: DynamicClient) -> DSCInitialization:
+    return DSCInitialization(client=admin_client, name=py_config["dsci_name"], ensure_exists=True)
+
+
 @pytest.fixture(scope="session")
 def dsc_resource(admin_client: DynamicClient) -> DataScienceCluster:
-    name = py_config["dsc_name"]
-    for dsc in DataScienceCluster.get(dyn_client=admin_client, name=name):
-        return dsc
-    raise ResourceNotFoundError(f"DSC resource {name} not found")
+    return DataScienceCluster(client=admin_client, name=py_config["dsc_name"], ensure_exists=True)
 
 
 @pytest.fixture(scope="module")
@@ -444,3 +448,33 @@ def minio_data_connection(
         },
     ) as minio_secret:
         yield minio_secret
+
+
+@pytest.fixture(scope="session")
+def nodes(admin_client: DynamicClient) -> Generator[list[Node], Any, Any]:
+    yield list(Node.get(dyn_client=admin_client))
+
+
+@pytest.fixture(scope="session")
+def junitxml_plugin(
+    request: FixtureRequest, record_testsuite_property: Callable[[str, object], None]
+) -> Callable[[str, object], None] | None:
+    return record_testsuite_property if request.config.pluginmanager.has_plugin("junitxml") else None
+
+
+@pytest.fixture(scope="session", autouse=True)
+@pytest.mark.early(order=0)
+def cluster_sanity_scope_session(
+    request: FixtureRequest,
+    nodes: list[Node],
+    dsci_resource: DSCInitialization,
+    dsc_resource: DataScienceCluster,
+    junitxml_plugin: Callable[[str, object], None],
+) -> None:
+    verify_cluster_sanity(
+        request=request,
+        nodes=nodes,
+        dsc_resource=dsc_resource,
+        dsci_resource=dsci_resource,
+        junitxml_property=junitxml_plugin,
+    )

tests/model_serving/model_server/multi_node/conftest.py

@@ -28,11 +28,6 @@
 from utilities.serving_runtime import ServingRuntimeFromTemplate
 
 
-@pytest.fixture(scope="session")
-def nodes(admin_client: DynamicClient) -> list[Node]:
-    return list(Node.get(dyn_client=admin_client))
-
-
 @pytest.fixture(scope="session")
 def nvidia_gpu_nodes(nodes: list[Node]) -> list[Node]:
     return [node for node in nodes if "nvidia.com/gpu.present" in node.labels.keys()]

utilities/exceptions.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from typing import Optional
 
 from ocp_resources.service import Service
@@ -112,3 +114,6 @@ class TooManyPodsError(Exception):
 
 class UnexpectedFailureError(Exception):
     pass
+
+class ResourceNotReadyError(Exception):
+    pass

utilities/infra.py

@@ -3,8 +3,11 @@
 import shlex
 from contextlib import contextmanager
 from functools import cache
-from typing import Any, Generator, Optional, Set, Callable
+
+from typing import Any, Callable, Generator, Optional, Set
+
 import kubernetes
+import pytest
 from _pytest.fixtures import FixtureRequest
 from kubernetes.dynamic import DynamicClient
 from kubernetes.dynamic.exceptions import ResourceNotFoundError, ResourceNotUniqueError
@@ -29,6 +32,7 @@
 from ocp_resources.service import Service
 from ocp_resources.service_account import ServiceAccount
 from ocp_resources.serving_runtime import ServingRuntime
+
 from pyhelper_utils.shell import run_command
 from pytest_testconfig import config as py_config
 from packaging.version import parse, Version
@@ -722,13 +726,12 @@ def get_product_version(admin_client: DynamicClient) -> Version:
     return Version.parse(operator_version)
 
 
-def get_dsci_applications_namespace(client: DynamicClient, dsci_name: str = "default-dsci") -> str:
+def get_dsci_applications_namespace(client: DynamicClient) -> str:
     """
     Get the namespace where DSCI applications are deployed.
 
     Args:
         client (DynamicClient): DynamicClient object
-        dsci_name (str): DSCI name
 
     Returns:
         str: Namespace where DSCI applications are deployed.
@@ -738,6 +741,7 @@ def get_dsci_applications_namespace(client: DynamicClient, dsci_name: str = "def
             MissingResourceError: If DSCI not found
 
     """
+    dsci_name = py_config["dsci_name"]
     dsci = DSCInitialization(client=client, name=dsci_name)
 
     if dsci.exists:
@@ -801,7 +805,11 @@ def wait_for_serverless_pods_deletion(resource: Project | Namespace, admin_clien
             pod.wait_deleted(timeout=Timeout.TIMEOUT_1MIN)
 
 
-@retry(wait_timeout=Timeout.TIMEOUT_30SEC, sleep=1, exceptions_dict={ResourceNotFoundError: []})
+@retry(
+    wait_timeout=Timeout.TIMEOUT_30SEC,
+    sleep=1,
+    exceptions_dict={ResourceNotFoundError: []},
+)
 def wait_for_isvc_pods(client: DynamicClient, isvc: InferenceService, runtime_name: str | None = None) -> list[Pod]:
     """
     Wait for ISVC pods.
@@ -903,6 +911,7 @@ def verify_cluster_sanity(
             wait_for_dsci_status_ready(dsci_resource=dsci_resource)
             wait_for_dsc_status_ready(dsc_resource=dsc_resource)
 
+
     except (ResourceNotReadyError, NodeUnschedulableError, NodeNotReadyError) as ex:
         error_msg = f"Cluster sanity check failed: {str(ex)}"
         # return_code set to 99 to not collide with https://docs.pytest.org/en/stable/reference/exit-codes.html