feat: Separate out cluster, operator and component health checks (#1133)

* feat: Separate out cluster, operator and component health checks

* fix: Addressed review comments

Author: dbasunag

pytest.ini

@@ -21,6 +21,8 @@ markers =
     ocp_interop: Interop testing with Openshift.
     downstream_only: Tests that are specific to downstream
     cluster_health: Tests that verifies that cluster is healthy to begin testing
+    operator_health: Tests that verifies that OpenDataHub/RHOAI operators are healthy and functioning correctly
+    component_health: Tests that verifies that OpenDataHub/RHOAI components are healthy and functioning correctly
     skip_must_gather: Tests that does not require must-gather for triaging
     install: Tests that are relevant for install scenario. To be used with upgrade marker, to indicate tests that are valid for both install and upgrade
 

tests/cluster_health/test_cluster_health.py

@@ -1,47 +1,15 @@
 import pytest
-from kubernetes.dynamic import DynamicClient
-from ocp_resources.data_science_cluster import DataScienceCluster
-from ocp_resources.dsc_initialization import DSCInitialization
 from ocp_resources.node import Node
 from ocp_utilities.infra import assert_nodes_in_healthy_condition, assert_nodes_schedulable
-from utilities.general import wait_for_pods_running
-from utilities.infra import wait_for_dsci_status_ready, wait_for_dsc_status_ready
-from pytest_testconfig import config as py_config
 from simple_logger.logger import get_logger
 
 LOGGER = get_logger(name=__name__)
 
 
 @pytest.mark.cluster_health
-def test_cluster_node_healthy(nodes: list[Node]):
+def test_cluster_node_healthy(nodes: list[Node]) -> None:
+    """
+    Tests if the cluster nodes are healthy
+    """
     assert_nodes_in_healthy_condition(nodes=nodes, healthy_node_condition_type={"KubeletReady": "True"})
     assert_nodes_schedulable(nodes=nodes)
-
-
-@pytest.mark.cluster_health
-def test_data_science_cluster_initialization_healthy(dsci_resource: DSCInitialization):
-    wait_for_dsci_status_ready(dsci_resource=dsci_resource)
-
-
-@pytest.mark.cluster_health
-def test_data_science_cluster_healthy(dsc_resource: DataScienceCluster):
-    wait_for_dsc_status_ready(dsc_resource=dsc_resource)
-
-
-@pytest.mark.parametrize(
-    "namespace_name",
-    [
-        pytest.param(
-            py_config["operator_namespace"],
-            id="test_operator_namespace_pod_healthy",
-        ),
-        pytest.param(
-            py_config["applications_namespace"],
-            id="test_application_namespace_pod_healthy",
-        ),
-    ],
-)
-@pytest.mark.cluster_health
-def test_pods_cluster_healthy(admin_client: DynamicClient, namespace_name: str):
-    LOGGER.info(f"Testing Pods in namespace {namespace_name} for cluster health")
-    wait_for_pods_running(admin_client=admin_client, namespace_name=namespace_name)

tests/cluster_health/test_operator_health.py

@@ -0,0 +1,48 @@
+import pytest
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.data_science_cluster import DataScienceCluster
+from ocp_resources.dsc_initialization import DSCInitialization
+from utilities.general import wait_for_pods_running
+from utilities.infra import wait_for_dsci_status_ready, wait_for_dsc_status_ready
+from pytest_testconfig import config as py_config
+from simple_logger.logger import get_logger
+
+LOGGER = get_logger(name=__name__)
+
+
+@pytest.mark.operator_health
+def test_data_science_cluster_initialization_healthy(dsci_resource: DSCInitialization) -> None:
+    """
+    Checks if a data science cluster initialization is healthy
+    """
+    wait_for_dsci_status_ready(dsci_resource=dsci_resource)
+
+
+@pytest.mark.operator_health
+def test_data_science_cluster_healthy(dsc_resource: DataScienceCluster) -> None:
+    """
+    Checks if a data science cluster is healthy
+    """
+    wait_for_dsc_status_ready(dsc_resource=dsc_resource)
+
+
+@pytest.mark.parametrize(
+    "namespace_name",
+    [
+        pytest.param(
+            py_config["operator_namespace"],
+            id="test_operator_namespace_pod_healthy",
+        ),
+        pytest.param(
+            py_config["applications_namespace"],
+            id="test_application_namespace_pod_healthy",
+        ),
+    ],
+)
+@pytest.mark.operator_health
+def test_pods_cluster_healthy(admin_client: DynamicClient, namespace_name: str) -> None:
+    """
+    Checks if pods in a given namespace are all healthy
+    """
+    LOGGER.info(f"Testing Pods in namespace {namespace_name} for cluster health")
+    wait_for_pods_running(admin_client=admin_client, namespace_name=namespace_name)

…odel_registry/cluster_health/__init__.py …el_registry/component_health/__init__.pytests/model_registry/cluster_health/__init__.py renamed to tests/model_registry/component_health/__init__.py tests/model_registry/cluster_health/__init__.py renamed to tests/model_registry/component_health/__init__.py

@@ -13,16 +13,18 @@
 LOGGER = get_logger(name=__name__)
 
 
-@pytest.mark.cluster_health
+@pytest.mark.component_health
 class TestMrDefault:
-    def test_mr_management_state(self, dsc_resource: DataScienceCluster):
+    def test_mr_management_state(self, dsc_resource: DataScienceCluster) -> None:
         """Verify MODELREGISTRY managementState is MANAGED in DSC."""
         assert (
             dsc_resource.instance.spec.components[DscComponents.MODELREGISTRY].managementState
             == DscComponents.ManagementState.MANAGED
         )
 
-    def test_mr_namespace_exists_and_active(self, admin_client: DynamicClient, dsc_resource: DataScienceCluster):
+    def test_mr_namespace_exists_and_active(
+        self, admin_client: DynamicClient, dsc_resource: DataScienceCluster
+    ) -> None:
         """Verify MR namespace exists and is in Active state."""
         namespace = Namespace(
             client=admin_client,
@@ -32,7 +34,7 @@ def test_mr_namespace_exists_and_active(self, admin_client: DynamicClient, dsc_r
         assert namespace.instance.status.phase == Namespace.Status.ACTIVE
         assert namespace.instance.metadata.name == py_config["model_registry_namespace"]
 
-    def test_mr_condition_in_dsc(self, dsc_resource: DataScienceCluster):
+    def test_mr_condition_in_dsc(self, dsc_resource: DataScienceCluster) -> None:
         """Verify MR ready condition is True in DSC."""
         for condition in dsc_resource.instance.status.conditions:
             if condition.type == DscComponents.COMPONENT_MAPPING[DscComponents.MODELREGISTRY]:
@@ -41,7 +43,7 @@ def test_mr_condition_in_dsc(self, dsc_resource: DataScienceCluster):
         else:
             pytest.fail("MR ready condition not found in DSC")
 
-    @pytest.mark.cluster_health
+    @pytest.mark.component_health
     def test_mr_pods_health(self, admin_client: DynamicClient):
         namespace = py_config["model_registry_namespace"]
         LOGGER.info(f"Testing Pods in namespace {namespace} for cluster health")