test: add cluster health and fix bug on image creation

Signed-off-by: Debarati Basu-Nag <dbasunag@redhat.com>

Author: dbasunag

.github/workflows/build-push-container-on-merge.yml

@@ -15,7 +15,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
+        with:
+          ref: ${{ github.event.pull_request.base.ref }}
       - name: Set env TAG
         run: |
           if [ ${{ github.event.pull_request.base.ref }} == "main" ]; then

pytest.ini

@@ -5,7 +5,8 @@ testpaths = tests
 markers =
     # General
     polarion: Store polarion test ID
-
+    cluster_health: Tests that verifies that cluster is healthy to begin testing
+    operator_health: Tests that verifies that OpenDataHub/RHOAI operators are healthy and functioning correctly
     # CI
     smoke: Mark tests as smoke tests; covers core functionality of the product. Aims to ensure that the build is stable enough for further testing.
     sanity: Mark tests as sanity tests. Aims to verify that specific functionality is working as expected.

tests/cluster_health/test_cluster_health.py

@@ -0,0 +1,15 @@
+import pytest
+from ocp_resources.node import Node
+from ocp_utilities.infra import assert_nodes_in_healthy_condition, assert_nodes_schedulable
+from simple_logger.logger import get_logger
+
+LOGGER = get_logger(name=__name__)
+
+
+@pytest.mark.cluster_health
+def test_cluster_node_healthy(nodes: list[Node]) -> None:
+    """
+    Tests if the cluster nodes are healthy
+    """
+    assert_nodes_in_healthy_condition(nodes=nodes, healthy_node_condition_type={"KubeletReady": "True"})
+    assert_nodes_schedulable(nodes=nodes)

tests/cluster_health/test_operator_health.py

@@ -0,0 +1,15 @@
+import pytest
+from ocp_resources.data_science_cluster import DataScienceCluster
+from simple_logger.logger import get_logger
+
+from tests.cluster_health.utils import wait_for_dsc_status_ready
+
+LOGGER = get_logger(name=__name__)
+
+
+@pytest.mark.operator_health
+def test_data_science_cluster_healthy(dsc_resource: DataScienceCluster) -> None:
+    """
+    Checks if a data science cluster is healthy
+    """
+    wait_for_dsc_status_ready(dsc_resource=dsc_resource)

tests/cluster_health/utils.py

@@ -0,0 +1,25 @@
+from simple_logger.logger import get_logger
+
+from ocp_resources.data_science_cluster import DataScienceCluster
+
+from timeout_sampler import retry
+
+LOGGER = get_logger(name=__name__)
+
+
+class ResourceNotReadyError(Exception):
+    pass
+
+
+@retry(
+    wait_timeout=120,
+    sleep=5,
+    exceptions_dict={ResourceNotReadyError: []},
+)
+def wait_for_dsc_status_ready(dsc_resource: DataScienceCluster) -> bool:
+    LOGGER.info(f"Wait for DSC {dsc_resource.name} are {dsc_resource.Status.READY}.")
+    if dsc_resource.status == dsc_resource.Status.READY:
+        return True
+    raise ResourceNotReadyError(
+        f"DSC {dsc_resource.name} is not ready.\nCurrent status: {dsc_resource.instance.status}"
+    )

tests/conftest.py

@@ -9,6 +9,7 @@
 import yaml
 from _pytest.tmpdir import TempPathFactory
 from ocp_resources.config_map import ConfigMap
+from ocp_resources.node import Node
 from ocp_resources.secret import Secret
 from pyhelper_utils.shell import run_command
 from pytest import FixtureRequest, Config
@@ -303,3 +304,8 @@ def cluster_monitoring_config(admin_client: DynamicClient) -> Generator[ConfigMa
         data=data,
     ) as cm:
         yield cm
+
+
+@pytest.fixture(scope="session")
+def nodes(admin_client: DynamicClient) -> list[Node]:
+    return list(Node.get(dyn_client=admin_client))

tests/model_serving/model_server/multi_node/conftest.py

@@ -19,11 +19,6 @@
 )
 
 
-@pytest.fixture(scope="session")
-def nodes(admin_client: DynamicClient) -> list[Node]:
-    return list(Node.get(dyn_client=admin_client))
-
-
 @pytest.fixture(scope="session")
 def nvidia_gpu_nodes(nodes: list[Node]) -> list[Node]:
     return [node for node in nodes if "nvidia.com/gpu.present" in node.labels.keys()]

utilities/jira.py

@@ -23,8 +23,8 @@ def get_jira_connection() -> JIRA:
 
     """
     return JIRA(
-        token_auth=os.getenv("PYTEST_JIRA_TOKEN"),
-        options={"server": os.getenv("PYTEST_JIRA_URL")},
+        server=os.getenv("PYTEST_JIRA_URL"),
+        basic_auth=(os.getenv("PYTEST_JIRA_USERNAME"), os.getenv("PYTEST_JIRA_TOKEN")),
     )