Test kubevirt_vmi_number_of_outdated

Test kubevirt_vmi_number_of_outdated metric.

Author: OhadRevah

tests/observability/conftest.py

@@ -14,8 +14,8 @@
     VIRT_OPERATOR,
 )
 from utilities.hco import ResourceEditorValidateHCOReconcile, get_installed_hco_csv
-from utilities.infra import get_deployment_by_name, scale_deployment_replicas
-from utilities.virt import get_all_virt_pods_with_running_status
+from utilities.infra import get_deployment_by_name, get_node_selector_dict, scale_deployment_replicas
+from utilities.virt import VirtualMachineForTests, fedora_vm_body, get_all_virt_pods_with_running_status, running_vm
 
 LOGGER = logging.getLogger(__name__)
 ANNOTATIONS_FOR_VIRT_OPERATOR_ENDPOINT = {
@@ -112,3 +112,16 @@ def initial_virt_operator_replicas_reverted(prometheus, initial_virt_operator_re
     validate_initial_virt_operator_replicas_reverted(
         prometheus=prometheus, initial_virt_operator_replicas=initial_virt_operator_replicas
     )
+
+
+@pytest.fixture()
+def vm_with_node_selector_for_upgrade(namespace, worker_node1):
+    name = "vm-with-node-selector"
+    with VirtualMachineForTests(
+        name=name,
+        namespace=namespace.name,
+        body=fedora_vm_body(name=name),
+        node_selector=get_node_selector_dict(node_selector=worker_node1.name),
+    ) as vm:
+        running_vm(vm=vm)
+        yield vm

tests/observability/constants.py

@@ -2,6 +2,7 @@
 SSP_HIGH_RATE_REJECTED_VMS = "SSPHighRateRejectedVms"
 BAD_HTTPGET_PATH = "/metrics-fake"
 SSP_COMMON_TEMPLATES_MODIFICATION_REVERTED = "SSPCommonTemplatesModificationReverted"
+KUBEVIRT_VMI_NUMBER_OF_OUTDATED = "kubevirt_vmi_number_of_outdated"
 VIRT_ALERTS_LIST = [
     "VirtOperatorDown",
     "NoReadyVirtOperator",

tests/observability/upgrade/__init__.py

@@ -0,0 +1,38 @@
+import pytest
+
+from tests.observability.constants import KUBEVIRT_VMI_NUMBER_OF_OUTDATED
+from tests.observability.utils import validate_metrics_value, wait_for_greater_than_zero_metric_value
+from tests.upgrade_params import IUO_UPGRADE_TEST_DEPENDENCY_NODE_ID
+from utilities.constants import DEPENDENCY_SCOPE_SESSION
+
+
+@pytest.mark.upgrade
+class TestUpgradeObservability:
+    TEST_METRIC_KUBEVIRT_VMI_NUMBER_OF_OUTDATED_BEFORE_UPGRADE = (
+        "test_metric_kubevirt_vmi_number_of_outdated_before_upgrade"
+    )
+    """Pre-upgrade tests"""
+
+    @pytest.mark.order(before=IUO_UPGRADE_TEST_DEPENDENCY_NODE_ID)
+    @pytest.mark.dependency(name=TEST_METRIC_KUBEVIRT_VMI_NUMBER_OF_OUTDATED_BEFORE_UPGRADE)
+    @pytest.mark.polarion("CNV-11749")
+    def test_metric_kubevirt_vmi_number_of_outdated_before_upgrade(self, prometheus, vm_with_node_selector_for_upgrade):
+        validate_metrics_value(
+            prometheus=prometheus,
+            metric_name=KUBEVIRT_VMI_NUMBER_OF_OUTDATED,
+            expected_value="0",
+        )
+
+    """Post-upgrade tests"""
+
+    @pytest.mark.polarion("CNV-11758")
+    @pytest.mark.order(after=IUO_UPGRADE_TEST_DEPENDENCY_NODE_ID)
+    @pytest.mark.dependency(
+        depends=[IUO_UPGRADE_TEST_DEPENDENCY_NODE_ID, TEST_METRIC_KUBEVIRT_VMI_NUMBER_OF_OUTDATED_BEFORE_UPGRADE],
+        scope=DEPENDENCY_SCOPE_SESSION,
+    )
+    def test_metric_kubevirt_vmi_number_of_outdated_after_upgrade(self, prometheus):
+        wait_for_greater_than_zero_metric_value(
+            prometheus=prometheus,
+            metric_name=KUBEVIRT_VMI_NUMBER_OF_OUTDATED,
+        )

tests/observability/upgrade/test_upgrade_observability.py

@@ -9,7 +9,9 @@
 from tests.observability.constants import SSP_COMMON_TEMPLATES_MODIFICATION_REVERTED
 from utilities.constants import (
     TIMEOUT_4MIN,
+    TIMEOUT_5MIN,
     TIMEOUT_15SEC,
+    TIMEOUT_30SEC,
 )
 from utilities.monitoring import get_metrics_value
 
@@ -62,3 +64,21 @@ def get_olm_namespace() -> Namespace:
     if olm_ns.exists:
         return olm_ns
     raise ResourceNotFoundError(f"Namespace: {olm_ns.name} not found.")
+
+
+def wait_for_greater_than_zero_metric_value(prometheus: Prometheus, metric_name: str) -> None:
+    samples = TimeoutSampler(
+        wait_timeout=TIMEOUT_5MIN,
+        sleep=TIMEOUT_30SEC,
+        func=get_metrics_value,
+        prometheus=prometheus,
+        metrics_name=metric_name,
+    )
+    sample = None
+    try:
+        for sample in samples:
+            if sample and int(sample) > 0:
+                return
+    except TimeoutExpiredError:
+        LOGGER.info(f"Metric value of: {metric_name} is: {sample}, expected value: non zero")
+        raise