From e588c1f6d80302edf1328995c067f888a8c8730b Mon Sep 17 00:00:00 2001 From: Ohad Date: Tue, 16 Jun 2026 13:34:40 +0300 Subject: [PATCH 1/2] Automation test for kubevirt_vmi_sync_totric metric Verify that kubevirt_vmi_sync_total metric is reported by both virt-controller and virt-handler, that after live migration the virt-controller value increases and the virt-handler pod changes, and that the metric is cleared after VM deletion. Signed-off-by: Ohad Assisted-by: claude code claude-opus-4-6 --- tests/observability/metrics/conftest.py | 15 ++++ tests/observability/metrics/constants.py | 1 + .../observability/metrics/test_vms_metrics.py | 31 +++++-- tests/observability/metrics/utils.py | 80 +++++++++++++++++++ 4 files changed, 121 insertions(+), 6 deletions(-) diff --git a/tests/observability/metrics/conftest.py b/tests/observability/metrics/conftest.py index 4330d744dd..a296c2764e 100644 --- a/tests/observability/metrics/conftest.py +++ b/tests/observability/metrics/conftest.py @@ -22,6 +22,7 @@ KUBEVIRT_VMI_PHASE_TRANSITION_TIME_FROM_DELETION_SECONDS_COUNT_SUCCEEDED, KUBEVIRT_VMI_PHASE_TRANSITION_TIME_FROM_DELETION_SECONDS_SUM_SUCCEEDED, KUBEVIRT_VMI_STATUS_ADDRESSES, + KUBEVIRT_VMI_SYNC_TOTAL, KUBEVIRT_VNC_ACTIVE_CONNECTIONS_BY_VMI, SUM_KUBEVIRT_VMI_PHASE_TRANSITION_TIME_FROM_DELETION_SECONDS_BUCKET_SUCCEEDED, ) @@ -34,6 +35,7 @@ get_vmi_guest_os_kernel_release_info_metric_from_vm, metric_result_output_dict_by_mountpoint, network_packets_received, + validate_vmi_sync_total_reported_and_positive, vnic_info_from_vm_or_vmi, ) from tests.observability.utils import validate_metrics_value @@ -661,3 +663,16 @@ def expected_cpu_affinity_metric_value(admin_client, vm_with_cpu_spec): # return multiplication for multi-CPU VMs return str(cpu_count_from_vm_node * cpu_count_from_vm) + + +@pytest.fixture(scope="class") +def initial_vmi_sync_total_values(prometheus, vm_for_migration_metrics_test): + metric_query = KUBEVIRT_VMI_SYNC_TOTAL.format(vm_name=vm_for_migration_metrics_test.name) + validate_vmi_sync_total_reported_and_positive(prometheus=prometheus, metric_query=metric_query) + results = prometheus.query_sampler(query=metric_query) + return {result["metric"]["pod"]: float(result["value"][1]) for result in results} + + +@pytest.fixture(scope="class") +def deleted_vmi_sync_total_vm(vm_for_migration_metrics_test): + vm_for_migration_metrics_test.delete(wait=True) diff --git a/tests/observability/metrics/constants.py b/tests/observability/metrics/constants.py index 4b24965d67..9b06d869cd 100644 --- a/tests/observability/metrics/constants.py +++ b/tests/observability/metrics/constants.py @@ -45,3 +45,4 @@ ] KUBEVIRT_VMI_NODE_CPU_AFFINITY = "kubevirt_vmi_node_cpu_affinity{{kubernetes_vmi_label_kubevirt_io_domain='{vm_name}'}}" +KUBEVIRT_VMI_SYNC_TOTAL = "kubevirt_vmi_sync_total{{name='{vm_name}'}}" diff --git a/tests/observability/metrics/test_vms_metrics.py b/tests/observability/metrics/test_vms_metrics.py index 5c3fdaeac3..00fa64c216 100644 --- a/tests/observability/metrics/test_vms_metrics.py +++ b/tests/observability/metrics/test_vms_metrics.py @@ -17,6 +17,7 @@ KUBEVIRT_VM_DISK_ALLOCATED_SIZE_BYTES, KUBEVIRT_VMI_PHASE_TRANSITION_TIME_FROM_DELETION_SECONDS_COUNT_SUCCEEDED, KUBEVIRT_VMI_PHASE_TRANSITION_TIME_FROM_DELETION_SECONDS_SUM_SUCCEEDED, + KUBEVIRT_VMI_SYNC_TOTAL, KUBEVIRT_VNC_ACTIVE_CONNECTIONS_BY_VMI, SUM_KUBEVIRT_VMI_PHASE_TRANSITION_TIME_FROM_DELETION_SECONDS_BUCKET_SUCCEEDED, ) @@ -24,7 +25,10 @@ compare_metric_file_system_values_with_vm_file_system_values, get_pvc_size_bytes, timestamp_to_seconds, + validate_metric_value_cleared, validate_metric_value_greater_than_initial_value, + validate_vmi_sync_total_after_migration, + validate_vmi_sync_total_reported_and_positive, validate_vnic_info, ) from tests.observability.utils import validate_metrics_value @@ -551,17 +555,15 @@ class TestVmiSyncTotal: """ Tests for kubevirt_vmi_sync_total metric. - Jira: https://redhat.atlassian.net/browse/CNV-80580 + Jira: https://redhat.atlassian.net/browse/CNV-80580 # Preconditions: - Running VM - Prometheus access configured """ - __test__ = False - @pytest.mark.polarion("CNV-16271") - def test_kubevirt_vmi_sync_total(self): + def test_kubevirt_vmi_sync_total(self, prometheus, vm_for_migration_metrics_test): """ Test that kubevirt_vmi_sync_total metric is reported by both virt-controller and virt-handler after a VM starts. @@ -574,9 +576,15 @@ def test_kubevirt_vmi_sync_total(self): - Two metric entries are returned — one from virt-controller and one from virt-handler — each with a value greater than 0 """ + validate_vmi_sync_total_reported_and_positive( + prometheus=prometheus, + metric_query=KUBEVIRT_VMI_SYNC_TOTAL.format(vm_name=vm_for_migration_metrics_test.name), + ) @pytest.mark.polarion("CNV-16272") - def test_kubevirt_vmi_sync_total_increases_after_migration(self): + def test_kubevirt_vmi_sync_total_increases_after_migration( + self, prometheus, initial_vmi_sync_total_values, vm_for_migration_metrics_test, migration_succeeded_scope_class + ): """ Test that kubevirt_vmi_sync_total metric value increases after a VM live migration. @@ -594,9 +602,16 @@ def test_kubevirt_vmi_sync_total_increases_after_migration(self): - Metric values from both virt-controller and virt-handler are greater than the values recorded before migration """ + validate_vmi_sync_total_after_migration( + prometheus=prometheus, + metric_query=KUBEVIRT_VMI_SYNC_TOTAL.format(vm_name=vm_for_migration_metrics_test.name), + initial_values=initial_vmi_sync_total_values, + ) @pytest.mark.polarion("CNV-16273") - def test_kubevirt_vmi_sync_total_cleared_after_vm_deletion(self): + def test_kubevirt_vmi_sync_total_cleared_after_vm_deletion( + self, prometheus, vm_for_migration_metrics_test, deleted_vmi_sync_total_vm + ): """ Test that kubevirt_vmi_sync_total metric entry is removed after the VM is deleted. @@ -612,3 +627,7 @@ def test_kubevirt_vmi_sync_total_cleared_after_vm_deletion(self): Expected: - Metric value is None """ + validate_metric_value_cleared( + prometheus=prometheus, + metric_name=KUBEVIRT_VMI_SYNC_TOTAL.format(vm_name=vm_for_migration_metrics_test.name), + ) diff --git a/tests/observability/metrics/utils.py b/tests/observability/metrics/utils.py index ee1931a746..2c87a1c7f2 100644 --- a/tests/observability/metrics/utils.py +++ b/tests/observability/metrics/utils.py @@ -51,6 +51,7 @@ TIMEOUT_30SEC, TIMEOUT_40MIN, USED, + VIRT_CONTROLLER, VIRT_HANDLER, Images, ) @@ -813,3 +814,82 @@ def _get_metric_values(): return metric_sample raise TimeoutError("Timed out waiting for Prometheus metrics to match expected values.") + + +def validate_vmi_sync_total_reported_and_positive( + prometheus: Prometheus, + metric_query: str, +) -> None: + samples = TimeoutSampler( + wait_timeout=TIMEOUT_4MIN, + sleep=TIMEOUT_15SEC, + func=prometheus.query_sampler, + query=metric_query, + ) + sample = None + try: + for sample in samples: + if not sample or len(sample) < 2: + continue + pods = {result["metric"]["pod"] for result in sample} + has_controller = any(pod.startswith(VIRT_CONTROLLER) for pod in pods) + has_handler = any(pod.startswith(VIRT_HANDLER) for pod in pods) + all_positive = all(float(result["value"][1]) > 0 for result in sample) + if has_controller and has_handler and all_positive: + return + except TimeoutExpiredError: + LOGGER.error(f"Expected entries from both virt-controller and virt-handler, got: {sample}") + raise + + +def validate_vmi_sync_total_after_migration( + prometheus: Prometheus, + metric_query: str, + initial_values: dict[str, float], +) -> None: + samples = TimeoutSampler( + wait_timeout=TIMEOUT_4MIN, + sleep=TIMEOUT_15SEC, + func=prometheus.query_sampler, + query=metric_query, + ) + current_values = None + try: + for sample in samples: + if sample: + current_values = {result["metric"]["pod"]: float(result["value"][1]) for result in sample} + controller_same_and_increased = all( + pod in current_values and current_values[pod] > value + for pod, value in initial_values.items() + if pod.startswith(VIRT_CONTROLLER) + ) + new_handler_with_value = any( + pod.startswith(VIRT_HANDLER) and pod not in initial_values and value > 0 + for pod, value in current_values.items() + ) + if controller_same_and_increased and new_handler_with_value: + return + except TimeoutExpiredError: + LOGGER.error(f"Post-migration validation failed. Initial: {initial_values}, current: {current_values}") + raise + + +def validate_metric_value_cleared( + prometheus: Prometheus, + metric_name: str, + timeout: int = TIMEOUT_4MIN, +) -> None: + samples = TimeoutSampler( + wait_timeout=timeout, + sleep=TIMEOUT_15SEC, + func=prometheus.query_sampler, + query=metric_name, + ) + sample = None + try: + for sample in samples: + if not sample or all(result["value"][1] == "0" for result in sample): + return + except TimeoutExpiredError: + LOGGER.error(f"Metric {metric_name} still has non-zero values: {sample}") + raise From f8e30e827cc4690d77291a81f4984e4f437ee547 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Jun 2026 14:41:43 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: Ohad --- tests/observability/metrics/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/observability/metrics/conftest.py b/tests/observability/metrics/conftest.py index 547398af73..d3fc8a8ca0 100644 --- a/tests/observability/metrics/conftest.py +++ b/tests/observability/metrics/conftest.py @@ -688,6 +688,9 @@ def initial_vmi_sync_total_values(prometheus, vm_for_migration_metrics_test): @pytest.fixture(scope="class") def deleted_vmi_sync_total_vm(vm_for_migration_metrics_test): vm_for_migration_metrics_test.delete(wait=True) + + +@pytest.fixture(scope="class") def vm_for_nad_swap_test( unprivileged_client, namespace,