From 12b725dc9b113935894efc46ee36bba20813e18a Mon Sep 17 00:00:00 2001 From: Samuel Albershtein Date: Fri, 20 Mar 2026 01:20:00 +0200 Subject: [PATCH] [VIRT] Refactor workload disruption migration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidate PostCopy and Paused migration tests into two parametrized classes split by OS (RHEL and Windows). Renamed from test_post_copy_migration.py to test_workload_disruption_migration.py. A single module-scoped MigrationPolicy is dynamically patched via ResourceEditor to toggle PostCopy/Paused mode per test class. Memory pressure uses built-in tools (Python on RHEL, PowerShell on Windows). Tests cover: - Migration mode verification (PostCopy and Paused) - Node drain with migration mode assertion - CPU hotplug after migration (RHEL and Windows) - Memory hotplug after migration (RHEL and Windows) - Background process survival across migrations (PID check) Consolidate VMIM cleanup for CNV-92094 workaround: move clean_up_migration_jobs to the vm_with_hotplug_support fixture teardown, guarded by is_jira_open("CNV-92094"). Removes scattered try/finally and per-fixture cleanup calls — cleanup runs once at class teardown and auto-disables when the bug is fixed. Signed-off-by: Samuel Albershtein Co-authored-by: AI (Claude) --- tests/virt/constants.py | 2 +- tests/virt/node/conftest.py | 15 +- .../node/hotplug/test_cpu_memory_hotplug.py | 65 ++--- .../test_post_copy_migration.py | 160 ------------ .../test_workload_disruption_migration.py | 247 ++++++++++++++++++ .../node/migration_and_maintenance/utils.py | 84 ++++++ tests/virt/upgrade/conftest.py | 7 +- 7 files changed, 380 insertions(+), 200 deletions(-) delete mode 100644 tests/virt/node/migration_and_maintenance/test_post_copy_migration.py create mode 100644 tests/virt/node/migration_and_maintenance/test_workload_disruption_migration.py create mode 100644 tests/virt/node/migration_and_maintenance/utils.py diff --git a/tests/virt/constants.py b/tests/virt/constants.py index 1cb1b4f6d2..6e62eaf763 100644 --- a/tests/virt/constants.py +++ b/tests/virt/constants.py @@ -31,7 +31,7 @@ # MigrationPolicy labels -VM_LABEL = {"post-copy-vm": "true"} +WORKLOAD_DISRUPTION_VM_LABEL = {"workload-disruption-vm": "true"} # BASH diff --git a/tests/virt/node/conftest.py b/tests/virt/node/conftest.py index c2ebcec7f3..fa0cf7bb15 100644 --- a/tests/virt/node/conftest.py +++ b/tests/virt/node/conftest.py @@ -70,10 +70,11 @@ def vmx_disabled_flag(nodes_cpu_architecture): @pytest.fixture(scope="class") -def hotplugged_vm( +def vm_with_hotplug_support( request, namespace, unprivileged_client, + admin_client, golden_image_data_volume_template_for_test_scope_class, modern_cpu_for_migration, vmx_disabled_flag, @@ -98,22 +99,24 @@ def hotplugged_vm( ) as vm: running_vm(vm=vm) yield vm + if is_jira_open(jira_id="CNV-92094"): + clean_up_migration_jobs(client=admin_client, vm=vm) @pytest.fixture() -def hotplugged_sockets_memory_guest(request, admin_client, hotplugged_vm, unprivileged_client): +def hotplugged_sockets_memory_guest(request, admin_client, vm_with_hotplug_support, unprivileged_client): param = request.param if param.get("skip_migration"): - hotplug_spec_vm(vm=hotplugged_vm, sockets=param.get("sockets"), memory_guest=param.get("memory_guest")) + hotplug_spec_vm( + vm=vm_with_hotplug_support, sockets=param.get("sockets"), memory_guest=param.get("memory_guest") + ) else: hotplug_spec_vm_and_verify_hotplug( - vm=hotplugged_vm, + vm=vm_with_hotplug_support, client=unprivileged_client, sockets=param.get("sockets"), memory_guest=param.get("memory_guest"), ) - yield - clean_up_migration_jobs(client=admin_client, vm=hotplugged_vm) @pytest.fixture() diff --git a/tests/virt/node/hotplug/test_cpu_memory_hotplug.py b/tests/virt/node/hotplug/test_cpu_memory_hotplug.py index 07d2c8e36f..e4355a3905 100644 --- a/tests/virt/node/hotplug/test_cpu_memory_hotplug.py +++ b/tests/virt/node/hotplug/test_cpu_memory_hotplug.py @@ -14,6 +14,7 @@ hotplug_spec_vm, wait_for_guest_os_cpu_count, ) +from utilities.constants.images import OS_FLAVOR_WINDOWS from utilities.constants.virt import ( FIVE_GI_MEMORY, FOUR_CPU_SOCKETS, @@ -41,24 +42,24 @@ @pytest.fixture() -def xfail_windows_memory_hotunplug(hotplugged_vm): - if "windows" in hotplugged_vm.name: +def xfail_windows_memory_hotunplug(vm_with_hotplug_support): + if vm_with_hotplug_support.os_flavor == OS_FLAVOR_WINDOWS: pytest.xfail(reason="Windows OS doesn't officially support memory hot unplug!") @pytest.fixture(scope="class") -def hotplug_vm_snapshot(hotplugged_vm): +def hotplug_vm_snapshot(vm_with_hotplug_support): with VirtualMachineSnapshot( - name=f"{hotplugged_vm.name}-snapshot", - namespace=hotplugged_vm.namespace, - vm_name=hotplugged_vm.name, + name=f"{vm_with_hotplug_support.name}-snapshot", + namespace=vm_with_hotplug_support.namespace, + vm_name=vm_with_hotplug_support.name, ) as snapshot: snapshot.wait_snapshot_done() yield snapshot @pytest.mark.parametrize( - "golden_image_data_source_for_test_scope_class, hotplugged_vm", + "golden_image_data_source_for_test_scope_class, vm_with_hotplug_support", [ pytest.param( {"os_dict": RHEL_LATEST}, @@ -81,19 +82,21 @@ class TestCPUHotPlug: ) @pytest.mark.dependency(name=f"{TESTS_CLASS_NAME}::hotplug_cpu") @pytest.mark.polarion("CNV-10695") - def test_hotplug_cpu(self, hotplugged_sockets_memory_guest, hotplugged_vm): - wait_for_guest_os_cpu_count(vm=hotplugged_vm, spec_cpu_amount=SIX_CPU_SOCKETS) + def test_hotplug_cpu(self, hotplugged_sockets_memory_guest, vm_with_hotplug_support): + wait_for_guest_os_cpu_count(vm=vm_with_hotplug_support, spec_cpu_amount=SIX_CPU_SOCKETS) @pytest.mark.dependency(depends=[f"{TESTS_CLASS_NAME}::hotplug_cpu"]) @pytest.mark.polarion("CNV-10696") @pytest.mark.usefixtures("hotplug_vm_snapshot") - def test_migrate_snapshot_hotplugged_vm(self, admin_client: DynamicClient, hotplugged_vm: VirtualMachineForTests): - migrate_vm_and_verify(vm=hotplugged_vm, client=admin_client, check_ssh_connectivity=True) + def test_migrate_snapshot_hotplugged_vm( + self, admin_client: DynamicClient, vm_with_hotplug_support: VirtualMachineForTests + ): + migrate_vm_and_verify(vm=vm_with_hotplug_support, client=admin_client, check_ssh_connectivity=True) @pytest.mark.dependency(depends=[f"{TESTS_CLASS_NAME}::hotplug_cpu"]) @pytest.mark.polarion("CNV-10697") - def test_restart_hotplugged_vm(self, hotplugged_vm): - restart_vm_wait_for_running_vm(vm=hotplugged_vm) + def test_restart_hotplug_vm(self, vm_with_hotplug_support): + restart_vm_wait_for_running_vm(vm=vm_with_hotplug_support) @pytest.mark.parametrize( "hotplugged_sockets_memory_guest", @@ -102,21 +105,21 @@ def test_restart_hotplugged_vm(self, hotplugged_vm): ) @pytest.mark.dependency(depends=[f"{TESTS_CLASS_NAME}::hotplug_cpu"]) @pytest.mark.polarion("CNV-10698") - def test_decrease_cpu_value(self, hotplugged_sockets_memory_guest, hotplugged_vm): + def test_decrease_cpu_value(self, hotplugged_sockets_memory_guest, vm_with_hotplug_support): assert_restart_required_condition( - vm=hotplugged_vm, expected_message="Reduction of CPU socket count requires a restart" + vm=vm_with_hotplug_support, expected_message="Reduction of CPU socket count requires a restart" ) @pytest.mark.dependency(depends=[f"{TESTS_CLASS_NAME}::hotplug_cpu"]) @pytest.mark.polarion("CNV-10699") - def test_hotplug_cpu_above_max_value(self, hotplugged_vm): + def test_hotplug_cpu_above_max_value(self, vm_with_hotplug_support): with pytest.raises(UnprocessibleEntityError): - hotplug_spec_vm(vm=hotplugged_vm, sockets=TEN_CPU_SOCKETS) + hotplug_spec_vm(vm=vm_with_hotplug_support, sockets=TEN_CPU_SOCKETS) pytest.fail("Socket value set higher than max value!") @pytest.mark.parametrize( - "golden_image_data_source_for_test_scope_class, hotplugged_vm", + "golden_image_data_source_for_test_scope_class, vm_with_hotplug_support", [ pytest.param( {"os_dict": RHEL_LATEST}, @@ -138,14 +141,16 @@ class TestMemoryHotPlug: ) @pytest.mark.dependency(name=f"{TESTS_CLASS_NAME}::hotplug_memory") @pytest.mark.polarion("CNV-10676") - def test_hotplug_memory(self, hotplugged_sockets_memory_guest, hotplugged_vm): - assert_guest_os_memory_amount(vm=hotplugged_vm, spec_memory_amount=SIX_GI_MEMORY) + def test_hotplug_memory(self, hotplugged_sockets_memory_guest, vm_with_hotplug_support): + assert_guest_os_memory_amount(vm=vm_with_hotplug_support, spec_memory_amount=SIX_GI_MEMORY) @pytest.mark.dependency(depends=[f"{TESTS_CLASS_NAME}::hotplug_memory"]) @pytest.mark.polarion("CNV-10677") @pytest.mark.usefixtures("hotplug_vm_snapshot") - def test_migrate_snapshot_hotplugged_vm(self, admin_client: DynamicClient, hotplugged_vm: VirtualMachineForTests): - migrate_vm_and_verify(vm=hotplugged_vm, client=admin_client, check_ssh_connectivity=True) + def test_migrate_snapshot_hotplugged_vm( + self, admin_client: DynamicClient, vm_with_hotplug_support: VirtualMachineForTests + ): + migrate_vm_and_verify(vm=vm_with_hotplug_support, client=admin_client, check_ssh_connectivity=True) @pytest.mark.parametrize( "hotplugged_sockets_memory_guest", [pytest.param({"memory_guest": FIVE_GI_MEMORY})], indirect=True @@ -153,20 +158,20 @@ def test_migrate_snapshot_hotplugged_vm(self, admin_client: DynamicClient, hotpl @pytest.mark.dependency(depends=[f"{TESTS_CLASS_NAME}::hotplug_memory"]) @pytest.mark.polarion("CNV-10679") def test_decrease_memory_value( - self, xfail_windows_memory_hotunplug, hotplugged_sockets_memory_guest, hotplugged_vm + self, xfail_windows_memory_hotunplug, hotplugged_sockets_memory_guest, vm_with_hotplug_support ): - assert_guest_os_memory_amount(vm=hotplugged_vm, spec_memory_amount=FIVE_GI_MEMORY) + assert_guest_os_memory_amount(vm=vm_with_hotplug_support, spec_memory_amount=FIVE_GI_MEMORY) @pytest.mark.dependency(depends=[f"{TESTS_CLASS_NAME}::hotplug_memory"]) @pytest.mark.polarion("CNV-10678") - def test_restart_hotplugged_vm(self, hotplugged_vm): - restart_vm_wait_for_running_vm(vm=hotplugged_vm) + def test_restart_hotplug_vm(self, vm_with_hotplug_support): + restart_vm_wait_for_running_vm(vm=vm_with_hotplug_support) @pytest.mark.dependency(depends=[f"{TESTS_CLASS_NAME}::hotplug_memory"]) @pytest.mark.polarion("CNV-10681") - def test_hotplug_memory_above_max_value(self, hotplugged_vm): + def test_hotplug_memory_above_max_value(self, vm_with_hotplug_support): with pytest.raises(UnprocessibleEntityError): - hotplug_spec_vm(vm=hotplugged_vm, memory_guest=TWELVE_GI_MEMORY) + hotplug_spec_vm(vm=vm_with_hotplug_support, memory_guest=TWELVE_GI_MEMORY) pytest.fail("Memory value set higher than max value!") @pytest.mark.parametrize( @@ -176,8 +181,8 @@ def test_hotplug_memory_above_max_value(self, hotplugged_vm): ) @pytest.mark.dependency(depends=[f"{TESTS_CLASS_NAME}::hotplug_memory"]) @pytest.mark.polarion("CNV-10682") - def test_reduce_memory_below_start_value(self, hotplugged_sockets_memory_guest, hotplugged_vm): + def test_reduce_memory_below_start_value(self, hotplugged_sockets_memory_guest, vm_with_hotplug_support): assert_restart_required_condition( - vm=hotplugged_vm, + vm=vm_with_hotplug_support, expected_message="memory updated in template spec to a value lower than what the VM started with", ) diff --git a/tests/virt/node/migration_and_maintenance/test_post_copy_migration.py b/tests/virt/node/migration_and_maintenance/test_post_copy_migration.py deleted file mode 100644 index 90bc5e29f5..0000000000 --- a/tests/virt/node/migration_and_maintenance/test_post_copy_migration.py +++ /dev/null @@ -1,160 +0,0 @@ -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING - -import pytest -from ocp_resources.migration_policy import MigrationPolicy - -from tests.os_params import RHEL_LATEST, RHEL_LATEST_LABELS, WINDOWS_LATEST, WINDOWS_LATEST_LABELS -from tests.utils import ( - assert_guest_os_memory_amount, - clean_up_migration_jobs, - wait_for_guest_os_cpu_count, -) -from tests.virt.constants import VM_LABEL -from tests.virt.utils import assert_migration_post_copy_mode -from utilities.constants.timeouts import ( - TIMEOUT_15MIN, - TIMEOUT_30MIN, -) -from utilities.constants.virt import ( - REGEDIT_PROC_NAME, - SIX_CPU_SOCKETS, - SIX_GI_MEMORY, -) -from utilities.virt import ( - check_migration_process_after_node_drain, - drain_node, - fetch_pid_from_linux_vm, - fetch_pid_from_windows_vm, - migrate_vm_and_verify, - start_and_fetch_processid_on_linux_vm, - start_and_fetch_processid_on_windows_vm, -) - -if TYPE_CHECKING: - from kubernetes.dynamic import DynamicClient - - from utilities.virt import VirtualMachineForTests - -pytestmark = [ - pytest.mark.rwx_default_storage, - pytest.mark.usefixtures("created_post_copy_migration_policy"), - pytest.mark.data_collector_scope(scope="module"), -] - - -LOGGER = logging.getLogger(__name__) -TESTS_CLASS_NAME = "TestPostCopyMigration" - - -def assert_same_pid_after_migration(orig_pid, vm): - if "windows" in vm.name: - new_pid = fetch_pid_from_windows_vm(vm=vm, process_name=REGEDIT_PROC_NAME) - else: - new_pid = fetch_pid_from_linux_vm(vm=vm, process_name="ping") - assert new_pid == orig_pid, f"PID mismatch after migration! orig_pid: {orig_pid}; new_pid: {new_pid}" - - -@pytest.fixture(scope="module") -def created_post_copy_migration_policy(): - with MigrationPolicy( - name="post-copy-migration-mp", - allow_auto_converge=True, - bandwidth_per_migration="100Mi", - completion_timeout_per_gb=1, - allow_post_copy=True, - vmi_selector=VM_LABEL, - ) as mp: - yield mp - - -@pytest.fixture(scope="class") -def vm_background_process_id(hotplugged_vm): - if "windows" in hotplugged_vm.name: - return start_and_fetch_processid_on_windows_vm(vm=hotplugged_vm, process_name=REGEDIT_PROC_NAME) - else: - return start_and_fetch_processid_on_linux_vm(vm=hotplugged_vm, process_name="ping", args="localhost") - - -@pytest.fixture() -def migrated_hotplugged_vm( - admin_client: DynamicClient, hotplugged_vm: VirtualMachineForTests -) -> VirtualMachineForTests: - migrate_vm_and_verify( - vm=hotplugged_vm, - client=admin_client, - timeout=TIMEOUT_30MIN if "windows" in hotplugged_vm.name else TIMEOUT_15MIN, - check_ssh_connectivity=True, - ) - return hotplugged_vm - - -@pytest.fixture() -def drained_node_with_hotplugged_vm(admin_client, hco_namespace, compact_cluster, hotplugged_vm): - with drain_node( - admin_client=admin_client, - node=hotplugged_vm.vmi.get_node(privileged_client=admin_client), - hco_namespace=hco_namespace, - compact_cluster=compact_cluster, - ): - check_migration_process_after_node_drain(client=admin_client, vm=hotplugged_vm, admin_client=admin_client) - clean_up_migration_jobs(client=admin_client, vm=hotplugged_vm) - - -@pytest.mark.parametrize( - "golden_image_data_source_for_test_scope_class, hotplugged_vm", - [ - pytest.param( - {"os_dict": RHEL_LATEST}, - { - "template_labels": RHEL_LATEST_LABELS, - "vm_name": "rhel-latest-post-copy-migration-vm", - "additional_labels": VM_LABEL, - }, - id="RHEL-VM", - ), - pytest.param( - {"os_dict": WINDOWS_LATEST}, - { - "template_labels": WINDOWS_LATEST_LABELS, - "vm_name": "windows-latest-post-copy-migration-vm", - "additional_labels": VM_LABEL, - }, - id="WIN-VM", - marks=[pytest.mark.special_infra, pytest.mark.high_resource_vm], - ), - ], - indirect=True, -) -class TestPostCopyMigration: - @pytest.mark.dependency(name=f"{TESTS_CLASS_NAME}::migrate_vm") - @pytest.mark.polarion("CNV-11421") - def test_migrate_vm(self, hotplugged_vm, vm_background_process_id, migrated_hotplugged_vm): - assert_migration_post_copy_mode(vm=hotplugged_vm) - assert_same_pid_after_migration(orig_pid=vm_background_process_id, vm=hotplugged_vm) - - @pytest.mark.dependency(name=f"{TESTS_CLASS_NAME}::node_drain", depends=[f"{TESTS_CLASS_NAME}::migrate_vm"]) - @pytest.mark.polarion("CNV-11422") - def test_node_drain(self, hotplugged_vm, vm_background_process_id, drained_node_with_hotplugged_vm): - assert_migration_post_copy_mode(vm=hotplugged_vm) - assert_same_pid_after_migration(orig_pid=vm_background_process_id, vm=hotplugged_vm) - - @pytest.mark.parametrize( - "hotplugged_sockets_memory_guest", [pytest.param({"sockets": SIX_CPU_SOCKETS})], indirect=True - ) - @pytest.mark.dependency(name=f"{TESTS_CLASS_NAME}::hotplug_cpu", depends=[f"{TESTS_CLASS_NAME}::node_drain"]) - @pytest.mark.polarion("CNV-11423") - def test_hotplug_cpu(self, hotplugged_sockets_memory_guest, hotplugged_vm, vm_background_process_id): - wait_for_guest_os_cpu_count(vm=hotplugged_vm, spec_cpu_amount=SIX_CPU_SOCKETS) - assert_same_pid_after_migration(orig_pid=vm_background_process_id, vm=hotplugged_vm) - - @pytest.mark.parametrize( - "hotplugged_sockets_memory_guest", [pytest.param({"memory_guest": SIX_GI_MEMORY})], indirect=True - ) - @pytest.mark.dependency(depends=[f"{TESTS_CLASS_NAME}::hotplug_cpu"]) - @pytest.mark.polarion("CNV-11424") - def test_hotplug_memory(self, hotplugged_sockets_memory_guest, hotplugged_vm, vm_background_process_id): - assert_guest_os_memory_amount(vm=hotplugged_vm, spec_memory_amount=SIX_GI_MEMORY) - assert_same_pid_after_migration(orig_pid=vm_background_process_id, vm=hotplugged_vm) diff --git a/tests/virt/node/migration_and_maintenance/test_workload_disruption_migration.py b/tests/virt/node/migration_and_maintenance/test_workload_disruption_migration.py new file mode 100644 index 0000000000..c0d2d5a5df --- /dev/null +++ b/tests/virt/node/migration_and_maintenance/test_workload_disruption_migration.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +import pytest +from ocp_resources.migration_policy import MigrationPolicy +from ocp_resources.resource import ResourceEditor + +from tests.os_params import RHEL_LATEST, RHEL_LATEST_LABELS, WINDOWS_LATEST, WINDOWS_LATEST_LABELS +from tests.utils import assert_guest_os_memory_amount, wait_for_guest_os_cpu_count +from tests.virt.constants import WORKLOAD_DISRUPTION_VM_LABEL +from tests.virt.node.migration_and_maintenance.utils import ( + assert_expected_migration_mode, + assert_same_pid_after_migration, + start_memory_pressure_on_vm, +) +from utilities.constants.images import OS_FLAVOR_WINDOWS +from utilities.constants.timeouts import TIMEOUT_15MIN, TIMEOUT_30MIN +from utilities.constants.virt import REGEDIT_PROC_NAME, SIX_CPU_SOCKETS, SIX_GI_MEMORY +from utilities.virt import ( + check_migration_process_after_node_drain, + drain_node, + migrate_vm_and_verify, + start_and_fetch_processid_on_linux_vm, + start_and_fetch_processid_on_windows_vm, +) + +pytestmark = [pytest.mark.rwx_default_storage] + + +RHEL_CLASS_NAME = "TestRhelWorkloadMigration" +WIN_CLASS_NAME = "TestWindowsWorkloadMigration" + + +@pytest.fixture(scope="module") +def workload_disruption_migration_policy(): + with MigrationPolicy( + name="workload-migration-mp", + allow_workload_disruption=True, + allow_auto_converge=False, + bandwidth_per_migration="70Mi", + completion_timeout_per_gb=10, + vmi_selector=WORKLOAD_DISRUPTION_VM_LABEL, + ) as mp: + yield mp + + +@pytest.fixture(scope="class") +def migration_mode(request, workload_disruption_migration_policy): + mode = request.param["mode"] + if mode == "PostCopy": + with ResourceEditor(patches={workload_disruption_migration_policy: {"spec": {"allowPostCopy": True}}}): + yield mode + else: + yield mode + + +@pytest.fixture(scope="class") +def vm_background_process_id(vm_with_hotplug_support): + if vm_with_hotplug_support.os_flavor == OS_FLAVOR_WINDOWS: + return start_and_fetch_processid_on_windows_vm(vm=vm_with_hotplug_support, process_name=REGEDIT_PROC_NAME) + return start_and_fetch_processid_on_linux_vm(vm=vm_with_hotplug_support, process_name="ping", args="localhost") + + +@pytest.fixture() +def vm_memory_pressure(vm_with_hotplug_support): + start_memory_pressure_on_vm(vm=vm_with_hotplug_support) + + +@pytest.fixture() +def migrated_vm_with_hotplug_support(admin_client, vm_with_hotplug_support): + migrate_vm_and_verify( + vm=vm_with_hotplug_support, + timeout=TIMEOUT_30MIN if vm_with_hotplug_support.os_flavor == OS_FLAVOR_WINDOWS else TIMEOUT_15MIN, + check_ssh_connectivity=True, + client=admin_client, + ) + + +@pytest.fixture() +def drained_node_for_hotplug_vm(admin_client, hco_namespace, vm_with_hotplug_support): + with drain_node( + admin_client=admin_client, + node=vm_with_hotplug_support.vmi.get_node(privileged_client=admin_client), + hco_namespace=hco_namespace, + ): + check_migration_process_after_node_drain( + client=admin_client, vm=vm_with_hotplug_support, admin_client=admin_client + ) + + +@pytest.mark.parametrize( + "golden_image_data_source_for_test_scope_class, vm_with_hotplug_support, migration_mode", + [ + pytest.param( + {"os_dict": RHEL_LATEST}, + { + "template_labels": RHEL_LATEST_LABELS, + "vm_name": "rhel-postcopy-vm", + "additional_labels": WORKLOAD_DISRUPTION_VM_LABEL, + }, + {"mode": "PostCopy"}, + id="RHEL-PostCopy", + ), + pytest.param( + {"os_dict": RHEL_LATEST}, + { + "template_labels": RHEL_LATEST_LABELS, + "vm_name": "rhel-paused-vm", + "additional_labels": WORKLOAD_DISRUPTION_VM_LABEL, + }, + {"mode": "Paused"}, + id="RHEL-Paused", + ), + ], + indirect=True, +) +@pytest.mark.usefixtures("vm_memory_pressure") +class TestRhelWorkloadMigration: + @pytest.mark.dependency(name=f"{RHEL_CLASS_NAME}::migrate_vm") + @pytest.mark.polarion("CNV-15225") + def test_awd_migration_mode( + self, + vm_with_hotplug_support, + vm_background_process_id, + migration_mode, + migrated_vm_with_hotplug_support, + ): + assert_expected_migration_mode(vm=vm_with_hotplug_support, expected_mode=migration_mode) + assert_same_pid_after_migration(orig_pid=vm_background_process_id, vm=vm_with_hotplug_support) + + @pytest.mark.dependency(name=f"{RHEL_CLASS_NAME}::node_drain", depends=[f"{RHEL_CLASS_NAME}::migrate_vm"]) + @pytest.mark.polarion("CNV-15245") + def test_awd_node_drain( + self, + vm_with_hotplug_support, + vm_background_process_id, + migration_mode, + drained_node_for_hotplug_vm, + ): + assert_expected_migration_mode(vm=vm_with_hotplug_support, expected_mode=migration_mode) + assert_same_pid_after_migration(orig_pid=vm_background_process_id, vm=vm_with_hotplug_support) + + @pytest.mark.parametrize( + "hotplugged_sockets_memory_guest", [pytest.param({"sockets": SIX_CPU_SOCKETS})], indirect=True + ) + @pytest.mark.dependency(name=f"{RHEL_CLASS_NAME}::hotplug_cpu", depends=[f"{RHEL_CLASS_NAME}::node_drain"]) + @pytest.mark.polarion("CNV-15234") + def test_awd_hotplug_cpu( + self, + vm_with_hotplug_support, + vm_background_process_id, + migration_mode, + hotplugged_sockets_memory_guest, + ): + assert_expected_migration_mode(vm=vm_with_hotplug_support, expected_mode=migration_mode) + wait_for_guest_os_cpu_count(vm=vm_with_hotplug_support, spec_cpu_amount=SIX_CPU_SOCKETS) + assert_same_pid_after_migration(orig_pid=vm_background_process_id, vm=vm_with_hotplug_support) + + @pytest.mark.parametrize( + "hotplugged_sockets_memory_guest", [pytest.param({"memory_guest": SIX_GI_MEMORY})], indirect=True + ) + @pytest.mark.dependency(depends=[f"{RHEL_CLASS_NAME}::hotplug_cpu"]) + @pytest.mark.polarion("CNV-15235") + def test_awd_hotplug_memory( + self, + vm_with_hotplug_support, + vm_background_process_id, + migration_mode, + hotplugged_sockets_memory_guest, + ): + assert_expected_migration_mode(vm=vm_with_hotplug_support, expected_mode=migration_mode) + assert_guest_os_memory_amount(vm=vm_with_hotplug_support, spec_memory_amount=SIX_GI_MEMORY) + assert_same_pid_after_migration(orig_pid=vm_background_process_id, vm=vm_with_hotplug_support) + + +@pytest.mark.parametrize( + "golden_image_data_source_for_test_scope_class, vm_with_hotplug_support, migration_mode", + [ + pytest.param( + {"os_dict": WINDOWS_LATEST}, + { + "template_labels": WINDOWS_LATEST_LABELS, + "vm_name": "windows-postcopy-vm", + "additional_labels": WORKLOAD_DISRUPTION_VM_LABEL, + }, + {"mode": "PostCopy"}, + id="WIN-PostCopy", + ), + pytest.param( + {"os_dict": WINDOWS_LATEST}, + { + "template_labels": WINDOWS_LATEST_LABELS, + "vm_name": "windows-paused-vm", + "additional_labels": WORKLOAD_DISRUPTION_VM_LABEL, + }, + {"mode": "Paused"}, + id="WIN-Paused", + ), + ], + indirect=True, +) +@pytest.mark.special_infra +@pytest.mark.high_resource_vm +@pytest.mark.usefixtures("vm_memory_pressure") +class TestWindowsWorkloadMigration: + @pytest.mark.dependency(name=f"{WIN_CLASS_NAME}::migrate_vm") + @pytest.mark.polarion("CNV-15246") + def test_awd_migration_mode( + self, + vm_with_hotplug_support, + vm_background_process_id, + migration_mode, + migrated_vm_with_hotplug_support, + ): + assert_expected_migration_mode(vm=vm_with_hotplug_support, expected_mode=migration_mode) + assert_same_pid_after_migration(orig_pid=vm_background_process_id, vm=vm_with_hotplug_support) + + @pytest.mark.parametrize( + "hotplugged_sockets_memory_guest", [pytest.param({"sockets": SIX_CPU_SOCKETS})], indirect=True + ) + @pytest.mark.dependency(name=f"{WIN_CLASS_NAME}::hotplug_cpu", depends=[f"{WIN_CLASS_NAME}::migrate_vm"]) + @pytest.mark.polarion("CNV-15247") + def test_awd_hotplug_cpu( + self, + vm_with_hotplug_support, + vm_background_process_id, + migration_mode, + hotplugged_sockets_memory_guest, + ): + assert_expected_migration_mode(vm=vm_with_hotplug_support, expected_mode=migration_mode) + wait_for_guest_os_cpu_count(vm=vm_with_hotplug_support, spec_cpu_amount=SIX_CPU_SOCKETS) + assert_same_pid_after_migration(orig_pid=vm_background_process_id, vm=vm_with_hotplug_support) + + @pytest.mark.parametrize( + "hotplugged_sockets_memory_guest", [pytest.param({"memory_guest": SIX_GI_MEMORY})], indirect=True + ) + @pytest.mark.dependency(depends=[f"{WIN_CLASS_NAME}::hotplug_cpu"]) + @pytest.mark.polarion("CNV-16312") + def test_awd_hotplug_memory( + self, + vm_with_hotplug_support, + vm_background_process_id, + migration_mode, + hotplugged_sockets_memory_guest, + ): + assert_expected_migration_mode(vm=vm_with_hotplug_support, expected_mode=migration_mode) + assert_guest_os_memory_amount(vm=vm_with_hotplug_support, spec_memory_amount=SIX_GI_MEMORY) + assert_same_pid_after_migration(orig_pid=vm_background_process_id, vm=vm_with_hotplug_support) diff --git a/tests/virt/node/migration_and_maintenance/utils.py b/tests/virt/node/migration_and_maintenance/utils.py new file mode 100644 index 0000000000..4e068681a3 --- /dev/null +++ b/tests/virt/node/migration_and_maintenance/utils.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from pyhelper_utils.shell import run_ssh_commands + +from utilities.constants.images import OS_FLAVOR_WINDOWS +from utilities.constants.timeouts import TCP_TIMEOUT_30SEC +from utilities.constants.virt import REGEDIT_PROC_NAME +from utilities.virt import fetch_pid_from_linux_vm, fetch_pid_from_windows_vm + +if TYPE_CHECKING: + from utilities.virt import VirtualMachineForTests + +LOGGER = logging.getLogger(__name__) + +# Allocate and continuously re-dirty memory pages without stress-ng (uses built-in PowerShell/Python). +WIN_MEM_LOAD_CMD = ( + "powershell.exe -NoProfile -WindowStyle Hidden -Command" + ' "$a0=New-Object byte[] (1GB);$a1=New-Object byte[] (1GB);' + ' while($true){ for($i=0;$i -lt $a0.Length;$i+=4096){$a0[$i]++;$a1[$i]++} }"' +) +RHEL_MEM_LOAD_CMD = ( + "import time; a = bytearray(2 * 1024**3); " + 'exec("while True: [a.__setitem__(i, (a[i]+1)%256) for i in range(0, len(a), 4096)]; time.sleep(0.5)")' +) + + +def assert_expected_migration_mode(vm: VirtualMachineForTests, expected_mode: str) -> None: + migration_state = vm.vmi.instance.status.migrationState + assert migration_state.mode == expected_mode, ( + f"Migration mode is not {expected_mode}! VMI MigrationState {migration_state}" + ) + + +def assert_same_pid_after_migration(orig_pid: str, vm: VirtualMachineForTests) -> None: + if vm.os_flavor == OS_FLAVOR_WINDOWS: + new_pid = fetch_pid_from_windows_vm(vm=vm, process_name=REGEDIT_PROC_NAME) + else: + new_pid = fetch_pid_from_linux_vm(vm=vm, process_name="ping") + assert new_pid == orig_pid, f"PID mismatch after migration! orig_pid: {orig_pid}; new_pid: {new_pid}" + + +def start_memory_pressure_on_vm(vm: VirtualMachineForTests) -> None: + """Continuously dirty memory pages to prevent pre-copy migration convergence. + + Kills any leftover stress processes before spawning new ones, since + previous processes may be degraded after migration. + + This function requires no external tools — uses built-in PowerShell on Windows + and Python on Linux. + + Args: + vm: Target VM to stress. + """ + is_windows = vm.os_flavor == OS_FLAVOR_WINDOWS + + LOGGER.info(f"Killing leftover memory pressure processes on VM {vm.name}") + if is_windows: + kill_cmd = ["powershell", "Stop-Process", "-Name", "powershell", "-Force"] + else: + kill_cmd = ["pkill", "-f", "python3 -c.*bytearray"] + run_ssh_commands(host=vm.ssh_exec, commands=kill_cmd, tcp_timeout=TCP_TIMEOUT_30SEC, check_rc=False) + + if is_windows: + cmd = [ + "powershell", + "Invoke-WmiMethod", + "-Class", + "Win32_Process", + "-Name", + "Create", + "-ArgumentList", + f"'{WIN_MEM_LOAD_CMD}'", + ] + else: + cmd = [ + "/bin/bash", + "-c", + f"nohup python3 -c '{RHEL_MEM_LOAD_CMD}' >/dev/null 2>&1 &", + ] + run_ssh_commands(host=vm.ssh_exec, commands=cmd, tcp_timeout=TCP_TIMEOUT_30SEC) + LOGGER.info(f"Started 2GB memory pressure on VM {vm.name}") diff --git a/tests/virt/upgrade/conftest.py b/tests/virt/upgrade/conftest.py index dd9cc05b5a..b3e7c078d6 100644 --- a/tests/virt/upgrade/conftest.py +++ b/tests/virt/upgrade/conftest.py @@ -12,7 +12,7 @@ from ocp_resources.virtual_machine_cluster_preference import VirtualMachineClusterPreference from pytest_testconfig import py_config -from tests.virt.constants import VM_LABEL +from tests.virt.constants import WORKLOAD_DISRUPTION_VM_LABEL from tests.virt.upgrade.utils import ( get_virt_launcher_images_from_csv, validate_vms_pod_updated, @@ -334,10 +334,11 @@ def post_copy_migration_policy_for_upgrade(admin_client): with MigrationPolicy( name="post-copy-migration-policy", allow_auto_converge=True, + allow_workload_disruption=True, bandwidth_per_migration="100Mi", completion_timeout_per_gb=1, allow_post_copy=True, - vmi_selector=VM_LABEL, + vmi_selector=WORKLOAD_DISRUPTION_VM_LABEL, client=admin_client, ) as mp: yield mp @@ -352,7 +353,7 @@ def vm_for_post_copy_upgrade(virt_upgrade_namespace, unprivileged_client, cpu_fo body=fedora_vm_body(name=vm_name), client=unprivileged_client, cpu_model=cpu_for_migration, - additional_labels=VM_LABEL, + additional_labels=WORKLOAD_DISRUPTION_VM_LABEL, ) as vm: running_vm(vm=vm) yield vm