Cherry-pick: 4.20 [VIRT] Stabilize node drain/cordon during maintenance (RedHatQE#3727)

cherry-pick
[3467](RedHatQE#3467)
into cnv-4.20

##### Short description:

##### More details:

##### What this PR does / why we need it:

##### Which issue(s) this PR fixes:

##### Special notes for reviewer:

##### jira-ticket:
<!-- full-ticket-url needs to be provided. This would add a link to the
pull request to the jira and close it when the pull request is merged
If the task is not tracked by a Jira ticket, just write "NONE".
-->

Signed-off-by: Samuel Albershtein <salbersh@redhat.com>

Author: SamAlber

tests/chaos/oadp/conftest.py

@@ -74,8 +74,8 @@ def rebooted_vm_source_node(rhel_vm_with_dv_running, oadp_backup_in_progress, wo
 
 
 @pytest.fixture()
-def drain_vm_source_node(rhel_vm_with_dv_running, oadp_backup_in_progress):
+def drain_vm_source_node(admin_client, rhel_vm_with_dv_running, oadp_backup_in_progress):
     vm_node = rhel_vm_with_dv_running.vmi.node
-    with node_mgmt_console(node=vm_node, node_mgmt="drain"):
+    with node_mgmt_console(admin_client=admin_client, node=vm_node, node_mgmt="drain"):
         wait_for_node_schedulable_status(node=vm_node, status=False)
         yield vm_node

tests/virt/cluster/migration_and_maintenance/test_evictionstrategy.py

@@ -42,9 +42,9 @@ def assert_vm_restarts_after_node_drain(source_node, vmi, vmi_old_uid):
 
 
 @pytest.fixture()
-def drained_node(vm_for_test_from_template_scope_class):
+def drained_node(admin_client, vm_for_test_from_template_scope_class):
     source_node = vm_for_test_from_template_scope_class.privileged_vmi.node
-    with node_mgmt_console(node=source_node, node_mgmt="drain"):
+    with node_mgmt_console(admin_client=admin_client, node=source_node, node_mgmt="drain"):
         yield source_node
 
 
@@ -125,9 +125,7 @@ class TestEvictionStrategy:
     def test_hco_evictionstrategy_livemigrate_vm_no_evictionstrategy(
         self, unprivileged_client, vm_for_test_from_template_scope_class, drained_node
     ):
-        check_migration_process_after_node_drain(
-            dyn_client=unprivileged_client, vm=vm_for_test_from_template_scope_class
-        )
+        check_migration_process_after_node_drain(client=unprivileged_client, vm=vm_for_test_from_template_scope_class)
 
     @pytest.mark.polarion("CNV-10088")
     def test_hco_evictionstrategy_none_vm_no_evictionstrategy(
@@ -169,6 +167,4 @@ def test_hco_evictionstrategy_none_vm_evictionstrategy_livemigrate(
         added_vm_evictionstrategy,
         drained_node,
     ):
-        check_migration_process_after_node_drain(
-            dyn_client=unprivileged_client, vm=vm_for_test_from_template_scope_class
-        )
+        check_migration_process_after_node_drain(client=unprivileged_client, vm=vm_for_test_from_template_scope_class)

tests/virt/node/descheduler/conftest.py

@@ -150,12 +150,13 @@ def node_to_drain(
 
 @pytest.fixture()
 def drain_uncordon_node(
+    admin_client,
     deployed_vms_for_descheduler_test,
     vms_orig_nodes_before_node_drain,
     node_to_drain,
 ):
     """Return when node is schedulable again after uncordon"""
-    with node_mgmt_console(node=node_to_drain, node_mgmt="drain"):
+    with node_mgmt_console(admin_client=admin_client, node=node_to_drain, node_mgmt="drain"):
         wait_for_node_schedulable_status(node=node_to_drain, status=False)
         for vm in deployed_vms_for_descheduler_test:
             if vms_orig_nodes_before_node_drain[vm.name].name == node_to_drain.name:

tests/virt/node/migration_and_maintenance/test_node_maintenance.py

@@ -3,7 +3,6 @@
 """
 
 import logging
-import random
 
 import pytest
 from ocp_resources.virtual_machine_instance_migration import (
@@ -37,20 +36,20 @@
 LOGGER = logging.getLogger(__name__)
 
 
-def drain_using_console(dyn_client, source_node, vm):
+def drain_using_console(admin_client, source_node, vm):
     with running_sleep_in_linux(vm=vm):
-        with node_mgmt_console(node=source_node, node_mgmt="drain"):
-            check_migration_process_after_node_drain(dyn_client=dyn_client, vm=vm)
+        with node_mgmt_console(admin_client=admin_client, node=source_node, node_mgmt="drain"):
+            check_migration_process_after_node_drain(client=admin_client, vm=vm)
 
 
-def drain_using_console_windows(dyn_client, source_node, vm):
+def drain_using_console_windows(admin_client, source_node, vm):
     process_name = OS_PROC_NAME["windows"]
     pre_migrate_processid = start_and_fetch_processid_on_windows_vm(
         vm=vm,
         process_name=process_name,
     )
-    with node_mgmt_console(node=source_node, node_mgmt="drain"):
-        check_migration_process_after_node_drain(dyn_client=dyn_client, vm=vm)
+    with node_mgmt_console(admin_client=admin_client, node=source_node, node_mgmt="drain"):
+        check_migration_process_after_node_drain(client=admin_client, vm=vm)
         post_migrate_processid = fetch_pid_from_windows_vm(vm=vm, process_name=process_name)
         assert post_migrate_processid == pre_migrate_processid, (
             f"Post migrate processid is: {post_migrate_processid}. Pre migrate processid is: {pre_migrate_processid}"
@@ -74,7 +73,7 @@ def vm_container_disk_fedora(
     namespace,
     unprivileged_client,
 ):
-    name = f"vm-nodemaintenance-{random.randrange(99999)}"
+    name = "vm-nodemaintenance"
     with VirtualMachineForTests(
         name=name,
         namespace=namespace.name,
@@ -120,7 +119,7 @@ def test_node_drain_using_console_fedora(
 ):
     privileged_virt_launcher_pod = vm_container_disk_fedora.privileged_vmi.virt_launcher_pod
     drain_using_console(
-        dyn_client=admin_client, source_node=privileged_virt_launcher_pod.node, vm=vm_container_disk_fedora
+        admin_client=admin_client, source_node=privileged_virt_launcher_pod.node, vm=vm_container_disk_fedora
     )
 
 
@@ -144,21 +143,20 @@ def test_node_drain_using_console_fedora(
 )
 @pytest.mark.usefixtures("cluster_cpu_model_scope_class", "golden_image_data_volume_multi_storage_scope_class")
 @pytest.mark.ibm_bare_metal
+@pytest.mark.usefixtures("no_migration_job")
 class TestNodeMaintenanceRHEL:
     @pytest.mark.polarion("CNV-2292")
     def test_node_drain_using_console_rhel(
         self,
-        no_migration_job,
         golden_image_vm_instance_from_template_multi_storage_scope_class,
         admin_client,
     ):
         vm = golden_image_vm_instance_from_template_multi_storage_scope_class
-        drain_using_console(dyn_client=admin_client, source_node=vm.privileged_vmi.virt_launcher_pod.node, vm=vm)
+        drain_using_console(admin_client=admin_client, source_node=vm.privileged_vmi.virt_launcher_pod.node, vm=vm)
 
     @pytest.mark.polarion("CNV-4995")
     def test_migration_when_multiple_nodes_unschedulable_using_console_rhel(
         self,
-        no_migration_job,
         golden_image_vm_instance_from_template_multi_storage_scope_class,
         schedulable_nodes,
         admin_client,
@@ -181,8 +179,8 @@ def test_migration_when_multiple_nodes_unschedulable_using_console_rhel(
             pod=vm.privileged_vmi.virt_launcher_pod,
             schedulable_nodes=schedulable_nodes,
         )
-        with node_mgmt_console(node=cordon_nodes[0], node_mgmt="cordon"):
-            drain_using_console(dyn_client=admin_client, source_node=vm.privileged_vmi.virt_launcher_pod.node, vm=vm)
+        with node_mgmt_console(admin_client=admin_client, node=cordon_nodes[0], node_mgmt="cordon"):
+            drain_using_console(admin_client=admin_client, source_node=vm.privileged_vmi.virt_launcher_pod.node, vm=vm)
 
 
 @pytest.mark.parametrize(
@@ -206,28 +204,31 @@ def test_migration_when_multiple_nodes_unschedulable_using_console_rhel(
 )
 @pytest.mark.usefixtures("cluster_modern_cpu_model_scope_class", "golden_image_data_volume_multi_storage_scope_class")
 @pytest.mark.ibm_bare_metal
+@pytest.mark.usefixtures("no_migration_job")
 class TestNodeCordonAndDrain:
     @pytest.mark.polarion("CNV-2048")
     def test_node_drain_template_windows(
         self,
-        no_migration_job,
         golden_image_vm_instance_from_template_multi_storage_scope_class,
         admin_client,
     ):
         vm = golden_image_vm_instance_from_template_multi_storage_scope_class
         drain_using_console_windows(
-            dyn_client=admin_client, source_node=vm.privileged_vmi.virt_launcher_pod.node, vm=vm
+            admin_client=admin_client, source_node=vm.privileged_vmi.virt_launcher_pod.node, vm=vm
         )
 
     @pytest.mark.polarion("CNV-4906")
     def test_node_cordon_template_windows(
         self,
-        no_migration_job,
         golden_image_vm_instance_from_template_multi_storage_scope_class,
         admin_client,
     ):
         vm = golden_image_vm_instance_from_template_multi_storage_scope_class
-        with node_mgmt_console(node=vm.privileged_vmi.virt_launcher_pod.node, node_mgmt="cordon"):
+        with node_mgmt_console(
+            admin_client=admin_client,
+            node=vm.privileged_vmi.virt_launcher_pod.node,
+            node_mgmt="cordon",
+        ):
             with pytest.raises(TimeoutExpiredError):
                 migration_job_sampler(
                     dyn_client=admin_client,

tests/virt/node/migration_and_maintenance/test_post_copy_migration.py

@@ -76,8 +76,8 @@ def migrated_hotplugged_vm(hotplugged_vm):
 
 @pytest.fixture()
 def drained_node_with_hotplugged_vm(admin_client, hotplugged_vm):
-    with node_mgmt_console(node=hotplugged_vm.privileged_vmi.node, node_mgmt="drain"):
-        check_migration_process_after_node_drain(dyn_client=admin_client, vm=hotplugged_vm)
+    with node_mgmt_console(admin_client=admin_client, node=hotplugged_vm.privileged_vmi.node, node_mgmt="drain"):
+        check_migration_process_after_node_drain(client=admin_client, vm=hotplugged_vm)
     clean_up_migration_jobs(client=admin_client, vm=hotplugged_vm)
 
 

tests/virt/node/migration_and_maintenance/test_vm_unscheduled_node.py

@@ -54,7 +54,7 @@ def unscheduled_node_vm(
     indirect=True,
 )
 @pytest.mark.polarion("CNV-4157")
-def test_schedule_vm_on_cordoned_node(worker_node1, data_volume_scope_function, unscheduled_node_vm):
+def test_schedule_vm_on_cordoned_node(admin_client, worker_node1, unscheduled_node_vm):
     """Test VM scheduling on a node under maintenance.
     1. Cordon the target node specified in the VM's nodeAffinity (worker_node1).
     2. Wait until the node status becomes 'Ready,SchedulingDisabled'.
@@ -65,7 +65,7 @@ def test_schedule_vm_on_cordoned_node(worker_node1, data_volume_scope_function,
     7. Verify that the VMI is running on the expected node (worker_node1).
     """
 
-    with node_mgmt_console(node=worker_node1, node_mgmt="cordon"):
+    with node_mgmt_console(admin_client=admin_client, node=worker_node1, node_mgmt="cordon"):
         wait_for_node_schedulable_status(node=worker_node1, status=False)
         unscheduled_node_vm.start()
         unscheduled_node_vm.vmi.wait_for_status(status=VirtualMachineInstance.Status.SCHEDULING, timeout=TIMEOUT_20SEC)

tests/virt/node/migration_and_maintenance/utils.py

@@ -132,10 +132,10 @@ def _get_source_migration_logs():
     assert len(matches) == 6, f"Not all migration logs found. Found {len(matches)} of 6"
 
 
-def assert_node_drain_and_vm_migration(dyn_client, vm, virt_handler_pods):
+def assert_node_drain_and_vm_migration(admin_client, vm, virt_handler_pods):
     source_node = vm.privileged_vmi.node
-    with node_mgmt_console(node=source_node, node_mgmt="drain"):
-        check_migration_process_after_node_drain(dyn_client=dyn_client, vm=vm)
+    with node_mgmt_console(admin_client=admin_client, node=source_node, node_mgmt="drain"):
+        check_migration_process_after_node_drain(client=admin_client, vm=vm)
         assert_vm_migrated_through_dedicated_network_with_logs(
             source_node=source_node, vm=vm, virt_handler_pods=virt_handler_pods
         )

utilities/virt.py

@@ -83,7 +83,7 @@
     Images,
 )
 from utilities.data_collector import collect_vnc_screenshot_for_vms
-from utilities.hco import wait_for_hco_conditions
+from utilities.hco import get_hco_namespace, wait_for_hco_conditions
 from utilities.storage import get_default_storage_class
 
 if TYPE_CHECKING:
@@ -2020,7 +2020,8 @@ def vm_instance_from_template(
 
 
 @contextmanager
-def node_mgmt_console(node, node_mgmt):
+def node_mgmt_console(admin_client, node, node_mgmt):
+    hco_namespace = get_hco_namespace(admin_client=admin_client)
     try:
         LOGGER.info(f"{node_mgmt.capitalize()} the node {node.name}")
         extra_opts = "--delete-emptydir-data --ignore-daemonsets=true --force" if node_mgmt == "drain" else ""
@@ -2030,9 +2031,15 @@ def node_mgmt_console(node, node_mgmt):
         )
         yield
     finally:
+        if node_mgmt == "drain":
+            LOGGER.info("Terminate drain process")
+            run(
+                shlex.split('pkill -f "oc adm drain"'),
+            )
         LOGGER.info(f"Uncordon node {node.name}")
         run(f"oc adm uncordon {node.name}", shell=True)
         wait_for_node_schedulable_status(node=node, status=True)
+        wait_for_kv_stabilize(admin_client=admin_client, hco_namespace=hco_namespace)
 
 
 @contextmanager
@@ -2212,15 +2219,15 @@ def get_created_migration_job(vm, timeout=TIMEOUT_1MIN, client=None):
         raise
 
 
-def check_migration_process_after_node_drain(dyn_client, vm):
+def check_migration_process_after_node_drain(client, vm):
     """
     Wait for migration process to succeed and verify that VM indeed moved to new node.
     """
     vmi_old_uid = vm.vmi.instance.metadata.uid
     source_node = vm.privileged_vmi.virt_launcher_pod.node
     LOGGER.info(f"The VMI was running on {source_node.name}")
     wait_for_node_schedulable_status(node=source_node, status=False)
-    vmim = get_created_migration_job(vm=vm, client=dyn_client, timeout=TIMEOUT_5MIN)
+    vmim = get_created_migration_job(vm=vm, client=client, timeout=TIMEOUT_5MIN)
     wait_for_migration_finished(
         namespace=vm.namespace, migration=vmim, timeout=TIMEOUT_30MIN if "windows" in vm.name else TIMEOUT_10MIN
     )