[Chaos][OADP] Add case CNV-12011: Reboot the worker node during OADP Backup (RedHatQE#1678)

Signed-off-by: Qixuan Wang <qwang@redhat.com>

Author: qwang1

tests/chaos/oadp/__init__.py

@@ -0,0 +1,66 @@
+import datetime
+import logging
+
+import pytest
+
+from utilities.constants import BACKUP_STORAGE_LOCATION, FILE_NAME_FOR_BACKUP, TEXT_TO_TEST, TIMEOUT_3MIN, TIMEOUT_10MIN
+from utilities.infra import ExecCommandOnPod, wait_for_node_status
+from utilities.oadp import VeleroBackup, create_rhel_vm
+from utilities.storage import write_file
+
+LOGGER = logging.getLogger(__name__)
+
+
+@pytest.fixture()
+def rhel_vm_with_dv_running(request, admin_client, chaos_namespace, snapshot_storage_class_name_scope_module):
+    """
+    Create a RHEL VM with a DataVolume.
+    """
+    vm_name = request.param["vm_name"]
+
+    with create_rhel_vm(
+        storage_class=snapshot_storage_class_name_scope_module,
+        namespace=chaos_namespace.name,
+        vm_name=vm_name,
+        dv_name=f"dv-{vm_name}",
+        client=admin_client,
+        wait_running=True,
+        rhel_image=request.param["rhel_image"],
+    ) as vm:
+        write_file(
+            vm=vm,
+            filename=FILE_NAME_FOR_BACKUP,
+            content=TEXT_TO_TEST,
+            stop_vm=False,
+        )
+        yield vm
+
+
+@pytest.fixture()
+def oadp_backup_in_progress(admin_client, chaos_namespace, rhel_vm_with_dv_running):
+    timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+    backup_name = f"backup-{timestamp}"
+
+    with VeleroBackup(
+        name=backup_name,
+        included_namespaces=[chaos_namespace.name],
+        client=admin_client,
+        snapshot_move_data=True,
+        storage_location=BACKUP_STORAGE_LOCATION,
+        wait_complete=False,
+    ) as backup:
+        backup.wait_for_status(status=backup.Backup.Status.INPROGRESS, timeout=TIMEOUT_3MIN)
+        yield backup
+
+
+@pytest.fixture()
+def rebooted_vm_source_node(rhel_vm_with_dv_running, oadp_backup_in_progress, workers_utility_pods):
+    vm_node = rhel_vm_with_dv_running.vmi.node
+
+    LOGGER.info(f"Rebooting node {vm_node.name}")
+    ExecCommandOnPod(utility_pods=workers_utility_pods, node=vm_node).exec(command="shutdown -r now", ignore_rc=True)
+    wait_for_node_status(node=vm_node, status=False, wait_timeout=TIMEOUT_10MIN)
+
+    LOGGER.info(f"Waiting for node {vm_node.name} to come back online")
+    wait_for_node_status(node=vm_node, status=True, wait_timeout=TIMEOUT_10MIN)
+    return

tests/chaos/oadp/conftest.py

@@ -0,0 +1,41 @@
+import logging
+
+import pytest
+
+from tests.os_params import RHEL_LATEST
+from utilities.constants import TIMEOUT_10MIN
+
+LOGGER = logging.getLogger(__name__)
+
+
+@pytest.mark.destructive
+@pytest.mark.chaos
+@pytest.mark.parametrize(
+    "rhel_vm_with_dv_running",
+    [
+        pytest.param(
+            {
+                "vm_name": "vm-node-reboot-12011",
+                "rhel_image": RHEL_LATEST["image_name"],
+            },
+            marks=pytest.mark.polarion("CNV-12011"),
+        ),
+    ],
+    indirect=True,
+)
+def test_reboot_vm_node_during_backup(
+    oadp_backup_in_progress,
+    rebooted_vm_source_node,
+):
+    """
+    Reboot the worker node where the VM is located during OADP backup using DataMover.
+    Validate that backup eventually PartiallyFailed.
+    """
+
+    LOGGER.info(
+        f"Waiting for backup to reach "
+        f"'{oadp_backup_in_progress.Backup.Status.PARTIALLYFAILED}' status after node recovery"
+    )
+    oadp_backup_in_progress.wait_for_status(
+        status=oadp_backup_in_progress.Backup.Status.PARTIALLYFAILED, timeout=TIMEOUT_10MIN
+    )

tests/chaos/oadp/test_oadp.py

@@ -3,15 +3,23 @@
 from ocp_resources.namespace import Namespace
 
 from tests.data_protection.oadp.utils import (
+    VeleroRestore,
+    is_storage_class_support_volume_mode,
+)
+from utilities.constants import (
+    BACKUP_STORAGE_LOCATION,
     FILE_NAME_FOR_BACKUP,
+    OS_FLAVOR_RHEL,
     TEXT_TO_TEST,
+    TIMEOUT_8MIN,
+    TIMEOUT_15MIN,
+    Images,
+)
+from utilities.infra import create_ns
+from utilities.oadp import (
     VeleroBackup,
-    VeleroRestore,
     create_rhel_vm,
-    is_storage_class_support_volume_mode,
 )
-from utilities.constants import OS_FLAVOR_RHEL, TIMEOUT_8MIN, TIMEOUT_15MIN, Images
-from utilities.infra import create_ns
 from utilities.storage import (
     check_upload_virtctl_result,
     create_dv,
@@ -132,7 +140,7 @@ def velero_backup_first_namespace_using_datamover(namespace_for_backup):
         ],
         name="datamover-backup-ns",
         snapshot_move_data=True,
-        storage_location="dpa-1",
+        storage_location=BACKUP_STORAGE_LOCATION,
     ) as backup:
         yield backup
 
@@ -226,7 +234,7 @@ def velero_backup_second_namespace_using_datamover(namespace_for_backup2):
         ],
         name="datamover-backup-ns2",
         snapshot_move_data=True,
-        storage_location="dpa-1",
+        storage_location=BACKUP_STORAGE_LOCATION,
     ) as backup:
         yield backup
 

tests/data_protection/oadp/conftest.py

@@ -1,89 +1,26 @@
 import logging
-import shlex
-from contextlib import contextmanager
 
-from ocp_resources.backup import Backup
-from ocp_resources.datavolume import DataVolume
 from ocp_resources.persistent_volume_claim import PersistentVolumeClaim
 from ocp_resources.restore import Restore
 from ocp_resources.storage_profile import StorageProfile
-from ocp_resources.virtual_machine import VirtualMachine
 
 from utilities import console
 from utilities.constants import (
+    ADP_NAMESPACE,
+    FILE_NAME_FOR_BACKUP,
     LS_COMMAND,
-    OS_FLAVOR_RHEL,
+    TEXT_TO_TEST,
     TIMEOUT_5MIN,
     TIMEOUT_10SEC,
     TIMEOUT_15SEC,
     TIMEOUT_20SEC,
-    Images,
 )
 from utilities.infra import (
-    cleanup_artifactory_secret_and_config_map,
-    get_artifactory_config_map,
-    get_artifactory_secret,
-    get_http_image_url,
-    get_pod_by_name_prefix,
     unique_name,
 )
-from utilities.virt import VirtualMachineForTests, running_vm
+from utilities.oadp import delete_velero_resource
 
-ADP_NAMESPACE = "openshift-adp"
-FILE_NAME_FOR_BACKUP = "file_before_backup.txt"
 LOGGER = logging.getLogger(__name__)
-TEXT_TO_TEST = "text"
-
-
-def delete_velero_resource(resource, client):
-    velero_pod = get_pod_by_name_prefix(dyn_client=client, pod_prefix="velero", namespace=ADP_NAMESPACE)
-    velero_pod.execute(
-        command=shlex.split(f"bash -c 'echo  Y | ./velero  delete {resource.kind.lower()} {resource.name}'")
-    )
-
-
-class VeleroBackup(Backup):
-    def __init__(
-        self,
-        name,
-        namespace=ADP_NAMESPACE,
-        included_namespaces=None,
-        client=None,
-        teardown=False,
-        yaml_file=None,
-        excluded_resources=None,
-        wait_complete=True,
-        snapshot_move_data=False,
-        storage_location=None,
-        timeout=TIMEOUT_5MIN,
-        **kwargs,
-    ):
-        super().__init__(
-            name=unique_name(name=name),
-            namespace=namespace,
-            included_namespaces=included_namespaces,
-            client=client,
-            teardown=teardown,
-            yaml_file=yaml_file,
-            excluded_resources=excluded_resources,
-            storage_location=storage_location,
-            snapshot_move_data=snapshot_move_data,
-            **kwargs,
-        )
-        self.wait_complete = wait_complete
-        self.timeout = timeout
-
-    def __enter__(self):
-        super().__enter__()
-        if self.wait_complete:
-            self.wait_for_status(
-                status=self.Status.COMPLETED,
-                timeout=self.timeout,
-            )
-        return self
-
-    def __exit__(self, exception_type, exception_value, traceback):
-        delete_velero_resource(resource=self, client=self.client)
 
 
 class VeleroRestore(Restore):
@@ -126,53 +63,6 @@ def __exit__(self, exception_type, exception_value, traceback):
         delete_velero_resource(resource=self, client=self.client)
 
 
-@contextmanager
-def create_rhel_vm(
-    storage_class,
-    namespace,
-    dv_name,
-    vm_name,
-    rhel_image,
-    client=None,
-    wait_running=True,
-    volume_mode=None,
-):
-    artifactory_secret = get_artifactory_secret(namespace=namespace)
-    artifactory_config_map = get_artifactory_config_map(namespace=namespace)
-    dv = DataVolume(
-        name=dv_name,
-        namespace=namespace,
-        source="http",
-        url=get_http_image_url(
-            image_directory=Images.Rhel.DIR,
-            image_name=rhel_image,
-        ),
-        storage_class=storage_class,
-        size=Images.Rhel.DEFAULT_DV_SIZE,
-        api_name="storage",
-        volume_mode=volume_mode,
-        secret=artifactory_secret,
-        cert_configmap=artifactory_config_map.name,
-    )
-    dv.to_dict()
-    dv_metadata = dv.res["metadata"]
-    with VirtualMachineForTests(
-        client=client,
-        name=vm_name,
-        namespace=dv_metadata["namespace"],
-        os_flavor=OS_FLAVOR_RHEL,
-        memory_guest=Images.Rhel.DEFAULT_MEMORY_SIZE,
-        data_volume_template={"metadata": dv_metadata, "spec": dv.res["spec"]},
-        run_strategy=VirtualMachine.RunStrategy.ALWAYS,
-    ) as vm:
-        if wait_running:
-            running_vm(vm=vm, wait_for_interfaces=True)
-        yield vm
-    cleanup_artifactory_secret_and_config_map(
-        artifactory_secret=artifactory_secret, artifactory_config_map=artifactory_config_map
-    )
-
-
 def check_file_in_vm(vm):
     with console.Console(vm=vm) as vm_console:
         vm_console.sendline(LS_COMMAND)

tests/data_protection/oadp/utils.py

@@ -902,3 +902,9 @@ class NamespacesNames:
 LATEST_RELEASE_STR = "latest_released"
 OS_VERSION_STR = "os_version"
 DATA_SOURCE_STR = "data_source"
+
+# OADP
+ADP_NAMESPACE = "openshift-adp"
+FILE_NAME_FOR_BACKUP = "file_before_backup.txt"
+TEXT_TO_TEST = "text"
+BACKUP_STORAGE_LOCATION = "dpa-1"