[4.20] [Storage] Use ssh retries in run_ssh_commands (#4471)

tests/storage/online_resize/utils.py

@@ -13,7 +13,7 @@
 from pyhelper_utils.shell import run_ssh_commands
 from timeout_sampler import TimeoutExpiredError, TimeoutSampler
 
-from utilities.constants import TIMEOUT_4MIN
+from utilities.constants import TIMEOUT_2MIN, TIMEOUT_4MIN, TIMEOUT_5SEC
 from utilities.storage import create_dv
 from utilities.virt import running_vm
 
@@ -61,9 +61,16 @@ def cksum_file(vm, filename, create=False):
         run_ssh_commands(
             host=vm.ssh_exec,
             commands=shlex.split(f"dd if=/dev/urandom of={filename} count=100 && sync"),
+            wait_timeout=TIMEOUT_2MIN,
+            sleep=TIMEOUT_5SEC,
         )
 
-    out = run_ssh_commands(host=vm.ssh_exec, commands=shlex.split(f"sha256sum {filename}"))[0]
+    out = run_ssh_commands(
+        host=vm.ssh_exec,
+        commands=shlex.split(f"sha256sum {filename}"),
+        wait_timeout=TIMEOUT_2MIN,
+        sleep=TIMEOUT_5SEC,
+    )[0]
     sha256sum = out.split()[0]
     LOGGER.info(f"File sha256sum is {sha256sum}")
     return sha256sum
@@ -104,7 +111,7 @@ def expand_pvc(dv, size_change):
 
 def get_resize_count(vm):
     commands = shlex.split("sudo dmesg | grep -c 'new size' || true")
-    result = run_ssh_commands(host=vm.ssh_exec, commands=commands)[0]
+    result = run_ssh_commands(host=vm.ssh_exec, commands=commands, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)[0]
 
     return int(result)
 
@@ -136,7 +143,9 @@ def wait_for_resize(vm, count=1):
             if current_resize_count in (desired_count, desired_count + 1):
                 break
     except TimeoutExpiredError:
-        dmesg = run_ssh_commands(host=vm.ssh_exec, commands=shlex.split("dmesg"))[0]
+        dmesg = run_ssh_commands(
+            host=vm.ssh_exec, commands=shlex.split("dmesg"), wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC
+        )[0]
         LOGGER.error(f"Failed to reach resize count {desired_count}.\ndmesg:\n{dmesg}")
         raise
 

tests/storage/snapshots/conftest.py

@@ -20,7 +20,7 @@
     create_windows_directory,
     set_permissions,
 )
-from utilities.constants import TIMEOUT_10MIN, UNPRIVILEGED_USER
+from utilities.constants import TIMEOUT_2MIN, TIMEOUT_5SEC, TIMEOUT_10MIN, UNPRIVILEGED_USER
 
 LOGGER = logging.getLogger(__name__)
 
@@ -87,7 +87,7 @@ def snapshot_dirctory_removed(windows_vm_for_snapshot, windows_snapshot):
     cmd = shlex.split(
         f'powershell -command "Remove-Item -Path {WINDOWS_DIRECTORY_PATH} -Recurse"',
     )
-    run_ssh_commands(host=windows_vm_for_snapshot.ssh_exec, commands=cmd)
+    run_ssh_commands(host=windows_vm_for_snapshot.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)
     assert_windows_directory_existence(
         expected_result=False,
         windows_vm=windows_vm_for_snapshot,
@@ -102,6 +102,6 @@ def file_created_during_snapshot(windows_vm_for_snapshot, windows_snapshot):
     cmd = shlex.split(
         f'powershell -command "for($i=1; $i -le 100; $i++){{$i| Out-File -FilePath {file} -Append}}"',
     )
-    run_ssh_commands(host=windows_vm_for_snapshot.ssh_exec, commands=cmd)
+    run_ssh_commands(host=windows_vm_for_snapshot.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)
     windows_snapshot.wait_snapshot_done(timeout=TIMEOUT_10MIN)
     windows_vm_for_snapshot.stop(wait=True)

tests/storage/snapshots/test_snapshots.py

@@ -22,11 +22,11 @@
 from tests.storage.snapshots.utils import (
     expected_output_after_restore,
     fail_to_create_snapshot_no_permissions,
-    run_command_on_vm_and_check_output,
     start_windows_vm_after_restore,
 )
 from tests.storage.utils import assert_windows_directory_existence
 from utilities.constants import LS_COMMAND, TIMEOUT_1MIN, TIMEOUT_10SEC
+from utilities.storage import run_command_on_vm_and_check_output
 from utilities.virt import restart_vm_wait_for_running_vm, running_vm
 
 LOGGER = logging.getLogger(__name__)

tests/storage/snapshots/utils.py

@@ -1,9 +1,6 @@
-import shlex
-
 import pytest
 from kubernetes.client.rest import ApiException
 from ocp_resources.virtual_machine_snapshot import VirtualMachineSnapshot
-from pyhelper_utils.shell import run_ssh_commands
 
 from tests.storage.snapshots.constants import ERROR_MSG_USER_CANNOT_CREATE_VM_SNAPSHOTS
 from utilities.constants import TIMEOUT_10MIN
@@ -47,22 +44,3 @@ def fail_to_create_snapshot_no_permissions(snapshot_name, namespace, vm_name, cl
 def start_windows_vm_after_restore(vm_restore, windows_vm):
     vm_restore.wait_restore_done(timeout=TIMEOUT_10MIN)
     running_vm(vm=windows_vm)
-
-
-def run_command_on_vm_and_check_output(vm, command, expected_result):
-    """Run command on RHEL VM via SSH and verify expected result is in output.
-
-    Args:
-        vm (VirtualMachineForTests): VM to run command on.
-        command (str): Command to run.
-        expected_result (str): Expected result to check.
-
-    Raises:
-        AssertionError: If expected result is not in output.
-    """
-    cmd_output = run_ssh_commands(
-        host=vm.ssh_exec,
-        commands=shlex.split(f"bash -c {shlex.quote(command)}"),
-    )[0].strip()
-    expected_result = expected_result.strip()
-    assert expected_result in cmd_output, f"Expected '{expected_result}' in output '{cmd_output}'"

tests/storage/storage_migration/conftest.py

@@ -27,6 +27,7 @@
     OS_FLAVOR_RHEL,
     OS_FLAVOR_WINDOWS,
     TIMEOUT_1MIN,
+    TIMEOUT_2MIN,
     TIMEOUT_5SEC,
     TIMEOUT_10MIN,
     TIMEOUT_30MIN,
@@ -355,6 +356,8 @@ def vm_with_mounted_hotplugged_disk(vm_for_storage_class_migration_with_hotplugg
                 f"sudo mount {HOTPLUGGED_DEVICE} {MOUNT_HOTPLUGGED_DEVICE_PATH}",
             ]
         ],
+        wait_timeout=TIMEOUT_2MIN,
+        sleep=TIMEOUT_5SEC,
     )
     yield vm_for_storage_class_migration_with_hotplugged_volume
 
@@ -366,6 +369,8 @@ def written_file_to_the_mounted_hotplugged_disk(vm_with_mounted_hotplugged_disk)
         commands=shlex.split(
             f"echo '{CONTENT}' | sudo tee {MOUNT_HOTPLUGGED_DEVICE_PATH}/{FILE_BEFORE_STORAGE_MIGRATION}"
         ),
+        wait_timeout=TIMEOUT_2MIN,
+        sleep=TIMEOUT_5SEC,
     )
     yield vm_with_mounted_hotplugged_disk
 
@@ -419,7 +424,7 @@ def written_file_to_windows_vms_before_migration(booted_vms_for_storage_class_mi
         cmd = shlex.split(
             f'powershell -command "\\"{CONTENT}\\" | Out-File -FilePath {WINDOWS_FILE_WITH_PATH} -Append"'
         )
-        run_ssh_commands(host=vm.ssh_exec, commands=cmd)
+        run_ssh_commands(host=vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)
     yield booted_vms_for_storage_class_migration
 
 

tests/storage/storage_migration/utils.py

@@ -11,7 +11,7 @@
     NO_STORAGE_CLASS_FAILURE_MESSAGE,
 )
 from utilities import console
-from utilities.constants import LS_COMMAND, TIMEOUT_20SEC
+from utilities.constants import LS_COMMAND, TIMEOUT_2MIN, TIMEOUT_5SEC, TIMEOUT_20SEC
 from utilities.virt import VirtualMachineForTests, get_vm_boot_time
 
 
@@ -96,12 +96,17 @@ def get_storage_class_for_storage_migration(storage_class: str, cluster_storage_
 
 def verify_file_in_hotplugged_disk(vm: VirtualMachineForTests, file_name: str, file_content: str) -> None:
     output = run_ssh_commands(
-        host=vm.ssh_exec, commands=shlex.split(f"cat {MOUNT_HOTPLUGGED_DEVICE_PATH}/{file_name}")
+        host=vm.ssh_exec,
+        commands=shlex.split(f"cat {MOUNT_HOTPLUGGED_DEVICE_PATH}/{file_name}"),
+        wait_timeout=TIMEOUT_2MIN,
+        sleep=TIMEOUT_5SEC,
     )[0]
     assert output.strip() == file_content, f"'{output}' does not equal '{file_content}'"
 
 
 def verify_file_in_windows_vm(windows_vm: VirtualMachineForTests, file_name_with_path: str, file_content: str) -> None:
     cmd = shlex.split(f'powershell -command "Get-Content {file_name_with_path}"')
-    out = run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd)[0].strip()
+    out = run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)[
+        0
+    ].strip()
     assert out.strip() == file_content, f"'{out}' does not equal '{file_content}'"

tests/storage/utils.py

@@ -27,6 +27,7 @@
 from utilities.constants import (
     CDI_UPLOADPROXY,
     TIMEOUT_2MIN,
+    TIMEOUT_5SEC,
     TIMEOUT_30MIN,
     Images,
 )
@@ -445,15 +446,17 @@ def assert_windows_directory_existence(
     expected_result: bool, windows_vm: VirtualMachineForTests, directory_path: str
 ) -> None:
     cmd = shlex.split(f'powershell -command "Test-Path -Path {directory_path}"')
-    out = run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd)[0].strip()
+    out = run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)[
+        0
+    ].strip()
     assert expected_result == ast.literal_eval(out), f"Directory exist: {out}, expected result: {expected_result}"
 
 
 def create_windows_directory(windows_vm: VirtualMachineForTests, directory_path: str) -> None:
     cmd = shlex.split(
-        f'powershell -command "New-Item -Path {directory_path} -ItemType Directory"',
+        f'powershell -command "New-Item -Path {directory_path} -ItemType Directory -Force"',
     )
-    run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd)
+    run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)
     assert_windows_directory_existence(
         expected_result=True,
         windows_vm=windows_vm,

tests/storage/vm_export/conftest.py

@@ -3,7 +3,6 @@
 """
 
 import base64
-import shlex
 from subprocess import check_output
 
 import pytest
@@ -18,13 +17,12 @@
 )
 from ocp_resources.virtual_machine_export import VirtualMachineExport
 from ocp_resources.virtual_machine_snapshot import VirtualMachineSnapshot
-from pyhelper_utils.shell import run_ssh_commands
 
 from tests.storage.vm_export.constants import VM_EXPORT_TEST_FILE_CONTENT, VM_EXPORT_TEST_FILE_NAME
 from tests.storage.vm_export.utils import create_blank_dv_by_specific_user, get_manifest_from_vmexport, get_manifest_url
 from utilities.constants import OS_FLAVOR_RHEL, U1_SMALL, UNPRIVILEGED_PASSWORD, UNPRIVILEGED_USER
 from utilities.infra import create_ns, login_with_user_password
-from utilities.storage import data_volume_template_with_source_ref_dict
+from utilities.storage import data_volume_template_with_source_ref_dict, write_file_via_ssh
 from utilities.virt import VirtualMachineForTests, running_vm
 
 
@@ -211,10 +209,7 @@ def rhel_vm_for_snapshot_with_content(
         ),
     ) as vm:
         running_vm(vm=vm)
-
-        cmd = shlex.split(f"echo '{VM_EXPORT_TEST_FILE_CONTENT}' > {VM_EXPORT_TEST_FILE_NAME} && sync")
-        run_ssh_commands(host=vm.ssh_exec, commands=cmd)
-
+        write_file_via_ssh(vm=vm, filename=VM_EXPORT_TEST_FILE_NAME, content=VM_EXPORT_TEST_FILE_CONTENT)
         yield vm
 
 

tests/storage/vm_export/test_vm_export.py

@@ -9,10 +9,10 @@
 from ocp_resources.persistent_volume_claim import PersistentVolumeClaim
 from ocp_resources.resource import Resource
 from ocp_resources.virtual_machine_export import VirtualMachineExport
-from pyhelper_utils.shell import run_ssh_commands
 
 from tests.storage.vm_export.constants import VM_EXPORT_TEST_FILE_CONTENT, VM_EXPORT_TEST_FILE_NAME
 from utilities.infra import run_virtctl_command
+from utilities.storage import run_command_on_vm_and_check_output
 from utilities.virt import running_vm
 
 VIRTUALMACHINEEXPORTS = "virtualmachineexports"
@@ -62,11 +62,9 @@ def test_vmexport_snapshot_manifests(
     vm_from_vmexport,
 ):
     running_vm(vm=vm_from_vmexport)
-
-    result = run_ssh_commands(host=vm_from_vmexport.ssh_exec, commands=shlex.split(f"cat {VM_EXPORT_TEST_FILE_NAME}"))
-    file_content = result[0].strip()
-
-    assert file_content == VM_EXPORT_TEST_FILE_CONTENT
+    run_command_on_vm_and_check_output(
+        vm=vm_from_vmexport, command=f"cat {VM_EXPORT_TEST_FILE_NAME}", expected_result=VM_EXPORT_TEST_FILE_CONTENT
+    )
 
 
 @pytest.mark.s390x

utilities/storage.py

@@ -674,15 +674,40 @@ def write_file(vm, filename, content, stop_vm=True):
 
 def write_file_via_ssh(vm: "VirtualMachineForTests", filename: str, content: str) -> None:
     """
-    Write content to a file in VM using SSH connection.
+    Write content to a file in VM using SSH connection with retry.
 
     Args:
         vm: VirtualMachine instance with SSH connectivity
         filename: Path to the file to write in the VM
         content: Content to write to the file
     """
     cmd = shlex.split(f"echo {shlex.quote(content)} > {shlex.quote(filename)} && sync")
-    run_ssh_commands(host=vm.ssh_exec, commands=cmd)
+    run_ssh_commands(host=vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)
+
+
+def run_command_on_vm_and_check_output(vm: "VirtualMachineForTests", command: str, expected_result: str) -> None:
+    """Run command on VM via SSH with retry and verify output matches expected result.
+
+    Command execution is retried with 2-minute timeout and 5-second intervals.
+
+    Args:
+        vm (VirtualMachineForTests): VM to run command on.
+        command (str): Command to run.
+        expected_result (str): Expected result to check.
+
+    Raises:
+        AssertionError: If command output differs from expected result.
+    """
+    cmd_output = run_ssh_commands(
+        host=vm.ssh_exec,
+        commands=shlex.split(f"bash -c {shlex.quote(command)}"),
+        wait_timeout=TIMEOUT_2MIN,
+        sleep=TIMEOUT_5SEC,
+    )[0].strip()
+    expected_result = expected_result.strip()
+    assert expected_result == cmd_output, (
+        f"Command output mismatch.\nCommand: {command}\nExpected: '{expected_result}'\nActual: '{cmd_output}'"
+    )
 
 
 def run_command_on_cirros_vm_and_check_output(vm, command, expected_result):
@@ -692,9 +717,10 @@ def run_command_on_cirros_vm_and_check_output(vm, command, expected_result):
 
 
 def assert_disk_serial(vm, command=shlex.split("sudo ls /dev/disk/by-id")):
-    assert HOTPLUG_DISK_SERIAL in run_ssh_commands(host=vm.ssh_exec, commands=command)[0], (
-        f"hotplug disk serial id {HOTPLUG_DISK_SERIAL} is not in VM"
-    )
+    assert (
+        HOTPLUG_DISK_SERIAL
+        in run_ssh_commands(host=vm.ssh_exec, commands=command, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)[0]
+    ), f"hotplug disk serial id {HOTPLUG_DISK_SERIAL} is not in VM"
 
 
 def assert_hotplugvolume_nonexist_optional_restart(vm, restart=False):
@@ -810,6 +836,8 @@ def check_disk_count_in_vm(vm):
     out = run_ssh_commands(
         host=vm.ssh_exec,
         commands=[shlex.split("lsblk | grep disk | grep -v SWAP| wc -l")],
+        wait_timeout=TIMEOUT_2MIN,
+        sleep=TIMEOUT_5SEC,
     )[0].strip()
     assert out == str(len(vm.instance.spec.template.spec.domain.devices.disks)), (
         "Failed to verify actual disk count against VMI"