[Storage] Use ssh retries in run_ssh_commands

Signed-off-by: Jenia Peimer <jpeimer@redhat.com>

Author: jpeimer

tests/storage/online_resize/utils.py

@@ -13,7 +13,7 @@
 from pyhelper_utils.shell import run_ssh_commands
 from timeout_sampler import TimeoutExpiredError, TimeoutSampler
 
-from utilities.constants import TIMEOUT_4MIN
+from utilities.constants import TIMEOUT_2MIN, TIMEOUT_4MIN, TIMEOUT_5SEC
 from utilities.storage import create_dv
 from utilities.virt import running_vm
 
@@ -61,9 +61,16 @@ def cksum_file(vm, filename, create=False):
         run_ssh_commands(
             host=vm.ssh_exec,
             commands=shlex.split(f"dd if=/dev/urandom of={filename} count=100 && sync"),
+            wait_timeout=TIMEOUT_2MIN,
+            sleep=TIMEOUT_5SEC,
         )
 
-    out = run_ssh_commands(host=vm.ssh_exec, commands=shlex.split(f"sha256sum {filename}"))[0]
+    out = run_ssh_commands(
+        host=vm.ssh_exec,
+        commands=shlex.split(f"sha256sum {filename}"),
+        wait_timeout=TIMEOUT_2MIN,
+        sleep=TIMEOUT_5SEC,
+    )[0]
     sha256sum = out.split()[0]
     LOGGER.info(f"File sha256sum is {sha256sum}")
     return sha256sum
@@ -104,7 +111,7 @@ def expand_pvc(dv, size_change):
 
 def get_resize_count(vm):
     commands = shlex.split("sudo dmesg | grep -c 'new size' || true")
-    result = run_ssh_commands(host=vm.ssh_exec, commands=commands)[0]
+    result = run_ssh_commands(host=vm.ssh_exec, commands=commands, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)[0]
 
     return int(result)
 
@@ -136,7 +143,9 @@ def wait_for_resize(vm, count=1):
             if current_resize_count in (desired_count, desired_count + 1):
                 break
     except TimeoutExpiredError:
-        dmesg = run_ssh_commands(host=vm.ssh_exec, commands=shlex.split("dmesg"))[0]
+        dmesg = run_ssh_commands(
+            host=vm.ssh_exec, commands=shlex.split("dmesg"), wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC
+        )[0]
         LOGGER.error(f"Failed to reach resize count {desired_count}.\ndmesg:\n{dmesg}")
         raise
 

tests/storage/snapshots/conftest.py

@@ -20,7 +20,7 @@
     create_windows_directory,
     set_permissions,
 )
-from utilities.constants import TIMEOUT_10MIN, UNPRIVILEGED_USER
+from utilities.constants import TIMEOUT_2MIN, TIMEOUT_5SEC, TIMEOUT_10MIN, UNPRIVILEGED_USER
 
 LOGGER = logging.getLogger(__name__)
 
@@ -88,7 +88,7 @@ def snapshot_dirctory_removed(windows_vm_for_snapshot, windows_snapshot):
     cmd = shlex.split(
         f'powershell -command "Remove-Item -Path {WINDOWS_DIRECTORY_PATH} -Recurse"',
     )
-    run_ssh_commands(host=windows_vm_for_snapshot.ssh_exec, commands=cmd)
+    run_ssh_commands(host=windows_vm_for_snapshot.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)
     assert_windows_directory_existence(
         expected_result=False,
         windows_vm=windows_vm_for_snapshot,
@@ -103,6 +103,6 @@ def file_created_during_snapshot(windows_vm_for_snapshot, windows_snapshot):
     cmd = shlex.split(
         f'powershell -command "for($i=1; $i -le 100; $i++){{$i| Out-File -FilePath {file} -Append}}"',
     )
-    run_ssh_commands(host=windows_vm_for_snapshot.ssh_exec, commands=cmd)
+    run_ssh_commands(host=windows_vm_for_snapshot.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)
     windows_snapshot.wait_snapshot_done(timeout=TIMEOUT_10MIN)
     windows_vm_for_snapshot.stop(wait=True)

tests/storage/snapshots/test_snapshots.py

@@ -22,11 +22,11 @@
 from tests.storage.snapshots.utils import (
     expected_output_after_restore,
     fail_to_create_snapshot_no_permissions,
-    run_command_on_vm_and_check_output,
     start_windows_vm_after_restore,
 )
 from tests.storage.utils import assert_windows_directory_existence
 from utilities.constants import LS_COMMAND, TIMEOUT_1MIN, TIMEOUT_10SEC
+from utilities.storage import run_command_on_vm_and_check_output
 from utilities.virt import restart_vm_wait_for_running_vm, running_vm
 
 LOGGER = logging.getLogger(__name__)

tests/storage/snapshots/utils.py

@@ -1,9 +1,6 @@
-import shlex
-
 import pytest
 from kubernetes.client.rest import ApiException
 from ocp_resources.virtual_machine_snapshot import VirtualMachineSnapshot
-from pyhelper_utils.shell import run_ssh_commands
 
 from tests.storage.snapshots.constants import ERROR_MSG_USER_CANNOT_CREATE_VM_SNAPSHOTS
 from utilities.constants import TIMEOUT_10MIN

tests/storage/storage_migration/conftest.py

@@ -30,6 +30,8 @@
     OS_FLAVOR_FEDORA,
     OS_FLAVOR_RHEL,
     OS_FLAVOR_WINDOWS,
+    TIMEOUT_2MIN,
+    TIMEOUT_5SEC,
     U1_SMALL,
     Images,
 )
@@ -336,6 +338,8 @@ def vm_with_mounted_hotplugged_disk(vm_for_storage_class_migration_with_hotplugg
                 f"sudo mount {HOTPLUGGED_DEVICE} {MOUNT_HOTPLUGGED_DEVICE_PATH}",
             ]
         ],
+        wait_timeout=TIMEOUT_2MIN,
+        sleep=TIMEOUT_5SEC,
     )
     yield vm_for_storage_class_migration_with_hotplugged_volume
 
@@ -347,6 +351,8 @@ def written_file_to_the_mounted_hotplugged_disk(vm_with_mounted_hotplugged_disk)
         commands=shlex.split(
             f"echo '{CONTENT}' | sudo tee {MOUNT_HOTPLUGGED_DEVICE_PATH}/{FILE_BEFORE_STORAGE_MIGRATION}"
         ),
+        wait_timeout=TIMEOUT_2MIN,
+        sleep=TIMEOUT_5SEC,
     )
     yield vm_with_mounted_hotplugged_disk
 
@@ -400,7 +406,7 @@ def written_file_to_windows_vms_before_migration(booted_vms_for_storage_class_mi
         cmd = shlex.split(
             f'powershell -command "\\"{CONTENT}\\" | Out-File -FilePath {WINDOWS_FILE_WITH_PATH} -Append"'
         )
-        run_ssh_commands(host=vm.ssh_exec, commands=cmd)
+        run_ssh_commands(host=vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)
     yield booted_vms_for_storage_class_migration
 
 

tests/storage/storage_migration/utils.py

@@ -13,7 +13,7 @@
     NO_STORAGE_CLASS_FAILURE_MESSAGE,
 )
 from tests.storage.utils import check_file_in_vm
-from utilities.constants import TIMEOUT_10MIN, TIMEOUT_10SEC
+from utilities.constants import TIMEOUT_2MIN, TIMEOUT_5SEC, TIMEOUT_10MIN, TIMEOUT_10SEC
 from utilities.exceptions import StorageMigrationError
 from utilities.virt import VirtualMachineForTests, get_vm_boot_time
 
@@ -89,14 +89,19 @@ def get_storage_class_for_storage_migration(storage_class: str, cluster_storage_
 
 def verify_file_in_hotplugged_disk(vm: VirtualMachineForTests, file_name: str, file_content: str) -> None:
     output = run_ssh_commands(
-        host=vm.ssh_exec, commands=shlex.split(f"cat {MOUNT_HOTPLUGGED_DEVICE_PATH}/{file_name}")
+        host=vm.ssh_exec,
+        commands=shlex.split(f"cat {MOUNT_HOTPLUGGED_DEVICE_PATH}/{file_name}"),
+        wait_timeout=TIMEOUT_2MIN,
+        sleep=TIMEOUT_5SEC,
     )[0]
     assert output.strip() == file_content, f"'{output}' does not equal '{file_content}'"
 
 
 def verify_file_in_windows_vm(windows_vm: VirtualMachineForTests, file_name_with_path: str, file_content: str) -> None:
     cmd = shlex.split(f'powershell -command "Get-Content {file_name_with_path}"')
-    out = run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd)[0].strip()
+    out = run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)[
+        0
+    ].strip()
     assert out.strip() == file_content, f"'{out}' does not equal '{file_content}'"
 
 

tests/storage/utils.py

@@ -36,6 +36,7 @@
     CDI_UPLOADPROXY,
     LS_COMMAND,
     TIMEOUT_2MIN,
+    TIMEOUT_5SEC,
     TIMEOUT_20SEC,
     TIMEOUT_30MIN,
     Images,
@@ -468,15 +469,17 @@ def assert_windows_directory_existence(
     expected_result: bool, windows_vm: VirtualMachineForTests, directory_path: str
 ) -> None:
     cmd = shlex.split(f'powershell -command "Test-Path -Path {directory_path}"')
-    out = run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd)[0].strip()
+    out = run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)[
+        0
+    ].strip()
     assert expected_result == ast.literal_eval(out), f"Directory exist: {out}, expected result: {expected_result}"
 
 
 def create_windows_directory(windows_vm: VirtualMachineForTests, directory_path: str) -> None:
     cmd = shlex.split(
         f'powershell -command "New-Item -Path {directory_path} -ItemType Directory"',
     )
-    run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd)
+    run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)
     assert_windows_directory_existence(
         expected_result=True,
         windows_vm=windows_vm,

tests/storage/vm_export/conftest.py

@@ -3,7 +3,6 @@
 """
 
 import base64
-import shlex
 from subprocess import check_output
 
 import pytest
@@ -18,13 +17,12 @@
 )
 from ocp_resources.virtual_machine_export import VirtualMachineExport
 from ocp_resources.virtual_machine_snapshot import VirtualMachineSnapshot
-from pyhelper_utils.shell import run_ssh_commands
 
 from tests.storage.constants import TEST_FILE_CONTENT, TEST_FILE_NAME
 from tests.storage.vm_export.utils import create_blank_dv_by_specific_user, get_manifest_from_vmexport, get_manifest_url
 from utilities.constants import OS_FLAVOR_RHEL, U1_SMALL, UNPRIVILEGED_PASSWORD, UNPRIVILEGED_USER
 from utilities.infra import create_ns, login_with_user_password
-from utilities.storage import data_volume_template_with_source_ref_dict
+from utilities.storage import data_volume_template_with_source_ref_dict, write_file_via_ssh
 from utilities.virt import VirtualMachineForTests, running_vm
 
 
@@ -218,10 +216,7 @@ def rhel_vm_for_snapshot_with_content(
         ),
     ) as vm:
         running_vm(vm=vm)
-
-        cmd = shlex.split(f"echo '{TEST_FILE_CONTENT}' > {TEST_FILE_NAME} && sync")
-        run_ssh_commands(host=vm.ssh_exec, commands=cmd)
-
+        write_file_via_ssh(vm=vm, filename=TEST_FILE_CONTENT, content=TEST_FILE_NAME)
         yield vm
 
 

tests/storage/vm_export/test_vm_export.py

@@ -9,10 +9,10 @@
 from ocp_resources.persistent_volume_claim import PersistentVolumeClaim
 from ocp_resources.resource import Resource
 from ocp_resources.virtual_machine_export import VirtualMachineExport
-from pyhelper_utils.shell import run_ssh_commands
 
 from tests.storage.constants import TEST_FILE_CONTENT, TEST_FILE_NAME
 from utilities.infra import run_virtctl_command
+from utilities.storage import run_command_on_vm_and_check_output
 from utilities.virt import running_vm
 
 VIRTUALMACHINEEXPORTS = "virtualmachineexports"
@@ -62,12 +62,8 @@ def test_vmexport_snapshot_manifests(
     vm_from_vmexport,
 ):
     running_vm(vm=vm_from_vmexport)
-
-    result = run_ssh_commands(host=vm_from_vmexport.ssh_exec, commands=shlex.split(f"cat {TEST_FILE_NAME}"))
-    file_content = result[0].strip()
-
-    assert file_content == TEST_FILE_CONTENT, (
-        f"Unexpected content in {TEST_FILE_NAME}: got '{file_content}', expected '{TEST_FILE_CONTENT}'"
+    run_command_on_vm_and_check_output(
+        vm=vm_from_vmexport, command=f"cat {TEST_FILE_NAME}", expected_result=TEST_FILE_CONTENT
     )
 
 

utilities/storage.py

@@ -643,8 +643,7 @@ def write_file_via_ssh(vm: virt_util.VirtualMachineForTests, filename: str, cont
         content: Content to write to the file
     """
     cmd = shlex.split(f"echo {shlex.quote(content)} > {shlex.quote(filename)} && sync")
-    run_ssh_commands(host=vm.ssh_exec, commands=cmd)
-
+    run_ssh_commands(host=vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)
 
 
 def run_command_on_vm_and_check_output(vm, command, expected_result):
@@ -661,6 +660,8 @@ def run_command_on_vm_and_check_output(vm, command, expected_result):
     cmd_output = run_ssh_commands(
         host=vm.ssh_exec,
         commands=shlex.split(f"bash -c {shlex.quote(command)}"),
+        wait_timeout=TIMEOUT_2MIN,
+        sleep=TIMEOUT_5SEC,
     )[0].strip()
     expected_result = expected_result.strip()
     assert expected_result in cmd_output, f"Expected '{expected_result}' in output '{cmd_output}'"
@@ -673,9 +674,10 @@ def run_command_on_cirros_vm_and_check_output(vm, command, expected_result):
 
 
 def assert_disk_serial(vm, command=shlex.split("sudo ls /dev/disk/by-id")):
-    assert HOTPLUG_DISK_SERIAL in run_ssh_commands(host=vm.ssh_exec, commands=command)[0], (
-        f"hotplug disk serial id {HOTPLUG_DISK_SERIAL} is not in VM"
-    )
+    assert (
+        HOTPLUG_DISK_SERIAL
+        in run_ssh_commands(host=vm.ssh_exec, commands=command, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)[0]
+    ), f"hotplug disk serial id {HOTPLUG_DISK_SERIAL} is not in VM"
 
 
 def assert_hotplugvolume_nonexist(vm):
@@ -811,6 +813,8 @@ def check_disk_count_in_vm(vm):
     out = run_ssh_commands(
         host=vm.ssh_exec,
         commands=[shlex.split("lsblk | grep disk | grep -v SWAP| wc -l")],
+        wait_timeout=TIMEOUT_2MIN,
+        sleep=TIMEOUT_5SEC,
     )[0].strip()
     assert out == str(len(vm.instance.spec.template.spec.domain.devices.disks)), (
         "Failed to verify actual disk count against VMI"