diff --git a/tests/storage/online_resize/utils.py b/tests/storage/online_resize/utils.py index b2962a8aba..450c518d5c 100644 --- a/tests/storage/online_resize/utils.py +++ b/tests/storage/online_resize/utils.py @@ -13,7 +13,7 @@ from pyhelper_utils.shell import run_ssh_commands from timeout_sampler import TimeoutExpiredError, TimeoutSampler -from utilities.constants import TIMEOUT_4MIN +from utilities.constants import TIMEOUT_2MIN, TIMEOUT_4MIN, TIMEOUT_5SEC from utilities.storage import create_dv from utilities.virt import running_vm @@ -61,9 +61,16 @@ def cksum_file(vm, filename, create=False): run_ssh_commands( host=vm.ssh_exec, commands=shlex.split(f"dd if=/dev/urandom of={filename} count=100 && sync"), + wait_timeout=TIMEOUT_2MIN, + sleep=TIMEOUT_5SEC, ) - out = run_ssh_commands(host=vm.ssh_exec, commands=shlex.split(f"sha256sum {filename}"))[0] + out = run_ssh_commands( + host=vm.ssh_exec, + commands=shlex.split(f"sha256sum {filename}"), + wait_timeout=TIMEOUT_2MIN, + sleep=TIMEOUT_5SEC, + )[0] sha256sum = out.split()[0] LOGGER.info(f"File sha256sum is {sha256sum}") return sha256sum @@ -104,7 +111,7 @@ def expand_pvc(dv, size_change): def get_resize_count(vm): commands = shlex.split("sudo dmesg | grep -c 'new size' || true") - result = run_ssh_commands(host=vm.ssh_exec, commands=commands)[0] + result = run_ssh_commands(host=vm.ssh_exec, commands=commands, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)[0] return int(result) @@ -136,7 +143,9 @@ def wait_for_resize(vm, count=1): if current_resize_count in (desired_count, desired_count + 1): break except TimeoutExpiredError: - dmesg = run_ssh_commands(host=vm.ssh_exec, commands=shlex.split("dmesg"))[0] + dmesg = run_ssh_commands( + host=vm.ssh_exec, commands=shlex.split("dmesg"), wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC + )[0] LOGGER.error(f"Failed to reach resize count {desired_count}.\ndmesg:\n{dmesg}") raise diff --git a/tests/storage/snapshots/conftest.py b/tests/storage/snapshots/conftest.py index b40bcf4ead..7112b0c81e 100644 --- a/tests/storage/snapshots/conftest.py +++ b/tests/storage/snapshots/conftest.py @@ -20,7 +20,7 @@ create_windows_directory, set_permissions, ) -from utilities.constants import TIMEOUT_10MIN, UNPRIVILEGED_USER +from utilities.constants import TIMEOUT_2MIN, TIMEOUT_5SEC, TIMEOUT_10MIN, UNPRIVILEGED_USER LOGGER = logging.getLogger(__name__) @@ -87,7 +87,7 @@ def snapshot_dirctory_removed(windows_vm_for_snapshot, windows_snapshot): cmd = shlex.split( f'powershell -command "Remove-Item -Path {WINDOWS_DIRECTORY_PATH} -Recurse"', ) - run_ssh_commands(host=windows_vm_for_snapshot.ssh_exec, commands=cmd) + run_ssh_commands(host=windows_vm_for_snapshot.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC) assert_windows_directory_existence( expected_result=False, windows_vm=windows_vm_for_snapshot, @@ -102,6 +102,6 @@ def file_created_during_snapshot(windows_vm_for_snapshot, windows_snapshot): cmd = shlex.split( f'powershell -command "for($i=1; $i -le 100; $i++){{$i| Out-File -FilePath {file} -Append}}"', ) - run_ssh_commands(host=windows_vm_for_snapshot.ssh_exec, commands=cmd) + run_ssh_commands(host=windows_vm_for_snapshot.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC) windows_snapshot.wait_snapshot_done(timeout=TIMEOUT_10MIN) windows_vm_for_snapshot.stop(wait=True) diff --git a/tests/storage/snapshots/test_snapshots.py b/tests/storage/snapshots/test_snapshots.py index f34a1ad566..4d5d81ab68 100644 --- a/tests/storage/snapshots/test_snapshots.py +++ b/tests/storage/snapshots/test_snapshots.py @@ -22,11 +22,11 @@ from tests.storage.snapshots.utils import ( expected_output_after_restore, fail_to_create_snapshot_no_permissions, - run_command_on_vm_and_check_output, start_windows_vm_after_restore, ) from tests.storage.utils import assert_windows_directory_existence from utilities.constants import LS_COMMAND, TIMEOUT_1MIN, TIMEOUT_10SEC +from utilities.storage import run_command_on_vm_and_check_output from utilities.virt import restart_vm_wait_for_running_vm, running_vm LOGGER = logging.getLogger(__name__) diff --git a/tests/storage/snapshots/utils.py b/tests/storage/snapshots/utils.py index 7b85eb72de..5a17f2b19a 100644 --- a/tests/storage/snapshots/utils.py +++ b/tests/storage/snapshots/utils.py @@ -1,9 +1,6 @@ -import shlex - import pytest from kubernetes.client.rest import ApiException from ocp_resources.virtual_machine_snapshot import VirtualMachineSnapshot -from pyhelper_utils.shell import run_ssh_commands from tests.storage.snapshots.constants import ERROR_MSG_USER_CANNOT_CREATE_VM_SNAPSHOTS from utilities.constants import TIMEOUT_10MIN @@ -47,22 +44,3 @@ def fail_to_create_snapshot_no_permissions(snapshot_name, namespace, vm_name, cl def start_windows_vm_after_restore(vm_restore, windows_vm): vm_restore.wait_restore_done(timeout=TIMEOUT_10MIN) running_vm(vm=windows_vm) - - -def run_command_on_vm_and_check_output(vm, command, expected_result): - """Run command on RHEL VM via SSH and verify expected result is in output. - - Args: - vm (VirtualMachineForTests): VM to run command on. - command (str): Command to run. - expected_result (str): Expected result to check. - - Raises: - AssertionError: If expected result is not in output. - """ - cmd_output = run_ssh_commands( - host=vm.ssh_exec, - commands=shlex.split(f"bash -c {shlex.quote(command)}"), - )[0].strip() - expected_result = expected_result.strip() - assert expected_result in cmd_output, f"Expected '{expected_result}' in output '{cmd_output}'" diff --git a/tests/storage/storage_migration/conftest.py b/tests/storage/storage_migration/conftest.py index 30fce8b0d8..8c677c3ed4 100644 --- a/tests/storage/storage_migration/conftest.py +++ b/tests/storage/storage_migration/conftest.py @@ -27,6 +27,7 @@ OS_FLAVOR_RHEL, OS_FLAVOR_WINDOWS, TIMEOUT_1MIN, + TIMEOUT_2MIN, TIMEOUT_5SEC, TIMEOUT_10MIN, TIMEOUT_30MIN, @@ -355,6 +356,8 @@ def vm_with_mounted_hotplugged_disk(vm_for_storage_class_migration_with_hotplugg f"sudo mount {HOTPLUGGED_DEVICE} {MOUNT_HOTPLUGGED_DEVICE_PATH}", ] ], + wait_timeout=TIMEOUT_2MIN, + sleep=TIMEOUT_5SEC, ) yield vm_for_storage_class_migration_with_hotplugged_volume @@ -366,6 +369,8 @@ def written_file_to_the_mounted_hotplugged_disk(vm_with_mounted_hotplugged_disk) commands=shlex.split( f"echo '{CONTENT}' | sudo tee {MOUNT_HOTPLUGGED_DEVICE_PATH}/{FILE_BEFORE_STORAGE_MIGRATION}" ), + wait_timeout=TIMEOUT_2MIN, + sleep=TIMEOUT_5SEC, ) yield vm_with_mounted_hotplugged_disk @@ -419,7 +424,7 @@ def written_file_to_windows_vms_before_migration(booted_vms_for_storage_class_mi cmd = shlex.split( f'powershell -command "\\"{CONTENT}\\" | Out-File -FilePath {WINDOWS_FILE_WITH_PATH} -Append"' ) - run_ssh_commands(host=vm.ssh_exec, commands=cmd) + run_ssh_commands(host=vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC) yield booted_vms_for_storage_class_migration diff --git a/tests/storage/storage_migration/utils.py b/tests/storage/storage_migration/utils.py index ebf8e6a415..fa99dc0e23 100644 --- a/tests/storage/storage_migration/utils.py +++ b/tests/storage/storage_migration/utils.py @@ -11,7 +11,7 @@ NO_STORAGE_CLASS_FAILURE_MESSAGE, ) from utilities import console -from utilities.constants import LS_COMMAND, TIMEOUT_20SEC +from utilities.constants import LS_COMMAND, TIMEOUT_2MIN, TIMEOUT_5SEC, TIMEOUT_20SEC from utilities.virt import VirtualMachineForTests, get_vm_boot_time @@ -96,12 +96,17 @@ def get_storage_class_for_storage_migration(storage_class: str, cluster_storage_ def verify_file_in_hotplugged_disk(vm: VirtualMachineForTests, file_name: str, file_content: str) -> None: output = run_ssh_commands( - host=vm.ssh_exec, commands=shlex.split(f"cat {MOUNT_HOTPLUGGED_DEVICE_PATH}/{file_name}") + host=vm.ssh_exec, + commands=shlex.split(f"cat {MOUNT_HOTPLUGGED_DEVICE_PATH}/{file_name}"), + wait_timeout=TIMEOUT_2MIN, + sleep=TIMEOUT_5SEC, )[0] assert output.strip() == file_content, f"'{output}' does not equal '{file_content}'" def verify_file_in_windows_vm(windows_vm: VirtualMachineForTests, file_name_with_path: str, file_content: str) -> None: cmd = shlex.split(f'powershell -command "Get-Content {file_name_with_path}"') - out = run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd)[0].strip() + out = run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)[ + 0 + ].strip() assert out.strip() == file_content, f"'{out}' does not equal '{file_content}'" diff --git a/tests/storage/utils.py b/tests/storage/utils.py index 95ed1bb345..4914538e86 100644 --- a/tests/storage/utils.py +++ b/tests/storage/utils.py @@ -27,6 +27,7 @@ from utilities.constants import ( CDI_UPLOADPROXY, TIMEOUT_2MIN, + TIMEOUT_5SEC, TIMEOUT_30MIN, Images, ) @@ -445,15 +446,17 @@ def assert_windows_directory_existence( expected_result: bool, windows_vm: VirtualMachineForTests, directory_path: str ) -> None: cmd = shlex.split(f'powershell -command "Test-Path -Path {directory_path}"') - out = run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd)[0].strip() + out = run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)[ + 0 + ].strip() assert expected_result == ast.literal_eval(out), f"Directory exist: {out}, expected result: {expected_result}" def create_windows_directory(windows_vm: VirtualMachineForTests, directory_path: str) -> None: cmd = shlex.split( - f'powershell -command "New-Item -Path {directory_path} -ItemType Directory"', + f'powershell -command "New-Item -Path {directory_path} -ItemType Directory -Force"', ) - run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd) + run_ssh_commands(host=windows_vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC) assert_windows_directory_existence( expected_result=True, windows_vm=windows_vm, diff --git a/tests/storage/vm_export/conftest.py b/tests/storage/vm_export/conftest.py index 9f61ebe20a..f2950e84cb 100644 --- a/tests/storage/vm_export/conftest.py +++ b/tests/storage/vm_export/conftest.py @@ -3,7 +3,6 @@ """ import base64 -import shlex from subprocess import check_output import pytest @@ -18,13 +17,12 @@ ) from ocp_resources.virtual_machine_export import VirtualMachineExport from ocp_resources.virtual_machine_snapshot import VirtualMachineSnapshot -from pyhelper_utils.shell import run_ssh_commands from tests.storage.vm_export.constants import VM_EXPORT_TEST_FILE_CONTENT, VM_EXPORT_TEST_FILE_NAME from tests.storage.vm_export.utils import create_blank_dv_by_specific_user, get_manifest_from_vmexport, get_manifest_url from utilities.constants import OS_FLAVOR_RHEL, U1_SMALL, UNPRIVILEGED_PASSWORD, UNPRIVILEGED_USER from utilities.infra import create_ns, login_with_user_password -from utilities.storage import data_volume_template_with_source_ref_dict +from utilities.storage import data_volume_template_with_source_ref_dict, write_file_via_ssh from utilities.virt import VirtualMachineForTests, running_vm @@ -211,10 +209,7 @@ def rhel_vm_for_snapshot_with_content( ), ) as vm: running_vm(vm=vm) - - cmd = shlex.split(f"echo '{VM_EXPORT_TEST_FILE_CONTENT}' > {VM_EXPORT_TEST_FILE_NAME} && sync") - run_ssh_commands(host=vm.ssh_exec, commands=cmd) - + write_file_via_ssh(vm=vm, filename=VM_EXPORT_TEST_FILE_NAME, content=VM_EXPORT_TEST_FILE_CONTENT) yield vm diff --git a/tests/storage/vm_export/test_vm_export.py b/tests/storage/vm_export/test_vm_export.py index c7e48a00fe..918f202f48 100644 --- a/tests/storage/vm_export/test_vm_export.py +++ b/tests/storage/vm_export/test_vm_export.py @@ -9,10 +9,10 @@ from ocp_resources.persistent_volume_claim import PersistentVolumeClaim from ocp_resources.resource import Resource from ocp_resources.virtual_machine_export import VirtualMachineExport -from pyhelper_utils.shell import run_ssh_commands from tests.storage.vm_export.constants import VM_EXPORT_TEST_FILE_CONTENT, VM_EXPORT_TEST_FILE_NAME from utilities.infra import run_virtctl_command +from utilities.storage import run_command_on_vm_and_check_output from utilities.virt import running_vm VIRTUALMACHINEEXPORTS = "virtualmachineexports" @@ -62,11 +62,9 @@ def test_vmexport_snapshot_manifests( vm_from_vmexport, ): running_vm(vm=vm_from_vmexport) - - result = run_ssh_commands(host=vm_from_vmexport.ssh_exec, commands=shlex.split(f"cat {VM_EXPORT_TEST_FILE_NAME}")) - file_content = result[0].strip() - - assert file_content == VM_EXPORT_TEST_FILE_CONTENT + run_command_on_vm_and_check_output( + vm=vm_from_vmexport, command=f"cat {VM_EXPORT_TEST_FILE_NAME}", expected_result=VM_EXPORT_TEST_FILE_CONTENT + ) @pytest.mark.s390x diff --git a/utilities/storage.py b/utilities/storage.py index 9cca5657e3..b88614f579 100644 --- a/utilities/storage.py +++ b/utilities/storage.py @@ -674,7 +674,7 @@ def write_file(vm, filename, content, stop_vm=True): def write_file_via_ssh(vm: "VirtualMachineForTests", filename: str, content: str) -> None: """ - Write content to a file in VM using SSH connection. + Write content to a file in VM using SSH connection with retry. Args: vm: VirtualMachine instance with SSH connectivity @@ -682,7 +682,32 @@ def write_file_via_ssh(vm: "VirtualMachineForTests", filename: str, content: str content: Content to write to the file """ cmd = shlex.split(f"echo {shlex.quote(content)} > {shlex.quote(filename)} && sync") - run_ssh_commands(host=vm.ssh_exec, commands=cmd) + run_ssh_commands(host=vm.ssh_exec, commands=cmd, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC) + + +def run_command_on_vm_and_check_output(vm: "VirtualMachineForTests", command: str, expected_result: str) -> None: + """Run command on VM via SSH with retry and verify output matches expected result. + + Command execution is retried with 2-minute timeout and 5-second intervals. + + Args: + vm (VirtualMachineForTests): VM to run command on. + command (str): Command to run. + expected_result (str): Expected result to check. + + Raises: + AssertionError: If command output differs from expected result. + """ + cmd_output = run_ssh_commands( + host=vm.ssh_exec, + commands=shlex.split(f"bash -c {shlex.quote(command)}"), + wait_timeout=TIMEOUT_2MIN, + sleep=TIMEOUT_5SEC, + )[0].strip() + expected_result = expected_result.strip() + assert expected_result == cmd_output, ( + f"Command output mismatch.\nCommand: {command}\nExpected: '{expected_result}'\nActual: '{cmd_output}'" + ) def run_command_on_cirros_vm_and_check_output(vm, command, expected_result): @@ -692,9 +717,10 @@ def run_command_on_cirros_vm_and_check_output(vm, command, expected_result): def assert_disk_serial(vm, command=shlex.split("sudo ls /dev/disk/by-id")): - assert HOTPLUG_DISK_SERIAL in run_ssh_commands(host=vm.ssh_exec, commands=command)[0], ( - f"hotplug disk serial id {HOTPLUG_DISK_SERIAL} is not in VM" - ) + assert ( + HOTPLUG_DISK_SERIAL + in run_ssh_commands(host=vm.ssh_exec, commands=command, wait_timeout=TIMEOUT_2MIN, sleep=TIMEOUT_5SEC)[0] + ), f"hotplug disk serial id {HOTPLUG_DISK_SERIAL} is not in VM" def assert_hotplugvolume_nonexist_optional_restart(vm, restart=False): @@ -810,6 +836,8 @@ def check_disk_count_in_vm(vm): out = run_ssh_commands( host=vm.ssh_exec, commands=[shlex.split("lsblk | grep disk | grep -v SWAP| wc -l")], + wait_timeout=TIMEOUT_2MIN, + sleep=TIMEOUT_5SEC, )[0].strip() assert out == str(len(vm.instance.spec.template.spec.domain.devices.disks)), ( "Failed to verify actual disk count against VMI"