diff --git a/lib/common.py b/lib/common.py index 76022bf86..797ad9b1f 100644 --- a/lib/common.py +++ b/lib/common.py @@ -6,6 +6,7 @@ import inspect import itertools import logging +import multiprocessing import os import random import string @@ -215,6 +216,28 @@ def wait_for_not( ) -> None: return wait_for(fn, msg, timeout_secs, retry_delay_secs, True) +def run_with_timeout(fn: Callable[[], Any], timeout_secs: int = 2 * 60) -> None: + queue: multiprocessing.Queue[Exception] = multiprocessing.Queue() + + def fn_wrapper() -> None: + try: + fn() + except Exception as e: + queue.put(e) + + proc = multiprocessing.Process(target=fn_wrapper) + proc.start() + proc.join(timeout=timeout_secs) + + if proc.is_alive(): + proc.terminate() + proc.join() + + raise TimeoutError(f"Timeout reached while waiting for fn call to return ({timeout_secs}s).") + + if not queue.empty(): + raise queue.get(block=False) + def is_uuid(maybe_uuid: str) -> bool: try: UUID(maybe_uuid, version=4) diff --git a/tests/storage/linstor/conftest.py b/tests/storage/linstor/conftest.py index 6f5857bff..70ca722de 100644 --- a/tests/storage/linstor/conftest.py +++ b/tests/storage/linstor/conftest.py @@ -2,22 +2,13 @@ import pytest -import functools import json import logging -import os from contextlib import contextmanager -from dataclasses import dataclass -import lib.commands as commands from lib import config from lib.common import safe_split -try: - from data import LINSTOR_REDUNDANCY # type: ignore -except ImportError: - LINSTOR_REDUNDANCY = 2 - # explicit import for package-scope fixtures from pkgfixtures import ( _xfs_config_on_hostA2, @@ -33,156 +24,14 @@ if TYPE_CHECKING: from lib.host import Host - from lib.pool import Pool from lib.sr import SR from lib.vdi import VDI from lib.vm import VM -GROUP_NAME = 'linstor_group' -STORAGE_POOL_NAME = f'{GROUP_NAME}/thin_device' -LINSTOR_RELEASE_PACKAGE = 'xcp-ng-release-linstor' -LINSTOR_PACKAGE = 'xcp-ng-linstor' - -@dataclass -class LinstorConfig: - uninstall_linstor: bool = True - -@pytest.fixture(scope='package') -def _linstor_config() -> LinstorConfig: - return LinstorConfig() - -@pytest.fixture(scope='package') -def lvm_disks( - pool_with_unused_512B_disk: Pool, - unused_512B_disks: dict[Host, list[Host.BlockDeviceInfo]], - provisioning_type: str, -) -> Generator[None, None, None]: - """ - Common LVM PVs on which a LV is created on each host of the pool. - - On each host in the pool, create PV on each of those disks whose - DEVICE NAME exists ACROSS THE WHOLE POOL. Then make a VG out of - all those, then a LV taking up the whole VG space. - - Return the list of device node paths for that list of devices - used in all hosts. - """ - hosts = pool_with_unused_512B_disk.hosts - - @functools.cache - def host_devices(host: Host) -> list[str]: - return [disk.path for disk in unused_512B_disks[host][0:1]] - - for host in hosts: - devices = host_devices(host) - for device in devices: - try: - host.ssh(f'pvcreate -ff -y {device}') - except commands.SSHCommandFailed as e: - if e.stdout.endswith('Mounted filesystem?'): - host.ssh(f'vgremove -f {GROUP_NAME} -y') - host.ssh(f'pvcreate -ff -y {device}') - elif e.stdout.endswith('excluded by a filter.'): - host.ssh(f'wipefs -a {device}') - host.ssh(f'pvcreate -ff -y {device}') - else: - raise e - - host.ssh(f'vgcreate {GROUP_NAME} ' + ' '.join(devices)) - if provisioning_type == 'thin': - host.ssh(f'lvcreate -l 100%FREE -T {STORAGE_POOL_NAME}') - - # FIXME ought to provide storage_pool_name and get rid of that other fixture - yield None - - for host in hosts: - host.ssh(f'vgremove -f {GROUP_NAME}') - for device in host_devices(host): - host.ssh(f'pvremove {device}') - -@pytest.fixture(scope="package") -def storage_pool_name(provisioning_type: str) -> str: - return GROUP_NAME if provisioning_type == "thick" else STORAGE_POOL_NAME - @pytest.fixture(params=["thin"], scope="session") def provisioning_type(request: pytest.FixtureRequest) -> str: return request.param -@pytest.fixture(scope='package') -def pool_with_linstor( - hostA2: Host, - lvm_disks: None, - pool_with_saved_yum_state: Pool, - _linstor_config: LinstorConfig -) -> Generator[Pool, None, None]: - import concurrent.futures - pool = pool_with_saved_yum_state - - def check_linstor_installed(host: Host) -> None: - if host.is_package_installed(LINSTOR_PACKAGE): - raise Exception( - f'{LINSTOR_PACKAGE} is already installed on host {host}. This should not be the case.' - ) - - with concurrent.futures.ThreadPoolExecutor() as executor: - executor.map(check_linstor_installed, pool.hosts) - - def install_linstor(host: Host) -> None: - logging.info(f"Installing {LINSTOR_PACKAGE} on host {host}...") - host.yum_install([LINSTOR_RELEASE_PACKAGE]) - host.yum_install([LINSTOR_PACKAGE], enablerepo="xcp-ng-linstor-testing") - # Needed because the linstor driver is not in the xapi sm-plugins list - # before installing the LINSTOR packages. - host.ssh('systemctl restart multipathd') - host.restart_toolstack(verify=True) - - with concurrent.futures.ThreadPoolExecutor() as executor: - executor.map(install_linstor, pool.hosts) - - yield pool - - def _disable_yum_rollback(host: Host) -> None: - host.saved_rollback_id = None - - if not _linstor_config.uninstall_linstor: - pool.exec_on_hosts_on_error_continue(_disable_yum_rollback) - return - - # Need to remove this package as we have separate run of `test_create_sr_without_linstor` - # for `thin` and `thick` `provisioning_type`. - def remove_linstor(host: Host) -> None: - logging.info(f"Cleaning up python-linstor from host {host}...") - host.yum_remove(["python-linstor"]) - host.restart_toolstack(verify=True) - - with concurrent.futures.ThreadPoolExecutor() as executor: - executor.map(remove_linstor, pool.hosts) - -@pytest.fixture(scope='package') -def linstor_redundancy(pool_with_linstor: Pool) -> int: - return min(len(pool_with_linstor.hosts), LINSTOR_REDUNDANCY) - -@pytest.fixture(scope='package') -def linstor_sr( - pool_with_linstor: Pool, - linstor_redundancy: int, - provisioning_type: str, - storage_pool_name: str, - lvm_disks: None, - _linstor_config: LinstorConfig -) -> Generator[SR, None, None]: - sr = pool_with_linstor.master.sr_create('linstor', 'LINSTOR-SR-test', { - 'group-name': storage_pool_name, - 'redundancy': str(linstor_redundancy), - 'provisioning': provisioning_type - }, shared=True) - yield sr - try: - sr.destroy() - except Exception as e: - _linstor_config.uninstall_linstor = False - raise pytest.fail("Could not destroy linstor SR, leaving packages in place for manual cleanup") from e - @pytest.fixture(scope='module') def vdi_on_linstor_sr(linstor_sr: SR) -> Generator[VDI, None, None]: vdi = linstor_sr.create_vdi('LINSTOR-VDI-test', virtual_size=config.volume_size) diff --git a/tests/storage/linstor/create_destroy/conftest.py b/tests/storage/linstor/create_destroy/conftest.py new file mode 100644 index 000000000..32277a625 --- /dev/null +++ b/tests/storage/linstor/create_destroy/conftest.py @@ -0,0 +1,10 @@ +# explicit import for package-scope fixtures +from tests.storage.linstor.pkgfixtures import ( + _linstor_config, + linstor_redundancy, + linstor_sr, + lvm_disk_paths, + lvm_disks, + pool_with_linstor, + storage_pool_name, +) diff --git a/tests/storage/linstor/pkgfixtures.py b/tests/storage/linstor/pkgfixtures.py new file mode 100644 index 000000000..fc5c4e8cd --- /dev/null +++ b/tests/storage/linstor/pkgfixtures.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import pytest + +import logging +from dataclasses import dataclass + +import lib.commands as commands + +try: + from data import LINSTOR_REDUNDANCY # type: ignore +except ImportError: + LINSTOR_REDUNDANCY = 2 + +from typing import TYPE_CHECKING, Generator + +if TYPE_CHECKING: + from lib.host import Host + from lib.pool import Pool + from lib.sr import SR + +# Due to a bug in the way pytest handles the setup and teardown of package-scoped fixtures, +# we moved the following fixtures out of this package's conftest.py. +# To workaround the bug, the fixture must be imported either in a package's own conftest.py, +# or directly in a test module. Then the fixtures will truly be handled as package-scoped. +# Reference: https://github.com/pytest-dev/pytest/issues/8189 + +GROUP_NAME = 'linstor_group' +STORAGE_POOL_NAME = f'{GROUP_NAME}/thin_device' +LINSTOR_RELEASE_PACKAGE = 'xcp-ng-release-linstor' +LINSTOR_PACKAGE = 'xcp-ng-linstor' + +@dataclass +class LinstorConfig: + uninstall_linstor: bool = True + +@pytest.fixture(scope='package') +def _linstor_config() -> LinstorConfig: + return LinstorConfig() + +@pytest.fixture(scope='package') +def lvm_disk_paths( + unused_512B_disks: dict[Host, list[Host.BlockDeviceInfo]], +) -> dict[Host, list[str]]: + return {host: [disk.path for disk in disks[0:1]] for (host, disks) in unused_512B_disks.items()} + +@pytest.fixture(scope='package') +def lvm_disks( + pool_with_unused_512B_disk: Pool, + lvm_disk_paths: dict[Host, list[str]], + provisioning_type: str, +) -> Generator[None, None, None]: + """ + Common LVM PVs on which a LV is created on each host of the pool. + + On each host in the pool, create PV on each of those disks whose + DEVICE NAME exists ACROSS THE WHOLE POOL. Then make a VG out of + all those, then a LV taking up the whole VG space. + + Return the list of device node paths for that list of devices + used in all hosts. + """ + hosts = pool_with_unused_512B_disk.hosts + + for host in hosts: + devices = lvm_disk_paths[host] + for device in devices: + try: + host.ssh(f'pvcreate -ff -y {device}') + except commands.SSHCommandFailed as e: + if e.stdout.endswith('Mounted filesystem?'): + host.ssh(f'vgremove -f {GROUP_NAME} -y') + host.ssh(f'pvcreate -ff -y {device}') + elif e.stdout.endswith('excluded by a filter.'): + host.ssh(f'wipefs -a {device}') + host.ssh(f'pvcreate -ff -y {device}') + else: + raise e + + host.ssh(f'vgcreate {GROUP_NAME} ' + ' '.join(devices)) + if provisioning_type == 'thin': + host.ssh(f'lvcreate -l 100%FREE -T {STORAGE_POOL_NAME}') + + # FIXME ought to provide storage_pool_name and get rid of that other fixture + yield None + + for host in hosts: + host.ssh(f'vgremove -f {GROUP_NAME}') + for device in lvm_disk_paths[host]: + host.ssh(f'pvremove {device}') + +@pytest.fixture(scope='package') +def storage_pool_name(provisioning_type: str) -> str: + return GROUP_NAME if provisioning_type == 'thick' else STORAGE_POOL_NAME + +@pytest.fixture(scope='package') +def pool_with_linstor( + hostA2: Host, + lvm_disks: None, + pool_with_saved_yum_state: Pool, + _linstor_config: LinstorConfig +) -> Generator[Pool, None, None]: + import concurrent.futures + pool = pool_with_saved_yum_state + + def check_linstor_installed(host: Host) -> None: + if host.is_package_installed(LINSTOR_PACKAGE): + raise Exception( + f'{LINSTOR_PACKAGE} is already installed on host {host}. This should not be the case.' + ) + + with concurrent.futures.ThreadPoolExecutor() as executor: + executor.map(check_linstor_installed, pool.hosts) + + def install_linstor(host: Host) -> None: + logging.info(f"Installing {LINSTOR_PACKAGE} on host {host}...") + host.yum_install([LINSTOR_RELEASE_PACKAGE]) + host.yum_install([LINSTOR_PACKAGE], enablerepo="xcp-ng-linstor-testing") + # Needed because the linstor driver is not in the xapi sm-plugins list + # before installing the LINSTOR packages. + host.ssh('systemctl restart multipathd') + host.restart_toolstack(verify=True) + + with concurrent.futures.ThreadPoolExecutor() as executor: + executor.map(install_linstor, pool.hosts) + + yield pool + + def _disable_yum_rollback(host: Host) -> None: + host.saved_rollback_id = None + + if not _linstor_config.uninstall_linstor: + pool.exec_on_hosts_on_error_continue(_disable_yum_rollback) + return + + # Need to remove this package as we have separate run of `test_create_sr_without_linstor` + # for `thin` and `thick` `provisioning_type`. + def remove_linstor(host: Host) -> None: + logging.info(f"Cleaning up python-linstor from host {host}...") + host.yum_remove(["python-linstor"]) + host.restart_toolstack(verify=True) + + with concurrent.futures.ThreadPoolExecutor() as executor: + executor.map(remove_linstor, pool.hosts) + +@pytest.fixture(scope='package') +def linstor_redundancy(pool_with_linstor: Pool) -> int: + return min(len(pool_with_linstor.hosts), LINSTOR_REDUNDANCY) + +@pytest.fixture(scope='package') +def linstor_sr( + pool_with_linstor: Pool, + linstor_redundancy: int, + provisioning_type: str, + storage_pool_name: str, + lvm_disks: None, + _linstor_config: LinstorConfig +) -> Generator[SR, None, None]: + sr = pool_with_linstor.master.sr_create('linstor', 'LINSTOR-SR-test', { + 'group-name': storage_pool_name, + 'redundancy': str(linstor_redundancy), + 'provisioning': provisioning_type + }, shared=True) + yield sr + try: + sr.destroy() + except Exception as e: + _linstor_config.uninstall_linstor = False + raise pytest.fail("Could not destroy linstor SR, leaving packages in place for manual cleanup") from e diff --git a/tests/storage/linstor/regular/__init__.py b/tests/storage/linstor/regular/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/storage/linstor/regular/conftest.py b/tests/storage/linstor/regular/conftest.py new file mode 100644 index 000000000..32277a625 --- /dev/null +++ b/tests/storage/linstor/regular/conftest.py @@ -0,0 +1,10 @@ +# explicit import for package-scope fixtures +from tests.storage.linstor.pkgfixtures import ( + _linstor_config, + linstor_redundancy, + linstor_sr, + lvm_disk_paths, + lvm_disks, + pool_with_linstor, + storage_pool_name, +) diff --git a/tests/storage/linstor/test_linstor_sr.py b/tests/storage/linstor/regular/test_linstor_sr.py similarity index 99% rename from tests/storage/linstor/test_linstor_sr.py rename to tests/storage/linstor/regular/test_linstor_sr.py index 2a07e05e5..7f5ee09a4 100644 --- a/tests/storage/linstor/test_linstor_sr.py +++ b/tests/storage/linstor/regular/test_linstor_sr.py @@ -13,8 +13,7 @@ from lib.vdi import VDI from lib.vm import VM from tests.storage import vdi_is_open - -from .conftest import GROUP_NAME, LINSTOR_PACKAGE +from tests.storage.linstor.pkgfixtures import GROUP_NAME, LINSTOR_PACKAGE from typing import Tuple diff --git a/tests/storage/linstor/test_linstor_sr_crosspool_migration.py b/tests/storage/linstor/regular/test_linstor_sr_crosspool_migration.py similarity index 100% rename from tests/storage/linstor/test_linstor_sr_crosspool_migration.py rename to tests/storage/linstor/regular/test_linstor_sr_crosspool_migration.py diff --git a/tests/storage/linstor/test_linstor_sr_intrapool_migration.py b/tests/storage/linstor/regular/test_linstor_sr_intrapool_migration.py similarity index 100% rename from tests/storage/linstor/test_linstor_sr_intrapool_migration.py rename to tests/storage/linstor/regular/test_linstor_sr_intrapool_migration.py diff --git a/tests/storage/linstor/unhealthy/__init__.py b/tests/storage/linstor/unhealthy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/storage/linstor/unhealthy/conftest.py b/tests/storage/linstor/unhealthy/conftest.py new file mode 100644 index 000000000..d4bcdfc98 --- /dev/null +++ b/tests/storage/linstor/unhealthy/conftest.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import pytest + +import json +import logging + +import lib.commands as commands + +# explicit import for package-scope fixtures +from tests.storage.linstor.pkgfixtures import ( + _linstor_config, + linstor_redundancy, + linstor_sr, + lvm_disks, + pool_with_linstor, + storage_pool_name, +) + +from typing import TYPE_CHECKING, Generator, List + +if TYPE_CHECKING: + from lib.host import Host + from lib.pool import Pool + +DM_FLAKEY_DEV_NAME = 'linfail' + +class FlakeyDisk: + def __init__( + self, + host: Host, + pool_hosts: List[Host], + device: Host.BlockDeviceInfo, + dm_dev_name: str, + ) -> None: + self._host = host + self._pool_hosts = pool_hosts + self._device = device + self._dm_dev_name = dm_dev_name + + @property + def path(self) -> str: + return f'/dev/mapper/{self._dm_dev_name}' + + def create(self) -> None: + self._host.ssh(f'dmsetup create {self._dm_dev_name} --table "{self._build_dm_table(False)}"') + + def remove(self) -> None: + self._host.ssh(f'dmsetup remove {self._dm_dev_name}') + + def fail(self) -> None: + logging.info(f'Failing device {self._device.path} on {self._host.hostname_or_ip}') + + self._apply_dm_table(self._build_dm_table(True)) + self._host.ssh('sync') + self._host.ssh('echo 3 > /proc/sys/vm/drop_caches') + + def repair(self) -> None: + logging.info(f'Repairing device {self._device.path} on {self._host.hostname_or_ip}') + + self._apply_dm_table(self._build_dm_table(False)) + cmd_res = self._host.ssh('linstor -m --controllers `xe host-list params=address --minimal` r l' + ' -n `hostname` --props DrbdOptions/SkipDisk') + failing_resources = json.loads(cmd_res) + + # Make sure we take care of the database first so repairing the other + # resources doesn't hang or fail. + failing_resource_names = [res['name'] for res in failing_resources[0]] + failing_resource_names.sort(key=lambda x: x != 'xcp-persistent-database') + + for failing_resource_name in failing_resource_names: + try: + self._host.ssh('linstor --controllers `xe host-list params=address --minimal` r sp' + f' `hostname` {failing_resource_name} DrbdOptions/SkipDisk') + except commands.SSHCommandFailed: + drbd_res_status = json.loads(self._host.ssh(f'drbdsetup status --json {failing_resource_name}')) + + if drbd_res_status[0]['devices'][0]['disk-state'] != 'Negotiating': + raise + + # The DRBD resource sometimes gets stuck in the "Negotiating" + # state when repairing the device. This can lead to a split-brain + # issue, so we try to fix it as one. + logging.warning('DRBD resource stuck in Negotiating state; attempting fix') + + self._host.ssh(f'drbdadm disconnect {failing_resource_name}') + self._host.ssh(f'drbdadm connect {failing_resource_name}') + + for other_host in filter(lambda x: x != self._host, self._pool_hosts): + drbd_res_status = json.loads(other_host.ssh(f'drbdsetup status --json {failing_resource_name}')) + + other_host.ssh(f'drbdadm disconnect {failing_resource_name}') + + if drbd_res_status[0]['role'] == 'Primary': + other_host.ssh(f'drbdadm connect {failing_resource_name}') + else: + other_host.ssh(f'drbdadm connect --discard-my-data {failing_resource_name}') + + self._host.ssh(f'drbdadm wait-sync {failing_resource_name}') + + def _build_dm_table(self, disk_failed: bool) -> str: + disk_size = self._device.size // 512 + + if disk_failed: + return f'0 {disk_size} flakey {self._device.path} 0 0 1' + else: + return f'0 {disk_size} flakey {self._device.path} 0 1 0' + + def _apply_dm_table(self, table: str) -> None: + self._host.ssh(f'dmsetup reload {self._dm_dev_name} --table "{table}"') + self._host.ssh(f'dmsetup resume {self._dm_dev_name}') + +@pytest.fixture(scope='package') +def flakey_unused_512B_disk( + pool_with_unused_512B_disk: Pool, + unused_512B_disks: dict[Host, list[Host.BlockDeviceInfo]], +) -> Generator[dict[Host, FlakeyDisk], None, None]: + flakey_disks: dict[Host, FlakeyDisk] = {} + hosts = pool_with_unused_512B_disk.hosts + + for host in hosts: + disk = unused_512B_disks[host][0] + flakey_disk = FlakeyDisk(host, hosts, disk, DM_FLAKEY_DEV_NAME) + flakey_disk.create() + + flakey_disks[host] = flakey_disk + + yield flakey_disks + + for flakey_disk in flakey_disks.values(): + flakey_disk.remove() + +@pytest.fixture(scope='package') +def lvm_disk_paths( + flakey_unused_512B_disk: dict[Host, FlakeyDisk], +) -> dict[Host, list[str]]: + # Overrides the `lvm_disk_paths` package-scoped fixture from the parent + # package so we can transparently use other fixtures that depend on it + # whilst having a dm-flakey device mapper underneath. + return {host: [disk.path] for (host, disk) in flakey_unused_512B_disk.items()} diff --git a/tests/storage/linstor/unhealthy/test_failed_disk_linstor_sr.py b/tests/storage/linstor/unhealthy/test_failed_disk_linstor_sr.py new file mode 100644 index 000000000..187acaac9 --- /dev/null +++ b/tests/storage/linstor/unhealthy/test_failed_disk_linstor_sr.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import pytest + +import logging +import random + +from lib.common import Defer, run_with_timeout +from lib.host import Host +from lib.sr import SR +from lib.vm import VM + +from .conftest import FlakeyDisk + +# Requirements: +# - three or more XCP-ng hosts >= 8.2 with additional unused disk(s) for the SR +# - LINSTOR redundancy set to at least 2 +# - access to XCP-ng RPM repository from the host + +class TestLinstorSRFailedDisk: + @pytest.mark.small_vm # run with a small VM to test the features + def test_linstor_sr_fail_disk( + self, + vm_on_linstor_sr: VM, + flakey_unused_512B_disk: dict[Host, FlakeyDisk], + linstor_sr: SR, + defer: Defer + ) -> None: + sr = linstor_sr + vm = vm_on_linstor_sr + failed_host = sr.pool.hosts[0] + + # Let xcp-persistent-database come in sync across the nodes. + failed_host.ssh('drbdadm wait-sync xcp-persistent-database') + + flakey_unused_512B_disk[failed_host].fail() + defer(lambda: flakey_unused_512B_disk[failed_host].repair()) + + for host in sr.pool.hosts: + logging.info(f'Checking VM on host {host.hostname_or_ip}') + + run_with_timeout(lambda: vm.start(on=host.uuid), timeout_secs=60) + vm.wait_for_os_booted() + vm.shutdown(verify=True)