From 237bce46bf5904d93cda7d816d29774e40a81a6c Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Mon, 18 Aug 2025 14:30:18 +0200 Subject: [PATCH 1/2] fix(linstorvolumemanager): robustify "usable size" getters Impacted functions: `_get_volumes_info` and `_get_volume_node_names_and_size`. Before this change "usable_size" validity was checked too early and which could lead to an exception for no good reason while the size could be known on at least one host despite an issue on other machines. Signed-off-by: Ronan Abhamon --- drivers/linstorvolumemanager.py | 40 ++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/drivers/linstorvolumemanager.py b/drivers/linstorvolumemanager.py index 5b7da10ac..1dddbf972 100755 --- a/drivers/linstorvolumemanager.py +++ b/drivers/linstorvolumemanager.py @@ -2099,7 +2099,7 @@ def _get_volumes_info(self, volume_name=None): if not self._volume_info_cache_dirty: return self._volume_info_cache - for resource in self._get_resource_cache().resources: + def process_resource(resource): if resource.name not in all_volume_info: current = all_volume_info[resource.name] = self.VolumeInfo( resource.name @@ -2131,15 +2131,21 @@ def _get_volumes_info(self, volume_name=None): ): current.virtual_size = usable_size - if current.virtual_size <= 0: - raise LinstorVolumeManagerError( - 'Failed to get usable size of `{}` on `{}`' - .format(resource.name, volume.storage_pool_name) - ) + try: + for resource in self._get_resource_cache().resources: + process_resource(resource) + for volume in all_volume_info.values(): + if volume.virtual_size <= 0: + raise LinstorVolumeManagerError( + 'Failed to get usable size of `{}`' + .format(volume.name) + ) - for current in all_volume_info.values(): - current.allocated_size *= 1024 - current.virtual_size *= 1024 + volume.allocated_size *= 1024 + volume.virtual_size *= 1024 + except LinstorVolumeManagerError: + self._mark_resource_cache_as_dirty() + raise self._volume_info_cache_dirty = False self._volume_info_cache = all_volume_info @@ -2158,16 +2164,14 @@ def _get_volume_node_names_and_size(self, volume_name): node_names.add(resource.node_name) current_size = volume.usable_size - if current_size < 0: - raise LinstorVolumeManagerError( - 'Failed to get usable size of `{}` on `{}`' - .format(resource.name, volume.storage_pool_name) - ) - - if size < 0: + if current_size > 0 and (current_size < size or size < 0): size = current_size - else: - size = min(size, current_size) + + if size < 0: + raise LinstorVolumeManagerError( + 'Failed to get usable size of `{}` on `{}`' + .format(resource.name, volume.storage_pool_name) + ) return (node_names, size * 1024) From 5e61b9aefe9561f2519d370bf90027412f8a25c4 Mon Sep 17 00:00:00 2001 From: Ronan Abhamon Date: Mon, 18 Aug 2025 17:32:14 +0200 Subject: [PATCH 2/2] feat(LinstorSR): improve DB volume robustness - Use specific DRBD options to detect failures in a small delay. - Use these options to control quorum with drbd-reactor. - Provide a better compromise in terms of availability. Signed-off-by: Ronan Abhamon --- drivers/LinstorSR.py | 9 ++++- drivers/linstorvolumemanager.py | 68 ++++++++++++++++----------------- 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/drivers/LinstorSR.py b/drivers/LinstorSR.py index dbfda35ee..4c154831c 100755 --- a/drivers/LinstorSR.py +++ b/drivers/LinstorSR.py @@ -25,6 +25,7 @@ from linstorvolumemanager import get_controller_node_name from linstorvolumemanager import LinstorVolumeManager from linstorvolumemanager import LinstorVolumeManagerError + from linstorvolumemanager import DATABASE_VOLUME_NAME from linstorvolumemanager import PERSISTENT_PREFIX LINSTOR_AVAILABLE = True @@ -618,7 +619,6 @@ def create(self, uuid, size) -> None: ips, self._redundancy, thin_provisioning=self._provisioning == 'thin', - auto_quorum=self._monitor_db_quorum, logger=util.SMlog ) self._vhdutil = LinstorVhdUtil(self.session, self._linstor) @@ -677,6 +677,8 @@ def delete(self, uuid) -> None: ) try: + if self._monitor_db_quorum: + self._linstor.set_drbd_ha_properties(DATABASE_VOLUME_NAME, enabled=False) self._update_drbd_reactor_on_all_hosts( controller_node_name=node_name, enabled=False ) @@ -692,6 +694,8 @@ def delete(self, uuid) -> None: self._update_drbd_reactor_on_all_hosts( controller_node_name=node_name, enabled=True ) + if self._monitor_db_quorum: + self._linstor.set_drbd_ha_properties(DATABASE_VOLUME_NAME, enabled=True) except Exception as e2: util.SMlog( 'Failed to restart drbd-reactor after destroy fail: {}' @@ -741,6 +745,9 @@ def attach(self, uuid) -> None: opterr='no such group: {}'.format(self._group_name) ) + if self._monitor_db_quorum and self.is_master(): + self._linstor.set_drbd_ha_properties(DATABASE_VOLUME_NAME) + @override @_locked_load def detach(self, uuid) -> None: diff --git a/drivers/linstorvolumemanager.py b/drivers/linstorvolumemanager.py index 1dddbf972..68c7b8e78 100755 --- a/drivers/linstorvolumemanager.py +++ b/drivers/linstorvolumemanager.py @@ -927,6 +927,34 @@ def set_auto_promote_timeout(self, volume_uuid, timeout): .format(volume_uuid, error_str) ) + def set_drbd_ha_properties(self, volume_name, enabled=True): + """ + Set or not HA DRBD properties required by drbd-reactor and + by specific volumes. + :param str volume_name: The volume to modify. + :param bool enabled: Enable or disable HA properties. + """ + + properties = { + 'DrbdOptions/auto-quorum': 'disabled', + 'DrbdOptions/Resource/auto-promote': 'no', + 'DrbdOptions/Resource/on-no-data-accessible': 'io-error', + 'DrbdOptions/Resource/on-no-quorum': 'io-error', + 'DrbdOptions/Resource/on-suspended-primary-outdated': 'force-secondary', + 'DrbdOptions/Resource/quorum': 'majority' + } + if enabled: + result = self._linstor.resource_dfn_modify(volume_name, properties) + else: + result = self._linstor.resource_dfn_modify(volume_name, {}, delete_props=list(properties.keys())) + + error_str = self._get_error_str(result) + if error_str: + raise LinstorVolumeManagerError( + 'Could not modify HA DRBD properties on volume `{}`: {}' + .format(volume_name, error_str) + ) + def get_volume_info(self, volume_uuid): """ Get the volume info of a particular volume. @@ -1735,18 +1763,13 @@ def get_all_group_names(cls, base_name): return [cls._build_group_name(base_name), cls._build_ha_group_name(base_name)] @classmethod - def create_sr( - cls, group_name, ips, redundancy, - thin_provisioning, auto_quorum, - logger=default_logger.__func__ - ): + def create_sr(cls, group_name, ips, redundancy, thin_provisioning, logger=default_logger.__func__): """ Create a new SR on the given nodes. :param str group_name: The SR group_name to use. :param set(str) ips: Node ips. :param int redundancy: How many copy of volumes should we store? :param bool thin_provisioning: Use thin or thick provisioning. - :param bool auto_quorum: DB quorum is monitored by LINSTOR. :param function logger: Function to log messages. :return: A new LinstorSr instance. :rtype: LinstorSr @@ -1754,14 +1777,7 @@ def create_sr( try: cls._start_controller(start=True) - sr = cls._create_sr( - group_name, - ips, - redundancy, - thin_provisioning, - auto_quorum, - logger - ) + sr = cls._create_sr(group_name, ips, redundancy, thin_provisioning, logger) finally: # Controller must be stopped and volume unmounted because # it is the role of the drbd-reactor daemon to do the right @@ -1775,11 +1791,7 @@ def create_sr( return sr @classmethod - def _create_sr( - cls, group_name, ips, redundancy, - thin_provisioning, auto_quorum, - logger=default_logger.__func__ - ): + def _create_sr(cls, group_name, ips, redundancy, thin_provisioning, logger=default_logger.__func__): # 1. Check if SR already exists. uri = 'linstor://localhost' @@ -1921,7 +1933,7 @@ def _create_sr( try: logger('Creating database volume...') volume_path = cls._create_database_volume( - lin, ha_group_name, storage_pool_name, node_names, redundancy, auto_quorum + lin, ha_group_name, storage_pool_name, node_names, redundancy ) except LinstorVolumeManagerError as e: if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS: @@ -2695,7 +2707,7 @@ def _request_database_path(cls, lin, activate=False): @classmethod def _create_database_volume( - cls, lin, group_name, storage_pool_name, node_names, redundancy, auto_quorum + cls, lin, group_name, storage_pool_name, node_names, redundancy ): try: dfns = lin.resource_dfn_list_raise().resource_definitions @@ -2779,20 +2791,6 @@ def _create_database_volume( ) ) - # We must modify the quorum. Otherwise we can't use correctly the - # drbd-reactor daemon. - if auto_quorum: - result = lin.resource_dfn_modify(DATABASE_VOLUME_NAME, { - 'DrbdOptions/auto-quorum': 'disabled', - 'DrbdOptions/Resource/quorum': 'majority' - }) - error_str = cls._get_error_str(result) - if error_str: - raise LinstorVolumeManagerError( - 'Could not activate quorum on database volume: {}' - .format(error_str) - ) - # Create database and ensure path exists locally and # on replicated devices. current_device_path = cls._request_database_path(lin, activate=True)