fix(linstor): Database Backup when Linstor Controller is not the Pool Master.

drivers/LinstorSR.py

@@ -14,9 +14,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from sm_typing import Any, Optional, override
+from sm_typing import Any, Optional, override, Literal
 
-from constants import CBTLOG_TAG
+from constants import CBTLOG_TAG, LINSTOR_AUTO_BACKUP_DELAY
 
 try:
     from linstorcowutil import LinstorCowUtil, MultiLinstorCowUtil
@@ -817,6 +817,24 @@ def is_master(self):
 
         return self._is_master
 
+    @override
+    def check_sr(self, sr_uuid) -> None:
+        # Note: check_sr is called on all hosts by the health check mechanism
+        # not by regular xapi calls such as scans.
+        # Applied only on the Linstor Controller, for reasons -> listed below.
+        if not LinstorVolumeManager.is_controller():
+            return
+        # Validate and clean previous backups if necessary.
+        # -> Needs access to backup files, available only on the Controller.
+        LinstorVolumeManager.database_invalidation()
+        # check_sr is launched on *all* hosts, but it turns out that
+        # we do not want all of them to blindly generate concurrencing backups.
+        # Hence we must choose one, either one is good, but there must be only one.
+        # Apply throttling: only backup if last one is >1h old.
+        # -> Needs access to backup files, available only on the Controller.
+        if LinstorVolumeManager.get_database_backup_age() > LINSTOR_AUTO_BACKUP_DELAY:
+            self.database_backup("auto")
+
     @override
     @_locked_load
     def vdi(self, uuid) -> VDI.VDI:
@@ -1570,6 +1588,22 @@ def _kick_gc(self):
         util.SMlog('Kicking GC')
         cleanup.start_gc_service(self.uuid)
 
+    def database_backup(self, name: Literal["auto", "create", "delete", "snapshot"]):
+        """
+        Generate a new database backup file.
+        This operation should not prevent the underlying action to be successful.
+        Hence all Exceptions are caught and logged.
+        """
+        if not self._linstor:
+            self._reconnect()
+        try:
+            self._linstor.database_backup(name)  # type: ignore
+        except Exception as e:
+            util.SMlog(
+                f"[database_backup] Error during creation: {e}",
+                priority=util.LOG_ERR,
+            )
+
 # ==============================================================================
 # LinstorSr VDI
 # ==============================================================================
@@ -1753,6 +1787,8 @@ def create(self, sr_uuid, vdi_uuid, size) -> str:
         self.ref = self._db_introduce()
         self.sr._update_stats(self.size)
 
+        self.sr.database_backup("create")
+
         return VDI.VDI.get_params(self)
 
     @override
@@ -1800,7 +1836,8 @@ def delete(self, sr_uuid, vdi_uuid, data_only=False) -> None:
         # TODO: Check size after delete.
         self.sr._update_stats(-self.size)
         self.sr._kick_gc()
-        return super(LinstorVDI, self).delete(sr_uuid, vdi_uuid, data_only)
+        super(LinstorVDI, self).delete(sr_uuid, vdi_uuid, data_only)
+        self.sr.database_backup("delete")
 
     @override
     def attach(self, sr_uuid, vdi_uuid) -> str:
@@ -2382,6 +2419,7 @@ def _do_snapshot(self, sr_uuid, vdi_uuid, snapType,
         finally:
             self.disable_leaf_on_secondary(vdi_uuid, secondary=secondary)
             blktap2.VDI.tap_unpause(self.session, sr_uuid, vdi_uuid, secondary)
+            self.sr.database_backup("snapshot")
 
     def _snapshot(self, snap_type, cbtlog=None, cbt_consistency=None):
         util.SMlog(

drivers/constants.py

@@ -11,3 +11,5 @@
 # Ref counting for VDI's: we need a ref count for LV activation/deactivation
 # on the master.
 NS_PREFIX_LVM: Final = "lvm-"
+
+LINSTOR_AUTO_BACKUP_DELAY = 3600

drivers/linstorvolumemanager.py

@@ -33,6 +33,10 @@
 import time
 import util
 import uuid
+from datetime import datetime
+from pathlib import Path
+import contextlib
+import zipfile
 
 # Persistent prefix to add to RAW persistent volumes.
 PERSISTENT_PREFIX = 'xcp-persistent-'
@@ -42,7 +46,11 @@
 DATABASE_SIZE = 1 << 30  # 1GB.
 DATABASE_PATH = '/var/lib/linstor'
 DATABASE_MKFS = 'mkfs.ext4'
-
+DATABASE_BACKUP_DIR_MAIN = Path(DATABASE_PATH)
+DATABASE_BACKUP_DIR_SPARE = Path("/var/lib/linstor.d/db-backups")
+DATABASE_BACKUP_NAME_FORMAT = "linstor_database_backup-{}-{}"
+DATABASE_BACKUP_RETENTION = 10
+DATABASE_BACKUP_DATE_FORMAT = "%Y%m%d_%H%M%S"
 LINSTOR_SATELLITE_PORT = 3366
 
 REG_DRBDADM_PRIMARY = re.compile("([^\\s]+)\\s+role:Primary")
@@ -251,6 +259,8 @@ def __init__(self, message, code=ERR_GENERIC):
     def code(self):
         return self._code
 
+class LinstorDatabaseBackupError(Exception):
+    pass
 
 # ==============================================================================
 
@@ -1774,6 +1784,51 @@ def get_database_path(self):
         """
         return self._request_database_path(self._linstor, activate=True)
 
+    @classmethod
+    def is_controller(cls):
+        return cls._is_mounted(DATABASE_PATH)
+
+    @classmethod
+    def get_database_backup_age(cls):
+        """
+        Return the latest backup age in seconds.
+        If not called on the Controller, since backups are not available,
+        returns a huge value (a timestamp of now).
+        """
+        return (datetime.now() - cls._get_latest_database_backup()[1]).total_seconds()
+
+    def database_backup(self, name=""):
+        # Create new backup
+        date = datetime.now().strftime(DATABASE_BACKUP_DATE_FORMAT)
+        filename = DATABASE_BACKUP_NAME_FORMAT.format(date, name)
+        self._linstor.controller_backupdb(filename)
+        # Relative path are ok for a secondary backup filename:
+        # https://github.com/LINBIT/linstor-server/blob/3e9306a9d8215606544c64c50ced150625ee4926/controller/src/main/java/com/linbit/linstor/api/rest/v1/Controller.java#L408
+        self._linstor.controller_backupdb(f"../linstor.d/db-backups/{filename}")
+        util.SMlog(f"[database_backup] Created: {filename}", priority=util.LOG_INFO)
+
+    @classmethod
+    def database_invalidation(cls):
+        """
+        Removes old backup based on two criterias:
+        - Validity of the zipfile done by self._check_database_backup.
+        - Number of valid files found, only the nth latest are kept.
+        """
+        for directory in (DATABASE_BACKUP_DIR_MAIN, DATABASE_BACKUP_DIR_SPARE):
+            valid_backup_count = 0
+            # Validate file and apply retention
+            for database_backup_path, _ in cls._get_sorted_database_backup(directory):
+                try:
+                    cls._check_database_backup(database_backup_path)
+                    valid_backup_count += 1
+                    if valid_backup_count < DATABASE_BACKUP_RETENTION:
+                        continue
+                except LinstorDatabaseBackupError as error:
+                    util.SMlog(f"[database_backup] Check failed `{error}` [{database_backup_path}]",
+                               priority=util.LOG_ERR)
+                with contextlib.suppress(OSError):
+                    os.unlink(database_backup_path)
+
     @classmethod
     def get_all_group_names(cls, base_name):
         """
@@ -2651,6 +2706,67 @@ def _get_volume_properties(self, volume_uuid):
         properties.namespace = self._build_volume_namespace(volume_uuid)
         return properties
 
+    @classmethod
+    def _list_database_backups(cls, database_backup_dir):
+        """
+        List all visible backup files in database_backup_dir.
+        DATABASE_BACKUP_DIR_MAIN is only available on the Linstor Controller.
+        DATABASE_BACKUP_DIR_SPARE will list backups previously made when the Host was the Linstor Controller.
+        This may not be useful information if it is not the Controller anymore.
+        """
+        for path in database_backup_dir.glob(DATABASE_BACKUP_NAME_FORMAT.format(
+                "[0-9]" * 8 + "_" + "[0-9]" * 6, "*") + ".zip"):
+            try:
+                yield path, datetime.strptime(path.name.split("-")[1], DATABASE_BACKUP_DATE_FORMAT)
+            except (ValueError, IndexError):
+                continue
+
+    @classmethod
+    def _get_sorted_database_backup(cls, database_backup_dir):
+        """
+        Return list of backups in database_backup_dir, alongside their creation date.
+        Sorted by date from the more recent to the older one.
+        """
+        return sorted(cls._list_database_backups(database_backup_dir),
+                      reverse=True,
+                      key=lambda p: p[1])
+
+    @classmethod
+    def _get_latest_database_backup(cls):
+        """
+        Return the latest backup in DATABASE_BACKUP_DIR_MAIN, and its creation date.
+        Returns (None, timestamp(0)) when none are found.
+        None will be found if it is not called on the Linstor Controller.
+        (cf _list_database_backups)
+        """
+        return max(cls._list_database_backups(DATABASE_BACKUP_DIR_MAIN),
+                   default=(None, datetime.fromtimestamp(0)),
+                   key=lambda p: p[1])
+
+    @classmethod
+    def _check_database_backup(cls, database_backup_path):
+        """
+        Make some validation of a database backup zip-file.
+        Check its a valid zipfile, and CRC-test its content.
+        Check it contains a non-empty linstordb.mv.db file.
+        Always raises a LinstorDatabaseBackupError if checks failed.
+        """
+        try:
+            with zipfile.ZipFile(database_backup_path, mode="r") as archive:
+                if archive.testzip() is not None:
+                    raise LinstorDatabaseBackupError("zip archive CRC failed")
+                linstordb = next((
+                    f
+                    for f in archive.filelist
+                    if f.filename == "linstordb.mv.db"
+                ), None)
+                if not linstordb:
+                    raise LinstorDatabaseBackupError("cannot find linstordb.mv.db")
+                if linstordb.file_size == 0:
+                    raise LinstorDatabaseBackupError("linstordb.mv.db is empty")
+        except (FileNotFoundError, zipfile.BadZipFile, zipfile.LargeZipFile) as e:
+            raise LinstorDatabaseBackupError(e) from e
+
     @classmethod
     def _build_sr_namespace(cls):
         return '/{}/'.format(cls.NAMESPACE_SR)