Linstor: (fix) Remove all operations on backup files since we cannot always access them from the pool master.

 - Backup are still being done through the Linstor API
 - Throttling is enforced using another method.
 - No retention is applied, no verifications of backup files are being made.
The removed operations needs to go somewhere where they'll be executed on the Linstor Master.
All SMlog have proper priority.
Failure (letting Exceptions be raised) only happens on non-interactive operations, preventing actions like snapshot from failing on an unreleted task, since the operation itself is finished and was successful.

Signed-off-by: Arnaud Garcia-Fernandez <arnaud.garcia-fernandez@vates.tech>

Author: Ythogtha

drivers/LinstorSR.py

@@ -811,7 +811,9 @@ def is_master(self):
 
     @override
     def check_sr(self, sr_uuid) -> None:
-        self.database_backup("auto", delay=3600)
+        # Automatic backup if there were no backups for the last hour.
+        # Let it fail if needed, so full Traceback is on SMLog.
+        self.database_backup("auto", delay=3600, fail=True)
 
 
     @override
@@ -1563,10 +1565,22 @@ def _kick_gc(self):
         util.SMlog('Kicking GC')
         cleanup.start_gc_service(self.uuid)
 
-    def database_backup(self, name="", *, delay=0):
+    def database_backup(self, name="", *, delay=0, fail=False):
+        """Generate a new database backup file.
+        This operation should not prevent the underlying action to be successful.
+        Hence all Exceptions are caught and re-raised only if asked to.
+        """
         if not self._linstor:
             self._reconnect()
-        self._linstor.database_backup(name, delay=delay)
+        try:
+            self._linstor.database_backup(name, delay=delay)
+        except Exception as e:
+            util.SMlog(
+                "[database_backup] Error during creation: {}".format(e),
+                priority=util.LOG_ERR,
+            )
+            if fail:
+                raise
 
 # ==============================================================================
 # LinstorSr VDI

drivers/linstorvolumemanager.py

@@ -44,11 +44,9 @@
 DATABASE_SIZE = 1 << 30  # 1GB.
 DATABASE_PATH = '/var/lib/linstor'
 DATABASE_MKFS = 'mkfs.ext4'
-DATABASE_BACKUP_DIR_MAIN = Path(DATABASE_PATH)
-DATABASE_BACKUP_DIR_SPARE = Path('/var/lib/linstor.d/db-backups')
+DATABASE_BACKUP_LOGDIR = Path('/var/lib/linstor.d/db-backups')
+DATABASE_BACKUP_LOGFILE = DATABASE_BACKUP_LOGDIR / "log.txt"
 DATABASE_BACKUP_NAME_FORMAT = "linstor_database_backup-{}-{}"
-DATABASE_BACKUP_NAME_LATEST = "linstor_database_backup-latest.zip"
-DATABASE_BACKUP_RETENTION = 10
 DATABASE_BACKUP_DATE_FORMAT = "%Y%m%d_%H%M%S"
 LINSTOR_SATELLITE_PORT = 3366
 
@@ -1768,32 +1766,19 @@ def get_database_path(self):
 
     def database_backup(self, name="", *, delay=0):
         now = datetime.now()
-        # Throttling to avoid too many backups of the same kind on a short period
+        # Throttling to avoid too many backups on a short period
         if delay:
-            _, date_latest = self._get_latest_database_backup(name)
-            if date_latest and ((now - date_latest).total_seconds() < delay):
+            latest = datetime.strptime(
+                self._get_latest_logged_database_backup_date(),
+                DATABASE_BACKUP_DATE_FORMAT)
+            if (now - latest).total_seconds() < delay:
                 return  # No backup for now
-
-        # Create new backup with link to latest
-        filename = DATABASE_BACKUP_NAME_FORMAT.format(now.strftime(DATABASE_BACKUP_DATE_FORMAT), name)
+        # Create new backup
+        date = now.strftime(DATABASE_BACKUP_DATE_FORMAT)
+        filename = DATABASE_BACKUP_NAME_FORMAT.format(date, name)
         self._linstor.controller_backupdb(filename)
-        # Copy to secondary backup location
-        with contextlib.suppress(OSError):
-            os.makedirs(DATABASE_BACKUP_DIR_SPARE, mode=0o755, exist_ok=True)
-            shutil.copy2(
-                (DATABASE_BACKUP_DIR_MAIN / filename).with_suffix(".zip"),
-                DATABASE_BACKUP_DIR_SPARE,
-            )
-        for directory in (DATABASE_BACKUP_DIR_MAIN, DATABASE_BACKUP_DIR_SPARE):
-            # Remove and set latest
-            with contextlib.suppress(OSError):
-                (directory / DATABASE_BACKUP_NAME_LATEST).unlink()
-            os.link(str((directory / filename).with_suffix(".zip")),
-                    str((directory / DATABASE_BACKUP_NAME_LATEST)))
-            # Apply retention
-            for old_file, _ in self._get_sorted_database_backup(directory)[DATABASE_BACKUP_RETENTION:]:
-                os.unlink(old_file)
-        util.SMlog("[database_backup] Created: {}".format(filename))
+        self._log_database_backup(date, name)
+        util.SMlog("[database_backup] Created: {}".format(filename), priority=util.LOG_INFO)
 
     @classmethod
     def get_all_group_names(cls, base_name):
@@ -2651,23 +2636,23 @@ def _get_volume_properties(self, volume_uuid):
         properties.namespace = self._build_volume_namespace(volume_uuid)
         return properties
 
-    def _list_database_backup(self, database_backup_dir, name="*"):
-        for path in database_backup_dir.glob(DATABASE_BACKUP_NAME_FORMAT.format(
-                "20[0-9][0-9][01][0-9][0-3][0-9]_[0-2][0-9][0-5][0-9][0-5][0-9]", name) + ".zip"):
-            try:
-                yield path, datetime.strptime(path.name.split("-")[1], DATABASE_BACKUP_DATE_FORMAT)
-            except (ValueError, IndexError):
-                continue
+    def _log_database_backup(self, date, name):
+        """Log a database backup operation: "date name"
+        We cannot assume the pool-master is the same as the linstor-master,
+        this file is on the pool-master, and serves for the throttling."""
+        os.makedirs(DATABASE_BACKUP_LOGDIR, mode=0o755, exist_ok=True)
+        with open(DATABASE_BACKUP_LOGFILE, "a", encoding="utf8") as f:
+            f.write(f"{date} {name}")
 
-    def _get_sorted_database_backup(self, database_backup_dir, name="*"):
-        return sorted(self._list_database_backup(database_backup_dir, name),
-                      reverse=True,
-                      key=lambda p: p[0].stat().st_mtime)
-
-    def _get_latest_database_backup(self, name="*"):
-        return max(self._list_database_backup(DATABASE_BACKUP_DIR_MAIN, name),
-                   default=(None, None),
-                   key=lambda p: p[0].stat().st_mtime)
+    def _get_latest_logged_database_backup_date(self):
+        # get last log line if it exists, and return the corresponding date
+        try:
+            with open(DATABASE_BACKUP_LOGFILE, "rb") as f:
+                # seek from the end, with 256 as a most-probable maximum line length
+                f.seek(-min(os.stat(DATABASE_BACKUP_LOGFILE).st_size, 256), os.SEEK_END)
+                return f.read().decode().splitlines()[-1].split()[0]
+        except FileNotFoundError:
+            return datetime.utcfromtimestamp(0)
 
     @classmethod
     def _build_sr_namespace(cls):