[WIP] Sanity checking around backing up files

medusa/backup_node.py

@@ -313,6 +313,18 @@ def update_monitoring(actual_backup_duration, backup_name, monitoring, node_back
 def backup_snapshots(storage, manifest, node_backup, node_backup_cache, snapshot):
     try:
         num_files = 0
+        _snapshot_file_dirs = snapshot.find_dirs()
+        _num_dirs = len(_snapshot_file_dirs)
+        logging.info("Found '{num_dir}' directories that may contain snapshot files.".format(num_dir=_num_dirs))
+
+        # Dump the list of directories if debugging
+        logging.debug("_snapshot_file_dirs: {}".format(_snapshot_file_dirs))
+
+        # Consider find_dirs() returning nothing to be a critical failure. At no point should a cassandra snapshot
+        #   exist in 0 directories on the host.
+        # We can't backup what we can't find and we consider any result other than a successful backup to be a failure
+        assert _num_dirs >= 1, "Could not identify any directories where snapshot files should reside."
+
         for snapshot_path in snapshot.find_dirs():
             logging.debug("Backing up {}".format(snapshot_path))
 

medusa/storage/__init__.py

@@ -351,8 +351,16 @@ def list_cluster_backups(self, backup_index=None):
 
     def latest_node_backup(self, *, fqdn):
         index_path = '{}index/latest_backup/{}/backup_name.txt'.format(self.prefix_path, fqdn)
+        logging.debug("Checking for latest backup in '{}'".format(index_path))
         try:
             latest_backup_name = self.storage_driver.get_blob_content_as_string(index_path)
+            # get_blob_content_as_string() uses None to indicate a 404 or other non-exception-worthy problem
+            assert latest_backup_name is not None, "Unable to determine the latest_backup_name from '{}'".format(
+                index_path)
+
+            # Otherwise, we know the name of the last backup
+            logging.info("Proceeding with latest_backup_name: '{}'".format(latest_backup_name))
+
             differential_blob = self.storage_driver.get_blob(
                 '{}{}/{}/meta/differential'.format(self.prefix_path, fqdn, latest_backup_name))
             # Should be removed after while. Here for backwards compatibility.