kopia snapshots improvements

Author: OriHoch

hasadna_k8s/ceph/cli.py

@@ -1,6 +1,7 @@
 import os
 import json
 import traceback
+import subprocess
 
 import click
 
@@ -31,7 +32,7 @@ def maintenance(**kwargs):
 @click.option('--with-weekly', is_flag=True)
 @click.option('--with-weekly-on-saturday', is_flag=True)
 def pvc_backup_all(with_full_maintenance, **kwargs):
-    from .pvc_backup import main_all
+    from .pvc_backup import main_all, CEPH_BACKUPS_HEARTBEAT_URL
     pvc_backup_failed = False
     try:
         main_all(**kwargs)
@@ -45,3 +46,8 @@ def pvc_backup_all(with_full_maintenance, **kwargs):
         from .kopia import maintenance
         maintenance(full=True)
         assert not pvc_backup_failed
+    if CEPH_BACKUPS_HEARTBEAT_URL:
+        print(f'Sending heartbeat to {CEPH_BACKUPS_HEARTBEAT_URL}...')
+        subprocess.check_call(['curl', CEPH_BACKUPS_HEARTBEAT_URL])
+    else:
+        raise Exception('CEPH_BACKUPS_HEARTBEAT_URL is not set, cannot send heartbeat.')

hasadna_k8s/ceph/pvc_backup.py

@@ -35,7 +35,7 @@ def main_block(namespace, pvc_name, pv):
                         kopia_snapshot_source = f'ceph@{namespace}:{pvc_name}'
                         print(f'Creating Kopia snapshot with source {kopia_snapshot_source}...')
                         subprocess.check_call(['kopia', 'snapshot', 'create', f'/tmp{device}', '--override-source', kopia_snapshot_source])
-                        print("Kopia snapshot created successfully.")
+                        return "Kopia snapshot created successfully."
                     finally:
                         if subprocess.call(['umount', f'/tmp{device}']) != 0:
                             print(f"ERROR! Failed to unmount device {device} for namespace '{namespace}', pvc '{pvc_name}'")
@@ -74,7 +74,7 @@ def main_shared(namespace, pvc_name, pv):
             kopia_snapshot_source = f'ceph@{namespace}:{pvc_name}'
             print(f'Creating Kopia snapshot with source {kopia_snapshot_source}...')
             subprocess.check_call(['kopia', 'snapshot', 'create', path, '--override-source', kopia_snapshot_source])
-            print("Kopia snapshot created successfully.")
+            return "Kopia snapshot created successfully."
         finally:
             if subprocess.call(['umount', '/mnt/ceph']) != 0:
                 print(f"ERROR! Failed to unmount /mnt/ceph after creating snapshot {backup_name} for namespace '{namespace}', pvc '{pvc_name}'")
@@ -83,17 +83,6 @@ def main_shared(namespace, pvc_name, pv):
             print(f"ERROR! Failed to remove snapshot {backup_name} for namespace '{namespace}', pvc '{pvc_name}'")
 
 
-def is_pvc_eligible_for_backup(namespace, pvc_name, pvc):
-    not_eligible_msg = None
-    if pvc['metadata']['labels'].get('hasadna/no-backup') == 'yes':
-        not_eligible_msg = 'has label hasadna/no-backup=yes'
-    if not_eligible_msg is None:
-        return True
-    else:
-        print(f'PVC {namespace}/{pvc_name} not eligible for backup: {not_eligible_msg}')
-        return False
-
-
 def main_pvc(namespace, pvc_name, pvc, with_weekly):
     backup_freq = pvc.get('metadata', {}).get('labels', {}).get('hasadna/iac-storage-backup-freq') or "weekly"
     assert backup_freq in ["none", "daily", "weekly"], f'Invalid backup frequency "{backup_freq}" for PVC {namespace}/{pvc_name}. Must be one of "none", "daily", "weekly".'
@@ -104,15 +93,13 @@ def main_pvc(namespace, pvc_name, pvc, with_weekly):
         print(f'Volume name: {volume_name}')
         pv = json.loads(subprocess.check_output(['kubectl', 'get', 'pv', volume_name, '-o', 'json']))
         if storage_class_name == 'rook-cephfs-shared':
-            main_shared(namespace, pvc_name, pv)
+            return main_shared(namespace, pvc_name, pv)
         elif storage_class_name == 'rook-ceph-block':
-            main_block(namespace, pvc_name, pv)
+            return main_block(namespace, pvc_name, pv)
         else:
             raise Exception(f'Unexpected storage class name: {storage_class_name}')
-        return True
     else:
-        print(f"Skipping backup for PVC {namespace}/{pvc_name} due to backup frequency '{backup_freq}' (with_weekly={with_weekly})")
-        return False
+        return f"Skipping backup for PVC {namespace}/{pvc_name} due to backup frequency '{backup_freq}' (with_weekly={with_weekly})"
 
 
 def main_all(with_weekly=False, with_weekly_on_saturday=False):
@@ -130,8 +117,9 @@ def main_all(with_weekly=False, with_weekly_on_saturday=False):
         storage_class_name = pvc['spec']['storageClassName']
         if phase == 'Bound':
             if storage_class_name in ['rook-cephfs-shared', 'rook-ceph-block']:
-                main_pvc(namespace, pvc_name, pvc, with_weekly)
-                backup_log.append(f'{namespace}/{pvc_name}: Backup completed successfully.')
+                msg = main_pvc(namespace, pvc_name, pvc, with_weekly)
+                print(msg)
+                backup_log.append(f'{namespace}/{pvc_name}: {msg}')
             else:
                 print(f'Skipping PVC {pvc_name} in namespace {namespace} with storage class {storage_class_name}. Only rook-cephfs-shared and rook-ceph-block are eligible for backup.')
                 backup_log.append(f'{namespace}/{pvc_name}: Skipped due to unsupported storage class {storage_class_name}.')
@@ -140,16 +128,11 @@ def main_all(with_weekly=False, with_weekly_on_saturday=False):
             backup_log.append(f'{namespace}/{pvc_name}: Skipped due to phase {phase}. Only Bound PVCs are eligible for backup.')
     print("Great Success! Backup log:")
     print('\n'.join(backup_log))
-    if CEPH_BACKUPS_HEARTBEAT_URL:
-        print(f'Sending heartbeat to {CEPH_BACKUPS_HEARTBEAT_URL}...')
-        subprocess.check_call(['curl', CEPH_BACKUPS_HEARTBEAT_URL])
-    else:
-        raise Exception('CEPH_BACKUPS_HEARTBEAT_URL is not set, cannot send heartbeat.')
 
 
 def main(namespace, pvc_name, with_weekly=False):
     print(f'Fetching PVC {pvc_name} in namespace {namespace}...')
     pvc = json.loads(subprocess.check_output([
         'kubectl', '-n', namespace, 'get', 'pvc', pvc_name, '-o', 'json'
     ]))
-    assert main_pvc(namespace, pvc_name, pvc, with_weekly)
+    print(main_pvc(namespace, pvc_name, pvc, with_weekly))