cleanup.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: cleanup-backups
data:
  cleanup.py: |
    import os
    import boto3
    import logging
    from datetime import datetime, timedelta
    from botocore.client import Config

    # Configure logging
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger()

    # Create S3 client for object storage
    def create_s3_client(endpoint, access_key, secret_key, region):
        try:
            session = boto3.session.Session()
            s3_client = session.client(
                's3',
                endpoint_url=endpoint,
                aws_access_key_id=access_key,
                aws_secret_access_key=secret_key,
                config=Config(signature_version='s3v4'),
                region_name=region
            )
            return s3_client
        except Exception as e:
            logger.error(f"Failed to create S3 client: {e}")
            raise

    # Generate S3 keys
    def generate_s3_keys(bucket, s3_client, prefix='/', delimiter='/', start_after=''):
        try:
            s3_paginator = s3_client.get_paginator('list_objects_v2')
            prefix = prefix.lstrip(delimiter)
            start_after = start_after or prefix
            for page in s3_paginator.paginate(Bucket=bucket, Prefix=prefix, StartAfter=start_after):
                for content in page.get('Contents', []):
                    yield content['Key']
        except Exception as e:
            logger.error(f"Failed to list objects: {e}")
            raise

    # Cleanup backups older than the threshold date
    def cleanup_backups(s3_client, bucket, obj, threshold_date):
        try:
            folder_name = obj.split('/')[2]
            creation_date_str = folder_name.split('_')[-2]
            creation_date = datetime.strptime(creation_date_str, '%Y-%m-%d')
            if creation_date < threshold_date:
                s3_client.delete_object(Bucket=bucket, Key=obj)
                logger.info(f"Deleted folder: {obj}", )
        except (IndexError, ValueError) as e:
            logger.warning(f"Skipping invalid folder name: {obj} {e}")

    def main():
        # Retrieve environment variables
        storage_endpoint = os.getenv('ENDPOINT')
        storage_access_key = os.getenv('S3_ACCESS_KEY_ID')
        storage_secret_key = os.getenv('S3_SECRET_ACCESS_KEY')
        bucket_name = os.getenv('BUCKET_NAME')
        folder_prefix = os.getenv('FOLDER_PREFIX')
        region_name = os.getenv('REGION_NAME')
        number_days_max_to_keep = int(os.getenv('NUMBER_DAYS_MAX_TO_KEEP'))
        number_backups_to_keep = int(os.getenv('NUMBER_BACKUPS_TO_KEEP'))

        # Validate essential environment variables
        if not all([storage_endpoint, storage_access_key, storage_secret_key, bucket_name, folder_prefix, region_name, number_days_max_to_keep, number_backups_to_keep]):
            logger.error("Missing one or more required environment variables")
            return

        # Calculate the threshold date
        threshold_date = datetime.now() - timedelta(days=number_days_max_to_keep)

        # Create S3 client
        s3_client = create_s3_client(storage_endpoint, storage_access_key, storage_secret_key, region_name)

        # Retrieve and process objects
        all_objects = []
        unique_folders = set()
        objects_generator = generate_s3_keys(bucket_name, s3_client, prefix=folder_prefix)

        for obj in objects_generator:
            all_objects.append(obj)
            unique_folders.add(obj.split('/')[2])

        folder_count = len(unique_folders)
        logger.info(f'Total number of backups in {folder_prefix}: {folder_count}')

        if folder_count > number_backups_to_keep: #56 represents a week of Dremio backups that runs every 3 hours 
            for obj in all_objects:
                cleanup_backups(s3_client, bucket_name, obj, threshold_date)
        else:
            logger.info(f'Due to low number of  backups {folder_count} no backups in {folder_prefix} have been deleted' )

    if __name__ == "__main__":
        main()

---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: dremio-backup-cleanup
  namespace: {{.Release.Namespace}}
spec:
  schedule: "* 22 * * *"
  failedJobsHistoryLimit: 1
  successfulJobsHistoryLimit: 1
  concurrencyPolicy: Forbid
  jobTemplate:
    spec:
      template:
        spec:
          containers:
            - name: cleanup-job
              image: python:3.12-alpine
              env:
                - name: ENDPOINT
                  value: "{{ $.Values.distStorage.aws.backup.host }}"
                - name: S3_ACCESS_KEY_ID
                  value: "{{ $.Values.distStorage.aws.credentials.accessKey }}"
                - name: S3_SECRET_ACCESS_KEY
                  value: "{{ $.Values.distStorage.aws.credentials.secret }}"
                - name: BUCKET_NAME
                  value: "{{ $.Values.distStorage.aws.bucketName }}"
                - name: FOLDER_PREFIX
                  value: "{{ $.Values.distStorage.aws.backup.path }}"
                - name: REGION_NAME
                  value: "{{ $.Values.distStorage.aws.backup.region }}"
                - name: NUMBER_DAYS_MAX_TO_KEEP
                  value: "{{ $.Values.distStorage.aws.backup.numberDaysMaxToKeep }}"
                - name: NUMBER_BACKUPS_TO_KEEP
                  value: "{{ $.Values.distStorage.aws.backup.numberBackupsToKeep }}"
              command: ["/bin/sh", "-c"]
              args:
                [
                  "pip install -q boto3 botocore && python /tmp/python/cleanup.py",
                ]
              volumeMounts:
                - name: scripts
                  mountPath: /tmp/python
          restartPolicy: Never
          volumes:
            - name: scripts
              configMap:
                name: cleanup-backups