Skip to content

fix: always take a fresh backup before uploading to S3 #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
iamimmanuelraj wants to merge 3 commits into from
Closed

fix: always take a fresh backup before uploading to S3 #28

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7af9e6f
fix: regenerate site config when reusing existing backup
iamimmanuelraj Jul 31, 2026
5dc95f7
fix: always take a fresh backup instead of reusing on-disk backups
iamimmanuelraj Jul 31, 2026
c0a61c8
chore: clean up old local backups after successful S3 upload
iamimmanuelraj Jul 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 22 additions & 42 deletions offsite_backups/offsite_backups/doctype/s3_backup_settings/s3_backup_settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,7 @@
from frappe.utils.background_jobs import enqueue
from rq.timeouts import JobTimeoutException

from offsite_backups.offsite_backups.offsite_backup_utils import (
generate_files_backup,
get_latest_backup_file,
send_email,
validate_file_size,
)
from offsite_backups.offsite_backups.offsite_backup_utils import send_email


class S3BackupSettings(Document):
Expand Down Expand Up @@ -107,7 +102,6 @@ def take_backups_if(freq):
@frappe.whitelist()
def take_backups_s3(retry_count=0):
try:
validate_file_size()
backup_to_s3()
send_email(True, "Amazon S3", "S3 Backup Settings", "notify_email")
except JobTimeoutException:
Expand All @@ -132,7 +126,7 @@ def notify():

def backup_to_s3():
from frappe.utils import get_backups_path
from frappe.utils.backups import new_backup
from frappe.utils.backups import delete_temp_backups, new_backup

doc = frappe.get_single("S3 Backup Settings")
bucket = doc.bucket
Expand All @@ -146,35 +140,18 @@ def backup_to_s3():
endpoint_url=doc.endpoint_url or "https://s3.amazonaws.com",
)

if frappe.flags.create_new_backup:
backup = new_backup(
ignore_files=False,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
force=True,
)
db_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
site_config = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_conf))
if backup_files:
files_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_files))
private_files = os.path.join(
get_backups_path(), os.path.basename(backup.backup_path_private_files)
)
else:
if backup_files:
db_filename, site_config, files_filename, private_files = get_latest_backup_file(
with_files=backup_files
)

if not files_filename or not private_files:
generate_files_backup()
db_filename, site_config, files_filename, private_files = get_latest_backup_file(
with_files=backup_files
)

else:
db_filename, site_config = get_latest_backup_file()
# Always take a fresh backup so the db, site config and files come from the
# same run. Reusing on-disk backups could stitch mismatched runs together and
# return a missing (None) site config, crashing the upload.
backup = new_backup(
ignore_files=not backup_files,
backup_path_db=None,
backup_path_files=None,
backup_path_private_files=None,
force=True,
)
db_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_db))
site_config = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_conf))

folder = path + os.path.basename(db_filename)[:15] + "/"
# for adding datetime to folder name
Expand All @@ -183,11 +160,14 @@ def backup_to_s3():
upload_file_to_s3(site_config, folder, conn, bucket)

if backup_files:
if private_files:
upload_file_to_s3(private_files, folder, conn, bucket)

if files_filename:
upload_file_to_s3(files_filename, folder, conn, bucket)
files_filename = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_files))
private_files = os.path.join(get_backups_path(), os.path.basename(backup.backup_path_private_files))
upload_file_to_s3(private_files, folder, conn, bucket)
upload_file_to_s3(files_filename, folder, conn, bucket)

# Clean up old local backups (older than keep_backups_for_hours) now that
# this run is safely offsite, so fresh full backups don't accumulate on disk.
delete_temp_backups()


def upload_file_to_s3(filename, folder, conn, bucket):
Expand Down