- added tmp directory & docker-compose .env file

Author: maZahaca

.gitignore

@@ -2,3 +2,4 @@ publisher/sanitized
 database/test/actual
 backups/backups
 sanitizer/files
+.env

backups/.env.example

@@ -0,0 +1,39 @@
+# DB to dump
+DB_NAME: test_db
+
+# DBs to dump
+DB_CONFIG_1: db-1:5432:*:postgres:password
+DB_CONFIG_2: db-2:5432:*:postgres:password
+
+# This dir will be created if it doesn't exist. This must be writable by the user the script is
+# running as.
+BACKUP_DIR: /backups/
+BACKUP_TMP_DIR: /tmp/
+
+# SETTINGS FOR ROTATED BACKUPS
+
+# Which day to take the weekly backup from (1-7 = Monday-Sunday)
+DAY_OF_WEEK_TO_KEEP: 5
+
+# Number of hours to keep daily backups
+DAYS_TO_KEEP_HOURLY: 1
+
+# Number of days to keep daily backups
+DAYS_TO_KEEP_DAILY: 7
+
+# How many weeks to keep weekly backups
+WEEKS_TO_KEEP_WEEKLY: 4
+
+# How many months to keep monthly backups
+MONTHS_TO_KEEP_MONTHLY: 6
+
+# Avoid dumping of existent roles to prevent password rewriting
+IGNORE_DUMP_ROLES: \(admin\|app\|postgres\)
+
+# Copy roles in a separate file
+PERFORM_BACKUP_ROLES: "1"
+
+#S3_BUCKET=***
+#S3_KEY: ***
+#S3_SECRET_KEY: ***
+#S3_BUCKET_URL: https://fra1.digitaloceanspaces.com

backups/docker-compose.yml

@@ -11,44 +11,8 @@ version: '3.4'
 services:
   backups:
     build: .
-    environment:
-      # DB to dump
-      DB_NAME: test_db
-
-      # DBs to dump
-      DB_CONFIG_1: db-1:5432:*:postgres:password
-      DB_CONFIG_2: db-2:5432:*:postgres:password
-
-      # This dir will be created if it doesn't exist. This must be writable by the user the script is
-      # running as.
-      BACKUP_DIR: /backups/
-
-      # SETTINGS FOR ROTATED BACKUPS
-
-      # Which day to take the weekly backup from (1-7 = Monday-Sunday)
-      DAY_OF_WEEK_TO_KEEP: 5
-
-      # Number of hours to keep daily backups
-      DAYS_TO_KEEP_HOURLY: 1
-
-      # Number of days to keep daily backups
-      DAYS_TO_KEEP_DAILY: 7
-
-      # How many weeks to keep weekly backups
-      WEEKS_TO_KEEP_WEEKLY: 4
-
-      # How many months to keep monthly backups
-      MONTHS_TO_KEEP_MONTHLY: 6
-
-      # Avoid dumping of existent roles to prevent password rewriting
-      IGNORE_DUMP_ROLES: \(admin\|app\|postgres\)
-
-      PERFORM_BACKUP_ROLES: "1"
-
-      S3_KEY: '*'
-      S3_SECRET_KEY: '*'
-      S3_BUCKET: plyo-db-backups-prod
-      S3_BUCKET_URL: https://fra1.digitaloceanspaces.com
+    env_file:
+      - .env
     volumes:
       - ./backups/:/backups/
       - ./src:/usr/src/app/

backups/src/pg_backup_rotated.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 # see https://wiki.postgresql.org/wiki/Automated_Backup_on_Linux
+BACKUP_TMP_DIR=${BACKUP_TMP_DIR:-/tmp}
 
 log () {
   echo "[pg_backup_rotated.sh]> $@"
@@ -17,7 +18,9 @@ function perform_backups()
 
     backup_date=`date +%Y-%m-%d`
     backup_file_path="${BACKUP_DIR}${backup_date}${suffix}".backup
+    backup_progress_file_path="${BACKUP_TMP_DIR}${backup_date}${suffix}".backup
     backup_roles_file_path="${backup_file_path}_roles.out"
+    backup_progress_roles_file_path="${backup_progress_file_path}_roles.out"
 
     if [[ -e ${backup_file_path} ]]; then
         log "${backup_file_path} already exists, skipping dump"
@@ -26,19 +29,19 @@ function perform_backups()
 
     log "Dumping custom backup for ${DB_NAME} database to ${backup_file_path}"
 
-    if ! pg_dump -Fc -h "$db_host" -p "$db_port" -U "$db_user" ${DB_NAME} -f ${backup_file_path}.in_progress; then
+    if ! pg_dump -Fc -h "$db_host" -p "$db_port" -U "$db_user" ${DB_NAME} -f ${backup_progress_file_path}.in_progress; then
         log "[ERROR] Failed to produce custom backup database ${DB_NAME}"
         exit 1
     else
         # finalize database backup
-        mv ${backup_file_path}.in_progress ${backup_file_path}
+        mv ${backup_progress_file_path}.in_progress ${backup_file_path}
 
         # perform backup of database roles
         if [[ "${PERFORM_BACKUP_ROLES}" == "1" ]]; then
           log "Dumping roles backup for ${DB_NAME} database to ${backup_roles_file_path}"
-          pg_dumpall -r -h "$db_host" -p "$db_port" -U postgres -f ${backup_roles_file_path}.in_progress
-          cat ${backup_roles_file_path}.in_progress | grep -v ${IGNORE_DUMP_ROLES} > "${backup_roles_file_path}"
-          rm -f ${backup_roles_file_path}.in_progress
+          pg_dumpall -r -h "$db_host" -p "$db_port" -U postgres -f ${backup_progress_roles_file_path}.in_progress
+          cat ${backup_progress_roles_file_path}.in_progress | grep -v ${IGNORE_DUMP_ROLES} > "${backup_roles_file_path}"
+          rm -f ${backup_progress_roles_file_path}.in_progress
         fi
 
         # perform copy backup to S3-compatible storage

sanitizer/.env.example

@@ -0,0 +1,20 @@
+ADMIN_ROLE_NAME: plyo
+EXTENSIONS: pgcrypto,ltree
+PRIVATE_SCHEMA_NAME: public
+SCHEMA_NAME: public
+POSTGRES_PASSWORD: password
+#ROLES_FILE_PATH: /roles/roles.out
+
+DB_NAME: database
+DB_HOST: postgres
+
+#S3_KEY: *
+#S3_SECRET_KEY: *
+#
+#S3_BUCKET: *
+#S3_BUCKET_URL: https://fra1.digitaloceanspaces.com
+#S3_BACKUP_MNT_POINT: /backups
+#
+#S3_SANITIZED_BUCKET: *
+#S3_SANITIZED_BUCKET_URL: https://fra1.digitaloceanspaces.com
+#S3_SANITIZED_BACKUP_MNT_POINT: /sanitized

sanitizer/docker-compose.yml

@@ -4,27 +4,8 @@ services:
     build:
       context: .
     working_dir: /usr/src/app
-    environment:
-      - ADMIN_ROLE_NAME=plyo
-      - EXTENSIONS=pgcrypto,ltree
-      - PRIVATE_SCHEMA_NAME=public
-      - SCHEMA_NAME=public
-      - POSTGRES_PASSWORD=password
-#      - ROLES_FILE_PATH=/roles/roles.out
-
-      - DB_NAME=plyo
-      - DB_HOST=plyo-db
-
-      - S3_KEY=*
-      - S3_SECRET_KEY=*
-
-      - S3_BUCKET=plyo-db-backups-prod
-      - S3_BUCKET_URL=https://fra1.digitaloceanspaces.com
-      - S3_BACKUP_MNT_POINT=/backups
-
-      - S3_SANITIZED_BUCKET=plyo-db-sanitized-prod
-      - S3_SANITIZED_BUCKET_URL=https://fra1.digitaloceanspaces.com
-      - S3_SANITIZED_BACKUP_MNT_POINT=/sanitized
+    env_file:
+      - .env
     volumes:
       - ./src/start.sh:/usr/src/app/start.sh
       - ./src/mount-s3.sh:/usr/src/app/mount-s3.sh