Merge pull request #125 from kartoza/monitor

capture pg_dump sucess or failure state and add monitor option

Author: NyakudyaA

README.md

@@ -158,6 +158,8 @@ For a typical usage of this look at the [docker-compose-s3.yml](https://github.c
 The image supports mounting the following configs:
 * `s3cfg` when backing to `S3` backend
 * backup-cron for any custom configuration you need to specify in the file.
+* `backup_monitoring.sh` - For any custom monitoring state on database dump completion or failure 
+i.e Add webhook/callback support for backup completion notifications 
 
 An environment variable `${EXTRA_CONFIG_DIR}` controls the location of the folder.
 
@@ -170,6 +172,21 @@ run the following:
 ```
 Where `s3cfg` is located in `/data`
 
+If you need to run i.e webhook you can implement your own custom hook logic
+```
+-e ${EXTRA_CONFIG_DIR}=/settings
+-v /data:/settings
+```
+
+Where `backup_monitoring.sh` is located in `/data` or in
+kubernetes you can mount this file as a config.
+
+or you can just run the env variable
+```bash
+MONITORING_ENDPOINT_COMMAND="""curl -D - -X POST -G 'https://appsignal-endpoint.net/check_ins/heartbeats' -d 'api_key=YOUR-APP-LEVEL-API-KEY' -d 'identifier=YOUR-CHECK-IN-IDENTIFIER'"""
+```
+to monitor success or failure of the backup.
+
 ## Restoring
 
 The image provides a simple restore script. There are two ways to restore files based on the

scripts/backups.sh

@@ -23,6 +23,22 @@ function s3_config() {
 
 }
 
+function backup_monitoring() {
+
+  if [[ -z "${MONITORING_ENDPOINT_COMMAND}" ]]; then
+    if [[ -f "${EXTRA_CONFIG_DIR}/backup_monitoring.sh" ]]; then
+      cp -f "${EXTRA_CONFIG_DIR}/backup_monitoring.sh" /backup-scripts/backup_monitoring.sh
+      chmod 0755 /backup-scripts/backup_monitoring.sh
+      /bin/bash /backup-scripts/backup_monitoring.sh
+    else
+      echo "No monitoring command or script found."
+    fi
+  else
+    eval "${MONITORING_ENDPOINT_COMMAND}"
+  fi
+}
+
+
 # Cleanup S3 bucket
 function clean_s3bucket() {
   S3_BUCKET="$1"
@@ -81,12 +97,14 @@ function dump_tables() {
             pg_dump ${PG_CONN_PARAMETERS} ${DUMP_ARGS} -d "${DATABASE}" -t "${DB_TABLE}" | openssl enc -aes-256-cbc -pass pass:${DB_DUMP_ENCRYPTION_PASS_PHRASE} -pbkdf2 -iter 10000 -md sha256 -out "${FILENAME}"
             if [[ $? -ne 0 ]];then
              echo -e "Backup of \e[0;32m ${DB_TABLE} \033[0m from DATABASE \e[0;32m ${DATABASE} \033[0m failed" >> ${CONSOLE_LOGGING_OUTPUT}
+             backup_monitoring
             fi
         else
             # Plain backup
             pg_dump ${PG_CONN_PARAMETERS} ${DUMP_ARGS} -d "${DATABASE}" -t "${DB_TABLE}" > "${FILENAME}"
             if [[ $? -ne 0 ]];then
              echo -e "Backup of \e[0;32m ${DB_TABLE} \033[0m from DATABASE \e[0;32m ${DATABASE} \033[0m failed" >> ${CONSOLE_LOGGING_OUTPUT}
+             backup_monitoring
             fi
         fi
 
@@ -96,54 +114,75 @@ function dump_tables() {
     done
 }
 
-
 function backup_db() {
   EXTRA_PARAMS=''
   if [ -n "$1" ]; then
     EXTRA_PARAMS=$1
   fi
+
   for DB in ${DBLIST}; do
     if [ -z "${ARCHIVE_FILENAME:-}" ]; then
-      export FILENAME=${MYBACKUPDIR}/${DUMPPREFIX}_${DB}.${MYDATE}.dmp
+      export FILENAME="${MYBACKUPDIR}/${DUMPPREFIX}_${DB}.${MYDATE}.dmp"
     else
-      export FILENAME=${MYBASEDIR}/"${ARCHIVE_FILENAME}.${DB}.dmp"
+      export FILENAME="${MYBASEDIR}/${ARCHIVE_FILENAME}.${DB}.dmp"
     fi
 
-    if [[ "${DB_TABLES}" =~ [Ff][Aa][Ll][Ss][Ee] ]]; then
-      export PGPASSWORD=${POSTGRES_PASS}
-      echo -e "Backup  of \e[1;31m ${DB} \033[0m starting at \e[1;31m $(date) \033[0m" >> ${CONSOLE_LOGGING_OUTPUT}
-      if [[ "${DB_DUMP_ENCRYPTION}" =~ [Tt][Rr][Uu][Ee] ]]; then
-        if ! ( set -o pipefail && pg_dump ${PG_CONN_PARAMETERS} ${DUMP_ARGS} -d "${DB}" | openssl enc -aes-256-cbc -pass pass:"${DB_DUMP_ENCRYPTION_PASS_PHRASE}" -pbkdf2 -iter 10000 -md sha256 -out "${FILENAME}" ); then
-          echo -e "\e[1;31mFailed to back up ${DB} at $(date)\033[0m" >> ${CONSOLE_LOGGING_OUTPUT}
-          rm ${FILENAME}
-          continue
-        fi
+    export PGPASSWORD="${POSTGRES_PASS}"
+
+    echo "[$(date)] Starting backup of ${DB}" >> "${CONSOLE_LOGGING_OUTPUT}"
+
+
+    if [[ "${DB_DUMP_ENCRYPTION}" =~ ^([Tt][Rr][Uu][Ee])$ ]]; then
+
+      set -o pipefail
+      if pg_dump ${PG_CONN_PARAMETERS} ${DUMP_ARGS} -d "${DB}" \
+        | openssl enc -aes-256-cbc \
+            -pass pass:"${DB_DUMP_ENCRYPTION_PASS_PHRASE}" \
+            -pbkdf2 -iter 10000 -md sha256 \
+            -out "${FILENAME}"
+      then
+        echo "[$(date)] Backup SUCCESS (encrypted) for ${DB}, saved to ${FILENAME}" \
+          >> "${CONSOLE_LOGGING_OUTPUT}"
+          backup_monitoring
       else
-        if ! pg_dump ${PG_CONN_PARAMETERS} ${DUMP_ARGS} -d ${DB} > ${FILENAME}; then
-          echo -e "\e[1;31mFailed to back up ${DB} at $(date)\033[0m" >> ${CONSOLE_LOGGING_OUTPUT}
-          rm ${FILENAME}
-          continue
-        fi
-      fi
-      echo -e "Backup of \e[1;33m ${DB} \033[0m completed at \e[1;33m $(date) \033[0m and dump located at \e[1;33m ${FILENAME} \033[0m" >> ${CONSOLE_LOGGING_OUTPUT}
-      if [[ ${STORAGE_BACKEND} == "S3" ]]; then
-        gzip ${FILENAME}
-        echo -e "Pushing database backup \e[1;31m ${FILENAME} \033[0m to \e[1;31m s3://${BUCKET}/ \033[0m" >> ${CONSOLE_LOGGING_OUTPUT}
-        ${EXTRA_PARAMS}
-        rm ${MYBACKUPDIR}/*.dmp.gz
+        echo "[$(date)] ERROR: Backup FAILED (encrypted) for ${DB}" \
+          >> "${CONSOLE_LOGGING_OUTPUT}"
+          backup_monitoring
+        rm -f "${FILENAME}"
+        continue
       fi
-    else
+      set +o pipefail
+
 
-      dump_tables ${DB}
-      if [[ ${STORAGE_BACKEND} == "S3" ]]; then
-        ${EXTRA_PARAMS}
-        rm ${MYBACKUPDIR}/*
+    else
+      if pg_dump ${PG_CONN_PARAMETERS} ${DUMP_ARGS} -d "${DB}" > "${FILENAME}"; then
+        echo "[$(date)] Backup SUCCESS for ${DB}, saved to ${FILENAME}" \
+          >> "${CONSOLE_LOGGING_OUTPUT}"
+          backup_monitoring
+      else
+        echo "[$(date)] ERROR: Backup FAILED for ${DB}" \
+          >> "${CONSOLE_LOGGING_OUTPUT}"
+          backup_monitoring
+        rm -f "${FILENAME}"
+        continue
       fi
     fi
-  done
 
+
+    if [[ "${STORAGE_BACKEND}" == "S3" ]]; then
+      gzip "${FILENAME}"
+
+      echo "[$(date)] Uploading ${FILENAME}.gz to s3://${BUCKET}/" \
+        >> "${CONSOLE_LOGGING_OUTPUT}"
+
+      ${EXTRA_PARAMS}
+
+      rm -f ${MYBACKUPDIR}/*.dmp.gz
+    fi
+  done
 }
 
+
 function remove_files() {
   TIME_MINUTES=$((REMOVE_BEFORE * 24 * 60))
   MIN_SAVED_FILE=${MIN_SAVED_FILE:-0}