Add email-on-fail to cron job (#1241)

* Add email-on-fail to cron job

* Update changelog

Author: CBroz1

CHANGELOG.md

@@ -6,7 +6,7 @@
 
 - Ensure merge tables are declared during file insertion #1205
 - Update URL for DANDI Docs #1210
-- Improve cron job documentation and script #1226
+- Improve cron job documentation and script #1226, #1241
 - Only add merge parts to `source_class_dict` if present in codebase #1237
 
 ### Pipelines
@@ -18,7 +18,8 @@
     - Allow population of missing `PositionIntervalMap` entries during population
         of `DLCPoseEstimation` #1208
 - Spikesorting
-    - Fix compatibility bug between v1 pipeline and `SortedSpikesGroup` unit filtering #1238
+    - Fix compatibility bug between v1 pipeline and `SortedSpikesGroup` unit
+        filtering #1238
 
 ## [0.5.4] (December 20, 2024)
 

maintenance_scripts/README.md

@@ -26,11 +26,40 @@ regularly as cron jobs.
 1. Clone the repository to the desired location.
 2. Set up a config file by copying `dj_local_conf_example.json` to
     `dj_local_conf.json` and filling in the necessary information.
-3. Set up a cron job to run `run_jobs.sh` at the desired interval by running
+3. Copy the `example.env` file to `.env` in the `maintenance_scripts` directory
+    and fill in the necessary information, including...
+    - `SPYGLASS_CONDA_ENV`: the name of the conda environment with Spyglass and
+        DataJoint installed.
+    - `SPYGLASS_REPO_PATH`: the path to the Spyglass repository.
+    - `SPYGLASS_LOG`: the path to the log file.
+    - Optional email settings. If not set, email notifications will not be sent.
+        - `SPYGLASS_EMAIL_SRC`: The email address from which to send notifications.
+        - `SPYGLASS_EMAIL_PASS`: the password for the email address.
+        - `SPYGLASS_EMAIL_DEST`: the email address to which to send notifications.
+4. Set up a cron job to run `run_jobs.sh` at the desired interval by running
     `crontab -e` and adding the script.
 
+Note that the log file will automatically be truncated to `SPYGLASS_MAX_LOG`
+lines on each run. 1000 lines should be sufficient.
+
+### Example Cron Job
+
 In the following example, the script is set to run every Monday at 4:00 AM.
 
 ```text
 0 4 * * 1 /path/to/run_jobs.sh
 ```
+
+### Email Service
+
+The script uses `curl` to send email notifications on failure. While this can
+work with
+[many email services](https://everything.curl.dev/usingcurl/smtp.html), Gmail is
+a common choice. To use Gmail, you will need to ...
+
+1. Turn on [2-step verification](https://myaccount.google.com/security-checkup)
+2. Turn on [less secure apps](https://myaccount.google.com/lesssecureapps)
+3. Create an [app password](https://myaccount.google.com/apppasswords)
+
+`curl` will not work with your master Gmail password, so you will need to use
+the app password instead.

maintenance_scripts/example.env

@@ -0,0 +1,8 @@
+# NOTE: Use quotes for strings with spaces
+export SPYGLASS_CONDA_ENV=spyglass # name of conda environment
+export SPYGLASS_REPO_PATH=/home/franklab/spyglass # path to spyglass repo
+export SPYGLASS_LOG=/home/franklab/spyglass.log # log file
+export SPYGLASS_EMAIL_SRC=[email protected] # optional, email sender
+export SPYGLASS_EMAIL_PASS="example password" # optional, email password
+export SPYGLASS_EMAIL_DEST=[email protected] # optional, email recipient
+export SPYGLASS_MAX_LOG=1000 # number of lines to keep in log

maintenance_scripts/run_jobs.sh

@@ -1,14 +1,52 @@
 #!/bin/bash
+# AUTHOR: Chris Brozdowski
+# DATE: 2025-02-20
+#
+# 1. Go to SPYGLASS_REPO_PATH and pull the latest changes from the master branch
+# 2. Test the SPYGLASS_CONDA_ENV conda environment
+# 3. Test the connection to the database
+# 4. Run the cleanup script
+#
+# This script is intended to be run as a cron job, weekly or more frequently.
+# It will store a log of its output in SPYGLASS_LOG.
+# If any of the operations fail, an email will be sent to SPYGLASS_EMAIL_DEST
 
-# SETUP:
-# 1. Create a conda environment with datajoint installed.
-# 2. Edit the variables below to match your setup.
-# 3. Set up the cron job (See README.md)
-# Note that the log file will be truncated to the last 1000 lines.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/.env" # load environment variables from this directory
 
-SPYGLASS_CONDA_ENV=spyglass
-SPYGLASS_REPO_PATH=/home/franklab/spyglass
-SPYGLASS_LOG=/home/franklab/spyglass/spyglass.log
+if [[ -z "${SPYGLASS_CONDA_ENV}" \
+  || -z "${SPYGLASS_REPO_PATH}" \
+  || -z "${SPYGLASS_LOG}" ]]; then
+  echo "Error: SPYGLASS_CONDA_ENV, SPYGLASS_REPO_PATH,
+        and SPYGLASS_LOG must be set in .env"
+  exit 1
+fi
+
+EMAIL_TEMPLATE=$(cat <<-EOF
+From: "Spyglass" <$SPYGLASS_EMAIL_SRC>
+To: $SPYGLASS_EMAIL_DEST
+Subject: cron fail - $(date "+%Y-%m-%d")
+
+%s
+EOF
+)
+
+on_fail() { # $1: error message. Echo message and send as email
+    echo "Error: $1"
+    if [ -z "$SPYGLASS_EMAIL_SRC" ]; then
+      return 1 # No email source, so don't send an email
+    fi
+    local error_msg="$1"
+    local content
+    content=$(printf "$EMAIL_TEMPLATE" "$error_msg")
+
+    curl -o /dev/null --ssl-reqd \
+      --url "smtps://smtp.gmail.com:465" \
+      --user "${SPYGLASS_EMAIL_SRC}:${SPYGLASS_EMAIL_PASS}" \
+      --mail-from "$SPYGLASS_EMAIL_SRC" \
+      --mail-rcpt "$SPYGLASS_EMAIL_DEST" \
+      -T <(echo "$content")
+}
 
 exec >> $SPYGLASS_LOG 2>&1
 
@@ -17,30 +55,33 @@ echo "SPYGLASS CRON JOB START: $(date +"%Y-%m-%d %H:%M:%S")"
 
 # Run from the root of the spyglass repository
 cd $SPYGLASS_REPO_PATH || \
-    { echo "Error: Could not change to the spyglass directory"; exit 1; }
+    { on_fail "Could not find repo path: $SPYGLASS_REPO_PATH"; exit 1; }
+
 
 # Update the spyglass repository
-git pull https://github.com/LorenFrankLab/spyglass.git master > /dev/null || \
-    { echo "Error: $PWD Could not update the spyglass repository"; exit 1; }
+git pull --quiet \
+  https://github.com/LorenFrankLab/spyglass.git master > /dev/null || \
+    { on_fail "Could not update the spyglass repo $PWD"; exit 1; }
 
 # Test conda environment
 if ! conda env list | grep -q $SPYGLASS_CONDA_ENV; then
-    echo "Error: Conda environment $SPYGLASS_CONDA_ENV not found"
-    exit 1
+  on_fail "Conda environment $SPYGLASS_CONDA_ENV not found"
+  exit 1
 fi
 
 # convenience function to run a command in the spyglass conda environment
 conda_run() { conda run --name $SPYGLASS_CONDA_ENV "$@"; }
 
 # Test connection to the database
-conda_run python -c "import datajoint as dj; dj.conn()" > /dev/null || \
-    { echo "Error: Could not connect to the database"; exit 1; }
+CONN_TEST="import datajoint as dj; dj.logger.setLevel('ERROR'); dj.conn()"
+conda_run python -c "$CONN_TEST" > /dev/null || \
+  { on_fail "Could not connect to the database"; exit 1; }
 
 # Run cleanup script
 conda_run python maintenance_scripts/cleanup.py
 
 echo "SPYGLASS CRON JOB END"
 
 # truncate long log file
-tail -n 1000 "$SPYGLASS_LOG" > "${SPYGLASS_LOG}.tmp" && \
-  mv "${SPYGLASS_LOG}.tmp" "$SPYGLASS_LOG"
+tail -n ${SPYGLASS_MAX_LOG:-1000} "$SPYGLASS_LOG" > "${SPYGLASS_LOG}.tmp" \
+  && mv "${SPYGLASS_LOG}.tmp" "$SPYGLASS_LOG"