pagerduty-trigger

#!/bin/bash
#
# pagerduty-trigger is a wrapper around the PagerDuty API to open a new
# incident from the command line. This script is designed to help integrate
# monit with PagerDuty.
#
# Copyright (c) 2012 Cold Brew Labs, Inc. See LICENSE for details.
#
# Uncomment and enter your service key here.
# PAGERDUTY_SERVICE_KEY="1234567890abcdef1234567890abcdef"

# http://mmonit.com/monit/documentation/monit.html
# Monit standard envirionment variables 
# MONIT_EVENT: The event that occurred on the service
# MONIT_DESCRIPTION : A description of the error condition
# MONIT_SERVICE : The name of the service (from monitrc) on which the event occurred.
# MONIT_DATE : The time and date (RFC 822 style) the event occurred
# MONIT_HOST : The host the event occurred on
# The following environment variables are only available for process service entries:
#  MONIT_PROCESS_PID: The process pid. This may be 0 if the process was (re)started,
#  MONIT_PROCESS_MEMORY : Process memory. This may be 0 if the process was (re)started,
#  MONIT_PROCESS_CHILDREN : Process children. This may be 0 if the process was (re)started,
#  MONIT_PROCESS_CPU_PERCENT : Process cpu%. This may be 0 if the process was (re)started,
if [ -z "$1" ]; then
  echo "Usage: $0 <event>"
  exit 1
elif [ -z "$PAGERDUTY_SERVICE_KEY" ]; then
  echo "Failed to trigger event: you must set the PAGERDUTY_SERVICE_KEY variable."
  exit 1
elif [ ! -x "/usr/local/bin/pagerduty" ]; then
  echo "Failed to trigger event: /usr/local/bin/pagerduty does not exist or is not executable."
  exit 1
fi

EVENT="$1"
HOST=$MONIT_HOST
if [ -z $HOST ] ; then
   HOST=`hostname -s`
fi 
INCIDENT_KEY=`echo "$HOST:$EVENT" | md5sum | cut -f 1 -d ' '`
TMP_FILE="/tmp/pagerduty-$INCIDENT_KEY"

if [ -f "$TMP_FILE" ]; then
  # re-trigger after 4 hrs 1 min (the extra minute is to ensure the incident auto-resolved with PagerDuty)
  if [ "$(( $(date +"%s") - $(stat -c "%Y" $TMP_FILE) ))" -lt "14460" ]; then
    echo "$TMP_FILE exists, aborting"
    exit 0
  else
    echo "$TMP_FILE exists but is older than 4 hours; re-triggering"
  fi
fi

DESCRIPTION=$MONIT_DESCRIPTION
if [ -z $DESCRIPTION ] then ;
   DESCRIPTION="$EVENT failed on $HOST"
fi
DATE=`date`

echo "$DATE: $DESCRIPTION" >> $TMP_FILE
/usr/local/bin/pagerduty -k "$PAGERDUTY_SERVICE_KEY" -i "$INCIDENT_KEY" --description="$DESCRIPTION" trigger

if [ "$?" -ne "0" ]; then
  echo "Failed to trigger incident"
  exit 1
fi

echo "Incident triggered successfully"
exit 0