rancher-mongo-conf/mongo-backup-verify at master · finboxio/rancher-mongo-conf · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
#! /bin/bash

# Verifies the collections in a snapshot

set -o pipefail

SNAPSHOT_ID=$1

BACKUP_MAX_DAYS=${BACKUP_MAX_DAYS}
BACKUP_MIN_COUNT=${BACKUP_MIN_COUNT:-3}

REGION=$(wget -qO- http://169.254.169.254/latest/dynamic/instance-identity/document | jq -r .region)
AZ=$(wget -qO- http://169.254.169.254/latest/meta-data/placement/availability-zone)
ID=$(wget -qO- http://169.254.169.254/latest/meta-data/instance-id)

volume=$(aws ec2 create-volume --region $REGION --availability-zone $AZ --snapshot-id $SNAPSHOT_ID)
VOLUME_ID=$(echo "$volume" | jq -r .VolumeId)
VOLUME_STATE=$(echo "$volume" | jq -r .State)
printf "Waiting for backup volume $VOLUME_ID..."
while [[ "$VOLUME_STATE" != "available" ]]; do
  printf "."
  sleep 10
  VOLUME_STATE=$(aws ec2 describe-volumes --region $REGION --volume-ids $VOLUME_ID | jq -r '.Volumes[0] | .State')
done
echo ""

devices=$(ebs-free-devices)
VOLUME_DEVICE=$(echo "$devices" | head -n 1)
attachment=$(aws ec2 attach-volume --region $REGION --volume-id $VOLUME_ID --instance-id $ID --device $VOLUME_DEVICE)
ATTACHMENT_STATE=$(echo "$attachment" | jq -r .State)
printf "Attaching volume $VOLUME_ID..."
while [[ "$ATTACHMENT_STATE" != "attached" ]]; do
  sleep 10
  printf "."
  ATTACHMENT_STATE=$(aws ec2 describe-volumes --region $REGION --volume-ids $VOLUME_ID | jq -r '.Volumes[0] | .Attachments[] | select(.InstanceId == "'$ID'" and .Device == "'$VOLUME_DEVICE'") | .State' 2>/dev/null)
done
echo ""

mkdir -p /backup
mount -t xfs -o noatime,nodiratime,noexec,nouuid /host/$VOLUME_DEVICE /backup 2>/dev/null

mongod --port 48240 --dbpath /backup/data --auth &
pid=$!

sleep 600
cat <<EOF > /tmp/validate.js
  dbs = db.getMongo().getDBNames()
  valid = true
  for (var i in dbs) {
    db = db.getMongo().getDB(dbs[i])
    collections = db.getCollectionNames()
    for (var j in collections) {
      valid = db.getCollection(collections[j]).validate().valid
      if (!valid) {
        print(dbs[i] + "." + collections[j] + " is not valid!")
        break
      } else {
        print(dbs[i] + "." + collections[j] + " is valid")
      }
    }
    if (!valid) break
  }
  print(valid)
EOF

mongo localhost:48240/admin -u ${MONGO_USER} -p${MONGO_PASSWORD} /tmp/validate.js | tee -a /tmp/valid
STATUS=$?
INVALID=$(grep "is not valid" /tmp/valid)
if [[ "$INVALID" != "" || "$STATUS" != "0" ]]; then
  echo ":thumbsdown: MongoDB backup $BACKUP_NAME failed: $INVALID" | slack -p -a 'danger'
  aws ec2 delete-snapshot --region $REGION --snapshot-id $SNAPSHOT_ID
else
  aws ec2 create-tags --region $REGION --resources $SNAPSHOT_ID --tags "Key=Verified,Value=true"
  echo ":thumbsup: MongoDB backup $BACKUP_NAME succeeded" | slack -p -a 'good'

  if [[ "$BACKUP_MAX_DAYS" != "" ]]; then
    ## Clean up old snapshots
    BACKUP_MAX_AGE=$(date -d "-${BACKUP_MAX_DAYS}day" +%s)
    aws ec2 describe-snapshots --region $REGION --filter "Name=volume-id,Values=$VOLUME_ID" \
      | jq -r ".Snapshots | sort_by(.StartTime) | reverse \
      | .[$BACKUP_MIN_COUNT:][] \
      | select((.StartTime | sub(\"[.][0-9][0-9][0-9]Z\"; \"Z\") | fromdate) < $BACKUP_MAX_AGE) \
      | .SnapshotId" \
      | xargs aws ec2 delete-snapshot --region $REGION --snapshot-id
  fi
fi

kill -9 $pid

sleep 30

umount /backup
attachment=$(aws ec2 detach-volume --region $REGION --volume-id $VOLUME_ID)
VOLUME_STATE=$(echo "$attachment" | jq -r .State)
printf "Detaching backup volume $VOLUME_ID..."
while [[ "$VOLUME_STATE" != "available" ]]; do
  sleep 10
  printf "."
  VOLUME_STATE=$(aws ec2 describe-volumes --region $REGION --volume-ids $VOLUME_ID | jq -r '.Volumes[0] | .State')
done
echo ""

echo "Deleting backup volume $VOLUME_ID..."
aws ec2 delete-volume --region $REGION --volume-id $VOLUME_ID

if [[ "$INVALID" != "" || "$STATUS" != "0" ]]; then
  exit 1
fi