feat: add node_exporter and disk/memory health-check alerting

Installs Prometheus node_exporter (enabled but unreachable externally --
UFW's default-deny-incoming policy blocks port 9100 same as everything
else; open it yourself, scoped to your monitoring server's IP, to scrape
remotely) and a disk/memory threshold check scheduled via cron.

In the Ansible role the cron schedule only activates once
health_check_alert_email is set, since alerting to nowhere is pointless.
In quick-setup.sh it defaults to the local root mailbox via ALERT_EMAIL.
Either way, actually delivering mail off-box requires a separately
configured mail transport (e.g. msmtp) -- this only wires up the local
script and schedule.

Author: saidalba

ansible/group_vars/all.yml

@@ -20,3 +20,14 @@ fail2ban_enabled: true
 fail2ban_bantime: 1h
 fail2ban_findtime: 10m
 fail2ban_maxretry: 5
+
+node_exporter_enabled: true
+
+# health_check_enabled installs the disk/memory check script regardless.
+# The cron schedule is only added once health_check_alert_email is set --
+# alerts also require a mail transport (e.g. msmtp) configured on the box
+# to actually be delivered anywhere; this only wires up the local piece.
+health_check_enabled: true
+health_check_alert_email: ""
+health_check_disk_threshold: 80
+health_check_mem_free_threshold: 10

ansible/roles/ubuntu_minimal/tasks/main.yml

@@ -166,3 +166,45 @@
 - name: Set default locale
   ansible.builtin.command: "update-locale LANG={{ locale }}"
   changed_when: false
+
+# node_exporter listens on 9100 but stays unreachable from the internet by
+# default -- the earlier "UFW default incoming policy: deny" task blocks it
+# same as everything else. Open 9100 yourself, restricted to your monitoring
+# server's IP, if you want to scrape it remotely.
+- name: Install Prometheus node_exporter
+  ansible.builtin.apt:
+    name: prometheus-node-exporter
+    state: present
+  when: node_exporter_enabled
+
+- name: Enable and start node_exporter
+  ansible.builtin.systemd:
+    name: prometheus-node-exporter
+    enabled: true
+    state: started
+  when: node_exporter_enabled
+
+- name: Install cron and mail utilities for health-check alerts
+  ansible.builtin.apt:
+    name:
+      - cron
+      - mailutils
+    state: present
+  when: health_check_enabled
+
+- name: Install disk/memory health-check script
+  ansible.builtin.template:
+    src: health-check.sh.j2
+    dest: /usr/local/bin/health-check.sh
+    owner: root
+    group: root
+    mode: "0755"
+  when: health_check_enabled
+
+- name: Schedule health-check via cron
+  ansible.builtin.cron:
+    name: "disk/memory health check"
+    minute: "*/15"
+    user: root
+    job: /usr/local/bin/health-check.sh
+  when: health_check_enabled and health_check_alert_email | length > 0

ansible/roles/ubuntu_minimal/templates/health-check.sh.j2

@@ -0,0 +1,6 @@
+#!/bin/bash
+# Managed by Ansible - ubuntu_minimal role.
+df -h | awk 'NR>1 && $5+0 > {{ health_check_disk_threshold }} {print "DISK ALERT: " $0}' | mail -s "Disk Alert: $(hostname)" {{ health_check_alert_email }}
+
+FREE=$(free | awk '/Mem/{printf "%.0f", $4/$2*100}')
+[ "$FREE" -lt {{ health_check_mem_free_threshold }} ] && echo "Low memory: ${FREE}% free" | mail -s "Memory Alert: $(hostname)" {{ health_check_alert_email }}

scripts/quick-setup.sh

@@ -13,7 +13,7 @@ apt-get update && apt-get upgrade -y
 apt-get autoremove -y
 
 echo "--> Installing utilities (fail2ban, monitoring, tools)..."
-apt-get install -y ufw fail2ban unattended-upgrades htop ncdu iotop nethogs tmux git micro
+apt-get install -y ufw fail2ban unattended-upgrades htop ncdu iotop nethogs tmux git micro cron mailutils
 
 # 1. Automate security & maintenance updates
 dpkg-reconfigure -f noninteractive unattended-upgrades
@@ -111,4 +111,23 @@ MaxRetentionSec=30day
 EOF
 systemctl restart systemd-journald
 
+# 7. Prometheus node_exporter (optional monitoring). Stays unreachable from
+# the internet by default -- UFW only allows SSH, same as everything else.
+# Open 9100 yourself, restricted to your monitoring server's IP, to scrape it.
+apt-get install -y prometheus-node-exporter
+systemctl enable --now prometheus-node-exporter
+
+# 8. Basic disk/memory health-check, mailed to ALERT_EMAIL (defaults to the
+# local root mailbox, readable on-box via `mail`). For alerts to leave the
+# server you need a real mail transport (e.g. msmtp) configured separately --
+# this only wires up the local script + schedule.
+cat << EOF > /usr/local/bin/health-check.sh
+#!/bin/bash
+df -h | awk 'NR>1 && \$5+0 > 80 {print "DISK ALERT: " \$0}' | mail -s "Disk Alert: \$(hostname)" "${ALERT_EMAIL:-root}"
+FREE=\$(free | awk '/Mem/{printf "%.0f", \$4/\$2*100}')
+[ "\$FREE" -lt 10 ] && echo "Low memory: \${FREE}% free" | mail -s "Memory Alert: \$(hostname)" "${ALERT_EMAIL:-root}"
+EOF
+chmod +x /usr/local/bin/health-check.sh
+echo "*/15 * * * * root /usr/local/bin/health-check.sh" > /etc/cron.d/health-check
+
 echo "--> Setup complete. Keep this terminal open and test SSH in a NEW terminal before closing it."