diff --git a/data/Dockerfiles/watchdog/watchdog.sh b/data/Dockerfiles/watchdog/watchdog.sh index 020f3f8388..48789bfa07 100755 --- a/data/Dockerfiles/watchdog/watchdog.sh +++ b/data/Dockerfiles/watchdog/watchdog.sh @@ -72,6 +72,11 @@ get_ipv6(){ echo ${IPV6} } +ipv6_enabled() { + local enabled="${ENABLE_IPV6:-true}" + [[ "${enabled,,}" =~ ^(true|yes|y|1)$ ]] +} + array_diff() { # https://stackoverflow.com/questions/2312762, Alex Offshore eval local ARR1=\(\"\${$2[@]}\"\) @@ -227,8 +232,164 @@ get_container_ip() { [[ ${LOOP_C} -gt 5 ]] && echo 240.0.0.0 || echo ${CONTAINER_IP} } +get_container_ipv6() { + local service="$1" + local attempt + local container_id + local container_ipv6 + + for attempt in {1..5}; do + container_id="$( + curl --silent --insecure \ + "https://dockerapi.${COMPOSE_PROJECT_NAME}_mailcow-network/containers/json" \ + | jq -r \ + --arg service "${service}" \ + --arg project "${COMPOSE_PROJECT_NAME,,}" \ + '.[] | + select(.Config.Labels["com.docker.compose.service"] == $service) | + select((.Config.Labels["com.docker.compose.project"] | ascii_downcase) == $project) | + .Id' \ + | shuf \ + | head -n 1 + )" + + if [[ -n "${container_id}" ]]; then + container_ipv6="$( + curl --silent --insecure \ + "https://dockerapi.${COMPOSE_PROJECT_NAME}_mailcow-network/containers/${container_id}/json" \ + | jq -r \ + '.NetworkSettings.Networks + | to_entries + | map(select(.key | endswith("_mailcow-network"))) + | map(.value.GlobalIPv6Address // empty) + | map(select(length > 0)) + | .[0] // empty' + )" + if [[ "${container_ipv6}" == *:* ]]; then + printf '%s\n' "${container_ipv6}" + return 0 + fi + fi + + [[ "${attempt}" -lt 5 ]] && sleep 0.5 + done + + return 1 +} + +dnssec_validation_mode() { + local config="${1:-/etc/mailcow/unbound.conf}" + + [[ -r "${config}" ]] || return 1 + awk -F: ' + /^[[:space:]]*#/ { next } + $1 ~ /^[[:space:]]*val-permissive-mode[[:space:]]*$/ { value=$2 } + END { + sub(/[[:space:]]*#.*/, "", value) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + if (tolower(value) == "yes") { + print "permissive" + } else { + print "strict" + } + } + ' "${config}" +} + +dnssec_validation_healthy() { + local family="$1" + local server="$2" + local transport="$3" + local mode="$4" + local signed_response + local bogus_response + + signed_response="$( + dig "-${family}" com. SOA +dnssec +timeout=2 +tries=1 @"${server}" 2>&1 + )" + if ! grep -Eq 'status: NOERROR' <<< "${signed_response}" \ + || ! grep -Eq 'flags:.+ ad[ ;]' <<< "${signed_response}"; then + echo "DNSSEC authenticated-answer failure over ${transport}" \ + 2>> /tmp/unbound-mailcow 1>&2 + return 1 + fi + + bogus_response="$( + dig "-${family}" dnssec-failed.org A +dnssec +timeout=2 +tries=1 @"${server}" 2>&1 + )" + + if grep -Eq 'flags:.+ ad[ ;]' <<< "${bogus_response}"; then + echo "DNSSEC bogus chain was marked authenticated over ${transport}" \ + 2>> /tmp/unbound-mailcow 1>&2 + return 1 + fi + + if [[ "${mode}" == "strict" ]] \ + && ! grep -Eq 'status: SERVFAIL' <<< "${bogus_response}"; then + echo "DNSSEC bogus-chain rejection failure over ${transport}" \ + 2>> /tmp/unbound-mailcow 1>&2 + return 1 + fi + + if [[ "${mode}" == "permissive" ]] \ + && ! grep -Eq 'status: (NOERROR|SERVFAIL)' <<< "${bogus_response}"; then + echo "DNSSEC permissive-mode response failure over ${transport}" \ + 2>> /tmp/unbound-mailcow 1>&2 + return 1 + fi + + echo "DNSSEC ${mode} validation succeeded over ${transport}" \ + 2>> /tmp/unbound-mailcow 1>&2 +} + +run_dns_check() { + /usr/lib/mailcow/check_dns.sh "$@" +} + +unbound_check_round() { + local failed=0 + local host_ip + local host_ipv6 + local mode + + host_ip="$(get_container_ip unbound-mailcow)" + if ! mode="$(dnssec_validation_mode)"; then + mode="strict" + echo "Cannot read the mounted Unbound configuration; using strict DNSSEC checks" \ + 2>> /tmp/unbound-mailcow 1>&2 + fi + + if ! run_dns_check -s "${host_ip}" -H stackoverflow.com \ + 2>> /tmp/unbound-mailcow 1>&2; then + failed=1 + fi + if ! dnssec_validation_healthy 4 "${host_ip}" IPv4 "${mode}"; then + failed=1 + fi + + if ipv6_enabled; then + if ! host_ipv6="$(get_container_ipv6 unbound-mailcow)"; then + echo "Cannot discover Unbound's IPv6 address on mailcow-network" \ + 2>> /tmp/unbound-mailcow 1>&2 + notify_error "ipv6-config" \ + "Watchdog cannot discover Unbound's IPv6 address on mailcow-network." \ + 3600 + else + if ! run_dns_check -s "${host_ipv6}" -H stackoverflow.com \ + 2>> /tmp/unbound-mailcow 1>&2; then + failed=1 + fi + if ! dnssec_validation_healthy 6 "${host_ipv6}" IPv6 "${mode}"; then + failed=1 + fi + fi + fi + + return "${failed}" +} + # One-time check -if grep -qi "$(echo ${IPV6_NETWORK} | cut -d: -f1-3)" <<< "$(ip a s)"; then +if ipv6_enabled; then if [[ -z "$(get_ipv6)" ]]; then notify_error "ipv6-config" "enable_ipv6 is true in docker-compose.yml, but an IPv6 link could not be established. Please verify your IPv6 connection." fi @@ -300,16 +461,11 @@ unbound_checks() { trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 while [ ${err_count} -lt ${THRESHOLD} ]; do touch /tmp/unbound-mailcow; echo "$(tail -50 /tmp/unbound-mailcow)" > /tmp/unbound-mailcow - host_ip=$(get_container_ip unbound-mailcow) err_c_cur=${err_count} - /usr/lib/mailcow/check_dns.sh -s ${host_ip} -H stackoverflow.com 2>> /tmp/unbound-mailcow 1>&2; err_count=$(( ${err_count} + $? )) - DNSSEC=$(dig com +dnssec | egrep 'flags:.+ad') - if [[ -z ${DNSSEC} ]]; then - echo "DNSSEC failure" 2>> /tmp/unbound-mailcow 1>&2 - err_count=$(( ${err_count} + 1)) - else - echo "DNSSEC check succeeded" 2>> /tmp/unbound-mailcow 1>&2 + if ! unbound_check_round; then + err_count=$(( ${err_count} + 1 )) fi + [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) progress "Unbound" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} @@ -1167,4 +1323,3 @@ while true; do kill -USR1 ${BACKGROUND_TASKS[*]} fi done - diff --git a/docker-compose.yml b/docker-compose.yml index 30a7ba96b6..bccba7713e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -526,7 +526,7 @@ services: - /lib/modules:/lib/modules:ro watchdog-mailcow: - image: ghcr.io/mailcow/watchdog:2.11 + image: ghcr.io/mailcow/watchdog:2.11a dns: - ${IPV4_NETWORK:-172.22.1}.254 tmpfs: @@ -536,6 +536,7 @@ services: - mysql-socket-vol-1:/var/run/mysqld/:z - postfix-vol-1:/var/spool/postfix - ./data/assets/ssl:/etc/ssl/mail/:ro,z + - ./data/conf/unbound/unbound.conf:/etc/mailcow/unbound.conf:ro,z restart: always depends_on: - postfix-mailcow @@ -544,6 +545,7 @@ services: - acme-mailcow - redis-mailcow environment: + - ENABLE_IPV6=${ENABLE_IPV6:-true} - IPV6_NETWORK=${IPV6_NETWORK:-fd4d:6169:6c63:6f77::/64} - LOG_LINES=${LOG_LINES:-9999} - TZ=${TZ} diff --git a/helper-scripts/tests/fixtures/unbound-permissive.conf b/helper-scripts/tests/fixtures/unbound-permissive.conf new file mode 100644 index 0000000000..87af39e16f --- /dev/null +++ b/helper-scripts/tests/fixtures/unbound-permissive.conf @@ -0,0 +1,2 @@ +server: + val-permissive-mode: yes diff --git a/helper-scripts/tests/fixtures/unbound-strict.conf b/helper-scripts/tests/fixtures/unbound-strict.conf new file mode 100644 index 0000000000..7262c9533a --- /dev/null +++ b/helper-scripts/tests/fixtures/unbound-strict.conf @@ -0,0 +1,2 @@ +server: + val-permissive-mode: no diff --git a/helper-scripts/tests/test_watchdog_dnssec.sh b/helper-scripts/tests/test_watchdog_dnssec.sh new file mode 100755 index 0000000000..ac1df6c30a --- /dev/null +++ b/helper-scripts/tests/test_watchdog_dnssec.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WATCHDOG="${ROOT}/data/Dockerfiles/watchdog/watchdog.sh" +COMPOSE="${ROOT}/docker-compose.yml" +STRICT_CONFIG="${ROOT}/helper-scripts/tests/fixtures/unbound-strict.conf" +PERMISSIVE_CONFIG="${ROOT}/helper-scripts/tests/fixtures/unbound-permissive.conf" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +require_pattern() { + local pattern="$1" + local file="$2" + grep -Eq -- "${pattern}" "${file}" || fail "${file} is missing ${pattern}" +} + +reject_pattern() { + local pattern="$1" + local file="$2" + if grep -Eq -- "${pattern}" "${file}"; then + fail "${file} still contains ${pattern}" + fi +} + +bash -n "${WATCHDOG}" + +require_pattern '^get_container_ipv6\(\)' "${WATCHDOG}" +require_pattern '^dnssec_validation_mode\(\)' "${WATCHDOG}" +require_pattern '^dnssec_validation_healthy\(\)' "${WATCHDOG}" +require_pattern '^unbound_check_round\(\)' "${WATCHDOG}" +require_pattern 'com\. SOA \+dnssec' "${WATCHDOG}" +require_pattern 'dnssec-failed\.org A \+dnssec' "${WATCHDOG}" +reject_pattern 'restart_container unbound' "${WATCHDOG}" + +require_pattern 'image: ghcr\.io/mailcow/watchdog:2\.11a' "${COMPOSE}" +require_pattern 'ENABLE_IPV6=' "${COMPOSE}" +require_pattern 'unbound\.conf:/etc/mailcow/unbound\.conf:ro,z' "${COMPOSE}" +reject_pattern 'UNBOUND_DNSSEC_MODE=' "${COMPOSE}" + +unbound_checks_body="$(sed -n '/^unbound_checks()/,/^}/p' "${WATCHDOG}")" +# The search string is intentionally literal source text. +# shellcheck disable=SC2016 +[[ "$(grep -Fc 'err_count=$(( ${err_count} + 1 ))' <<< "${unbound_checks_body}")" -eq 1 ]] \ + || fail "one Unbound polling round can add more than one health error" + +eval "$( + sed -n \ + -e '/^get_container_ipv6()/,/^}/p' \ + -e '/^dnssec_validation_mode()/,/^}/p' \ + -e '/^dnssec_validation_healthy()/,/^}/p' \ + -e '/^run_dns_check()/,/^}/p' \ + -e '/^unbound_check_round()/,/^}/p' \ + "${WATCHDOG}" +)" + +export COMPOSE_PROJECT_NAME="mailcowdockerized" +curl() { + case "$*" in + */containers/json) + printf '%s\n' \ + '[{"Id":"unbound-id","Config":{"Labels":{"com.docker.compose.service":"unbound-mailcow","com.docker.compose.project":"mailcowdockerized"}}}]' + ;; + */containers/unbound-id/json) + printf '%s\n' \ + '{"NetworkSettings":{"Networks":{"mailcowdockerized_mailcow-network":{"GlobalIPv6Address":"fd00::53"}}}}' + ;; + *) + return 1 + ;; + esac +} + +[[ "$(get_container_ipv6 unbound-mailcow)" == "fd00::53" ]] \ + || fail "Docker API IPv6 address selection failed" +[[ "$(dnssec_validation_mode "${STRICT_CONFIG}")" == "strict" ]] \ + || fail "strict Unbound configuration was misclassified" +[[ "$(dnssec_validation_mode "${PERMISSIVE_CONFIG}")" == "permissive" ]] \ + || fail "permissive Unbound configuration was misclassified" + +DIG_CAPTURE="$(mktemp)" +trap 'rm -f "${DIG_CAPTURE}"' EXIT +MOCK_BOGUS_STATUS="SERVFAIL" +MOCK_BOGUS_AD="" +MOCK_SIGNED_AD="yes" +dig() { + printf '%s\n' "$*" >> "${DIG_CAPTURE}" + case "$*" in + *"com. SOA"*) + printf '%s\n' ';; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 1' + if [[ -n "${MOCK_SIGNED_AD}" ]]; then + printf '%s\n' ';; flags: qr rd ra ad; QUERY: 1, ANSWER: 1' + else + printf '%s\n' ';; flags: qr rd ra; QUERY: 1, ANSWER: 1' + fi + ;; + *dnssec-failed.org*) + printf '%s\n' ";; ->>HEADER<<- opcode: QUERY, status: ${MOCK_BOGUS_STATUS}, id: 2" + [[ -z "${MOCK_BOGUS_AD}" ]] \ + || printf '%s\n' ';; flags: qr rd ra ad; QUERY: 1, ANSWER: 1' + ;; + *) + return 1 + ;; + esac +} + +dnssec_validation_healthy 6 "fd00::53" IPv6 strict \ + || fail "valid strict DNSSEC behavior was rejected" +grep -Fq -- '-6 com. SOA +dnssec +timeout=2 +tries=1 @fd00::53' "${DIG_CAPTURE}" \ + || fail "signed lookup did not use the IPv6 Unbound transport" +grep -Fq -- '-6 dnssec-failed.org A +dnssec +timeout=2 +tries=1 @fd00::53' "${DIG_CAPTURE}" \ + || fail "bogus lookup did not use the IPv6 Unbound transport" + +MOCK_BOGUS_STATUS="NOERROR" +if dnssec_validation_healthy 4 "192.0.2.53" IPv4 strict; then + fail "strict mode accepted a bogus DNSSEC chain" +fi + +dnssec_validation_healthy 4 "192.0.2.53" IPv4 permissive \ + || fail "valid permissive DNSSEC behavior was rejected" + +MOCK_BOGUS_AD="yes" +if dnssec_validation_healthy 4 "192.0.2.53" IPv4 permissive; then + fail "permissive mode accepted a bogus chain as authenticated" +fi + +get_container_ip() { + printf '%s\n' "192.0.2.53" +} +# Invoked by the function extracted above. +# shellcheck disable=SC2329 +get_container_ipv6() { + printf '%s\n' "fd00::53" +} +ipv6_enabled() { + return 0 +} +dnssec_validation_mode() { + printf '%s\n' "strict" +} +# Invoked by the function extracted above. +# shellcheck disable=SC2329 +run_dns_check() { + return 1 +} +# Invoked by the function extracted above. +# shellcheck disable=SC2329 +notify_error() { + : +} + +MOCK_SIGNED_AD="" +if unbound_check_round; then + fail "a failed polling round was reported as healthy" +fi + +MOCK_SIGNED_AD="yes" +MOCK_BOGUS_STATUS="SERVFAIL" +MOCK_BOGUS_AD="" +# Invoked by the function extracted above. +# shellcheck disable=SC2329 +run_dns_check() { + return 0 +} +# Invoked by the function extracted above. +# shellcheck disable=SC2329 +get_container_ipv6() { + return 1 +} +IPV6_NOTICE=0 +# Invoked by the function extracted above. +# shellcheck disable=SC2329 +notify_error() { + IPV6_NOTICE=1 +} + +unbound_check_round \ + || fail "IPv6 discovery trouble was misclassified as an Unbound process failure" +[[ "${IPV6_NOTICE}" -eq 1 ]] \ + || fail "IPv6 discovery trouble did not trigger a configuration notification" + +echo "Watchdog IPv6 and DNSSEC tests passed."