From a8fb1d3f4fe88a136bc8e4dbc410af64c09a1162 Mon Sep 17 00:00:00 2001 From: andryyy Date: Wed, 20 Sep 2017 10:56:49 +0200 Subject: [PATCH] Add experimental watchdog --- data/Dockerfiles/watchdog/Dockerfile | 22 ++ data/Dockerfiles/watchdog/watchdog.sh | 281 +++++++++++++++++++++++ data/conf/rspamd/dynmaps/index.html | 2 + data/conf/rspamd/local.d/composites.conf | 4 + data/conf/rspamd/local.d/metrics.conf | 17 ++ data/conf/rspamd/local.d/mx_check.conf | 2 +- docker-compose.yml | 17 ++ generate_config.sh | 3 + update.sh | 2 +- 9 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 data/Dockerfiles/watchdog/Dockerfile create mode 100755 data/Dockerfiles/watchdog/watchdog.sh create mode 100644 data/conf/rspamd/dynmaps/index.html create mode 100644 data/conf/rspamd/local.d/composites.conf diff --git a/data/Dockerfiles/watchdog/Dockerfile b/data/Dockerfiles/watchdog/Dockerfile new file mode 100644 index 00000000..b6d52fe9 --- /dev/null +++ b/data/Dockerfiles/watchdog/Dockerfile @@ -0,0 +1,22 @@ +FROM alpine:3.6 + +LABEL maintainer "André Peters " + +# Installation +RUN apk add --update \ + && apk add --no-cache nagios-plugins-smtp \ + nagios-plugins-tcp \ + nagios-plugins-http \ + nagios-plugins-ping \ + curl \ + bash \ + jq \ + fcgi \ + nagios-plugins-mysql \ + nagios-plugins-dns \ + bind-tools + +COPY watchdog.sh /watchdog.sh + +# Less verbose +CMD /watchdog.sh 2> /dev/null diff --git a/data/Dockerfiles/watchdog/watchdog.sh b/data/Dockerfiles/watchdog/watchdog.sh new file mode 100755 index 00000000..3abd13b8 --- /dev/null +++ b/data/Dockerfiles/watchdog/watchdog.sh @@ -0,0 +1,281 @@ +#!/bin/bash + +trap "exit" INT TERM +trap "kill 0" EXIT +PARENT_PID=$$ + +# Prepare +BACKGROUND_TASKS=() + +# Skip watchdog? +if [[ "${USE_WATCHDOG}" =~ ^([nN][oO])+$ ]]; then + echo "Skipping watchdog, sleeping..." + sleep 365d + exit 0 +fi + +# Checks pipe their corresponding container name in this pipe +if [[ ! -p /tmp/com_pipe ]]; then + mkfifo /tmp/com_pipe +fi + +# Common functions +progress() { + SERVICE=${1} + TOTAL=${2} + CURRENT=${3} + DIFF=${4} + [[ -z ${DIFF} ]] && DIFF=0 + [[ -z ${TOTAL} || -z ${CURRENT} ]] && return + [[ ${CURRENT} -gt ${TOTAL} ]] && return + [[ ${CURRENT} -lt 0 ]] && CURRENT=0 + percent=$(( 200 * ${CURRENT} / ${TOTAL} % 2 + 100 * ${CURRENT} / ${TOTAL} )) + completed=$(( ${percent} / 2 )) + remaining=$(( 50 - ${completed} )) + echo -ne "$(date) Health level: " + echo -n "[" + printf "%0.s>" $(seq ${completed}) + [[ ${remaining} != 0 ]] && printf "%0.s." $(seq ${remaining}) + echo -en "] ${percent}% - Service: ${SERVICE}, health trend: " + [[ ${DIFF} =~ ^-[1-9] ]] && echo -en "\e[31mnegative \e[0m" || echo -en "\e[32mpositive \e[0m" + echo "(${DIFF})" +} + +# Check functions +nginx_checks() { + err_count=0 + diff_c=0 + THRESHOLD=16 + # Reduce error count by 2 after restarting an unhealthy container + trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 + while [ ${err_count} -lt ${THRESHOLD} ]; do + err_c_cur=${err_count} + /usr/lib/nagios/plugins/check_ping -H nginx-mailcow -w 2000,10% -c 4000,100% -p2 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_http -H nginx-mailcow -u / -p 8081 1>&2; err_count=$(( ${err_count} + $? )) + sleep $(( ( RANDOM % 30 ) + 10 )) + [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 + [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) + progress "Nginx" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} + done + return 1 +} + +mysql_checks() { + err_count=0 + diff_c=0 + THRESHOLD=12 + # Reduce error count by 2 after restarting an unhealthy container + trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 + while [ ${err_count} -lt ${THRESHOLD} ]; do + err_c_cur=${err_count} + /usr/lib/nagios/plugins/check_mysql -H mysql-mailcow -P 3306 -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_mysql_query -H mysql-mailcow -P 3306 -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} -q "SELECT COUNT(*) FROM information_schema.tables" 1>&2; err_count=$(( ${err_count} + $? )) + sleep $(( ( RANDOM % 30 ) + 10 )) + [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 + [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) + progress "MySQL/MariaDB" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} + done + return 1 +} + +sogo_checks() { + err_count=0 + diff_c=0 + THRESHOLD=20 + # Reduce error count by 2 after restarting an unhealthy container + trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 + while [ ${err_count} -lt ${THRESHOLD} ]; do + err_c_cur=${err_count} + /usr/lib/nagios/plugins/check_http -H sogo-mailcow -u /WebServerResources/css/theme-default.css -p 9192 -R md-default-theme 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_http -H sogo-mailcow -u /SOGo.index/ -p 20000 -R "SOGo\sGroupware" 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_http -H nginx-mailcow -u /SOGo/ -p 443 --ssl -R "Bad Gateway" --invert-regex 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_http -H nginx-mailcow -u /SOGo/ -p 80 -R "Bad Gateway" --invert-regex 1>&2; err_count=$(( ${err_count} + $? )) + sleep $(( ( RANDOM % 30 ) + 10 )) + [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 + [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) + progress "SOGo" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} + done + return 1 +} + +postfix_checks() { + err_count=0 + diff_c=0 + THRESHOLD=16 + # Reduce error count by 2 after restarting an unhealthy container + trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 + while [ ${err_count} -lt ${THRESHOLD} ]; do + err_c_cur=${err_count} + /usr/lib/nagios/plugins/check_smtp -H postfix-mailcow -p 25 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_smtp -H postfix-mailcow -p 588 -f watchdog -C "RCPT TO:null@localhost" -C DATA -C . -R 250 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_smtp -H postfix-mailcow -p 587 -S 1>&2; err_count=$(( ${err_count} + $? )) + sleep $(( ( RANDOM % 30 ) + 10 )) + [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 + [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) + progress "Postfix" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} + done + return 1 +} + +dovecot_checks() { + err_count=0 + diff_c=0 + THRESHOLD=24 + # Reduce error count by 2 after restarting an unhealthy container + trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 + while [ ${err_count} -lt ${THRESHOLD} ]; do + err_c_cur=${err_count} + /usr/lib/nagios/plugins/check_smtp -H dovecot-mailcow -p 24 -f "watchdog" -C "RCPT TO:" -L -R "User doesn't exist" 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_imap -H dovecot-mailcow -p 993 -S -e "OK " 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_imap -H dovecot-mailcow -p 143 -e "OK " 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_tcp -H dovecot-mailcow -p 10001 -e "VERSION" 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_tcp -H dovecot-mailcow -p 4190 -e "Dovecot ready" 1>&2; err_count=$(( ${err_count} + $? )) + sleep $(( ( RANDOM % 30 ) + 10 )) + [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 + [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) + progress "Dovecot" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} + done + return 1 +} + +phpfpm_checks() { + err_count=0 + diff_c=0 + THRESHOLD=12 + # Reduce error count by 2 after restarting an unhealthy container + trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 + while [ ${err_count} -lt ${THRESHOLD} ]; do + err_c_cur=${err_count} + cgi-fcgi -bind -connect php-fpm-mailcow:9000 | grep PHP 1>&2; err_count=$(( ${err_count} + ($? * 2))) + /usr/lib/nagios/plugins/check_ping -H php-fpm-mailcow -w 2000,10% -c 4000,100% -p2 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_http -H nginx-mailcow -u /settings.php -p 8081 -r "settings \{" 1>&2; err_count=$(( ${err_count} + ($? * 2))) + sleep $(( ( RANDOM % 30 ) + 10 )) + [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 + [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) + progress "PHP-FPM" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} + done + return 1 +} + +dns_checks() { + err_count=0 + diff_c=0 + THRESHOLD=28 + # Reduce error count by 2 after restarting an unhealthy container + trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 + while [ ${err_count} -lt ${THRESHOLD} ]; do + err_c_cur=${err_count} + /usr/lib/nagios/plugins/check_dns -H google.com 1>&2; err_count=$(( ${err_count} + ($? * 2))) + /usr/lib/nagios/plugins/check_dns -s $(dig unbound-mailcow +short A) -H google.com 1>&2; err_count=$(( ${err_count} + ($? * 2))) + /usr/lib/nagios/plugins/check_dns -s $(dig unbound-mailcow +short AAAA) -H google.com 1>&2; err_count=$(( ${err_count} + ($? * 2))) + dig +dnssec org. @172.22.1.254 | grep -E 'flags:.+ad' 1>&2; err_count=$(( ${err_count} + ($? * 2))) + sleep $(( ( RANDOM % 30 ) + 10 )) + [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 + [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) + progress "Unbound" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} + done + return 1 +} + +# Create watchdog agents +( +while true; do + if ! nginx_checks; then + echo -e "\e[31m$(date) - Nginx hit error limit\e[0m" + echo nginx-mailcow > /tmp/com_pipe + fi +done +) & +BACKGROUND_TASKS+=($!) + +( +while true; do + if ! mysql_checks; then + echo -e "\e[31m$(date) - MySQL hit error limit\e[0m" + echo mysql-mailcow > /tmp/com_pipe + fi +done +) & +BACKGROUND_TASKS+=($!) + +( +while true; do + if ! phpfpm_checks; then + echo -e "\e[31m$(date) - PHP-FPM hit error limit\e[0m" + echo php-fpm-mailcow > /tmp/com_pipe + fi +done +) & +BACKGROUND_TASKS+=($!) + +( +while true; do + if ! sogo_checks; then + echo -e "\e[31m$(date) - SOGo hit error limit\e[0m" + echo sogo-mailcow > /tmp/com_pipe + fi +done +) & +BACKGROUND_TASKS+=($!) + +( +while true; do + if ! postfix_checks; then + echo -e "\e[31m$(date) - Postfix hit error limit\e[0m" + echo postfix-mailcow > /tmp/com_pipe + fi +done +) & +BACKGROUND_TASKS+=($!) + +( +while true; do + if ! dovecot_checks; then + echo -e "\e[31m$(date) - Dovecot hit error limit\e[0m" + echo dovecot-mailcow > /tmp/com_pipe + fi +done +) & +BACKGROUND_TASKS+=($!) + +( +while true; do + if ! dns_checks; then + echo -e "\e[31m$(date) - Unbound hit error limit\e[0m" + echo unbound-mailcow > /tmp/com_pipe + fi +done +) & +BACKGROUND_TASKS+=($!) + +# Monitor watchdog agents, stop script when agents fails and wait for respawn by Docker (restart:always:n) +( +while true; do + for bg_task in ${BACKGROUND_TASKS[*]}; do + if ! kill -0 ${bg_task} 21>&2; then + echo "Worker ${bg_task} died, stopping watchdog and waiting for respawn..." + kill -TERM ${PARENT_PID} + fi + sleep 1 + done +done +) & + +# Restart container when threshold limit reached +while true; do + CONTAINER_ID= + read com_pipe_answer + diff --git a/data/conf/rspamd/local.d/composites.conf b/data/conf/rspamd/local.d/composites.conf new file mode 100644 index 00000000..e895fb0f --- /dev/null +++ b/data/conf/rspamd/local.d/composites.conf @@ -0,0 +1,4 @@ +MX_IMPLICIT { + expression = "MX_GOOD and MX_MISSING"; + score = -0.01; +} diff --git a/data/conf/rspamd/local.d/metrics.conf b/data/conf/rspamd/local.d/metrics.conf index 4fd0953c..b3afa78c 100644 --- a/data/conf/rspamd/local.d/metrics.conf +++ b/data/conf/rspamd/local.d/metrics.conf @@ -17,3 +17,20 @@ group "bayes" { description = "Message probably ham, probability: "; } } +group "MX" { + symbol "MX_INVALID" { + score = 0.5; + description = "No connectable MX"; + one_shot = "true"; + } + symbol "MX_MISSING" { + score = 2.0; + description = "No MX record"; + one_shot = "true"; + } + symbol "MX_GOOD" { + score = -0.01; + description = "MX was ok"; + one_shot = "true"; + } +} diff --git a/data/conf/rspamd/local.d/mx_check.conf b/data/conf/rspamd/local.d/mx_check.conf index 6a775a21..22fcedf1 100644 --- a/data/conf/rspamd/local.d/mx_check.conf +++ b/data/conf/rspamd/local.d/mx_check.conf @@ -1,4 +1,4 @@ -timeout = 1.0; +timeout = 8.0; symbol_bad_mx = "MX_INVALID"; symbol_no_mx = "MX_MISSING"; symbol_good_mx = "MX_GOOD"; diff --git a/docker-compose.yml b/docker-compose.yml index f0cf0e30..2ec2d8ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -307,6 +307,23 @@ services: volumes: - /lib/modules:/lib/modules:ro + watchdog-mailcow: + image: mailcow/watchdog:1.0 + build: ./data/Dockerfiles/watchdog + init: false + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + restart: always + environment: + - DBNAME=${DBNAME} + - DBUSER=${DBUSER} + - DBPASS=${DBPASS} + - USE_WATCHDOG=${USE_WATCHDOG:-n} + networks: + mailcow-network: + aliases: + - watchdog + ipv6nat: image: robbertkl/ipv6nat restart: always diff --git a/generate_config.sh b/generate_config.sh index 2d03891e..a7e52ebe 100755 --- a/generate_config.sh +++ b/generate_config.sh @@ -96,6 +96,9 @@ SKIP_FAIL2BAN=n # To never run clamd-mailcow SKIP_CLAMD=n +# Enable watchdog to restart unhealthy containers (experimental) +USE_WATCHDOG=n + EOF mkdir -p data/assets/ssl diff --git a/update.sh b/update.sh index 4e9a1a96..f38c47a0 100755 --- a/update.sh +++ b/update.sh @@ -4,7 +4,7 @@ for bin in curl docker-compose docker git awk sha1sum; do if [[ -z $(which ${bin}) ]]; then echo "Cannot find ${bin}, exiting..."; exit 1; fi done -CONFIG_ARRAY=("SKIP_LETS_ENCRYPT" "SKIP_CLAMD" "SKIP_IP_CHECK" "SKIP_FAIL2BAN" "ADDITIONAL_SAN" "DOVEADM_PORT") +CONFIG_ARRAY=("SKIP_LETS_ENCRYPT" "USE_WATCHDOG" "SKIP_CLAMD" "SKIP_IP_CHECK" "SKIP_FAIL2BAN" "ADDITIONAL_SAN" "DOVEADM_PORT") echo >> mailcow.conf for option in ${CONFIG_ARRAY[@]}; do if [[ ${option} == "ADDITIONAL_SAN" ]]; then