diff --git a/data/Dockerfiles/watchdog/watchdog.sh b/data/Dockerfiles/watchdog/watchdog.sh index b45efb85..bedbc882 100755 --- a/data/Dockerfiles/watchdog/watchdog.sh +++ b/data/Dockerfiles/watchdog/watchdog.sh @@ -43,20 +43,22 @@ log_msg() { function mail_error() { [[ -z ${1} ]] && return 1 [[ -z ${2} ]] && return 2 - [[ -z ${3} ]] && BODY="Service was restarted, please check your mailcow installation." || BODY="${3}" + [[ -z ${3} ]] && BODY="Service was restarted on $(date), please check your mailcow installation." || BODY="$(date) - ${3}" RCPT_DOMAIN=$(echo ${1} | awk -F @ {'print $NF'}) RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}') if [[ -z ${RCPT_MX} ]]; then log_msg "Cannot determine MX for ${1}, skipping email notification..." return 1 fi + [ -f "/tmp/${2}" ] && ATTACH="--attach /tmp/${2}@text/plain" || ATTACH= ./smtp-cli --missing-modules-ok \ --subject="Watchdog: ${2} hit the error rate limit" \ --body-plain="${BODY}" \ --to=${1} \ --from="watchdog@${MAILCOW_HOSTNAME}" \ --server="${RCPT_MX}" \ - --hello-host=${MAILCOW_HOSTNAME} + --hello-host=${MAILCOW_HOSTNAME} \ + ${ATTACH} log_msg "Sent notification email to ${1}" } @@ -106,9 +108,10 @@ nginx_checks() { # Reduce error count by 2 after restarting an unhealthy container trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 while [ ${err_count} -lt ${THRESHOLD} ]; do + cat /dev/null > /tmp/nginx-mailcow host_ip=$(get_container_ip nginx-mailcow) err_c_cur=${err_count} - /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u / -p 8081 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u / -p 8081 2>> /tmp/nginx-mailcow 1>&2; err_count=$(( ${err_count} + $? )) [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) progress "Nginx" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} @@ -125,15 +128,16 @@ unbound_checks() { # Reduce error count by 2 after restarting an unhealthy container trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 while [ ${err_count} -lt ${THRESHOLD} ]; do + cat /dev/null > /tmp/unbound-mailcow host_ip=$(get_container_ip unbound-mailcow) err_c_cur=${err_count} - /usr/lib/nagios/plugins/check_dns -s ${host_ip} -H google.com 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_dns -s ${host_ip} -H google.com 2>> /tmp/unbound-mailcow 1>&2; err_count=$(( ${err_count} + $? )) DNSSEC=$(dig com +dnssec | egrep 'flags:.+ad') if [[ -z ${DNSSEC} ]]; then - echo "DNSSEC failure" 1>&2 + echo "DNSSEC failure" 2>> /tmp/unbound-mailcow 1>&2 err_count=$(( ${err_count} + 1)) else - echo "DNSSEC check succeeded" 1>&2 + echo "DNSSEC check succeeded" 2>> /tmp/unbound-mailcow 1>&2 fi [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) @@ -151,10 +155,11 @@ mysql_checks() { # Reduce error count by 2 after restarting an unhealthy container trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 while [ ${err_count} -lt ${THRESHOLD} ]; do + cat /dev/null > /tmp/mysql-mailcow host_ip=$(get_container_ip mysql-mailcow) err_c_cur=${err_count} - /usr/lib/nagios/plugins/check_mysql -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} 1>&2; err_count=$(( ${err_count} + $? )) - /usr/lib/nagios/plugins/check_mysql_query -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} -q "SELECT COUNT(*) FROM information_schema.tables" 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_mysql -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_mysql_query -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} -q "SELECT COUNT(*) FROM information_schema.tables" 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? )) [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) progress "MySQL/MariaDB" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} @@ -171,9 +176,10 @@ sogo_checks() { # Reduce error count by 2 after restarting an unhealthy container trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 while [ ${err_count} -lt ${THRESHOLD} ]; do + cat /dev/null > /tmp/sogo-mailcow host_ip=$(get_container_ip sogo-mailcow) err_c_cur=${err_count} - /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /SOGo.index/ -p 20000 -R "SOGo\.MainUI" 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /SOGo.index/ -p 20000 -R "SOGo\.MainUI" 2>> /tmp/sogo-mailcow 1>&2; err_count=$(( ${err_count} + $? )) [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) progress "SOGo" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} @@ -190,10 +196,11 @@ postfix_checks() { # Reduce error count by 2 after restarting an unhealthy container trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 while [ ${err_count} -lt ${THRESHOLD} ]; do - host_ip=$(get_container_ip postfix-mailcow) + cat /dev/null > /tmp/postfix-mailcow + host_ip=$(get_container_ip postfix-mailcow) err_c_cur=${err_count} - /usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 589 -f "watchdog@invalid" -C "RCPT TO:null@localhost" -C DATA -C . -R 250 1>&2; err_count=$(( ${err_count} + $? )) - /usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 589 -S 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 589 -f "watchdog@invalid" -C "RCPT TO:null@localhost" -C DATA -C . -R 250 2>> /tmp/postfix-mailcow 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 589 -S 2>> /tmp/postfix-mailcow 1>&2; err_count=$(( ${err_count} + $? )) [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) progress "Postfix" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} @@ -210,9 +217,10 @@ clamd_checks() { # Reduce error count by 2 after restarting an unhealthy container trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 while [ ${err_count} -lt ${THRESHOLD} ]; do - host_ip=$(get_container_ip clamd-mailcow) + cat /dev/null > /tmp/clamd-mailcow + host_ip=$(get_container_ip clamd-mailcow) err_c_cur=${err_count} - /usr/lib/nagios/plugins/check_clamd -4 -H ${host_ip} 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_clamd -4 -H ${host_ip} 2>> /tmp/clamd-mailcow 1>&2; err_count=$(( ${err_count} + $? )) [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) progress "Clamd" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} @@ -230,13 +238,14 @@ dovecot_checks() { # Reduce error count by 2 after restarting an unhealthy container trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 while [ ${err_count} -lt ${THRESHOLD} ]; do + cat /dev/null > /tmp/dovecot-mailcow host_ip=$(get_container_ip dovecot-mailcow) err_c_cur=${err_count} - /usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 24 -f "watchdog@invalid" -C "RCPT TO:" -L -R "User doesn't exist" 1>&2; err_count=$(( ${err_count} + $? )) - /usr/lib/nagios/plugins/check_imap -4 -H ${host_ip} -p 993 -S -e "OK " 1>&2; err_count=$(( ${err_count} + $? )) - /usr/lib/nagios/plugins/check_imap -4 -H ${host_ip} -p 143 -e "OK " 1>&2; err_count=$(( ${err_count} + $? )) - /usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 10001 -e "VERSION" 1>&2; err_count=$(( ${err_count} + $? )) - /usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 4190 -e "Dovecot ready" 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 24 -f "watchdog@invalid" -C "RCPT TO:" -L -R "User doesn't exist" 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_imap -4 -H ${host_ip} -p 993 -S -e "OK " 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_imap -4 -H ${host_ip} -p 143 -e "OK " 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 10001 -e "VERSION" 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 4190 -e "Dovecot ready" 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? )) [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) progress "Dovecot" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} @@ -253,10 +262,11 @@ phpfpm_checks() { # Reduce error count by 2 after restarting an unhealthy container trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 while [ ${err_count} -lt ${THRESHOLD} ]; do + cat /dev/null > /tmp/php-fpm-mailcow host_ip=$(get_container_ip php-fpm-mailcow) err_c_cur=${err_count} - nc -z ${host_ip} 9001 ; err_count=$(( ${err_count} + ($? * 2))) - nc -z ${host_ip} 9002 ; err_count=$(( ${err_count} + ($? * 2))) + /usr/lib/nagios/plugins/check_tcp -H ${host_ip} -p 9001 2>> /tmp/php-fpm-mailcow 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_tcp -H ${host_ip} -p 9002 2>> /tmp/php-fpm-mailcow 1>&2; err_count=$(( ${err_count} + $? )) [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) progress "PHP-FPM" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} @@ -266,26 +276,6 @@ phpfpm_checks() { return 1 } -cert_checks() { - err_count=0 - diff_c=0 - THRESHOLD=1 - # Reduce error count by 2 after restarting an unhealthy container - trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 - while [ ${err_count} -lt ${THRESHOLD} ]; do - host_ip=$(get_container_ip nginx-mailcow) - err_c_cur=${err_count} - /usr/lib/nagios/plugins/check_http -H ${host_ip} -p ${HTTPS_PORT} -C 15 1>&2; err_count=$(( ${err_count} + $? )) - [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 - [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) - progress "TLS certificate" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} - diff_c=0 - # Sleep 1 day, fixme: 1 day lag - sleep 86400 - done - return 1 -} - rspamd_checks() { err_count=0 diff_c=0 @@ -293,6 +283,7 @@ rspamd_checks() { # Reduce error count by 2 after restarting an unhealthy container trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 while [ ${err_count} -lt ${THRESHOLD} ]; do + cat /dev/null > /tmp/rspamd-mailcow host_ip=$(get_container_ip rspamd-mailcow) err_c_cur=${err_count} SCORE=$(/usr/bin/curl -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/scan -d ' @@ -302,10 +293,10 @@ From: watchdog@localhost Empty ' | jq -rc .required_score) if [[ ${SCORE} != "9999" ]]; then - echo "Rspamd settings check failed" 1>&2 + echo "Rspamd settings check failed" 2>> /tmp/rspamd-mailcow 1>&2 err_count=$(( ${err_count} + 1)) else - echo "Rspamd settings check succeeded" 1>&2 + echo "Rspamd settings check succeeded" 2>> /tmp/rspamd-mailcow 1>&2 fi [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) @@ -387,16 +378,6 @@ done BACKGROUND_TASKS+=($!) fi -( -while true; do - if ! cert_checks; then - log_msg "TLS certificate hit error limit" - [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "TLS check" "TLS certificate expires soon!" - fi -done -) & -BACKGROUND_TASKS+=($!) - ( while true; do if ! postfix_checks; then diff --git a/docker-compose.yml b/docker-compose.yml index 3fede481..11abe688 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -337,7 +337,7 @@ services: - /lib/modules:/lib/modules:ro watchdog-mailcow: - image: mailcow/watchdog:1.25 + image: mailcow/watchdog:1.26 # Debug #command: /watchdog.sh build: ./data/Dockerfiles/watchdog