[Watchdog] Remove cert check (wip)
[Watchdog] Append last check loop as attachment to mail [Watchdog] Print time and date in mail alertsmaster
André
parent
e74f33615f
commit
86b92f68cb
|
|
@ -43,20 +43,22 @@ log_msg() {
|
|||
function mail_error() {
|
||||
[[ -z ${1} ]] && return 1
|
||||
[[ -z ${2} ]] && return 2
|
||||
[[ -z ${3} ]] && BODY="Service was restarted, please check your mailcow installation." || BODY="${3}"
|
||||
[[ -z ${3} ]] && BODY="Service was restarted on $(date), please check your mailcow installation." || BODY="$(date) - ${3}"
|
||||
RCPT_DOMAIN=$(echo ${1} | awk -F @ {'print $NF'})
|
||||
RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}')
|
||||
if [[ -z ${RCPT_MX} ]]; then
|
||||
log_msg "Cannot determine MX for ${1}, skipping email notification..."
|
||||
return 1
|
||||
fi
|
||||
[ -f "/tmp/${2}" ] && ATTACH="--attach /tmp/${2}@text/plain" || ATTACH=
|
||||
./smtp-cli --missing-modules-ok \
|
||||
--subject="Watchdog: ${2} hit the error rate limit" \
|
||||
--body-plain="${BODY}" \
|
||||
--to=${1} \
|
||||
--from="watchdog@${MAILCOW_HOSTNAME}" \
|
||||
--server="${RCPT_MX}" \
|
||||
--hello-host=${MAILCOW_HOSTNAME}
|
||||
--hello-host=${MAILCOW_HOSTNAME} \
|
||||
${ATTACH}
|
||||
log_msg "Sent notification email to ${1}"
|
||||
}
|
||||
|
||||
|
|
@ -106,9 +108,10 @@ nginx_checks() {
|
|||
# Reduce error count by 2 after restarting an unhealthy container
|
||||
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
|
||||
while [ ${err_count} -lt ${THRESHOLD} ]; do
|
||||
cat /dev/null > /tmp/nginx-mailcow
|
||||
host_ip=$(get_container_ip nginx-mailcow)
|
||||
err_c_cur=${err_count}
|
||||
/usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u / -p 8081 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u / -p 8081 2>> /tmp/nginx-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
|
||||
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
|
||||
progress "Nginx" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
|
||||
|
|
@ -125,15 +128,16 @@ unbound_checks() {
|
|||
# Reduce error count by 2 after restarting an unhealthy container
|
||||
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
|
||||
while [ ${err_count} -lt ${THRESHOLD} ]; do
|
||||
cat /dev/null > /tmp/unbound-mailcow
|
||||
host_ip=$(get_container_ip unbound-mailcow)
|
||||
err_c_cur=${err_count}
|
||||
/usr/lib/nagios/plugins/check_dns -s ${host_ip} -H google.com 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_dns -s ${host_ip} -H google.com 2>> /tmp/unbound-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
DNSSEC=$(dig com +dnssec | egrep 'flags:.+ad')
|
||||
if [[ -z ${DNSSEC} ]]; then
|
||||
echo "DNSSEC failure" 1>&2
|
||||
echo "DNSSEC failure" 2>> /tmp/unbound-mailcow 1>&2
|
||||
err_count=$(( ${err_count} + 1))
|
||||
else
|
||||
echo "DNSSEC check succeeded" 1>&2
|
||||
echo "DNSSEC check succeeded" 2>> /tmp/unbound-mailcow 1>&2
|
||||
fi
|
||||
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
|
||||
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
|
||||
|
|
@ -151,10 +155,11 @@ mysql_checks() {
|
|||
# Reduce error count by 2 after restarting an unhealthy container
|
||||
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
|
||||
while [ ${err_count} -lt ${THRESHOLD} ]; do
|
||||
cat /dev/null > /tmp/mysql-mailcow
|
||||
host_ip=$(get_container_ip mysql-mailcow)
|
||||
err_c_cur=${err_count}
|
||||
/usr/lib/nagios/plugins/check_mysql -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_mysql_query -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} -q "SELECT COUNT(*) FROM information_schema.tables" 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_mysql -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_mysql_query -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} -q "SELECT COUNT(*) FROM information_schema.tables" 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
|
||||
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
|
||||
progress "MySQL/MariaDB" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
|
||||
|
|
@ -171,9 +176,10 @@ sogo_checks() {
|
|||
# Reduce error count by 2 after restarting an unhealthy container
|
||||
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
|
||||
while [ ${err_count} -lt ${THRESHOLD} ]; do
|
||||
cat /dev/null > /tmp/sogo-mailcow
|
||||
host_ip=$(get_container_ip sogo-mailcow)
|
||||
err_c_cur=${err_count}
|
||||
/usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /SOGo.index/ -p 20000 -R "SOGo\.MainUI" 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /SOGo.index/ -p 20000 -R "SOGo\.MainUI" 2>> /tmp/sogo-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
|
||||
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
|
||||
progress "SOGo" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
|
||||
|
|
@ -190,10 +196,11 @@ postfix_checks() {
|
|||
# Reduce error count by 2 after restarting an unhealthy container
|
||||
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
|
||||
while [ ${err_count} -lt ${THRESHOLD} ]; do
|
||||
host_ip=$(get_container_ip postfix-mailcow)
|
||||
cat /dev/null > /tmp/postfix-mailcow
|
||||
host_ip=$(get_container_ip postfix-mailcow)
|
||||
err_c_cur=${err_count}
|
||||
/usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 589 -f "watchdog@invalid" -C "RCPT TO:null@localhost" -C DATA -C . -R 250 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 589 -S 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 589 -f "watchdog@invalid" -C "RCPT TO:null@localhost" -C DATA -C . -R 250 2>> /tmp/postfix-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 589 -S 2>> /tmp/postfix-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
|
||||
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
|
||||
progress "Postfix" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
|
||||
|
|
@ -210,9 +217,10 @@ clamd_checks() {
|
|||
# Reduce error count by 2 after restarting an unhealthy container
|
||||
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
|
||||
while [ ${err_count} -lt ${THRESHOLD} ]; do
|
||||
host_ip=$(get_container_ip clamd-mailcow)
|
||||
cat /dev/null > /tmp/clamd-mailcow
|
||||
host_ip=$(get_container_ip clamd-mailcow)
|
||||
err_c_cur=${err_count}
|
||||
/usr/lib/nagios/plugins/check_clamd -4 -H ${host_ip} 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_clamd -4 -H ${host_ip} 2>> /tmp/clamd-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
|
||||
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
|
||||
progress "Clamd" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
|
||||
|
|
@ -230,13 +238,14 @@ dovecot_checks() {
|
|||
# Reduce error count by 2 after restarting an unhealthy container
|
||||
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
|
||||
while [ ${err_count} -lt ${THRESHOLD} ]; do
|
||||
cat /dev/null > /tmp/dovecot-mailcow
|
||||
host_ip=$(get_container_ip dovecot-mailcow)
|
||||
err_c_cur=${err_count}
|
||||
/usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 24 -f "watchdog@invalid" -C "RCPT TO:<watchdog@invalid>" -L -R "User doesn't exist" 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_imap -4 -H ${host_ip} -p 993 -S -e "OK " 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_imap -4 -H ${host_ip} -p 143 -e "OK " 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 10001 -e "VERSION" 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 4190 -e "Dovecot ready" 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 24 -f "watchdog@invalid" -C "RCPT TO:<watchdog@invalid>" -L -R "User doesn't exist" 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_imap -4 -H ${host_ip} -p 993 -S -e "OK " 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_imap -4 -H ${host_ip} -p 143 -e "OK " 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 10001 -e "VERSION" 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 4190 -e "Dovecot ready" 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
|
||||
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
|
||||
progress "Dovecot" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
|
||||
|
|
@ -253,10 +262,11 @@ phpfpm_checks() {
|
|||
# Reduce error count by 2 after restarting an unhealthy container
|
||||
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
|
||||
while [ ${err_count} -lt ${THRESHOLD} ]; do
|
||||
cat /dev/null > /tmp/php-fpm-mailcow
|
||||
host_ip=$(get_container_ip php-fpm-mailcow)
|
||||
err_c_cur=${err_count}
|
||||
nc -z ${host_ip} 9001 ; err_count=$(( ${err_count} + ($? * 2)))
|
||||
nc -z ${host_ip} 9002 ; err_count=$(( ${err_count} + ($? * 2)))
|
||||
/usr/lib/nagios/plugins/check_tcp -H ${host_ip} -p 9001 2>> /tmp/php-fpm-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
/usr/lib/nagios/plugins/check_tcp -H ${host_ip} -p 9002 2>> /tmp/php-fpm-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
|
||||
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
|
||||
progress "PHP-FPM" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
|
||||
|
|
@ -266,26 +276,6 @@ phpfpm_checks() {
|
|||
return 1
|
||||
}
|
||||
|
||||
cert_checks() {
|
||||
err_count=0
|
||||
diff_c=0
|
||||
THRESHOLD=1
|
||||
# Reduce error count by 2 after restarting an unhealthy container
|
||||
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
|
||||
while [ ${err_count} -lt ${THRESHOLD} ]; do
|
||||
host_ip=$(get_container_ip nginx-mailcow)
|
||||
err_c_cur=${err_count}
|
||||
/usr/lib/nagios/plugins/check_http -H ${host_ip} -p ${HTTPS_PORT} -C 15 1>&2; err_count=$(( ${err_count} + $? ))
|
||||
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
|
||||
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
|
||||
progress "TLS certificate" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
|
||||
diff_c=0
|
||||
# Sleep 1 day, fixme: 1 day lag
|
||||
sleep 86400
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
rspamd_checks() {
|
||||
err_count=0
|
||||
diff_c=0
|
||||
|
|
@ -293,6 +283,7 @@ rspamd_checks() {
|
|||
# Reduce error count by 2 after restarting an unhealthy container
|
||||
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
|
||||
while [ ${err_count} -lt ${THRESHOLD} ]; do
|
||||
cat /dev/null > /tmp/rspamd-mailcow
|
||||
host_ip=$(get_container_ip rspamd-mailcow)
|
||||
err_c_cur=${err_count}
|
||||
SCORE=$(/usr/bin/curl -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/scan -d '
|
||||
|
|
@ -302,10 +293,10 @@ From: watchdog@localhost
|
|||
Empty
|
||||
' | jq -rc .required_score)
|
||||
if [[ ${SCORE} != "9999" ]]; then
|
||||
echo "Rspamd settings check failed" 1>&2
|
||||
echo "Rspamd settings check failed" 2>> /tmp/rspamd-mailcow 1>&2
|
||||
err_count=$(( ${err_count} + 1))
|
||||
else
|
||||
echo "Rspamd settings check succeeded" 1>&2
|
||||
echo "Rspamd settings check succeeded" 2>> /tmp/rspamd-mailcow 1>&2
|
||||
fi
|
||||
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
|
||||
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
|
||||
|
|
@ -387,16 +378,6 @@ done
|
|||
BACKGROUND_TASKS+=($!)
|
||||
fi
|
||||
|
||||
(
|
||||
while true; do
|
||||
if ! cert_checks; then
|
||||
log_msg "TLS certificate hit error limit"
|
||||
[[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "TLS check" "TLS certificate expires soon!"
|
||||
fi
|
||||
done
|
||||
) &
|
||||
BACKGROUND_TASKS+=($!)
|
||||
|
||||
(
|
||||
while true; do
|
||||
if ! postfix_checks; then
|
||||
|
|
|
|||
|
|
@ -337,7 +337,7 @@ services:
|
|||
- /lib/modules:/lib/modules:ro
|
||||
|
||||
watchdog-mailcow:
|
||||
image: mailcow/watchdog:1.25
|
||||
image: mailcow/watchdog:1.26
|
||||
# Debug
|
||||
#command: /watchdog.sh
|
||||
build: ./data/Dockerfiles/watchdog
|
||||
|
|
|
|||
Loading…
Reference in New Issue