[Watchdog] Add Watchdog verbose logging (#4299)

* [Watchdog] Add verbose logging

* [Watchdog] More verbose debugging

* [Watchdog] Enable MX check for recipients

Co-authored-by: andryyy <andre.peters@debinux.de>
master
Peter 2021-10-29 06:48:49 +02:00 committed by GitHub
parent 15c0b3f7b7
commit 6bf70cf846
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 40 additions and 15 deletions

View File

@ -36,4 +36,4 @@ RUN apk add --update \
COPY watchdog.sh /watchdog.sh COPY watchdog.sh /watchdog.sh
COPY check_mysql_slavestatus.sh /usr/lib/nagios/plugins/check_mysql_slavestatus.sh COPY check_mysql_slavestatus.sh /usr/lib/nagios/plugins/check_mysql_slavestatus.sh
CMD /watchdog.sh 2> /dev/null CMD /watchdog.sh

View File

@ -6,7 +6,10 @@ trap "kill 0" EXIT
# Prepare # Prepare
BACKGROUND_TASKS=() BACKGROUND_TASKS=()
echo "Waiting for containers to settle..." echo "Waiting for containers to settle..."
sleep 30 for i in {30..1}; do
echo "${i}"
sleep 1
done
if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then
echo -e "$(date) - USE_WATCHDOG=n, skipping watchdog..." echo -e "$(date) - USE_WATCHDOG=n, skipping watchdog..."
@ -14,6 +17,14 @@ if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then
exec $(readlink -f "$0") exec $(readlink -f "$0")
fi fi
if [[ "${WATCHDOG_VERBOSE}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
SMTP_VERBOSE="--verbose"
set -xv
else
SMTP_VERBOSE=""
exec 2>/dev/null
fi
# Checks pipe their corresponding container name in this pipe # Checks pipe their corresponding container name in this pipe
if [[ ! -p /tmp/com_pipe ]]; then if [[ ! -p /tmp/com_pipe ]]; then
mkfifo /tmp/com_pipe mkfifo /tmp/com_pipe
@ -114,16 +125,16 @@ function mail_error() {
IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}" IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}"
for rcpt in "${MAIL_RCPTS[@]}"; do for rcpt in "${MAIL_RCPTS[@]}"; do
RCPT_DOMAIN= RCPT_DOMAIN=
#RCPT_MX= RCPT_MX=
RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'}) RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'})
# Latest smtp-cli looks up mx via dns CHECK_FOR_VALID_MX=$(dig +short ${RCPT_DOMAIN} mx)
#RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}') if [[ -z ${CHECK_FOR_VALID_MX} ]]; then
#if [[ -z ${RCPT_MX} ]]; then log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
# log_msg "Cannot determine MX for ${rcpt}, skipping email notification..." return 1
# return 1 fi
#fi
[ -f "/tmp/${1}" ] && BODY="/tmp/${1}" [ -f "/tmp/${1}" ] && BODY="/tmp/${1}"
timeout 10s ./smtp-cli --missing-modules-ok \ timeout 10s ./smtp-cli --missing-modules-ok \
"${SMTP_VERBOSE}" \
--charset=UTF-8 \ --charset=UTF-8 \
--subject="${SUBJECT}" \ --subject="${SUBJECT}" \
--body-plain="${BODY}" \ --body-plain="${BODY}" \
@ -132,8 +143,15 @@ function mail_error() {
--from="watchdog@${MAILCOW_HOSTNAME}" \ --from="watchdog@${MAILCOW_HOSTNAME}" \
--hello-host=${MAILCOW_HOSTNAME} \ --hello-host=${MAILCOW_HOSTNAME} \
--ipv4 --ipv4
#--server="${RCPT_MX}" if [[ $? -eq 1 ]]; then # exit code 1 is fine
log_msg "Sent notification email to ${rcpt}" log_msg "Sent notification email to ${rcpt}"
else
if [[ "${SMTP_VERBOSE}" == "" ]]; then
log_msg "Error while sending notification email to ${rcpt}. You can enable verbose logging by setting 'WATCHDOG_VERBOSE=y' in mailcow.conf."
else
log_msg "Error while sending notification email to ${rcpt}."
fi
fi
done done
} }

View File

@ -443,9 +443,7 @@ services:
- /lib/modules:/lib/modules:ro - /lib/modules:/lib/modules:ro
watchdog-mailcow: watchdog-mailcow:
image: mailcow/watchdog:1.94 image: mailcow/watchdog:1.95
# Debug
#command: /watchdog.sh
dns: dns:
- ${IPV4_NETWORK:-172.22.1}.254 - ${IPV4_NETWORK:-172.22.1}.254
tmpfs: tmpfs:
@ -470,6 +468,7 @@ services:
- WATCHDOG_SUBJECT=${WATCHDOG_SUBJECT:-Watchdog ALERT} - WATCHDOG_SUBJECT=${WATCHDOG_SUBJECT:-Watchdog ALERT}
- WATCHDOG_EXTERNAL_CHECKS=${WATCHDOG_EXTERNAL_CHECKS:-n} - WATCHDOG_EXTERNAL_CHECKS=${WATCHDOG_EXTERNAL_CHECKS:-n}
- WATCHDOG_MYSQL_REPLICATION_CHECKS=${WATCHDOG_MYSQL_REPLICATION_CHECKS:-n} - WATCHDOG_MYSQL_REPLICATION_CHECKS=${WATCHDOG_MYSQL_REPLICATION_CHECKS:-n}
- WATCHDOG_VERBOSE=${WATCHDOG_VERBOSE:-n}
- MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME} - MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
- COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mailcow-dockerized} - COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mailcow-dockerized}
- IPV4_NETWORK=${IPV4_NETWORK:-172.22.1} - IPV4_NETWORK=${IPV4_NETWORK:-172.22.1}

View File

@ -287,6 +287,9 @@ WATCHDOG_NOTIFY_BAN=n
# Will only work with unmodified mailcow setups. # Will only work with unmodified mailcow setups.
WATCHDOG_EXTERNAL_CHECKS=n WATCHDOG_EXTERNAL_CHECKS=n
# Enable watchdog verbose logging
WATCHDOG_VERBOSE=n
# Max log lines per service to keep in Redis logs # Max log lines per service to keep in Redis logs
LOG_LINES=9999 LOG_LINES=9999

View File

@ -513,6 +513,11 @@ for option in ${CONFIG_ARRAY[@]}; do
echo '# https://mailcow.github.io/mailcow-dockerized-docs/debug-reset-tls/' >> mailcow.conf echo '# https://mailcow.github.io/mailcow-dockerized-docs/debug-reset-tls/' >> mailcow.conf
echo 'ACME_CONTACT=' >> mailcow.conf echo 'ACME_CONTACT=' >> mailcow.conf
fi fi
elif [[ ${option} == "WATCHDOG_VERBOSE" ]]; then
if ! grep -q ${option} mailcow.conf; then
echo '# Enable watchdog verbose logging' >> mailcow.conf
echo 'WATCHDOG_VERBOSE=n' >> mailcow.conf
fi
elif ! grep -q ${option} mailcow.conf; then elif ! grep -q ${option} mailcow.conf; then
echo "Adding new option \"${option}\" to mailcow.conf" echo "Adding new option \"${option}\" to mailcow.conf"
echo "${option}=n" >> mailcow.conf echo "${option}=n" >> mailcow.conf