From 6bf70cf84621be2926f1550cfc71bfbe4ef08c5f Mon Sep 17 00:00:00 2001 From: Peter Date: Fri, 29 Oct 2021 06:48:49 +0200 Subject: [PATCH] [Watchdog] Add Watchdog verbose logging (#4299) * [Watchdog] Add verbose logging * [Watchdog] More verbose debugging * [Watchdog] Enable MX check for recipients Co-authored-by: andryyy --- data/Dockerfiles/watchdog/Dockerfile | 2 +- data/Dockerfiles/watchdog/watchdog.sh | 40 +++++++++++++++++++-------- docker-compose.yml | 5 ++-- generate_config.sh | 3 ++ update.sh | 5 ++++ 5 files changed, 40 insertions(+), 15 deletions(-) diff --git a/data/Dockerfiles/watchdog/Dockerfile b/data/Dockerfiles/watchdog/Dockerfile index 8378edd7..00cb8e99 100644 --- a/data/Dockerfiles/watchdog/Dockerfile +++ b/data/Dockerfiles/watchdog/Dockerfile @@ -36,4 +36,4 @@ RUN apk add --update \ COPY watchdog.sh /watchdog.sh COPY check_mysql_slavestatus.sh /usr/lib/nagios/plugins/check_mysql_slavestatus.sh -CMD /watchdog.sh 2> /dev/null +CMD /watchdog.sh diff --git a/data/Dockerfiles/watchdog/watchdog.sh b/data/Dockerfiles/watchdog/watchdog.sh index cd3a2881..231d0ecd 100755 --- a/data/Dockerfiles/watchdog/watchdog.sh +++ b/data/Dockerfiles/watchdog/watchdog.sh @@ -6,7 +6,10 @@ trap "kill 0" EXIT # Prepare BACKGROUND_TASKS=() echo "Waiting for containers to settle..." -sleep 30 +for i in {30..1}; do + echo "${i}" + sleep 1 +done if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then echo -e "$(date) - USE_WATCHDOG=n, skipping watchdog..." @@ -14,6 +17,14 @@ if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then exec $(readlink -f "$0") fi +if [[ "${WATCHDOG_VERBOSE}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then + SMTP_VERBOSE="--verbose" + set -xv +else + SMTP_VERBOSE="" + exec 2>/dev/null +fi + # Checks pipe their corresponding container name in this pipe if [[ ! -p /tmp/com_pipe ]]; then mkfifo /tmp/com_pipe @@ -114,16 +125,16 @@ function mail_error() { IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}" for rcpt in "${MAIL_RCPTS[@]}"; do RCPT_DOMAIN= - #RCPT_MX= + RCPT_MX= RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'}) - # Latest smtp-cli looks up mx via dns - #RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}') - #if [[ -z ${RCPT_MX} ]]; then - # log_msg "Cannot determine MX for ${rcpt}, skipping email notification..." - # return 1 - #fi + CHECK_FOR_VALID_MX=$(dig +short ${RCPT_DOMAIN} mx) + if [[ -z ${CHECK_FOR_VALID_MX} ]]; then + log_msg "Cannot determine MX for ${rcpt}, skipping email notification..." + return 1 + fi [ -f "/tmp/${1}" ] && BODY="/tmp/${1}" timeout 10s ./smtp-cli --missing-modules-ok \ + "${SMTP_VERBOSE}" \ --charset=UTF-8 \ --subject="${SUBJECT}" \ --body-plain="${BODY}" \ @@ -132,8 +143,15 @@ function mail_error() { --from="watchdog@${MAILCOW_HOSTNAME}" \ --hello-host=${MAILCOW_HOSTNAME} \ --ipv4 - #--server="${RCPT_MX}" - log_msg "Sent notification email to ${rcpt}" + if [[ $? -eq 1 ]]; then # exit code 1 is fine + log_msg "Sent notification email to ${rcpt}" + else + if [[ "${SMTP_VERBOSE}" == "" ]]; then + log_msg "Error while sending notification email to ${rcpt}. You can enable verbose logging by setting 'WATCHDOG_VERBOSE=y' in mailcow.conf." + else + log_msg "Error while sending notification email to ${rcpt}." + fi + fi done } @@ -154,7 +172,7 @@ get_container_ip() { CONTAINER_ID=($(printf "%s\n" "${CONTAINER_ID[@]}" | shuf)) if [[ ! -z ${CONTAINER_ID} ]]; then for matched_container in "${CONTAINER_ID[@]}"; do - CONTAINER_IPS=($(curl --silent --insecure https://dockerapi/containers/${matched_container}/json | jq -r '.NetworkSettings.Networks[].IPAddress')) + CONTAINER_IPS=($(curl --silent --insecure https://dockerapi/containers/${matched_container}/json | jq -r '.NetworkSettings.Networks[].IPAddress')) for ip_match in "${CONTAINER_IPS[@]}"; do # grep will do nothing if one of these vars is empty [[ -z ${ip_match} ]] && continue diff --git a/docker-compose.yml b/docker-compose.yml index 7a63f497..033a6685 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -443,9 +443,7 @@ services: - /lib/modules:/lib/modules:ro watchdog-mailcow: - image: mailcow/watchdog:1.94 - # Debug - #command: /watchdog.sh + image: mailcow/watchdog:1.95 dns: - ${IPV4_NETWORK:-172.22.1}.254 tmpfs: @@ -470,6 +468,7 @@ services: - WATCHDOG_SUBJECT=${WATCHDOG_SUBJECT:-Watchdog ALERT} - WATCHDOG_EXTERNAL_CHECKS=${WATCHDOG_EXTERNAL_CHECKS:-n} - WATCHDOG_MYSQL_REPLICATION_CHECKS=${WATCHDOG_MYSQL_REPLICATION_CHECKS:-n} + - WATCHDOG_VERBOSE=${WATCHDOG_VERBOSE:-n} - MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME} - COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mailcow-dockerized} - IPV4_NETWORK=${IPV4_NETWORK:-172.22.1} diff --git a/generate_config.sh b/generate_config.sh index 9aea784e..93343818 100755 --- a/generate_config.sh +++ b/generate_config.sh @@ -287,6 +287,9 @@ WATCHDOG_NOTIFY_BAN=n # Will only work with unmodified mailcow setups. WATCHDOG_EXTERNAL_CHECKS=n +# Enable watchdog verbose logging +WATCHDOG_VERBOSE=n + # Max log lines per service to keep in Redis logs LOG_LINES=9999 diff --git a/update.sh b/update.sh index 1bbc1c5b..cf093eab 100755 --- a/update.sh +++ b/update.sh @@ -513,6 +513,11 @@ for option in ${CONFIG_ARRAY[@]}; do echo '# https://mailcow.github.io/mailcow-dockerized-docs/debug-reset-tls/' >> mailcow.conf echo 'ACME_CONTACT=' >> mailcow.conf fi +elif [[ ${option} == "WATCHDOG_VERBOSE" ]]; then + if ! grep -q ${option} mailcow.conf; then + echo '# Enable watchdog verbose logging' >> mailcow.conf + echo 'WATCHDOG_VERBOSE=n' >> mailcow.conf + fi elif ! grep -q ${option} mailcow.conf; then echo "Adding new option \"${option}\" to mailcow.conf" echo "${option}=n" >> mailcow.conf