[Watchdog] Add Watchdog verbose logging (#4299)

* [Watchdog] Add verbose logging

* [Watchdog] More verbose debugging

* [Watchdog] Enable MX check for recipients

Co-authored-by: andryyy <andre.peters@debinux.de>

data/Dockerfiles/watchdog/Dockerfile

@@ -36,4 +36,4 @@ RUN apk add --update \
 COPY watchdog.sh /watchdog.sh
 COPY check_mysql_slavestatus.sh /usr/lib/nagios/plugins/check_mysql_slavestatus.sh
 
-CMD /watchdog.sh 2> /dev/null
+CMD /watchdog.sh

data/Dockerfiles/watchdog/watchdog.sh

@@ -6,7 +6,10 @@ trap "kill 0" EXIT
 # Prepare
 BACKGROUND_TASKS=()
 echo "Waiting for containers to settle..."
-sleep 30
+for i in {30..1}; do
+  echo "${i}"
+  sleep 1
+done
 
 if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then
   echo -e "$(date) - USE_WATCHDOG=n, skipping watchdog..."
@@ -14,6 +17,14 @@ if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then
   exec $(readlink -f "$0")
 fi
 
+if [[ "${WATCHDOG_VERBOSE}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  SMTP_VERBOSE="--verbose"
+  set -xv
+else
+  SMTP_VERBOSE=""
+  exec 2>/dev/null
+fi
+
 # Checks pipe their corresponding container name in this pipe
 if [[ ! -p /tmp/com_pipe ]]; then
   mkfifo /tmp/com_pipe
@@ -114,16 +125,16 @@ function mail_error() {
   IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}"
   for rcpt in "${MAIL_RCPTS[@]}"; do
     RCPT_DOMAIN=
-    #RCPT_MX=
+    RCPT_MX=
     RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'})
-    # Latest smtp-cli looks up mx via dns
+    CHECK_FOR_VALID_MX=$(dig +short ${RCPT_DOMAIN} mx)
-    #RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}')
+    if [[ -z ${CHECK_FOR_VALID_MX} ]]; then
-    #if [[ -z ${RCPT_MX} ]]; then
+      log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
-    #  log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
+      return 1
-    #  return 1
+    fi
-    #fi
     [ -f "/tmp/${1}" ] && BODY="/tmp/${1}"
     timeout 10s ./smtp-cli --missing-modules-ok \
+      "${SMTP_VERBOSE}" \
       --charset=UTF-8 \
       --subject="${SUBJECT}" \
       --body-plain="${BODY}" \
@@ -132,8 +143,15 @@ function mail_error() {
       --from="watchdog@${MAILCOW_HOSTNAME}" \
       --hello-host=${MAILCOW_HOSTNAME} \
       --ipv4
-      #--server="${RCPT_MX}"
+    if [[ $? -eq 1 ]]; then # exit code 1 is fine
-    log_msg "Sent notification email to ${rcpt}"
+      log_msg "Sent notification email to ${rcpt}"
+    else
+      if [[ "${SMTP_VERBOSE}" == "" ]]; then
+        log_msg "Error while sending notification email to ${rcpt}. You can enable verbose logging by setting 'WATCHDOG_VERBOSE=y' in mailcow.conf."
+      else
+        log_msg "Error while sending notification email to ${rcpt}."
+      fi
+    fi
   done
 }
 
@@ -154,7 +172,7 @@ get_container_ip() {
       CONTAINER_ID=($(printf "%s\n" "${CONTAINER_ID[@]}" | shuf))
       if [[ ! -z ${CONTAINER_ID} ]]; then
         for matched_container in "${CONTAINER_ID[@]}"; do
-          CONTAINER_IPS=($(curl --silent --insecure https://dockerapi/containers/${matched_container}/json | jq -r '.NetworkSettings.Networks[].IPAddress')) 
+          CONTAINER_IPS=($(curl --silent --insecure https://dockerapi/containers/${matched_container}/json | jq -r '.NetworkSettings.Networks[].IPAddress'))
           for ip_match in "${CONTAINER_IPS[@]}"; do
             # grep will do nothing if one of these vars is empty
             [[ -z ${ip_match} ]] && continue

docker-compose.yml

@@ -443,9 +443,7 @@ services:
         - /lib/modules:/lib/modules:ro
 
     watchdog-mailcow:
-      image: mailcow/watchdog:1.94
+      image: mailcow/watchdog:1.95
-      # Debug
-      #command: /watchdog.sh
       dns:
         - ${IPV4_NETWORK:-172.22.1}.254
       tmpfs:
@@ -470,6 +468,7 @@ services:
         - WATCHDOG_SUBJECT=${WATCHDOG_SUBJECT:-Watchdog ALERT}
         - WATCHDOG_EXTERNAL_CHECKS=${WATCHDOG_EXTERNAL_CHECKS:-n}
         - WATCHDOG_MYSQL_REPLICATION_CHECKS=${WATCHDOG_MYSQL_REPLICATION_CHECKS:-n}
+        - WATCHDOG_VERBOSE=${WATCHDOG_VERBOSE:-n}
         - MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
         - COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mailcow-dockerized}
         - IPV4_NETWORK=${IPV4_NETWORK:-172.22.1}

generate_config.sh

@@ -287,6 +287,9 @@ WATCHDOG_NOTIFY_BAN=n
 # Will only work with unmodified mailcow setups.
 WATCHDOG_EXTERNAL_CHECKS=n
 
+# Enable watchdog verbose logging
+WATCHDOG_VERBOSE=n
+
 # Max log lines per service to keep in Redis logs
 
 LOG_LINES=9999

update.sh

@@ -513,6 +513,11 @@ for option in ${CONFIG_ARRAY[@]}; do
       echo '# https://mailcow.github.io/mailcow-dockerized-docs/debug-reset-tls/' >> mailcow.conf
       echo 'ACME_CONTACT=' >> mailcow.conf
   fi
+elif [[ ${option} == "WATCHDOG_VERBOSE" ]]; then
+    if ! grep -q ${option} mailcow.conf; then
+      echo '# Enable watchdog verbose logging' >> mailcow.conf
+      echo 'WATCHDOG_VERBOSE=n' >> mailcow.conf
+  fi
   elif ! grep -q ${option} mailcow.conf; then
     echo "Adding new option \"${option}\" to mailcow.conf"
     echo "${option}=n" >> mailcow.conf