[Watchdog] Add Watchdog verbose logging (#4299)
* [Watchdog] Add verbose logging * [Watchdog] More verbose debugging * [Watchdog] Enable MX check for recipients Co-authored-by: andryyy <andre.peters@debinux.de>master
parent
15c0b3f7b7
commit
6bf70cf846
|
@ -36,4 +36,4 @@ RUN apk add --update \
|
||||||
COPY watchdog.sh /watchdog.sh
|
COPY watchdog.sh /watchdog.sh
|
||||||
COPY check_mysql_slavestatus.sh /usr/lib/nagios/plugins/check_mysql_slavestatus.sh
|
COPY check_mysql_slavestatus.sh /usr/lib/nagios/plugins/check_mysql_slavestatus.sh
|
||||||
|
|
||||||
CMD /watchdog.sh 2> /dev/null
|
CMD /watchdog.sh
|
||||||
|
|
|
@ -6,7 +6,10 @@ trap "kill 0" EXIT
|
||||||
# Prepare
|
# Prepare
|
||||||
BACKGROUND_TASKS=()
|
BACKGROUND_TASKS=()
|
||||||
echo "Waiting for containers to settle..."
|
echo "Waiting for containers to settle..."
|
||||||
sleep 30
|
for i in {30..1}; do
|
||||||
|
echo "${i}"
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then
|
if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then
|
||||||
echo -e "$(date) - USE_WATCHDOG=n, skipping watchdog..."
|
echo -e "$(date) - USE_WATCHDOG=n, skipping watchdog..."
|
||||||
|
@ -14,6 +17,14 @@ if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then
|
||||||
exec $(readlink -f "$0")
|
exec $(readlink -f "$0")
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [[ "${WATCHDOG_VERBOSE}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
|
||||||
|
SMTP_VERBOSE="--verbose"
|
||||||
|
set -xv
|
||||||
|
else
|
||||||
|
SMTP_VERBOSE=""
|
||||||
|
exec 2>/dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
# Checks pipe their corresponding container name in this pipe
|
# Checks pipe their corresponding container name in this pipe
|
||||||
if [[ ! -p /tmp/com_pipe ]]; then
|
if [[ ! -p /tmp/com_pipe ]]; then
|
||||||
mkfifo /tmp/com_pipe
|
mkfifo /tmp/com_pipe
|
||||||
|
@ -114,16 +125,16 @@ function mail_error() {
|
||||||
IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}"
|
IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}"
|
||||||
for rcpt in "${MAIL_RCPTS[@]}"; do
|
for rcpt in "${MAIL_RCPTS[@]}"; do
|
||||||
RCPT_DOMAIN=
|
RCPT_DOMAIN=
|
||||||
#RCPT_MX=
|
RCPT_MX=
|
||||||
RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'})
|
RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'})
|
||||||
# Latest smtp-cli looks up mx via dns
|
CHECK_FOR_VALID_MX=$(dig +short ${RCPT_DOMAIN} mx)
|
||||||
#RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}')
|
if [[ -z ${CHECK_FOR_VALID_MX} ]]; then
|
||||||
#if [[ -z ${RCPT_MX} ]]; then
|
log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
|
||||||
# log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
|
return 1
|
||||||
# return 1
|
fi
|
||||||
#fi
|
|
||||||
[ -f "/tmp/${1}" ] && BODY="/tmp/${1}"
|
[ -f "/tmp/${1}" ] && BODY="/tmp/${1}"
|
||||||
timeout 10s ./smtp-cli --missing-modules-ok \
|
timeout 10s ./smtp-cli --missing-modules-ok \
|
||||||
|
"${SMTP_VERBOSE}" \
|
||||||
--charset=UTF-8 \
|
--charset=UTF-8 \
|
||||||
--subject="${SUBJECT}" \
|
--subject="${SUBJECT}" \
|
||||||
--body-plain="${BODY}" \
|
--body-plain="${BODY}" \
|
||||||
|
@ -132,8 +143,15 @@ function mail_error() {
|
||||||
--from="watchdog@${MAILCOW_HOSTNAME}" \
|
--from="watchdog@${MAILCOW_HOSTNAME}" \
|
||||||
--hello-host=${MAILCOW_HOSTNAME} \
|
--hello-host=${MAILCOW_HOSTNAME} \
|
||||||
--ipv4
|
--ipv4
|
||||||
#--server="${RCPT_MX}"
|
if [[ $? -eq 1 ]]; then # exit code 1 is fine
|
||||||
log_msg "Sent notification email to ${rcpt}"
|
log_msg "Sent notification email to ${rcpt}"
|
||||||
|
else
|
||||||
|
if [[ "${SMTP_VERBOSE}" == "" ]]; then
|
||||||
|
log_msg "Error while sending notification email to ${rcpt}. You can enable verbose logging by setting 'WATCHDOG_VERBOSE=y' in mailcow.conf."
|
||||||
|
else
|
||||||
|
log_msg "Error while sending notification email to ${rcpt}."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -443,9 +443,7 @@ services:
|
||||||
- /lib/modules:/lib/modules:ro
|
- /lib/modules:/lib/modules:ro
|
||||||
|
|
||||||
watchdog-mailcow:
|
watchdog-mailcow:
|
||||||
image: mailcow/watchdog:1.94
|
image: mailcow/watchdog:1.95
|
||||||
# Debug
|
|
||||||
#command: /watchdog.sh
|
|
||||||
dns:
|
dns:
|
||||||
- ${IPV4_NETWORK:-172.22.1}.254
|
- ${IPV4_NETWORK:-172.22.1}.254
|
||||||
tmpfs:
|
tmpfs:
|
||||||
|
@ -470,6 +468,7 @@ services:
|
||||||
- WATCHDOG_SUBJECT=${WATCHDOG_SUBJECT:-Watchdog ALERT}
|
- WATCHDOG_SUBJECT=${WATCHDOG_SUBJECT:-Watchdog ALERT}
|
||||||
- WATCHDOG_EXTERNAL_CHECKS=${WATCHDOG_EXTERNAL_CHECKS:-n}
|
- WATCHDOG_EXTERNAL_CHECKS=${WATCHDOG_EXTERNAL_CHECKS:-n}
|
||||||
- WATCHDOG_MYSQL_REPLICATION_CHECKS=${WATCHDOG_MYSQL_REPLICATION_CHECKS:-n}
|
- WATCHDOG_MYSQL_REPLICATION_CHECKS=${WATCHDOG_MYSQL_REPLICATION_CHECKS:-n}
|
||||||
|
- WATCHDOG_VERBOSE=${WATCHDOG_VERBOSE:-n}
|
||||||
- MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
|
- MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
|
||||||
- COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mailcow-dockerized}
|
- COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mailcow-dockerized}
|
||||||
- IPV4_NETWORK=${IPV4_NETWORK:-172.22.1}
|
- IPV4_NETWORK=${IPV4_NETWORK:-172.22.1}
|
||||||
|
|
|
@ -287,6 +287,9 @@ WATCHDOG_NOTIFY_BAN=n
|
||||||
# Will only work with unmodified mailcow setups.
|
# Will only work with unmodified mailcow setups.
|
||||||
WATCHDOG_EXTERNAL_CHECKS=n
|
WATCHDOG_EXTERNAL_CHECKS=n
|
||||||
|
|
||||||
|
# Enable watchdog verbose logging
|
||||||
|
WATCHDOG_VERBOSE=n
|
||||||
|
|
||||||
# Max log lines per service to keep in Redis logs
|
# Max log lines per service to keep in Redis logs
|
||||||
|
|
||||||
LOG_LINES=9999
|
LOG_LINES=9999
|
||||||
|
|
|
@ -513,6 +513,11 @@ for option in ${CONFIG_ARRAY[@]}; do
|
||||||
echo '# https://mailcow.github.io/mailcow-dockerized-docs/debug-reset-tls/' >> mailcow.conf
|
echo '# https://mailcow.github.io/mailcow-dockerized-docs/debug-reset-tls/' >> mailcow.conf
|
||||||
echo 'ACME_CONTACT=' >> mailcow.conf
|
echo 'ACME_CONTACT=' >> mailcow.conf
|
||||||
fi
|
fi
|
||||||
|
elif [[ ${option} == "WATCHDOG_VERBOSE" ]]; then
|
||||||
|
if ! grep -q ${option} mailcow.conf; then
|
||||||
|
echo '# Enable watchdog verbose logging' >> mailcow.conf
|
||||||
|
echo 'WATCHDOG_VERBOSE=n' >> mailcow.conf
|
||||||
|
fi
|
||||||
elif ! grep -q ${option} mailcow.conf; then
|
elif ! grep -q ${option} mailcow.conf; then
|
||||||
echo "Adding new option \"${option}\" to mailcow.conf"
|
echo "Adding new option \"${option}\" to mailcow.conf"
|
||||||
echo "${option}=n" >> mailcow.conf
|
echo "${option}=n" >> mailcow.conf
|
||||||
|
|
Loading…
Reference in New Issue