paxton
/
mailcow
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Watchdog] Do also log errors to Redis if availble

master
andryyy 2017-10-04 23:18:51 +02:00
parent 073c6c6e73
commit 9860d44d04
2 changed files with 16 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
data/Dockerfiles/watchdog/Dockerfile
View File

@ -15,7 +15,8 @@ RUN apk add --update \
nagios-plugins-mysql \ nagios-plugins-mysql \
nagios-plugins-dns \ nagios-plugins-dns \
nagios-plugins-disk \ nagios-plugins-disk \
bind-tools bind-tools \
redis
COPY watchdog.sh /watchdog.sh COPY watchdog.sh /watchdog.sh

14
data/Dockerfiles/watchdog/watchdog.sh
View File

@ -34,6 +34,10 @@ progress() {
echo "(${DIFF})" echo "(${DIFF})"
} }
log_to_redis() {
redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}")\"}"
}
get_container_ip() { get_container_ip() {
# ${1} is container # ${1} is container
CONTAINER_ID= CONTAINER_ID=
@ -228,6 +232,7 @@ dns_checks() {
( (
while true; do while true; do
if ! nginx_checks; then if ! nginx_checks; then
log_to_redis "Nginx hit error limit"
echo -e "\e[31m$(date) - Nginx hit error limit\e[0m" echo -e "\e[31m$(date) - Nginx hit error limit\e[0m"
echo nginx-mailcow > /tmp/com_pipe echo nginx-mailcow > /tmp/com_pipe
fi fi
@ -238,6 +243,7 @@ BACKGROUND_TASKS+=($!)
( (
while true; do while true; do
if ! mysql_checks; then if ! mysql_checks; then
log_to_redis "MySQL hit error limit"
echo -e "\e[31m$(date) - MySQL hit error limit\e[0m" echo -e "\e[31m$(date) - MySQL hit error limit\e[0m"
echo mysql-mailcow > /tmp/com_pipe echo mysql-mailcow > /tmp/com_pipe
fi fi
@ -248,6 +254,7 @@ BACKGROUND_TASKS+=($!)
( (
while true; do while true; do
if ! phpfpm_checks; then if ! phpfpm_checks; then
log_to_redis "PHP-FPM hit error limit"
echo -e "\e[31m$(date) - PHP-FPM hit error limit\e[0m" echo -e "\e[31m$(date) - PHP-FPM hit error limit\e[0m"
echo php-fpm-mailcow > /tmp/com_pipe echo php-fpm-mailcow > /tmp/com_pipe
fi fi
@ -258,6 +265,7 @@ BACKGROUND_TASKS+=($!)
( (
while true; do while true; do
if ! sogo_checks; then if ! sogo_checks; then
log_to_redis "SOGo hit error limit"
echo -e "\e[31m$(date) - SOGo hit error limit\e[0m" echo -e "\e[31m$(date) - SOGo hit error limit\e[0m"
echo sogo-mailcow > /tmp/com_pipe echo sogo-mailcow > /tmp/com_pipe
fi fi
@ -268,6 +276,7 @@ BACKGROUND_TASKS+=($!)
( (
while true; do while true; do
if ! postfix_checks; then if ! postfix_checks; then
log_to_redis "Postfix hit error limit"
echo -e "\e[31m$(date) - Postfix hit error limit\e[0m" echo -e "\e[31m$(date) - Postfix hit error limit\e[0m"
echo postfix-mailcow > /tmp/com_pipe echo postfix-mailcow > /tmp/com_pipe
fi fi
@ -278,6 +287,7 @@ BACKGROUND_TASKS+=($!)
( (
while true; do while true; do
if ! dovecot_checks; then if ! dovecot_checks; then
log_to_redis "Dovecot hit error limit"
echo -e "\e[31m$(date) - Dovecot hit error limit\e[0m" echo -e "\e[31m$(date) - Dovecot hit error limit\e[0m"
echo dovecot-mailcow > /tmp/com_pipe echo dovecot-mailcow > /tmp/com_pipe
fi fi
@ -288,6 +298,7 @@ BACKGROUND_TASKS+=($!)
( (
while true; do while true; do
if ! dns_checks; then if ! dns_checks; then
log_to_redis "Unbound hit error limit"
echo -e "\e[31m$(date) - Unbound hit error limit\e[0m" echo -e "\e[31m$(date) - Unbound hit error limit\e[0m"
#echo unbound-mailcow > /tmp/com_pipe #echo unbound-mailcow > /tmp/com_pipe
fi fi
@ -298,6 +309,7 @@ BACKGROUND_TASKS+=($!)
( (
while true; do while true; do
if ! rspamd_checks; then if ! rspamd_checks; then
log_to_redis "Rspamd hit error limit"
echo -e "\e[31m$(date) - Rspamd hit error limit\e[0m" echo -e "\e[31m$(date) - Rspamd hit error limit\e[0m"
echo rspamd-mailcow > /tmp/com_pipe echo rspamd-mailcow > /tmp/com_pipe
fi fi
@ -311,6 +323,7 @@ while true; do
for bg_task in ${BACKGROUND_TASKS[*]}; do for bg_task in ${BACKGROUND_TASKS[*]}; do
if ! kill -0 ${bg_task} 21>&2; then if ! kill -0 ${bg_task} 21>&2; then
echo "Worker ${bg_task} died, stopping watchdog and waiting for respawn..." echo "Worker ${bg_task} died, stopping watchdog and waiting for respawn..."
log_to_redis "Worker ${bg_task} died, stopping watchdog and waiting for respawn..."
kill -TERM ${PARENT_PID} kill -TERM ${PARENT_PID}
fi fi
sleep 1 sleep 1
@ -327,6 +340,7 @@ while true; do
sleep 3 sleep 3
CONTAINER_ID=$(curl --silent --unix-socket /var/run/docker.sock http/containers/json?all=1 | jq -rc "map(select(.Names[] | contains (\"${com_pipe_answer}\"))) | .[] .Id") CONTAINER_ID=$(curl --silent --unix-socket /var/run/docker.sock http/containers/json?all=1 | jq -rc "map(select(.Names[] | contains (\"${com_pipe_answer}\"))) | .[] .Id")
if [[ ! -z ${CONTAINER_ID} ]]; then if [[ ! -z ${CONTAINER_ID} ]]; then
log_to_redis "Sending restart command to ${CONTAINER_ID}..."
echo "Sending restart command to ${CONTAINER_ID}..." echo "Sending restart command to ${CONTAINER_ID}..."
curl --silent --unix-socket /var/run/docker.sock -XPOST http/containers/${CONTAINER_ID}/restart curl --silent --unix-socket /var/run/docker.sock -XPOST http/containers/${CONTAINER_ID}/restart
fi fi