diff --git a/data/Dockerfiles/watchdog/watchdog.sh b/data/Dockerfiles/watchdog/watchdog.sh index a76d5d9c..932f8ef5 100755 --- a/data/Dockerfiles/watchdog/watchdog.sh +++ b/data/Dockerfiles/watchdog/watchdog.sh @@ -19,6 +19,17 @@ if [[ ! -p /tmp/com_pipe ]]; then mkfifo /tmp/com_pipe fi +# Wait for containers +while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do + echo "Waiting for SQL..." + sleep 2 +done + +until [[ $(redis-cli -h redis-mailcow PING) == "PONG" ]]; do + echo "Waiting for Redis..." + sleep 2 +done + redis-cli -h redis-mailcow DEL F2B_RES > /dev/null # Common functions @@ -184,6 +195,31 @@ unbound_checks() { return 1 } +redis_checks() { + err_count=0 + diff_c=0 + THRESHOLD=5 + # Reduce error count by 2 after restarting an unhealthy container + trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 + while [ ${err_count} -lt ${THRESHOLD} ]; do + touch /tmp/redis-mailcow; echo "$(tail -50 /tmp/redis-mailcow)" > /tmp/redis-mailcow + host_ip=$(get_container_ip redis-mailcow) + err_c_cur=${err_count} + /usr/lib/nagios/plugins/check_tcp -4 -H redis-mailcow -p 6379 -E -s "PING\n" -q "QUIT" -e "PONG" 2>> /tmp/redis-mailcow 1>&2; err_count=$(( ${err_count} + $? )) + [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 + [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) + progress "Redis" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} + if [[ $? == 10 ]]; then + diff_c=0 + sleep 1 + else + diff_c=0 + sleep $(( ( RANDOM % 60 ) + 20 )) + fi + done + return 1 +} + mysql_checks() { err_count=0 diff_c=0 @@ -559,6 +595,18 @@ PID=$! echo "Spawned mysql_checks with PID ${PID}" BACKGROUND_TASKS+=(${PID}) +( +while true; do + if ! redis_checks; then + log_msg "Redis hit error limit" + echo redis-mailcow > /tmp/com_pipe + fi +done +) & +PID=$! +echo "Spawned redis_checks with PID ${PID}" +BACKGROUND_TASKS+=(${PID}) + ( while true; do if ! phpfpm_checks; then diff --git a/docker-compose.yml b/docker-compose.yml index c52d63f4..0563cbdb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -360,7 +360,7 @@ services: - /lib/modules:/lib/modules:ro watchdog-mailcow: - image: mailcow/watchdog:1.57 + image: mailcow/watchdog:1.58 # Debug #command: /watchdog.sh build: ./data/Dockerfiles/watchdog