From 1ad469a24a461b19352767af3e90d780a327aec4 Mon Sep 17 00:00:00 2001 From: andryyy Date: Wed, 5 Feb 2020 11:01:09 +0100 Subject: [PATCH] [Watchdog] Use Redis master for write operations --- data/Dockerfiles/watchdog/watchdog.sh | 30 +++++++++++++++++---------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/data/Dockerfiles/watchdog/watchdog.sh b/data/Dockerfiles/watchdog/watchdog.sh index 6b0875b3..c563f5a3 100755 --- a/data/Dockerfiles/watchdog/watchdog.sh +++ b/data/Dockerfiles/watchdog/watchdog.sh @@ -30,7 +30,14 @@ until [[ $(redis-cli -h redis-mailcow PING) == "PONG" ]]; do sleep 2 done -redis-cli -h redis-mailcow DEL F2B_RES > /dev/null +# Do not attempt to write to slave +if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then + REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT}" +else + REDIS_CMDLINE="redis-cli -h redis -p 6379" +fi + +${REDIS_CMDLINE} DEL F2B_RES > /dev/null # Common functions get_ipv6(){ @@ -65,7 +72,7 @@ progress() { [[ ${CURRENT} -gt ${TOTAL} ]] && return [[ ${CURRENT} -lt 0 ]] && CURRENT=0 PERCENT=$(( 200 * ${CURRENT} / ${TOTAL} % 2 + 100 * ${CURRENT} / ${TOTAL} )) - redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null + ${REDIS_CMDLINE} LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}" no_redis # Return 10 to indicate a dead service [ ${CURRENT} -le 0 ] && return 10 @@ -73,7 +80,7 @@ progress() { log_msg() { if [[ ${2} != "no_redis" ]]; then - redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \ + ${REDIS_CMDLINE} LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \ tr '\r\n%&;$"_[]{}-' ' ')\"}" > /dev/null fi echo $(date) $(printf '%s\n' "${1}") @@ -252,6 +259,7 @@ unbound_checks() { } redis_checks() { + # A check for the local redis container err_count=0 diff_c=0 THRESHOLD=5 @@ -465,20 +473,20 @@ fail2ban_checks() { err_count=0 diff_c=0 THRESHOLD=1 - F2B_LOG_STATUS=($(redis-cli -h redis-mailcow --raw HKEYS F2B_ACTIVE_BANS)) + F2B_LOG_STATUS=($(${REDIS_CMDLINE} --raw HKEYS F2B_ACTIVE_BANS)) F2B_RES= # Reduce error count by 2 after restarting an unhealthy container trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 while [ ${err_count} -lt ${THRESHOLD} ]; do err_c_cur=${err_count} F2B_LOG_STATUS_PREV=(${F2B_LOG_STATUS[@]}) - F2B_LOG_STATUS=($(redis-cli -h redis-mailcow --raw HKEYS F2B_ACTIVE_BANS)) + F2B_LOG_STATUS=($(${REDIS_CMDLINE} --raw HKEYS F2B_ACTIVE_BANS)) array_diff F2B_RES F2B_LOG_STATUS F2B_LOG_STATUS_PREV if [[ ! -z "${F2B_RES}" ]]; then err_count=$(( ${err_count} + 1 )) - echo -n "${F2B_RES[@]}" | tr -cd "[a-fA-F0-9.:/] " | timeout 3s redis-cli -x -h redis-mailcow SET F2B_RES > /dev/null + echo -n "${F2B_RES[@]}" | tr -cd "[a-fA-F0-9.:/] " | timeout 3s ${REDIS_CMDLINE} -x SET F2B_RES > /dev/null if [ $? -ne 0 ]; then - redis-cli -x -h redis-mailcow DEL F2B_RES + ${REDIS_CMDLINE} -x DEL F2B_RES fi fi [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 @@ -501,7 +509,7 @@ acme_checks() { THRESHOLD=1 ACME_LOG_STATUS=$(redis-cli -h redis GET ACME_FAIL_TIME) if [[ -z "${ACME_LOG_STATUS}" ]]; then - redis-cli -h redis SET ACME_FAIL_TIME 0 + ${REDIS_CMDLINE} SET ACME_FAIL_TIME 0 ACME_LOG_STATUS=0 fi # Reduce error count by 2 after restarting an unhealthy container @@ -673,7 +681,7 @@ BACKGROUND_TASKS+=(${PID}) ( while true; do if ! redis_checks; then - log_msg "Redis hit error limit" + log_msg "Local Redis hit error limit" echo redis-mailcow > /tmp/com_pipe fi done @@ -877,9 +885,9 @@ while true; do log_msg "acme-mailcow did not complete successfully" [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check acme-mailcow for further information." elif [[ ${com_pipe_answer} == "fail2ban" ]]; then - F2B_RES=($(timeout 4s redis-cli -h redis-mailcow --raw GET F2B_RES 2> /dev/null)) + F2B_RES=($(timeout 4s ${REDIS_CMDLINE} --raw GET F2B_RES 2> /dev/null)) if [[ ! -z "${F2B_RES}" ]]; then - redis-cli -h redis-mailcow DEL F2B_RES > /dev/null + ${REDIS_CMDLINE} DEL F2B_RES > /dev/null host= for host in "${F2B_RES[@]}"; do log_msg "Banned ${host}"