[Watchdog] Use Redis master for write operations

master
andryyy 2020-02-05 11:01:09 +01:00
parent 9dec216261
commit 1ad469a24a
No known key found for this signature in database
GPG Key ID: 8EC34FF2794E25EF
1 changed files with 19 additions and 11 deletions

View File

@ -30,7 +30,14 @@ until [[ $(redis-cli -h redis-mailcow PING) == "PONG" ]]; do
sleep 2 sleep 2
done done
redis-cli -h redis-mailcow DEL F2B_RES > /dev/null # Do not attempt to write to slave
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT}"
else
REDIS_CMDLINE="redis-cli -h redis -p 6379"
fi
${REDIS_CMDLINE} DEL F2B_RES > /dev/null
# Common functions # Common functions
get_ipv6(){ get_ipv6(){
@ -65,7 +72,7 @@ progress() {
[[ ${CURRENT} -gt ${TOTAL} ]] && return [[ ${CURRENT} -gt ${TOTAL} ]] && return
[[ ${CURRENT} -lt 0 ]] && CURRENT=0 [[ ${CURRENT} -lt 0 ]] && CURRENT=0
PERCENT=$(( 200 * ${CURRENT} / ${TOTAL} % 2 + 100 * ${CURRENT} / ${TOTAL} )) PERCENT=$(( 200 * ${CURRENT} / ${TOTAL} % 2 + 100 * ${CURRENT} / ${TOTAL} ))
redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null ${REDIS_CMDLINE} LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null
log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}" no_redis log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}" no_redis
# Return 10 to indicate a dead service # Return 10 to indicate a dead service
[ ${CURRENT} -le 0 ] && return 10 [ ${CURRENT} -le 0 ] && return 10
@ -73,7 +80,7 @@ progress() {
log_msg() { log_msg() {
if [[ ${2} != "no_redis" ]]; then if [[ ${2} != "no_redis" ]]; then
redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \ ${REDIS_CMDLINE} LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
tr '\r\n%&;$"_[]{}-' ' ')\"}" > /dev/null tr '\r\n%&;$"_[]{}-' ' ')\"}" > /dev/null
fi fi
echo $(date) $(printf '%s\n' "${1}") echo $(date) $(printf '%s\n' "${1}")
@ -252,6 +259,7 @@ unbound_checks() {
} }
redis_checks() { redis_checks() {
# A check for the local redis container
err_count=0 err_count=0
diff_c=0 diff_c=0
THRESHOLD=5 THRESHOLD=5
@ -465,20 +473,20 @@ fail2ban_checks() {
err_count=0 err_count=0
diff_c=0 diff_c=0
THRESHOLD=1 THRESHOLD=1
F2B_LOG_STATUS=($(redis-cli -h redis-mailcow --raw HKEYS F2B_ACTIVE_BANS)) F2B_LOG_STATUS=($(${REDIS_CMDLINE} --raw HKEYS F2B_ACTIVE_BANS))
F2B_RES= F2B_RES=
# Reduce error count by 2 after restarting an unhealthy container # Reduce error count by 2 after restarting an unhealthy container
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
while [ ${err_count} -lt ${THRESHOLD} ]; do while [ ${err_count} -lt ${THRESHOLD} ]; do
err_c_cur=${err_count} err_c_cur=${err_count}
F2B_LOG_STATUS_PREV=(${F2B_LOG_STATUS[@]}) F2B_LOG_STATUS_PREV=(${F2B_LOG_STATUS[@]})
F2B_LOG_STATUS=($(redis-cli -h redis-mailcow --raw HKEYS F2B_ACTIVE_BANS)) F2B_LOG_STATUS=($(${REDIS_CMDLINE} --raw HKEYS F2B_ACTIVE_BANS))
array_diff F2B_RES F2B_LOG_STATUS F2B_LOG_STATUS_PREV array_diff F2B_RES F2B_LOG_STATUS F2B_LOG_STATUS_PREV
if [[ ! -z "${F2B_RES}" ]]; then if [[ ! -z "${F2B_RES}" ]]; then
err_count=$(( ${err_count} + 1 )) err_count=$(( ${err_count} + 1 ))
echo -n "${F2B_RES[@]}" | tr -cd "[a-fA-F0-9.:/] " | timeout 3s redis-cli -x -h redis-mailcow SET F2B_RES > /dev/null echo -n "${F2B_RES[@]}" | tr -cd "[a-fA-F0-9.:/] " | timeout 3s ${REDIS_CMDLINE} -x SET F2B_RES > /dev/null
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
redis-cli -x -h redis-mailcow DEL F2B_RES ${REDIS_CMDLINE} -x DEL F2B_RES
fi fi
fi fi
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
@ -501,7 +509,7 @@ acme_checks() {
THRESHOLD=1 THRESHOLD=1
ACME_LOG_STATUS=$(redis-cli -h redis GET ACME_FAIL_TIME) ACME_LOG_STATUS=$(redis-cli -h redis GET ACME_FAIL_TIME)
if [[ -z "${ACME_LOG_STATUS}" ]]; then if [[ -z "${ACME_LOG_STATUS}" ]]; then
redis-cli -h redis SET ACME_FAIL_TIME 0 ${REDIS_CMDLINE} SET ACME_FAIL_TIME 0
ACME_LOG_STATUS=0 ACME_LOG_STATUS=0
fi fi
# Reduce error count by 2 after restarting an unhealthy container # Reduce error count by 2 after restarting an unhealthy container
@ -673,7 +681,7 @@ BACKGROUND_TASKS+=(${PID})
( (
while true; do while true; do
if ! redis_checks; then if ! redis_checks; then
log_msg "Redis hit error limit" log_msg "Local Redis hit error limit"
echo redis-mailcow > /tmp/com_pipe echo redis-mailcow > /tmp/com_pipe
fi fi
done done
@ -877,9 +885,9 @@ while true; do
log_msg "acme-mailcow did not complete successfully" log_msg "acme-mailcow did not complete successfully"
[[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check acme-mailcow for further information." [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check acme-mailcow for further information."
elif [[ ${com_pipe_answer} == "fail2ban" ]]; then elif [[ ${com_pipe_answer} == "fail2ban" ]]; then
F2B_RES=($(timeout 4s redis-cli -h redis-mailcow --raw GET F2B_RES 2> /dev/null)) F2B_RES=($(timeout 4s ${REDIS_CMDLINE} --raw GET F2B_RES 2> /dev/null))
if [[ ! -z "${F2B_RES}" ]]; then if [[ ! -z "${F2B_RES}" ]]; then
redis-cli -h redis-mailcow DEL F2B_RES > /dev/null ${REDIS_CMDLINE} DEL F2B_RES > /dev/null
host= host=
for host in "${F2B_RES[@]}"; do for host in "${F2B_RES[@]}"; do
log_msg "Banned ${host}" log_msg "Banned ${host}"