[Watchdog] Use Redis master for write operations
parent
9dec216261
commit
1ad469a24a
|
@ -30,7 +30,14 @@ until [[ $(redis-cli -h redis-mailcow PING) == "PONG" ]]; do
|
||||||
sleep 2
|
sleep 2
|
||||||
done
|
done
|
||||||
|
|
||||||
redis-cli -h redis-mailcow DEL F2B_RES > /dev/null
|
# Do not attempt to write to slave
|
||||||
|
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
|
||||||
|
REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT}"
|
||||||
|
else
|
||||||
|
REDIS_CMDLINE="redis-cli -h redis -p 6379"
|
||||||
|
fi
|
||||||
|
|
||||||
|
${REDIS_CMDLINE} DEL F2B_RES > /dev/null
|
||||||
|
|
||||||
# Common functions
|
# Common functions
|
||||||
get_ipv6(){
|
get_ipv6(){
|
||||||
|
@ -65,7 +72,7 @@ progress() {
|
||||||
[[ ${CURRENT} -gt ${TOTAL} ]] && return
|
[[ ${CURRENT} -gt ${TOTAL} ]] && return
|
||||||
[[ ${CURRENT} -lt 0 ]] && CURRENT=0
|
[[ ${CURRENT} -lt 0 ]] && CURRENT=0
|
||||||
PERCENT=$(( 200 * ${CURRENT} / ${TOTAL} % 2 + 100 * ${CURRENT} / ${TOTAL} ))
|
PERCENT=$(( 200 * ${CURRENT} / ${TOTAL} % 2 + 100 * ${CURRENT} / ${TOTAL} ))
|
||||||
redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null
|
${REDIS_CMDLINE} LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null
|
||||||
log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}" no_redis
|
log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}" no_redis
|
||||||
# Return 10 to indicate a dead service
|
# Return 10 to indicate a dead service
|
||||||
[ ${CURRENT} -le 0 ] && return 10
|
[ ${CURRENT} -le 0 ] && return 10
|
||||||
|
@ -73,7 +80,7 @@ progress() {
|
||||||
|
|
||||||
log_msg() {
|
log_msg() {
|
||||||
if [[ ${2} != "no_redis" ]]; then
|
if [[ ${2} != "no_redis" ]]; then
|
||||||
redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
|
${REDIS_CMDLINE} LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
|
||||||
tr '\r\n%&;$"_[]{}-' ' ')\"}" > /dev/null
|
tr '\r\n%&;$"_[]{}-' ' ')\"}" > /dev/null
|
||||||
fi
|
fi
|
||||||
echo $(date) $(printf '%s\n' "${1}")
|
echo $(date) $(printf '%s\n' "${1}")
|
||||||
|
@ -252,6 +259,7 @@ unbound_checks() {
|
||||||
}
|
}
|
||||||
|
|
||||||
redis_checks() {
|
redis_checks() {
|
||||||
|
# A check for the local redis container
|
||||||
err_count=0
|
err_count=0
|
||||||
diff_c=0
|
diff_c=0
|
||||||
THRESHOLD=5
|
THRESHOLD=5
|
||||||
|
@ -465,20 +473,20 @@ fail2ban_checks() {
|
||||||
err_count=0
|
err_count=0
|
||||||
diff_c=0
|
diff_c=0
|
||||||
THRESHOLD=1
|
THRESHOLD=1
|
||||||
F2B_LOG_STATUS=($(redis-cli -h redis-mailcow --raw HKEYS F2B_ACTIVE_BANS))
|
F2B_LOG_STATUS=($(${REDIS_CMDLINE} --raw HKEYS F2B_ACTIVE_BANS))
|
||||||
F2B_RES=
|
F2B_RES=
|
||||||
# Reduce error count by 2 after restarting an unhealthy container
|
# Reduce error count by 2 after restarting an unhealthy container
|
||||||
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
|
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
|
||||||
while [ ${err_count} -lt ${THRESHOLD} ]; do
|
while [ ${err_count} -lt ${THRESHOLD} ]; do
|
||||||
err_c_cur=${err_count}
|
err_c_cur=${err_count}
|
||||||
F2B_LOG_STATUS_PREV=(${F2B_LOG_STATUS[@]})
|
F2B_LOG_STATUS_PREV=(${F2B_LOG_STATUS[@]})
|
||||||
F2B_LOG_STATUS=($(redis-cli -h redis-mailcow --raw HKEYS F2B_ACTIVE_BANS))
|
F2B_LOG_STATUS=($(${REDIS_CMDLINE} --raw HKEYS F2B_ACTIVE_BANS))
|
||||||
array_diff F2B_RES F2B_LOG_STATUS F2B_LOG_STATUS_PREV
|
array_diff F2B_RES F2B_LOG_STATUS F2B_LOG_STATUS_PREV
|
||||||
if [[ ! -z "${F2B_RES}" ]]; then
|
if [[ ! -z "${F2B_RES}" ]]; then
|
||||||
err_count=$(( ${err_count} + 1 ))
|
err_count=$(( ${err_count} + 1 ))
|
||||||
echo -n "${F2B_RES[@]}" | tr -cd "[a-fA-F0-9.:/] " | timeout 3s redis-cli -x -h redis-mailcow SET F2B_RES > /dev/null
|
echo -n "${F2B_RES[@]}" | tr -cd "[a-fA-F0-9.:/] " | timeout 3s ${REDIS_CMDLINE} -x SET F2B_RES > /dev/null
|
||||||
if [ $? -ne 0 ]; then
|
if [ $? -ne 0 ]; then
|
||||||
redis-cli -x -h redis-mailcow DEL F2B_RES
|
${REDIS_CMDLINE} -x DEL F2B_RES
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
|
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
|
||||||
|
@ -501,7 +509,7 @@ acme_checks() {
|
||||||
THRESHOLD=1
|
THRESHOLD=1
|
||||||
ACME_LOG_STATUS=$(redis-cli -h redis GET ACME_FAIL_TIME)
|
ACME_LOG_STATUS=$(redis-cli -h redis GET ACME_FAIL_TIME)
|
||||||
if [[ -z "${ACME_LOG_STATUS}" ]]; then
|
if [[ -z "${ACME_LOG_STATUS}" ]]; then
|
||||||
redis-cli -h redis SET ACME_FAIL_TIME 0
|
${REDIS_CMDLINE} SET ACME_FAIL_TIME 0
|
||||||
ACME_LOG_STATUS=0
|
ACME_LOG_STATUS=0
|
||||||
fi
|
fi
|
||||||
# Reduce error count by 2 after restarting an unhealthy container
|
# Reduce error count by 2 after restarting an unhealthy container
|
||||||
|
@ -673,7 +681,7 @@ BACKGROUND_TASKS+=(${PID})
|
||||||
(
|
(
|
||||||
while true; do
|
while true; do
|
||||||
if ! redis_checks; then
|
if ! redis_checks; then
|
||||||
log_msg "Redis hit error limit"
|
log_msg "Local Redis hit error limit"
|
||||||
echo redis-mailcow > /tmp/com_pipe
|
echo redis-mailcow > /tmp/com_pipe
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
@ -877,9 +885,9 @@ while true; do
|
||||||
log_msg "acme-mailcow did not complete successfully"
|
log_msg "acme-mailcow did not complete successfully"
|
||||||
[[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check acme-mailcow for further information."
|
[[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check acme-mailcow for further information."
|
||||||
elif [[ ${com_pipe_answer} == "fail2ban" ]]; then
|
elif [[ ${com_pipe_answer} == "fail2ban" ]]; then
|
||||||
F2B_RES=($(timeout 4s redis-cli -h redis-mailcow --raw GET F2B_RES 2> /dev/null))
|
F2B_RES=($(timeout 4s ${REDIS_CMDLINE} --raw GET F2B_RES 2> /dev/null))
|
||||||
if [[ ! -z "${F2B_RES}" ]]; then
|
if [[ ! -z "${F2B_RES}" ]]; then
|
||||||
redis-cli -h redis-mailcow DEL F2B_RES > /dev/null
|
${REDIS_CMDLINE} DEL F2B_RES > /dev/null
|
||||||
host=
|
host=
|
||||||
for host in "${F2B_RES[@]}"; do
|
for host in "${F2B_RES[@]}"; do
|
||||||
log_msg "Banned ${host}"
|
log_msg "Banned ${host}"
|
||||||
|
|
Loading…
Reference in New Issue