paxton
/
mailcow
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Watchdog] Use Redis master for write operations

master
andryyy 2020-02-05 11:01:09 +01:00
parent 9dec216261
commit 1ad469a24a
No known key found for this signature in database
GPG Key ID: 8EC34FF2794E25EF
1 changed files with 19 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
data/Dockerfiles/watchdog/watchdog.sh
View File

@ -30,7 +30,14 @@ until [[ $(redis-cli -h redis-mailcow PING) == "PONG" ]]; do
sleep 2
done
redis-cli -h redis-mailcow DEL F2B_RES > /dev/null
# Do not attempt to write to slave
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT}"
else
REDIS_CMDLINE="redis-cli -h redis -p 6379"
fi
${REDIS_CMDLINE} DEL F2B_RES > /dev/null
# Common functions
get_ipv6(){
@ -65,7 +72,7 @@ progress() {
[[ ${CURRENT} -gt ${TOTAL} ]] && return
[[ ${CURRENT} -lt 0 ]] && CURRENT=0
PERCENT=$(( 200 * ${CURRENT} / ${TOTAL} % 2 + 100 * ${CURRENT} / ${TOTAL} ))
redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null
${REDIS_CMDLINE} LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null
log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}" no_redis
# Return 10 to indicate a dead service
[ ${CURRENT} -le 0 ] && return 10
@ -73,7 +80,7 @@ progress() {
log_msg() {
if [[ ${2} != "no_redis" ]]; then
redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
${REDIS_CMDLINE} LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
tr '\r\n%&;$"_[]{}-' ' ')\"}" > /dev/null
fi
echo $(date) $(printf '%s\n' "${1}")
@ -252,6 +259,7 @@ unbound_checks() {
}
redis_checks() {
# A check for the local redis container
err_count=0
diff_c=0
THRESHOLD=5
@ -465,20 +473,20 @@ fail2ban_checks() {
err_count=0
diff_c=0
THRESHOLD=1
F2B_LOG_STATUS=($(redis-cli -h redis-mailcow --raw HKEYS F2B_ACTIVE_BANS))
F2B_LOG_STATUS=($(${REDIS_CMDLINE} --raw HKEYS F2B_ACTIVE_BANS))
F2B_RES=
# Reduce error count by 2 after restarting an unhealthy container
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
while [ ${err_count} -lt ${THRESHOLD} ]; do
err_c_cur=${err_count}
F2B_LOG_STATUS_PREV=(${F2B_LOG_STATUS[@]})
F2B_LOG_STATUS=($(redis-cli -h redis-mailcow --raw HKEYS F2B_ACTIVE_BANS))
F2B_LOG_STATUS=($(${REDIS_CMDLINE} --raw HKEYS F2B_ACTIVE_BANS))
array_diff F2B_RES F2B_LOG_STATUS F2B_LOG_STATUS_PREV
if [[ ! -z "${F2B_RES}" ]]; then
err_count=$(( ${err_count} + 1 ))
echo -n "${F2B_RES[@]}" | tr -cd "[a-fA-F0-9.:/] " | timeout 3s redis-cli -x -h redis-mailcow SET F2B_RES > /dev/null
echo -n "${F2B_RES[@]}" | tr -cd "[a-fA-F0-9.:/] " | timeout 3s ${REDIS_CMDLINE} -x SET F2B_RES > /dev/null
if [ $? -ne 0 ]; then
redis-cli -x -h redis-mailcow DEL F2B_RES
${REDIS_CMDLINE} -x DEL F2B_RES
fi
fi
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
@ -501,7 +509,7 @@ acme_checks() {
THRESHOLD=1
ACME_LOG_STATUS=$(redis-cli -h redis GET ACME_FAIL_TIME)
if [[ -z "${ACME_LOG_STATUS}" ]]; then
redis-cli -h redis SET ACME_FAIL_TIME 0
${REDIS_CMDLINE} SET ACME_FAIL_TIME 0
ACME_LOG_STATUS=0
fi
# Reduce error count by 2 after restarting an unhealthy container
@ -673,7 +681,7 @@ BACKGROUND_TASKS+=(${PID})
(
while true; do
if ! redis_checks; then
log_msg "Redis hit error limit"
log_msg "Local Redis hit error limit"
echo redis-mailcow > /tmp/com_pipe
fi
done
@ -877,9 +885,9 @@ while true; do
log_msg "acme-mailcow did not complete successfully"
[[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check acme-mailcow for further information."
elif [[ ${com_pipe_answer} == "fail2ban" ]]; then
F2B_RES=($(timeout 4s redis-cli -h redis-mailcow --raw GET F2B_RES 2> /dev/null))
F2B_RES=($(timeout 4s ${REDIS_CMDLINE} --raw GET F2B_RES 2> /dev/null))
if [[ ! -z "${F2B_RES}" ]]; then
redis-cli -h redis-mailcow DEL F2B_RES > /dev/null
${REDIS_CMDLINE} DEL F2B_RES > /dev/null
host=
for host in "${F2B_RES[@]}"; do
log_msg "Banned ${host}"