[Watchdog] Send mail when IP was banned

data/Dockerfiles/watchdog/watchdog.sh

@@ -19,7 +19,17 @@ if [[ ! -p /tmp/com_pipe ]]; then
   mkfifo /tmp/com_pipe
 fi
 
+redis-cli -h redis-mailcow DEL F2B_RES > /dev/null
+
 # Common functions
+array_diff() {
+  # https://stackoverflow.com/questions/2312762, Alex Offshore
+  eval local ARR1=\(\"\${$2[@]}\"\)
+  eval local ARR2=\(\"\${$3[@]}\"\)
+  local IFS=$'\n'
+  mapfile -t $1 < <(comm -23 <(echo "${ARR1[*]}" | sort) <(echo "${ARR2[*]}" | sort))
+}
+
 progress() {
   SERVICE=${1}
   TOTAL=${2}
@@ -58,7 +68,15 @@ function mail_error() {
       log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
       return 1
     fi
-    [[ ${1} == "watchdog-mailcow" ]] && SUBJECT="Watchdog started" || SUBJECT="Watchdog: ${1} hit the error rate limit"
+    # Some exceptions for subject and body formats
+    if [[ ${1} == "watchdog-mailcow" ]]; then
+      SUBJECT="Watchdog started"
+    elif [[ ${1} == "fail2ban" ]]; then
+      SUBJECT="${BODY}"
+      BODY="Please see netfilter-mailcow for more details and triggered rules."
+    else
+      SUBJECT="Watchdog: ${1} triggered an event"
+    fi
     [ -f "/tmp/${1}" ] && ATTACH="--attach /tmp/${1}@text/plain" || ATTACH=
     ./smtp-cli --missing-modules-ok \
       --subject="${SUBJECT}" \
@@ -353,6 +371,37 @@ ratelimit_checks() {
   return 1
 }
 
+fail2ban_checks() {
+  err_count=0
+  diff_c=0
+  THRESHOLD=1
+  F2B_LOG_STATUS=($(redis-cli -h redis-mailcow --raw HKEYS F2B_ACTIVE_BANS))
+  F2B_RES=
+  # Reduce error count by 2 after restarting an unhealthy container
+  trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
+  while [ ${err_count} -lt ${THRESHOLD} ]; do
+    err_c_cur=${err_count}
+    F2B_LOG_STATUS_PREV=(${F2B_LOG_STATUS[@]})
+    F2B_LOG_STATUS=($(redis-cli -h redis-mailcow --raw HKEYS F2B_ACTIVE_BANS))
+    array_diff F2B_RES F2B_LOG_STATUS F2B_LOG_STATUS_PREV
+    if [[ ! -z "${F2B_RES}" ]]; then
+      err_count=$(( ${err_count} + 1 ))
+      echo -n "${F2B_RES[@]}" | redis-cli -x -h redis-mailcow SET F2B_RES > /dev/null
+    fi
+    [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
+    [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
+    progress "Fail2ban" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
+    if [[ $? == 10 ]]; then
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 30 )  + 10 ))
+    fi
+  done
+  return 1
+}
+
 acme_checks() {
   err_count=0
   diff_c=0
@@ -442,8 +491,13 @@ Empty
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "Rspamd" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
-    diff_c=0
+    if [[ $? == 10 ]]; then
-    sleep $(( ( RANDOM % 30 )  + 10 ))
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 30 )  + 10 ))
+    fi
   done
   return 1
 }
@@ -556,6 +610,16 @@ done
 ) &
 BACKGROUND_TASKS+=($!)
 
+(
+while true; do
+  if ! fail2ban_checks; then
+    log_msg "Fail2ban hit error limit"
+    echo fail2ban > /tmp/com_pipe
+  fi
+done
+) &
+BACKGROUND_TASKS+=($!)
+
 (
 while true; do
   if ! acme_checks; then
@@ -619,6 +683,14 @@ while true; do
   elif [[ ${com_pipe_answer} == "acme-mailcow" ]]; then
     log_msg "acme-mailcow did not complete successfully"
     [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check acme-mailcow for further information."
+  elif [[ ${com_pipe_answer} == "fail2ban" ]]; then
+    F2B_RES=($(redis-cli -h redis-mailcow --raw GET F2B_RES))
+    redis-cli -h redis-mailcow DEL F2B_RES > /dev/null
+    host=
+    for host in "${F2B_RES[@]}"; do
+      log_msg "Banned ${F2B_RES}"
+      [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "IP ban: ${host}"
+    done
   elif [[ ${com_pipe_answer} =~ .+-mailcow ]] || [[ ${com_pipe_answer} == "ipv6nat-mailcow" ]]; then
     kill -STOP ${BACKGROUND_TASKS[*]}
     sleep 3