[Compose, DockerAPI, Web, Watchdog] Watchdog may send notification mails (todo: docs), DockerAPI via Flesk for limited access

data/Dockerfiles/dockerapi/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:2-alpine
+LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
+
+RUN apk add -U --no-cache iptables ip6tables
+RUN pip install docker flask flask-restful
+
+COPY server.py /
+CMD ["python2", "-u", "/server.py"]

data/Dockerfiles/dockerapi/server.py

@@ -0,0 +1,41 @@
+from flask import Flask
+from flask_restful import Resource, Api
+from docker import APIClient
+
+dockercli = APIClient(base_url='unix://var/run/docker.sock')
+app = Flask(__name__)
+api = Api(app)
+
+class Containers(Resource):
+    def get(self):
+        return dockercli.containers(all=True)
+
+class ContainerInfo(Resource):
+    def get(self, container_id):
+        return dockercli.containers(all=True, filters={"id": container_id})
+
+class ContainerStart(Resource):
+    def post(self, container_id):
+        try:
+            dockercli.start(container_id);
+        except:
+            return 'Error'
+        else:
+            return 'OK'
+
+class ContainerStop(Resource):
+    def post(self, container_id):
+        try:
+            dockercli.stop(container_id);
+        except: 
+            return 'Error'
+        else:
+            return 'OK'
+
+api.add_resource(Containers, '/info/container/all')
+api.add_resource(ContainerInfo, '/info/container/<string:container_id>')
+api.add_resource(ContainerStop, '/stop/container/<string:container_id>')
+api.add_resource(ContainerStart, '/start/container/<string:container_id>')
+
+if __name__ == '__main__':
+    app.run(debug=False, host='0.0.0.0', port='8080')

data/Dockerfiles/watchdog/Dockerfile

@@ -16,7 +16,15 @@ RUN apk add --update \
 	nagios-plugins-dns \
     nagios-plugins-disk \
 	bind-tools \
-	redis
+	redis \
+	perl \
+	perl-io-socket-ssl \
+	perl-socket \
+	perl-socket6 \
+	perl-mime-lite \
+	perl-term-readkey \
+	&& curl https://raw.githubusercontent.com/mludvig/smtp-cli/v3.8/smtp-cli -o /smtp-cli \
+	&& chmod +x smtp-cli
 
 COPY watchdog.sh /watchdog.sh
 

data/Dockerfiles/watchdog/watchdog.sh

@@ -38,6 +38,26 @@ log_to_redis() {
 	redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}")\"}"
 }
 
+function mail_error() {
+  [[ -z ${1} ]] && return 1
+  [[ -z ${2} ]] && return 2
+  RCPT_DOMAIN=$(echo ${1} | awk -F @ {'print $NF'})
+  RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}')
+  if [[ -z ${RCPT_MX} ]]; then
+    log_to_redis "Cannot determine MX for ${1}, skipping email notification..."
+    echo "Cannot determine MX for ${1}"
+    return 1
+  fi
+  ./smtp-cli --missing-modules-ok \
+    --subject="Watchdog: ${2} service hit the error rate limit" \
+    --body-plain="Service was restarted, please check your mailcow installation." \
+    --to=${1} \
+    --from="watchdog@${MAILCOW_HOSTNAME}" \
+    --server="${RCPT_MX}" \
+    --hello-host=${MAILCOW_HOSTNAME}
+}
+
+
 get_container_ip() {
   # ${1} is container
   CONTAINER_ID=
@@ -233,6 +253,7 @@ dns_checks() {
 while true; do
   if ! nginx_checks; then
     log_to_redis "Nginx hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "nginx-mailcow"
     echo -e "\e[31m$(date) - Nginx hit error limit\e[0m"
     echo nginx-mailcow > /tmp/com_pipe
   fi
@@ -244,6 +265,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
   if ! mysql_checks; then
     log_to_redis "MySQL hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "mysql-mailcow"
     echo -e "\e[31m$(date) - MySQL hit error limit\e[0m"
     echo mysql-mailcow > /tmp/com_pipe
   fi
@@ -255,6 +277,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
   if ! phpfpm_checks; then
     log_to_redis "PHP-FPM hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "php-fpm-mailcow"
     echo -e "\e[31m$(date) - PHP-FPM hit error limit\e[0m"
     echo php-fpm-mailcow > /tmp/com_pipe
   fi
@@ -266,6 +289,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
   if ! sogo_checks; then
     log_to_redis "SOGo hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "sogo-mailcow"
     echo -e "\e[31m$(date) - SOGo hit error limit\e[0m"
     echo sogo-mailcow > /tmp/com_pipe
   fi
@@ -277,6 +301,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
   if ! postfix_checks; then
     log_to_redis "Postfix hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "postfix-mailcow"
     echo -e "\e[31m$(date) - Postfix hit error limit\e[0m"
     echo postfix-mailcow > /tmp/com_pipe
   fi
@@ -288,6 +313,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
   if ! dovecot_checks; then
     log_to_redis "Dovecot hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "dovecot-mailcow"
     echo -e "\e[31m$(date) - Dovecot hit error limit\e[0m"
     echo dovecot-mailcow > /tmp/com_pipe
   fi
@@ -299,6 +325,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
   if ! dns_checks; then
     log_to_redis "Unbound hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "unbound-mailcow"
     echo -e "\e[31m$(date) - Unbound hit error limit\e[0m"
     #echo unbound-mailcow > /tmp/com_pipe
   fi
@@ -310,6 +337,7 @@ BACKGROUND_TASKS+=($!)
 while true; do
   if ! rspamd_checks; then
     log_to_redis "Rspamd hit error limit"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "rspamd-mailcow"
     echo -e "\e[31m$(date) - Rspamd hit error limit\e[0m"
     echo rspamd-mailcow > /tmp/com_pipe
   fi

data/conf/rspamd/local.d/multimap.conf

@@ -23,7 +23,7 @@ KEEP_SPAM {
 
 SPOOFED_SENDER {
   type = "rcpt";
-  filter = "email:domain";
+  filter = "email:domain:tld";
   map = "redis://DOMAIN_MAP";
   require_symbols = "AUTH_NA | !RCVD_VIA_SMTP_AUTH";
 }

data/web/api.php

@@ -0,0 +1,32 @@
+<?php
+set_time_limit (0);
+
+$address = '0.0.0.0';
+
+$port = 7777;
+$con = 1;
+$word = "";
+
+$sock = socket_create(AF_INET, SOCK_STREAM, 0);
+$bind = socket_bind($sock, $address, $port);
+
+socket_listen($sock);
+
+while ($con == 1)
+{
+    $client = socket_accept($sock);
+    $input = socket_read($client, 2024);
+
+    if ($input == 'exit') 
+    {
+        $close = socket_close($sock);
+        $con = 0;
+    }
+
+    if($con == 1)
+    {
+        $word .= $input;
+    }
+}
+
+echo $word;

data/web/inc/call_sogo_ctrl.php

@@ -11,8 +11,7 @@ function docker($service_name, $action, $post_action = null, $post_fields = null
   curl_setopt($curl, CURLOPT_HTTPHEADER,array( 'Content-Type: application/json' ));
   switch($action) {
     case 'get_id':
-      curl_setopt($curl, CURLOPT_UNIX_SOCKET_PATH, "/var/run/docker.sock");
+      curl_setopt($curl, CURLOPT_URL, 'http://dockerapi:8080/info/container/all');
-      curl_setopt($curl, CURLOPT_URL, 'http:/v1.26/containers/json?all=1');
       curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
       curl_setopt($curl, CURLOPT_POST, 0);
       $response = curl_exec($curl);
@@ -35,13 +34,11 @@ function docker($service_name, $action, $post_action = null, $post_fields = null
       return false;
     break;
     case 'info':
-      curl_setopt($curl, CURLOPT_UNIX_SOCKET_PATH, "/var/run/docker.sock");
       $container_id = docker($service_name, 'get_id');
       if (ctype_xdigit($container_id)) {
-        curl_setopt($curl, CURLOPT_UNIX_SOCKET_PATH, "/var/run/docker.sock");
+        curl_setopt($curl, CURLOPT_URL, 'http://dockerapi:8080/info/container/' . $container_id);
-        curl_setopt($curl, CURLOPT_URL, 'http/containers/' . $container_id . '/json');
-        curl_setopt($curl, CURLOPT_POST, 0);
         curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
+        curl_setopt($curl, CURLOPT_POST, 0);
         $response = curl_exec($curl);
         if ($response === false) {
           $err = curl_error($curl);
@@ -65,9 +62,8 @@ function docker($service_name, $action, $post_action = null, $post_fields = null
     case 'post':
       if (!empty($post_action)) {
         $container_id = docker($service_name, 'get_id');
-        if (ctype_xdigit($container_id)) {
+        if (ctype_xdigit($container_id) && ctype_alnum($post_action)) {
-          curl_setopt($curl, CURLOPT_UNIX_SOCKET_PATH, "/var/run/docker.sock");
+          curl_setopt($curl, CURLOPT_URL, 'http://dockerapi:8080/' . $post_action . '/container/' . $container_id);
-          curl_setopt($curl, CURLOPT_URL, 'http/containers/' . $container_id . '/' . $post_action);
           curl_setopt($curl, CURLOPT_POST, 1);
           if (!empty($post_fields)) {
             curl_setopt( $curl, CURLOPT_POSTFIELDS, json_encode($post_fields));
@@ -96,30 +92,30 @@ function docker($service_name, $action, $post_action = null, $post_fields = null
 
 if ($_GET['ACTION'] == "start") {
   $retry = 0;
-  while (!docker('sogo-mailcow', 'info')['State']['Running'] && $retry <= 3) {
+  while (docker('sogo-mailcow', 'info')[0]['State'] != "running" && $retry <= 3) {
     $response = docker('sogo-mailcow', 'post', 'start');
-    $last_response = ($response === true) ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-warning">Error: ' . $response . '</span></b>';
+    $last_response = (trim($response) == "\"OK\"") ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-danger">Error: ' . $response . '</span></b>';
-    if ($response === true) {
+    if (trim($response) == "\"OK\"") {
       break;
     }
     usleep(1500000);
     $retry++;
   }
-  echo $last_response;
+  echo (!isset($last_response)) ? '<b><span class="pull-right text-warning">Not running</span></b>' : $last_response;
 }
 
 if ($_GET['ACTION'] == "stop") {
   $retry = 0;
-  while (docker('sogo-mailcow', 'info')['State']['Running'] && $retry <= 3) {
+  while (docker('sogo-mailcow', 'info')[0]['State'] == "running" && $retry <= 3) {
     $response = docker('sogo-mailcow', 'post', 'stop');
-    $last_response = ($response === true) ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-warning">Error: ' . $response . '</span></b>';
+    $last_response = (trim($response) == "\"OK\"") ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-danger">Error: ' . $response . '</span></b>';
-    if ($response === true) {
+    if (trim($response) == "\"OK\"") {
       break;
     }
     usleep(1500000);
     $retry++;
   }
-  echo $last_response;
+  echo (!isset($last_response)) ? '<b><span class="pull-right text-warning">Not running</span></b>' : $last_response;
 }
 
 ?>

docker-compose.yml

@@ -105,7 +105,6 @@ services:
         - ./data/web:/web:ro
         - ./data/conf/rspamd/dynmaps:/dynmaps:ro
         - dkim-vol-1:/data/dkim
-        - /var/run/docker.sock:/var/run/docker.sock:ro
       environment:
         - DBNAME=${DBNAME}
         - DBUSER=${DBUSER}
@@ -297,7 +296,7 @@ services:
         - /lib/modules:/lib/modules:ro
 
     watchdog-mailcow:
-      image: mailcow/watchdog:1.3
+      image: mailcow/watchdog:1.4
       build: ./data/Dockerfiles/watchdog
       init: false
       volumes:
@@ -309,11 +308,24 @@ services:
         - DBUSER=${DBUSER}
         - DBPASS=${DBPASS}
         - USE_WATCHDOG=${USE_WATCHDOG:-n}
+        - WATCHDOG_NOTIFY_EMAIL=${WATCHDOG_NOTIFY_EMAIL}
+        - MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
       networks:
         mailcow-network:
           aliases:
             - watchdog
 
+    dockerapi-mailcow:
+      image: mailcow/dockerapi:1.0
+      build: ./data/Dockerfiles/dockerapi
+      init: false
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock:ro
+      networks:
+        mailcow-network:
+          aliases:
+            - dockerapi
+
     ipv6nat:
       image: robbertkl/ipv6nat
       restart: always

update.sh

@@ -4,7 +4,7 @@ for bin in curl docker-compose docker git awk sha1sum; do
 	if [[ -z $(which ${bin}) ]]; then echo "Cannot find ${bin}, exiting..."; exit 1; fi
 done
 
-CONFIG_ARRAY=("SKIP_LETS_ENCRYPT" "USE_WATCHDOG" "SKIP_CLAMD" "SKIP_IP_CHECK" "SKIP_FAIL2BAN" "ADDITIONAL_SAN" "DOVEADM_PORT")
+CONFIG_ARRAY=("SKIP_LETS_ENCRYPT" "USE_WATCHDOG" "WATCHDOG_NOTIFY_EMAIL" "SKIP_CLAMD" "SKIP_IP_CHECK" "SKIP_FAIL2BAN" "ADDITIONAL_SAN" "DOVEADM_PORT")
 echo >> mailcow.conf
 for option in ${CONFIG_ARRAY[@]}; do
 	if [[ ${option} == "ADDITIONAL_SAN" ]]; then
@@ -22,6 +22,11 @@ for option in ${CONFIG_ARRAY[@]}; do
 			echo "Adding new option \"${option}\" to mailcow.conf"
 			echo "DOVEADM_PORT=127.0.0.1:19991" >> mailcow.conf
 		fi
+	elif [[ ${option} == "WATCHDOG_NOTIFY_EMAIL" ]]; then
+		if ! grep -q ${option} mailcow.conf; then
+			echo "Adding new option \"${option}\" to mailcow.conf"
+			echo "WATCHDOG_NOTIFY_EMAIL=" >> mailcow.conf
+		fi
 	elif ! grep -q ${option} mailcow.conf; then
 		echo "Adding new option \"${option}\" to mailcow.conf"
 		echo "${option}=n" >> mailcow.conf