Merge pull request #1 from mailcow/master

Merge change since fork creation

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -0,0 +1,30 @@
+---
+name: Bug report
+about: Report a bug for this project
+
+---
+
+**README and remove me**
+For community support and other discussion, you are welcome to visit and stay with us @ Freenode, #mailcow
+Answering can take a few seconds up to many hours, please be patient.
+Commercial support, including a ticket system, can be found @ https://www.servercow.de/mailcow#support - we are also available via Telegram. \o/
+
+**Describe the bug, try to make it reproducible**
+A clear and concise description of what the bug is. How can it be reproduced? 
+If applicable, add screenshots to help explain your problem. Very useful for bugs in mailcow UI.
+
+**System information and quick debugging**
+General logs:
+- Please take a look at the [documentation](https://mailcow.github.io/mailcow-dockerized-docs/debug-logs/).
+
+Further information (where applicable):
+ - Your OS (is Apparmor or SELinux active?)
+ - Your virtualization technology (KVM/QEMU, Xen, VMware, VirtualBox etc.)
+ - Your server/VM specifications (Memory, CPU Cores)
+ - Don't try to run mailcow on a Synology or QNAP NAS, do you?
+ - Docker and Docker Compose versions
+ - Output of `git diff origin/master`, any other changes to the code?
+ - All third-party firewalls and custom iptables rules are unsupported. Please check the Docker docs about how to use Docker with your own ruleset. Nevertheless, iptabels output can help _us_ to help _you_: `iptables -L -vn`, `ip6tables -L -vn`, `iptables -L -vn -t nat` and `ip6tables -L -vn -t nat `
+ - Reverse proxy? If you think this problem is related to your reverse proxy, please post your configuration.
+ - Browser (if it's a Web UI issue) - please clean your browser cache and try again, problem persists?
+ - Check `docker exec -it $(docker ps -qf name=acme-mailcow) dig +short stackoverflow.com @172.22.1.254` (set the IP accordingly, if you changed the internal mailcow network) and `docker exec -it $(docker ps -qf name=acme-mailcow) dig +short stackoverflow.com @1.1.1.1` - output? Timeout?

.github/ISSUE_TEMPLATE/Feature_request.md

@@ -0,0 +1,14 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Additional context**
+Add any other context or screenshots about the feature request here or remove this section

.github/stale.yml

@@ -0,0 +1,18 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 60
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+  - enhancement
+# Label to use when marking an issue as stale
+staleLabel: dunno
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

.gitignore

@@ -1,17 +1,20 @@
 rebuild-images.sh
 data/conf/sogo/sieve.creds
+data/conf/phpfpm/sogo-sso/sogo-sso.pass
 data/conf/dovecot/dovecot-master.passwd
+data/conf/dovecot/dovecot-master.userdb
 mailcow.conf
 mailcow.conf_backup
 data/conf/nginx/*.active
 data/conf/postfix/sql
+data/conf/postfix/allow_mailcow_local.regexp
 data/conf/dovecot/sql
 data/conf/nextcloud-*.bak
 data/web/inc/vars.local.inc.php
 data/assets/ssl/*
 .vscode/*
 data/web/.well-known/acme-challenge
-data/web/nextcloud/
+data/web/nextcloud*/
 data/conf/rspamd/local.d/*
 data/conf/rspamd/override.d/*
 !data/conf/nginx/dynmaps.conf
@@ -20,4 +23,14 @@ data/conf/rspamd/override.d/*
 data/conf/nginx/*.conf
 data/conf/nginx/*.custom
 data/conf/nginx/*.bak
+data/conf/dovecot/acl_anyone
+data/conf/dovecot/mail_plugins*
+data/conf/dovecot/sogo-sso.conf
 data/conf/dovecot/extra.conf
+data/conf/rspamd/custom/*
+data/conf/portainer/
+data/gitea/
+data/gogs/
+data/conf/sogo/plist_ldap
+.github/
+docker-compose.override.yml

README.md

@@ -1,8 +1,12 @@
 # mailcow: dockerized - 🐮 + 🐋 = 💕
 
-[![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=JWBSYHF4SMC68)
+## Want to support mailcow?
 
-**mailcow Bitcoin donations:** 1E5rgzgA1sS3QH7r1ToWxRC3GEavfsGMrx
+Donate via **PayPal** [![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=JWBSYHF4SMC68) or via **Liberapay** [![Liberapay.com](https://mailcow.email/img/lp.png)](https://liberapay.com/mailcow)
+
+Or just spread the word: moo.
+
+## Info and documentation
 
 Please see [the official documentation](https://mailcow.github.io/mailcow-dockerized-docs/) for instructions.
 

data/Dockerfiles/acme/Dockerfile

@@ -1,10 +1,9 @@
-FROM alpine:3.6
+FROM alpine:3.9
 
 LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
 
 RUN apk add --update --no-cache \
   bash \
-  acme-client \
   curl \
   openssl \
   bind-tools \
@@ -12,7 +11,10 @@ RUN apk add --update --no-cache \
   mariadb-client \
   redis \
   tini \
-  tzdata
+  tzdata \
+  py-pip \
+  && pip install --upgrade pip \
+  && pip install acme-tiny
 
 COPY docker-entrypoint.sh /srv/docker-entrypoint.sh
 COPY expand6.sh /srv/expand6.sh

data/Dockerfiles/acme/docker-entrypoint.sh

@@ -5,6 +5,16 @@ exec 5>&1
 # Thanks to https://github.com/cvmiller -> https://github.com/cvmiller/expand6
 source /srv/expand6.sh
 
+# Skipping IP check when we like to live dangerously
+if [[ "${SKIP_IP_CHECK}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  SKIP_IP_CHECK=y
+fi
+
+# Skipping HTTP check when we like to live dangerously
+if [[ "${SKIP_HTTP_VERIFICATION}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  SKIP_HTTP_VERIFICATION=y
+fi
+
 log_f() {
   if [[ ${2} == "no_nl" ]]; then
     echo -n "$(date) - ${1}"
@@ -13,8 +23,12 @@ log_f() {
   elif [[ ${2} != "redis_only" ]]; then
     echo "$(date) - ${1}"
   fi
-  redis-cli -h redis LPUSH ACME_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
+  if [[ ${3} == "b64" ]]; then
-    tr '%&;$"_[]{}-\r\n' ' ')\"}" > /dev/null
+    redis-cli -h redis LPUSH ACME_LOG "{\"time\":\"$(date +%s)\",\"message\":\"base64,$(printf '%s' "${1}")\"}" > /dev/null
+  else
+    redis-cli -h redis LPUSH ACME_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
+      tr '%&;$"[]{}-\r\n' ' ')\"}" > /dev/null
+  fi
 }
 
 if [[ "${SKIP_LETS_ENCRYPT}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
@@ -32,12 +46,34 @@ log_f "OK" no_date
 ACME_BASE=/var/lib/acme
 SSL_EXAMPLE=/var/lib/ssl-example
 
-mkdir -p ${ACME_BASE}/acme/private
+mkdir -p ${ACME_BASE}/acme
 
-restart_containers(){
+# Migrate
+[[ -f ${ACME_BASE}/acme/private/privkey.pem ]] && mv ${ACME_BASE}/acme/private/privkey.pem ${ACME_BASE}/acme/key.pem
+[[ -f ${ACME_BASE}/acme/private/account.key ]] && mv ${ACME_BASE}/acme/private/account.key ${ACME_BASE}/acme/account.pem
+
+reload_configurations(){
+  # Reading container IDs
+  # Wrapping as array to ensure trimmed content when calling $NGINX etc.
+  local NGINX=($(curl --silent --insecure https://dockerapi/containers/json | jq -r '.[] | {name: .Config.Labels["com.docker.compose.service"], id: .Id}' | jq -rc 'select( .name | tostring | contains("nginx-mailcow")) | .id' | tr "\n" " "))
+  local DOVECOT=($(curl --silent --insecure https://dockerapi/containers/json | jq -r '.[] | {name: .Config.Labels["com.docker.compose.service"], id: .Id}' | jq -rc 'select( .name | tostring | contains("dovecot-mailcow")) | .id' | tr "\n" " "))
+  local POSTFIX=($(curl --silent --insecure https://dockerapi/containers/json | jq -r '.[] | {name: .Config.Labels["com.docker.compose.service"], id: .Id}' | jq -rc 'select( .name | tostring | contains("postfix-mailcow")) | .id' | tr "\n" " "))
+  # Reloading
+  echo "Reloading Nginx..."
+  NGINX_RELOAD_RET=$(curl -X POST --insecure https://dockerapi/containers/${NGINX}/exec -d '{"cmd":"reload", "task":"nginx"}' --silent -H 'Content-type: application/json' | jq -r .type)
+  [[ ${NGINX_RELOAD_RET} != 'success' ]] && { echo "Could not reload Nginx, restarting container..."; restart_container ${NGINX} ; }
+  echo "Reloading Dovecot..."
+  DOVECOT_RELOAD_RET=$(curl -X POST --insecure https://dockerapi/containers/${DOVECOT}/exec -d '{"cmd":"reload", "task":"dovecot"}' --silent -H 'Content-type: application/json' | jq -r .type)
+  [[ ${DOVECOT_RELOAD_RET} != 'success' ]] && { echo "Could not reload Dovecot, restarting container..."; restart_container ${DOVECOT} ; }
+  echo "Reloading Postfix..."
+  POSTFIX_RELOAD_RET=$(curl -X POST --insecure https://dockerapi/containers/${POSTFIX}/exec -d '{"cmd":"reload", "task":"postfix"}' --silent -H 'Content-type: application/json' | jq -r .type)
+  [[ ${POSTFIX_RELOAD_RET} != 'success' ]] && { echo "Could not reload Postfix, restarting container..."; restart_container ${POSTFIX} ; }
+}
+
+restart_container(){
   for container in $*; do
     log_f "Restarting ${container}..." no_nl
-    C_REST_OUT=$(curl -X POST http://dockerapi:8080/containers/${container}/restart | jq -r '.msg')
+    C_REST_OUT=$(curl -X POST --insecure https://dockerapi/containers/${container}/restart | jq -r '.msg')
     log_f "${C_REST_OUT}" no_date
   done
 }
@@ -90,28 +126,37 @@ get_ipv6(){
   echo ${IPV6}
 }
 
+verify_challenge_path(){
+  # verify_challenge_path URL 4|6
+  RAND_FILE=${RANDOM}${RANDOM}${RANDOM}
+  touch /var/www/acme/${RAND_FILE}
+  if [[ ${SKIP_HTTP_VERIFICATION} == "y" ]]; then
+    echo '(skipping check, returning 0)'
+    return 0
+  elif [[ "$(curl -${2} http://${1}/.well-known/acme-challenge/${RAND_FILE} --write-out %{http_code} --silent --output /dev/null)" =~ ^(2|3)  ]]; then
+    rm /var/www/acme/${RAND_FILE}
+    return 0
+  else
+    rm /var/www/acme/${RAND_FILE}
+    return 1
+  fi
+}
+
 [[ ! -f ${ACME_BASE}/dhparams.pem ]] && cp ${SSL_EXAMPLE}/dhparams.pem ${ACME_BASE}/dhparams.pem
 
 if [[ -f ${ACME_BASE}/cert.pem ]] && [[ -f ${ACME_BASE}/key.pem ]]; then
   ISSUER=$(openssl x509 -in ${ACME_BASE}/cert.pem -noout -issuer)
-  if [[ ${ISSUER} != *"Let's Encrypt"* && ${ISSUER} != *"mailcow"* ]]; then
+  if [[ ${ISSUER} != *"Let's Encrypt"* && ${ISSUER} != *"mailcow"* && ${ISSUER} != *"Fake LE Intermediate"* ]]; then
     log_f "Found certificate with issuer other than mailcow snake-oil CA and Let's Encrypt, skipping ACME client..."
     sleep 3650d
     exec $(readlink -f "$0")
-  else
-    declare -a SAN_ARRAY_NOW
-    SAN_NAMES=$(openssl x509 -noout -text -in ${ACME_BASE}/cert.pem | awk '/X509v3 Subject Alternative Name/ {getline;gsub(/ /, "", $0); print}' | tr -d "DNS:")
-    if [[ ! -z ${SAN_NAMES} ]]; then
-      IFS=',' read -a SAN_ARRAY_NOW <<< ${SAN_NAMES}
-      log_f "Found Let's Encrypt or mailcow snake-oil CA issued certificate with SANs: ${SAN_ARRAY_NOW[*]}"
-    fi
   fi
 else
-  if [[ -f ${ACME_BASE}/acme/fullchain.pem ]] && [[ -f ${ACME_BASE}/acme/private/privkey.pem ]]; then
+  if [[ -f ${ACME_BASE}/acme/cert.pem ]] && [[ -f ${ACME_BASE}/acme/key.pem ]]; then
-    if verify_hash_match ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/acme/private/privkey.pem; then
+    if verify_hash_match ${ACME_BASE}/acme/cert.pem ${ACME_BASE}/acme/key.pem; then
       log_f "Restoring previous acme certificate and restarting script..."
-      cp ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem
+      cp ${ACME_BASE}/acme/cert.pem ${ACME_BASE}/cert.pem
-      cp ${ACME_BASE}/acme/private/privkey.pem ${ACME_BASE}/key.pem
+      cp ${ACME_BASE}/acme/key.pem ${ACME_BASE}/key.pem
       # Restarting with env var set to trigger a restart,
       exec env TRIGGER_RESTART=1 $(readlink -f "$0")
     fi
@@ -123,25 +168,80 @@ else
     exec env TRIGGER_RESTART=1 $(readlink -f "$0")
   fi
 fi
+chmod 600 ${ACME_BASE}/key.pem
 
-log_f "Waiting for database... "
+log_f "Waiting for database... " no_nl
-while ! mysqladmin ping --host mysql -u${DBUSER} -p${DBPASS} --silent; do
+while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
   sleep 2
 done
+log_f "OK" no_date
+
+log_f "Waiting for Nginx... " no_nl
+until $(curl --output /dev/null --silent --head --fail http://nginx:8081); do
+  sleep 2
+done
+log_f "OK" no_date
+
+# Waiting for domain table
+log_f "Waiting for domain table... " no_nl
+while [[ -z ${DOMAIN_TABLE} ]]; do
+  curl --silent http://nginx/ >/dev/null 2>&1
+  DOMAIN_TABLE=$(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SHOW TABLES LIKE 'domain'" -Bs)
+  [[ -z ${DOMAIN_TABLE} ]] && sleep 10
+done
+log_f "OK" no_date
+
 log_f "Initializing, please wait... "
 
-
 while true; do
-  if [[ "${SKIP_IP_CHECK}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+
-    SKIP_IP_CHECK=y
+  # Re-using previous acme-mailcow account and domain keys
+  if [[ ! -f ${ACME_BASE}/acme/key.pem ]]; then
+    log_f "Generating missing domain private key..."
+    openssl genrsa 4096 > ${ACME_BASE}/acme/key.pem
+  else
+    log_f "Using existing domain key ${ACME_BASE}/acme/key.pem"
   fi
+  if [[ ! -f ${ACME_BASE}/acme/account.pem ]]; then
+    log_f "Generating missing Lets Encrypt account key..."
+    openssl genrsa 4096 > ${ACME_BASE}/acme/account.pem
+  else
+    log_f "Using existing Lets Encrypt account key ${ACME_BASE}/acme/account.pem"
+  fi
+
+  chmod 600 ${ACME_BASE}/acme/key.pem
+  chmod 600 ${ACME_BASE}/acme/account.pem
+
+  # Cleaning up and init validation arrays
   unset SQL_DOMAIN_ARR
   unset VALIDATED_CONFIG_DOMAINS
   unset ADDITIONAL_VALIDATED_SAN
+  unset ADDITIONAL_WC_ARR
+  unset ADDITIONAL_SAN_ARR
+  unset SAN_CHANGE
+  unset SAN_ARRAY_NOW
+  unset ORPHANED_SAN
+  unset ADDED_SAN
+  SAN_CHANGE=0
+  declare -a SAN_ARRAY_NOW
+  declare -a ORPHANED_SAN
+  declare -a ADDED_SAN
   declare -a SQL_DOMAIN_ARR
   declare -a VALIDATED_CONFIG_DOMAINS
   declare -a ADDITIONAL_VALIDATED_SAN
-  IFS=',' read -r -a ADDITIONAL_SAN_ARR <<< "${ADDITIONAL_SAN}"
+  declare -a ADDITIONAL_WC_ARR
+  declare -a ADDITIONAL_SAN_ARR
+  IFS=',' read -r -a TMP_ARR <<< "${ADDITIONAL_SAN}"
+  for i in "${TMP_ARR[@]}" ; do
+    if [[ "$i" =~ \.\*$ ]]; then
+      ADDITIONAL_WC_ARR+=(${i::-2})
+    else
+      ADDITIONAL_SAN_ARR+=($i)
+    fi
+  done
+  ADDITIONAL_WC_ARR+=('autodiscover')
+
+  # Start IP detection
   log_f "Detecting IP addresses... " no_nl
   IPV4=$(get_ipv4)
   IPV6=$(get_ipv6)
@@ -160,73 +260,50 @@ while true; do
     fi
   fi
 
-  # Container ids may have changed
+  #########################################
-  CONTAINERS_RESTART=($(curl --silent http://dockerapi:8080/containers/json | jq -r '.[] | {name: .Config.Labels["com.docker.compose.service"], id: .Id}' | jq -rc 'select( .name | tostring | contains("nginx-mailcow") or contains("postfix-mailcow") or contains("dovecot-mailcow")) | .id' | tr "\n" " "))
+  # IP and webroot challenge verification #
-
-  log_f "Waiting for domain table... " no_nl
-  while [[ -z ${DOMAIN_TABLE} ]]; do
-    curl --silent http://nginx/ >/dev/null 2>&1
-    DOMAIN_TABLE=$(mysql -h mysql-mailcow -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SHOW TABLES LIKE 'domain'" -Bs)
-    [[ -z ${DOMAIN_TABLE} ]] && sleep 10
-  done
-  log_f "OK" no_date
-
   while read domains; do
     SQL_DOMAIN_ARR+=("${domains}")
-  done < <(mysql -h mysql-mailcow -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT domain FROM domain WHERE backupmx=0 UNION SELECT alias_domain FROM alias_domain" -Bs)
+  done < <(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT domain FROM domain WHERE backupmx=0" -Bs)
 
   for SQL_DOMAIN in "${SQL_DOMAIN_ARR[@]}"; do
-    A_CONFIG=$(dig A autoconfig.${SQL_DOMAIN} +short | tail -n 1)
+    for SUBDOMAIN in "${ADDITIONAL_WC_ARR[@]}"; do
-    AAAA_CONFIG=$(dig AAAA autoconfig.${SQL_DOMAIN} +short | tail -n 1)
+      if [[  "${SUBDOMAIN}.${SQL_DOMAIN}" != "${MAILCOW_HOSTNAME}" ]]; then
-    # Check if CNAME without v6 enabled target
+        A_SUBDOMAIN=$(dig A ${SUBDOMAIN}.${SQL_DOMAIN} +short | tail -n 1)
-    if [[ ! -z ${AAAA_CONFIG} ]] && [[ -z $(echo ${AAAA_CONFIG} | grep "^\([0-9a-fA-F]\{0,4\}:\)\{1,7\}[0-9a-fA-F]\{0,4\}$") ]]; then
+        AAAA_SUBDOMAIN=$(dig AAAA ${SUBDOMAIN}.${SQL_DOMAIN} +short | tail -n 1)
-      AAAA_CONFIG=
+        # Check if CNAME without v6 enabled target
-    fi
+        if [[ ! -z ${AAAA_SUBDOMAIN} ]] && [[ -z $(echo ${AAAA_SUBDOMAIN} | grep "^\([0-9a-fA-F]\{0,4\}:\)\{1,7\}[0-9a-fA-F]\{0,4\}$") ]]; then
-    if [[ ! -z ${AAAA_CONFIG} ]]; then
+          AAAA_SUBDOMAIN=
-      log_f "Found AAAA record for autoconfig.${SQL_DOMAIN}: ${AAAA_CONFIG} - skipping A record check"
+        fi
-      if [[ $(expand ${IPV6:-"0000:0000:0000:0000:0000:0000:0000:0000"}) == $(expand ${AAAA_CONFIG}) ]] || [[ ${SKIP_IP_CHECK} == "y" ]]; then
+        if [[ ! -z ${AAAA_SUBDOMAIN} ]]; then
-        log_f "Confirmed AAAA record autoconfig.${SQL_DOMAIN}"
+          log_f "Found AAAA record for ${SUBDOMAIN}.${SQL_DOMAIN}: ${AAAA_SUBDOMAIN} - skipping A record check"
-        VALIDATED_CONFIG_DOMAINS+=("autoconfig.${SQL_DOMAIN}")
+          if [[ $(expand ${IPV6:-"0000:0000:0000:0000:0000:0000:0000:0000"}) == $(expand ${AAAA_SUBDOMAIN}) ]] || [[ ${SKIP_IP_CHECK} == "y" ]]; then
-      else
+            if verify_challenge_path "${SUBDOMAIN}.${SQL_DOMAIN}" 6; then
-        log_f "Cannot match your IP ${IPV6:-NO_IPV6_LINK} against hostname autoconfig.${SQL_DOMAIN} ($(expand ${AAAA_CONFIG}))"
+              log_f "Confirmed AAAA record ${AAAA_SUBDOMAIN}"
+              VALIDATED_CONFIG_DOMAINS+=("${SUBDOMAIN}.${SQL_DOMAIN}")
+            else
+              log_f "Confirmed AAAA record ${AAAA_SUBDOMAIN}, but HTTP validation failed"
+            fi
+          else
+            log_f "Cannot match your IP ${IPV6:-NO_IPV6_LINK} against hostname ${SUBDOMAIN}.${SQL_DOMAIN} ($(expand ${AAAA_SUBDOMAIN}))"
+          fi
+        elif [[ ! -z ${A_SUBDOMAIN} ]]; then
+          log_f "Found A record for ${SUBDOMAIN}.${SQL_DOMAIN}: ${A_SUBDOMAIN}"
+          if [[ ${IPV4:-ERR} == ${A_SUBDOMAIN} ]] || [[ ${SKIP_IP_CHECK} == "y" ]]; then
+            if verify_challenge_path "${SUBDOMAIN}.${SQL_DOMAIN}" 4; then
+              log_f "Confirmed A record ${A_SUBDOMAIN}"
+              VALIDATED_CONFIG_DOMAINS+=("${SUBDOMAIN}.${SQL_DOMAIN}")
+            else
+              log_f "Confirmed AAAA record ${A_SUBDOMAIN}, but HTTP validation failed"
+            fi
+          else
+            log_f "Cannot match your IP ${IPV4} against hostname ${SUBDOMAIN}.${SQL_DOMAIN} (${A_SUBDOMAIN})"
+          fi
+        else
+          log_f "No A or AAAA record found for hostname ${SUBDOMAIN}.${SQL_DOMAIN}"
+        fi
       fi
-    elif [[ ! -z ${A_CONFIG} ]]; then
+    done
-      log_f "Found A record for autoconfig.${SQL_DOMAIN}: ${A_CONFIG}"
-      if [[ ${IPV4:-ERR} == ${A_CONFIG} ]] || [[ ${SKIP_IP_CHECK} == "y" ]]; then
-        log_f "Confirmed A record autoconfig.${SQL_DOMAIN}"
-        VALIDATED_CONFIG_DOMAINS+=("autoconfig.${SQL_DOMAIN}")
-      else
-        log_f "Cannot match your IP ${IPV4} against hostname autoconfig.${SQL_DOMAIN} (${A_CONFIG})"
-      fi
-    else
-      log_f "No A or AAAA record found for hostname autoconfig.${SQL_DOMAIN}"
-    fi
-
-    A_DISCOVER=$(dig A autodiscover.${SQL_DOMAIN} +short | tail -n 1)
-    AAAA_DISCOVER=$(dig AAAA autodiscover.${SQL_DOMAIN} +short | tail -n 1)
-    # Check if CNAME without v6 enabled target
-    if [[ ! -z ${AAAA_DISCOVER} ]] && [[ -z $(echo ${AAAA_DISCOVER} | grep "^\([0-9a-fA-F]\{0,4\}:\)\{1,7\}[0-9a-fA-F]\{0,4\}$") ]]; then
-      AAAA_DISCOVER=
-    fi
-    if [[ ! -z ${AAAA_DISCOVER} ]]; then
-      log_f "Found AAAA record for autodiscover.${SQL_DOMAIN}: ${AAAA_DISCOVER} - skipping A record check"
-      if [[ $(expand ${IPV6:-"0000:0000:0000:0000:0000:0000:0000:0000"}) == $(expand ${AAAA_DISCOVER}) ]] || [[ ${SKIP_IP_CHECK} == "y" ]]; then
-        log_f "Confirmed AAAA record autodiscover.${SQL_DOMAIN}"
-        VALIDATED_CONFIG_DOMAINS+=("autodiscover.${SQL_DOMAIN}")
-      else
-        log_f "Cannot match your IP ${IPV6:-NO_IPV6_LINK} against hostname autodiscover.${SQL_DOMAIN} ($(expand ${AAAA_DISCOVER}))"
-      fi
-    elif [[ ! -z ${A_DISCOVER} ]]; then
-      log_f "Found A record for autodiscover.${SQL_DOMAIN}: ${A_DISCOVER}"
-      if [[ ${IPV4:-ERR} == ${A_DISCOVER} ]] || [[ ${SKIP_IP_CHECK} == "y" ]]; then
-        log_f "Confirmed A record autodiscover.${SQL_DOMAIN}"
-        VALIDATED_CONFIG_DOMAINS+=("autodiscover.${SQL_DOMAIN}")
-      else
-        log_f "Cannot match your IP ${IPV4} against hostname autodiscover.${SQL_DOMAIN} (${A_DISCOVER})"
-      fi
-    else
-      log_f "No A or AAAA record found for hostname autodiscover.${SQL_DOMAIN}"
-    fi
   done
 
   A_MAILCOW_HOSTNAME=$(dig A ${MAILCOW_HOSTNAME} +short | tail -n 1)
@@ -238,16 +315,24 @@ while true; do
   if [[ ! -z ${AAAA_MAILCOW_HOSTNAME} ]]; then
     log_f "Found AAAA record for ${MAILCOW_HOSTNAME}: ${AAAA_MAILCOW_HOSTNAME} - skipping A record check"
     if [[ $(expand ${IPV6:-"0000:0000:0000:0000:0000:0000:0000:0000"}) == $(expand ${AAAA_MAILCOW_HOSTNAME}) ]] || [[ ${SKIP_IP_CHECK} == "y" ]]; then
-      log_f "Confirmed AAAA record ${MAILCOW_HOSTNAME}"
+      if verify_challenge_path "${MAILCOW_HOSTNAME}" 6; then
-      VALIDATED_MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
+        log_f "Confirmed AAAA record ${AAAA_MAILCOW_HOSTNAME}"
+        VALIDATED_MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
+      else
+        log_f "Confirmed AAAA record ${A_MAILCOW_HOSTNAME}, but HTTP validation failed"
+      fi
     else
       log_f "Cannot match your IP ${IPV6:-NO_IPV6_LINK} against hostname ${MAILCOW_HOSTNAME} ($(expand ${AAAA_MAILCOW_HOSTNAME}))"
     fi
   elif [[ ! -z ${A_MAILCOW_HOSTNAME} ]]; then
     log_f "Found A record for ${MAILCOW_HOSTNAME}: ${A_MAILCOW_HOSTNAME}"
     if [[ ${IPV4:-ERR} == ${A_MAILCOW_HOSTNAME} ]] || [[ ${SKIP_IP_CHECK} == "y" ]]; then
-      log_f "Confirmed A record ${A_MAILCOW_HOSTNAME}"
+      if verify_challenge_path "${MAILCOW_HOSTNAME}" 4; then
-      VALIDATED_MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
+        log_f "Confirmed A record ${A_MAILCOW_HOSTNAME}"
+        VALIDATED_MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
+      else
+        log_f "Confirmed A record ${A_MAILCOW_HOSTNAME}, but HTTP validation failed"
+      fi
     else
       log_f "Cannot match your IP ${IPV4} against hostname ${MAILCOW_HOSTNAME} (${A_MAILCOW_HOSTNAME})"
     fi
@@ -279,16 +364,24 @@ while true; do
     if [[ ! -z ${AAAA_SAN} ]]; then
       log_f "Found AAAA record for ${SAN}: ${AAAA_SAN} - skipping A record check"
       if [[ $(expand ${IPV6:-"0000:0000:0000:0000:0000:0000:0000:0000"}) == $(expand ${AAAA_SAN}) ]] || [[ ${SKIP_IP_CHECK} == "y" ]]; then
-        log_f "Confirmed AAAA record ${SAN}"
+        if verify_challenge_path "${SAN}" 6; then
-        ADDITIONAL_VALIDATED_SAN+=("${SAN}")
+          log_f "Confirmed AAAA record ${AAAA_SAN}"
+          ADDITIONAL_VALIDATED_SAN+=("${SAN}")
+        else
+          log_f "Confirmed AAAA record ${AAAA_SAN}, but HTTP validation failed"
+        fi
       else
         log_f "Cannot match your IP ${IPV6:-NO_IPV6_LINK} against hostname ${SAN} ($(expand ${AAAA_SAN}))"
       fi
     elif [[ ! -z ${A_SAN} ]]; then
       log_f "Found A record for ${SAN}: ${A_SAN}"
       if [[ ${IPV4:-ERR} == ${A_SAN} ]] || [[ ${SKIP_IP_CHECK} == "y" ]]; then
-        log_f "Confirmed A record ${A_SAN}"
+        if verify_challenge_path "${SAN}" 4; then
-        ADDITIONAL_VALIDATED_SAN+=("${SAN}")
+          log_f "Confirmed A record ${A_SAN}"
+          ADDITIONAL_VALIDATED_SAN+=("${SAN}")
+        else
+          log_f "Confirmed A record ${A_SAN}, but HTTP validation failed"
+        fi
       else
         log_f "Cannot match your IP ${IPV4} against hostname ${SAN} (${A_SAN})"
       fi
@@ -306,113 +399,98 @@ while true; do
     exec $(readlink -f "$0")
   fi
 
-  array_diff ORPHANED_SAN SAN_ARRAY_NOW ALL_VALIDATED
+  # Collecting SANs from active certificate
-  if [[ ! -z ${ORPHANED_SAN[*]} ]] && [[ ${ISSUER} != *"mailcow"* ]]; then
+  SAN_NAMES=$(openssl x509 -noout -text -in ${ACME_BASE}/cert.pem | awk '/X509v3 Subject Alternative Name/ {getline;gsub(/ /, "", $0); print}' | tr -d "DNS:")
-    DATE=$(date +%Y-%m-%d_%H_%M_%S)
+  if [[ ! -z ${SAN_NAMES} ]]; then
-    log_f "Found orphaned SAN ${ORPHANED_SAN[*]} in certificate, moving old files to ${ACME_BASE}/acme/private/${DATE}.bak/, keeping key file..."
+    IFS=',' read -a SAN_ARRAY_NOW <<< ${SAN_NAMES}
-    mkdir -p ${ACME_BASE}/acme/private/${DATE}.bak/
-    [[ -f ${ACME_BASE}/acme/private/account.key ]] && mv ${ACME_BASE}/acme/private/account.key ${ACME_BASE}/acme/private/${DATE}.bak/
-    [[ -f ${ACME_BASE}/acme/fullchain.pem ]] && mv ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/acme/private/${DATE}.bak/
-    [[ -f ${ACME_BASE}/acme/cert.pem ]] && mv ${ACME_BASE}/acme/cert.pem ${ACME_BASE}/acme/private/${DATE}.bak/
-    cp ${ACME_BASE}/acme/private/privkey.pem ${ACME_BASE}/acme/private/${DATE}.bak/ # Keep key for TLSA 3 1 1 records
   fi
 
-  ACME_RESPONSE=$(acme-client \
+  # Finding difference in SAN array now vs. SAN array by current configuration
-    -v -e -b -N -n \
+  array_diff ORPHANED_SAN SAN_ARRAY_NOW ALL_VALIDATED
-    -a 'https://letsencrypt.org/documents/LE-SA-v1.2-November-15-2017.pdf' \
+  if [[ ! -z ${ORPHANED_SAN[*]} ]]; then
-    -f ${ACME_BASE}/acme/private/account.key \
+    log_f "Found orphaned SANs ${ORPHANED_SAN[*]}"
-    -k ${ACME_BASE}/acme/private/privkey.pem \
+    SAN_CHANGE=1
-    -c ${ACME_BASE}/acme \
+  fi
-    ${ALL_VALIDATED[*]} 2>&1 | tee /dev/fd/5)
+  array_diff ADDED_SAN ALL_VALIDATED SAN_ARRAY_NOW
+  if [[ ! -z ${ADDED_SAN[*]} ]]; then
+    log_f "Found new SANs ${ADDED_SAN[*]}"
+    SAN_CHANGE=1
+  fi
+
+  if [[ ${SAN_CHANGE} == 0 ]]; then
+    # Certificate did not change but could be due for renewal (4 weeks)
+    if ! openssl x509 -checkend 1209600 -noout -in ${ACME_BASE}/cert.pem; then
+      log_f "Certificate is due for renewal (< 2 weeks)"
+    else
+      log_f "Certificate validation done, neither changed nor due for renewal, sleeping for another day."
+      sleep 1d
+      continue
+    fi
+  fi
+
+  DATE=$(date +%Y-%m-%d_%H_%M_%S)
+  log_f "Creating backups in ${ACME_BASE}/backups/${DATE}/ ..."
+  mkdir -p ${ACME_BASE}/backups/${DATE}/
+  [[ -f ${ACME_BASE}/acme/acme.csr ]] && cp ${ACME_BASE}/acme/acme.csr ${ACME_BASE}/backups/${DATE}/
+  [[ -f ${ACME_BASE}/acme/cert.pem ]] && cp ${ACME_BASE}/acme/cert.pem ${ACME_BASE}/backups/${DATE}/
+  [[ -f ${ACME_BASE}/acme/key.pem ]] && cp ${ACME_BASE}/acme/key.pem ${ACME_BASE}/backups/${DATE}/
+  [[ -f ${ACME_BASE}/acme/account.pem ]] && cp ${ACME_BASE}/acme/account.pem ${ACME_BASE}/backups/${DATE}/
+
+  # Generating CSR
+  printf "[SAN]\nsubjectAltName=" > /tmp/_SAN
+  printf "DNS:%s," "${ALL_VALIDATED[@]}" >> /tmp/_SAN
+  sed -i '$s/,$//' /tmp/_SAN
+  openssl req -new -sha256 -key ${ACME_BASE}/acme/key.pem -subj "/" -reqexts SAN -config <(cat /etc/ssl/openssl.cnf /tmp/_SAN) > ${ACME_BASE}/acme/acme.csr
+
+  if [[ "${LE_STAGING}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+    log_f "Using Let's Encrypt staging servers"
+    STAGING_PARAMETER='--directory-url https://acme-staging-v02.api.letsencrypt.org/directory'
+  else
+    STAGING_PARAMETER=
+  fi
+
+  # acme-tiny writes info to stderr and ceritifcate to stdout
+  # The redirects will do the following:
+  # - redirect stdout to temp certificate file
+  # - redirect acme-tiny stderr to stdout (logs to variable ACME_RESPONSE)
+  # - tee stderr to get live output and log to dockerd
+
+  ACME_RESPONSE=$(acme-tiny ${STAGING_PARAMETER} \
+    --account-key ${ACME_BASE}/acme/account.pem \
+    --disable-check \
+    --csr ${ACME_BASE}/acme/acme.csr \
+    --acme-dir /var/www/acme/ 2>&1 > /tmp/_cert.pem | tee /dev/fd/5)
 
   case "$?" in
-    0) # new certs
+    0) # cert requested
-      log_f "${ACME_RESPONSE}" redis_only
+      ACME_RESPONSE_B64=$(echo "${ACME_RESPONSE}" | openssl enc -e -A -base64)
-      # cp the new certificates and keys
+      log_f "${ACME_RESPONSE_B64}" redis_only b64
-      cp ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem
+      log_f "Deploying..."
-      cp ${ACME_BASE}/acme/private/privkey.pem ${ACME_BASE}/key.pem
+      # Deploy the new certificate and key
-
+      # Moving temp cert to acme/cert.pem
-      # restart docker containers
+      if verify_hash_match /tmp/_cert.pem ${ACME_BASE}/acme/key.pem; then
-      if ! verify_hash_match ${ACME_BASE}/cert.pem ${ACME_BASE}/key.pem; then
+        mv /tmp/_cert.pem ${ACME_BASE}/acme/cert.pem
-        log_f "Certificate was successfully requested, but key and certificate have non-matching hashes, restoring mailcow snake-oil and restarting containers..."
+        cp ${ACME_BASE}/acme/cert.pem ${ACME_BASE}/cert.pem
-        cp ${SSL_EXAMPLE}/cert.pem ${ACME_BASE}/cert.pem
+        cp ${ACME_BASE}/acme/key.pem ${ACME_BASE}/key.pem
-        cp ${SSL_EXAMPLE}/key.pem ${ACME_BASE}/key.pem
+        reload_configurations
-      fi
+        rm /var/www/acme/*
-      restart_containers ${CONTAINERS_RESTART[*]}
+        log_f "Certificate successfully deployed, removing backup, sleeping 1d"
-      ;;
+        sleep 1d
-    1) # failure
+      else
-      log_f "${ACME_RESPONSE}" redis_only
+        log_f "Certificate was successfully requested, but key and certificate have non-matching hashes, ignoring certificate"
-      if [[ $ACME_RESPONSE =~ "No registration exists" ]]; then
+        log_f "Retrying in 30 minutes..."
-        log_f "Registration keys are invalid, deleting old keys and restarting..."
+        sleep 30m
-        rm ${ACME_BASE}/acme/private/account.key
         exec $(readlink -f "$0")
       fi
-      if [[ -f ${ACME_BASE}/acme/private/${DATE}.bak/fullchain.pem ]] && [[ -f ${ACME_BASE}/acme/private/${DATE}.bak/privkey.pem ]]; then
-        log_f "Error requesting certificate, restoring previous certificate from backup and restarting containers...."
-        cp ${ACME_BASE}/acme/private/${DATE}.bak/fullchain.pem ${ACME_BASE}/cert.pem
-        cp ${ACME_BASE}/acme/private/${DATE}.bak/privkey.pem ${ACME_BASE}/key.pem
-        TRIGGER_RESTART=1
-      elif [[ -f ${ACME_BASE}/acme/fullchain.pem ]] && [[ -f ${ACME_BASE}/acme/private/privkey.pem ]]; then
-        log_f "Error requesting certificate, restoring from previous acme request and restarting containers..."
-        cp ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem
-        cp ${ACME_BASE}/acme/private/privkey.pem ${ACME_BASE}/key.pem
-        TRIGGER_RESTART=1
-      fi
-      if ! verify_hash_match ${ACME_BASE}/cert.pem ${ACME_BASE}/key.pem; then
-        log_f "Error verifying certificates, restoring mailcow snake-oil and restarting containers..."
-        cp ${SSL_EXAMPLE}/cert.pem ${ACME_BASE}/cert.pem
-        cp ${SSL_EXAMPLE}/key.pem ${ACME_BASE}/key.pem
-        TRIGGER_RESTART=1
-      fi
-      [[ ${TRIGGER_RESTART} == 1 ]] && restart_containers ${CONTAINERS_RESTART[*]}
-      log_f "Retrying in 30 minutes..."
-      sleep 30m
-      exec $(readlink -f "$0")
       ;;
-    2) # no change
+    *) # non-zero is non-fun
-      log_f "${ACME_RESPONSE}" redis_only
+      ACME_RESPONSE_B64=$(echo "${ACME_RESPONSE}" | openssl enc -e -A -base64)
-      if ! diff ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem; then
+      log_f "${ACME_RESPONSE_B64}" redis_only b64
-        log_f "Certificate was not changed, but active certificate does not match the verified certificate, fixing and restarting containers..."
-        cp ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem
-        cp ${ACME_BASE}/acme/private/privkey.pem ${ACME_BASE}/key.pem
-        TRIGGER_RESTART=1
-      fi
-      if ! verify_hash_match ${ACME_BASE}/cert.pem ${ACME_BASE}/key.pem; then
-        log_f "Certificate was not changed, but hashes do not match, restoring from previous acme request and restarting containers..."
-        cp ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem
-        cp ${ACME_BASE}/acme/private/privkey.pem ${ACME_BASE}/key.pem
-        TRIGGER_RESTART=1
-      fi
-      log_f "Certificate was not changed"
-      [[ ${TRIGGER_RESTART} == 1 ]] && restart_containers ${CONTAINERS_RESTART[*]}
-      ;;
-    *) # unspecified
-      log_f "${ACME_RESPONSE}" redis_only
-      if [[ -f ${ACME_BASE}/acme/private/${DATE}.bak/fullchain.pem ]] && [[ -f ${ACME_BASE}/acme/private/${DATE}.bak/privkey.pem ]]; then
-        log_f "Error requesting certificate, restoring previous certificate from backup and restarting containers...."
-        cp ${ACME_BASE}/acme/private/${DATE}.bak/fullchain.pem ${ACME_BASE}/cert.pem
-        cp ${ACME_BASE}/acme/private/${DATE}.bak/privkey.pem ${ACME_BASE}/key.pem
-        TRIGGER_RESTART=1
-            elif [[ -f ${ACME_BASE}/acme/fullchain.pem ]] && [[ -f ${ACME_BASE}/acme/private/privkey.pem ]]; then
-        log_f "Error requesting certificate, restoring from previous acme request and restarting containers..."
-        cp ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem
-        cp ${ACME_BASE}/acme/private/privkey.pem ${ACME_BASE}/key.pem
-        TRIGGER_RESTART=1
-      fi
-      if ! verify_hash_match ${ACME_BASE}/cert.pem ${ACME_BASE}/key.pem; then
-        log_f "Error verifying certificates, restoring mailcow snake-oil..."
-        cp ${SSL_EXAMPLE}/cert.pem ${ACME_BASE}/cert.pem
-        cp ${SSL_EXAMPLE}/key.pem ${ACME_BASE}/key.pem
-        TRIGGER_RESTART=1
-      fi
-      [[ ${TRIGGER_RESTART} == 1 ]] && restart_containers ${CONTAINERS_RESTART[*]}
       log_f "Retrying in 30 minutes..."
+      redis-cli -h redis SET ACME_FAIL_TIME "$(date +%s)"
       sleep 30m
       exec $(readlink -f "$0")
       ;;
   esac
 
-  log_f "ACME certificate validation done. Sleeping for another day."
-  sleep 1d
-
 done

data/Dockerfiles/clamd/Dockerfile

@@ -1,24 +1,37 @@
-FROM alpine:3.8
+FROM debian:stretch-slim
 
 LABEL maintainer "André Peters <andre.peters@servercow.de>"
 
-# Add scripts
-COPY dl_files.sh bootstrap.sh ./
-
 # Installation
-ENV CLAMAV 0.100.1
+ENV CLAMAV 0.101.1
 
-RUN apk add --no-cache --virtual build-dependencies alpine-sdk ncurses-dev zlib-dev bzip2-dev pcre-dev linux-headers fts-dev libxml2-dev libressl-dev \
+RUN apt-get update && apt-get install -y --no-install-recommends \
-  && apk add --no-cache curl bash tini libxml2 libbz2 pcre fts libressl tzdata \
+  ca-certificates \
+  zlib1g-dev \
+  libncurses5-dev \
+  libzip-dev \
+  libpcre2-dev \
+  libxml2-dev \
+  libssl-dev \
+  build-essential \
+  libjson-c-dev \
+  curl \
+  bash \
+  wget \
+  tzdata \
+  dnsutils \
+  rsync \
+  dos2unix \
+  netcat \
+  && rm -rf /var/lib/apt/lists/* \
   && wget -O - https://www.clamav.net/downloads/production/clamav-${CLAMAV}.tar.gz | tar xfvz - \
   && cd clamav-${CLAMAV} \
-  && LIBS=-lfts ./configure \
+  && ./configure \
   --prefix=/usr \
   --libdir=/usr/lib \
   --sysconfdir=/etc/clamav \
   --mandir=/usr/share/man \
   --infodir=/usr/share/info \
-  --without-iconv \
   --disable-llvm \
   --with-user=clamav \
   --with-group=clamav \
@@ -30,18 +43,19 @@ RUN apk add --no-cache --virtual build-dependencies alpine-sdk ncurses-dev zlib-
   && make install \
   && make clean \
   && cd .. && rm -rf clamav-${CLAMAV} \
-  && apk del build-dependencies \
+  && apt-get -y --auto-remove purge build-essential \
-  && addgroup -S clamav \
+  && apt-get -y purge zlib1g-dev \
-  && adduser -S -D -h /var/lib/clamav -s /sbin/nologin -G clamav -g clamav clamav \
+  libncurses5-dev \
-  && adduser clamav tty \
+  libzip-dev \
-  && mkdir -p /run/clamav \
+  libpcre2-dev \
-  && chown clamav:clamav /run/clamav \
+  libxml2-dev \
-  && chmod +x /dl_files.sh \
+  libssl-dev \
-  && set -ex; /bin/bash /dl_files.sh \
+  libjson-c-dev \
-  && chmod 750 /run/clamav
+  && addgroup --system --gid 700 clamav \
+  && adduser --system --no-create-home --home /var/lib/clamav --uid 700 --gid 700 --disabled-login clamav \
+  && rm -rf /tmp/* /var/tmp/*
 
-# Port provision
+COPY bootstrap.sh ./
-EXPOSE 3310
+COPY tini /sbin/tini
 
-# AV daemon bootstrapping
 CMD ["/sbin/tini", "-g", "--", "/bootstrap.sh"]

data/Dockerfiles/clamd/bootstrap.sh

@@ -6,16 +6,30 @@ if [[ "${SKIP_CLAMD}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
   exit 0
 fi
 
-# Create log pipes
+# Prepare whitelist
-mkdir -p /var/log/clamav
+
-touch /var/log/clamav/clamd.log /var/log/clamav/freshclam.log
+mkdir -p /run/clamav /var/lib/clamav
-chown -R clamav:clamav /var/log/clamav/
+
-chown root:tty /dev/console
+if [[ -s /etc/clamav/whitelist.ign2 ]]; then
-chmod g+rw /dev/console
+  echo "Copying non-empty whitelist.ign2 to /var/lib/clamav/whitelist.ign2"
+  cp /etc/clamav/whitelist.ign2 /var/lib/clamav/whitelist.ign2
+fi
+if [[ ! -f /var/lib/clamav/whitelist.ign2 ]]; then
+  echo "Creating /var/lib/clamav/whitelist.ign2"
+  echo "Example-Signature.Ignore-1" > /var/lib/clamav/whitelist.ign2
+fi
+
+chown clamav:clamav -R /var/lib/clamav /run/clamav
+
+chmod 755 /var/lib/clamav
+chmod 644 -R /var/lib/clamav/*
+chmod 750 /run/clamav
+
+echo "Stating whitelist.ign2"
+stat /var/lib/clamav/whitelist.ign2
 
-# Prepare
-[[ ! -f /var/lib/clamav/whitelist.ign2 ]] && touch /var/lib/clamav/whitelist.ign2
 dos2unix /var/lib/clamav/whitelist.ign2
+
 sed -i '/^\s*$/d' /var/lib/clamav/whitelist.ign2
 
 BACKGROUND_TASKS=()
@@ -29,7 +43,35 @@ done
 ) &
 BACKGROUND_TASKS+=($!)
 
-clamd &
+(
+while true; do
+  sleep 2m
+  SANE_MIRRORS="$(dig +ignore +short rsync.sanesecurity.net)"
+  for sane_mirror in ${SANE_MIRRORS}; do
+    rsync -avp --chown=clamav:clamav --chmod=Du=rwx,Dgo=rx,Fu=rw,Fog=r --timeout=5 rsync://${sane_mirror}/sanesecurity/ \
+      --include 'blurl.ndb' \
+      --include 'junk.ndb' \
+      --include 'jurlbl.ndb' \
+      --include 'jurbla.ndb' \
+      --include 'phishtank.ndb' \
+      --include 'phish.ndb' \
+      --include 'spamimg.hdb' \
+      --include 'scam.ndb' \
+      --include 'rogue.hdb' \
+      --include 'sanesecurity.ftm' \
+      --include 'sigwhitelist.ign2' \
+      --exclude='*' /var/lib/clamav/
+    if [ $? -eq 0 ]; then
+      echo RELOAD | nc localhost 3310
+      break
+    fi
+  done
+  sleep 30h
+done
+) &
+BACKGROUND_TASKS+=($!)
+
+nice -n10 clamd &
 BACKGROUND_TASKS+=($!)
 
 while true; do

data/Dockerfiles/clamd/dl_files.sh

@@ -1,32 +0,0 @@
-#!/bin/bash
-
-declare -a DB_MIRRORS=(
-  "switch.clamav.net"
-  "clamavdb.heanet.ie"
-  "clamav.iol.cz"
-  "clamav.univ-nantes.fr"
-  "clamav.easynet.fr"
-  "clamav.begi.net"
-)
-declare -a DB_MIRRORS=( $(shuf -e "${DB_MIRRORS[@]}") )
-
-DB_FILES=(
-  "bytecode.cvd"
-  "daily.cvd"
-  "main.cvd"
-)
-
-for i in "${DB_MIRRORS[@]}"; do
-  for j in "${DB_FILES[@]}"; do
-  [[ -f "/var/lib/clamav/${j}" && -s "/var/lib/clamav/${j}" ]] && continue;
-  if [[ $(curl -o /dev/null --connect-timeout 1 \
-    --max-time 1 \
-    --silent \
-    --head \
-    --write-out "%{http_code}\n" "${i}/${j}") == 200 ]]; then
-    curl "${i}/${j}" -o "/var/lib/clamav/${j}" -#
-  fi
-  done
-done
-
-chown clamav:clamav /var/lib/clamav/*.cvd

data/Dockerfiles/dockerapi/Dockerfile

@@ -1,8 +1,11 @@
-FROM python:2-alpine
+FROM alpine:3.9
 LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
 
-RUN apk add -U --no-cache iptables ip6tables tzdata
+RUN apk add -U --no-cache python2 python-dev py-pip gcc musl-dev tzdata openssl-dev libffi-dev \
-RUN pip install docker==3.0.1 flask flask-restful
+  && pip2 install --upgrade pip \
+  && pip2 install --upgrade docker==3.0.1 flask flask-restful pyOpenSSL \
+  && apk del python-dev py2-pip gcc
 
 COPY server.py /
+
 CMD ["python2", "-u", "/server.py"]

data/Dockerfiles/dockerapi/server.py

@@ -1,14 +1,19 @@
 from flask import Flask
 from flask_restful import Resource, Api
 from flask import jsonify
+from flask import Response
 from flask import request
 from threading import Thread
+from OpenSSL import crypto
 import docker
+import uuid
 import signal
 import time
 import os
 import re
 import sys
+import ssl
+import socket
 
 docker_client = docker.DockerClient(base_url='unix://var/run/docker.sock', version='auto')
 app = Flask(__name__)
@@ -62,65 +67,228 @@ class container_post(Resource):
         except Exception as e:
           return jsonify(type='danger', msg=str(e))
 
+      elif post_action == 'top':
+        try:
+          for container in docker_client.containers.list(all=True, filters={"id": container_id}):
+            return jsonify(type='success', msg=container.top())
+        except Exception as e:
+          return jsonify(type='danger', msg=str(e))
+
+      elif post_action == 'stats':
+        try:
+          for container in docker_client.containers.list(all=True, filters={"id": container_id}):
+            return jsonify(type='success', msg=container.stats(decode=True, stream=False))
+        except Exception as e:
+          return jsonify(type='danger', msg=str(e))
+
       elif post_action == 'exec':
 
         if not request.json or not 'cmd' in request.json:
           return jsonify(type='danger', msg='cmd is missing')
 
-        if request.json['cmd'] == 'df' and request.json['dir']:
+        if request.json['cmd'] == 'mailq':
-          try:
+          if 'items' in request.json:
-            for container in docker_client.containers.list(filters={"id": container_id}):
+            r = re.compile("^[0-9a-fA-F]+$")
-              # Should be changed to be able to validate a path
+            filtered_qids = filter(r.match, request.json['items'])
-              directory = re.sub('[^0-9a-zA-Z/]+', '', request.json['dir'])
+            if filtered_qids:
-              df_return = container.exec_run(["/bin/bash", "-c", "/bin/df -H " + directory + " | /usr/bin/tail -n1 | /usr/bin/tr -s [:blank:] | /usr/bin/tr ' ' ','"], user='nobody')
+              if request.json['task'] == 'delete':
-              if df_return.exit_code == 0:
+                flagged_qids = ['-d %s' % i for i in filtered_qids]
-                return df_return.output.rstrip()
+                sanitized_string = str(' '.join(flagged_qids));
-              else:
+                try:
-                return "0,0,0,0,0,0"
+                  for container in docker_client.containers.list(filters={"id": container_id}):
-          except Exception as e:
+                    postsuper_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postsuper " + sanitized_string])
-            return jsonify(type='danger', msg=str(e))
+                    return exec_run_handler('generic', postsuper_r)
-        elif request.json['cmd'] == 'sieve_list' and request.json['username']:
+                except Exception as e:
-          try:
+                  return jsonify(type='danger', msg=str(e))
-            for container in docker_client.containers.list(filters={"id": container_id}):
+              if request.json['task'] == 'hold':
-              sieve_return = container.exec_run(["/bin/bash", "-c", "/usr/local/bin/doveadm sieve list -u '" + request.json['username'].replace("'", "'\\''") + "'"], user='vmail')
+                flagged_qids = ['-h %s' % i for i in filtered_qids]
-              return sieve_return.output
+                sanitized_string = str(' '.join(flagged_qids));
-          except Exception as e:
+                try:
-            return jsonify(type='danger', msg=str(e))
+                  for container in docker_client.containers.list(filters={"id": container_id}):
-        elif request.json['cmd'] == 'sieve_print' and request.json['script_name'] and request.json['username']:
+                    postsuper_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postsuper " + sanitized_string])
-          try:
+                    return exec_run_handler('generic', postsuper_r)
-            for container in docker_client.containers.list(filters={"id": container_id}):
+                except Exception as e:
-              sieve_return = container.exec_run(["/bin/bash", "-c", "/usr/local/bin/doveadm sieve get -u '" + request.json['username'].replace("'", "'\\''") + "' '" + request.json['script_name'].replace("'", "'\\''") + "'"], user='vmail')
+                  return jsonify(type='danger', msg=str(e))
-              return sieve_return.output
+              if request.json['task'] == 'unhold':
-          except Exception as e:
+                flagged_qids = ['-H %s' % i for i in filtered_qids]
-            return jsonify(type='danger', msg=str(e))
+                sanitized_string = str(' '.join(flagged_qids));
-        elif request.json['cmd'] == 'worker_password' and request.json['raw']:
+                try:
-          try:
+                  for container in docker_client.containers.list(filters={"id": container_id}):
-            for container in docker_client.containers.list(filters={"id": container_id}):
+                    postsuper_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postsuper " + sanitized_string])
-              hash = container.exec_run(["/bin/bash", "-c", "/usr/bin/rspamadm pw -e -p '" + request.json['raw'].replace("'", "'\\''") + "' 2> /dev/null"], user='_rspamd')
+                    return exec_run_handler('generic', postsuper_r)
-              if hash.exit_code == 0:
+                except Exception as e:
-                hash_stdout = str(hash.output)
+                  return jsonify(type='danger', msg=str(e))
-                for line in hash_stdout.split("\n"):
+              if request.json['task'] == 'deliver':
-                  if '$2$' in line:
+                flagged_qids = ['-i %s' % i for i in filtered_qids]
-                    hash = line.strip()
+                try:
-                f = open("/access.inc", "w")
+                  for container in docker_client.containers.list(filters={"id": container_id}):
-                f.write('enable_password = "' + re.sub('[^0-9a-zA-Z\$]+', '', hash.rstrip()) + '";\n')
+                    for i in flagged_qids:
-                f.close()
+                      postqueue_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postqueue " + i], user='postfix')
-                container.restart()
+                      # todo: check each exit code
-                return jsonify(type='success', msg='command completed successfully')
+                    return jsonify(type='success', msg=str("Scheduled immediate delivery"))
-              else:
+                except Exception as e:
-                return jsonify(type='danger', msg='command did not complete, exit code was ' + int(hash.exit_code))
+                  return jsonify(type='danger', msg=str(e))
-          except Exception as e:
+          elif request.json['task'] == 'list':
-            return jsonify(type='danger', msg=str(e))
+            try:
-        elif request.json['cmd'] == 'mailman_password' and request.json['email'] and request.json['passwd']:
+              for container in docker_client.containers.list(filters={"id": container_id}):
-          try:
+                mailq_return = container.exec_run(["/usr/sbin/postqueue", "-j"], user='postfix')
-            for container in docker_client.containers.list(filters={"id": container_id}):
+                return exec_run_handler('utf8_text_only', mailq_return)
-              add_su = container.exec_run(["/bin/bash", "-c", "/opt/mm_web/add_su.py '" + request.json['passwd'].replace("'", "'\\''") + "' '" + request.json['email'].replace("'", "'\\''") + "'"], user='mailman')
+            except Exception as e:
-              if add_su.exit_code == 0:
+              return jsonify(type='danger', msg=str(e))
-                return jsonify(type='success', msg='command completed successfully')
+          elif request.json['task'] == 'flush':
-              else:
+            try:
-                return jsonify(type='danger', msg='command did not complete, exit code was ' + int(add_su.exit_code))
+              for container in docker_client.containers.list(filters={"id": container_id}):
-          except Exception as e:
+                postqueue_r = container.exec_run(["/usr/sbin/postqueue", "-f"], user='postfix')
-            return jsonify(type='danger', msg=str(e))
+                return exec_run_handler('generic', postqueue_r)
+            except Exception as e:
+              return jsonify(type='danger', msg=str(e))
+          elif request.json['task'] == 'super_delete':
+            try:
+              for container in docker_client.containers.list(filters={"id": container_id}):
+                postsuper_r = container.exec_run(["/usr/sbin/postsuper", "-d", "ALL"])
+                return exec_run_handler('generic', postsuper_r)
+            except Exception as e:
+              return jsonify(type='danger', msg=str(e))
+
+        elif request.json['cmd'] == 'system':
+          if request.json['task'] == 'fts_rescan':
+            if 'username' in request.json:
+              try:
+                for container in docker_client.containers.list(filters={"id": container_id}):
+                  rescan_return = container.exec_run(["/bin/bash", "-c", "/usr/local/bin/doveadm fts rescan -u '" + request.json['username'].replace("'", "'\\''") + "'"], user='vmail')
+                  if rescan_return.exit_code == 0:
+                    return jsonify(type='success', msg='fts_rescan: rescan triggered')
+                  else:
+                    return jsonify(type='warning', msg='fts_rescan error')
+              except Exception as e:
+                return jsonify(type='danger', msg=str(e))
+            if 'all' in request.json:
+              try:
+                for container in docker_client.containers.list(filters={"id": container_id}):
+                  rescan_return = container.exec_run(["/bin/bash", "-c", "/usr/local/bin/doveadm fts rescan -A"], user='vmail')
+                  if rescan_return.exit_code == 0:
+                    return jsonify(type='success', msg='fts_rescan: rescan triggered')
+                  else:
+                    return jsonify(type='warning', msg='fts_rescan error')
+              except Exception as e:
+                return jsonify(type='danger', msg=str(e))
+          elif request.json['task'] == 'df':
+            if 'dir' in request.json:
+              try:
+                for container in docker_client.containers.list(filters={"id": container_id}):
+                  df_return = container.exec_run(["/bin/bash", "-c", "/bin/df -H '" + request.json['dir'].replace("'", "'\\''") + "' | /usr/bin/tail -n1 | /usr/bin/tr -s [:blank:] | /usr/bin/tr ' ' ','"], user='nobody')
+                  if df_return.exit_code == 0:
+                    return df_return.output.rstrip()
+                  else:
+                    return "0,0,0,0,0,0"
+              except Exception as e:
+                return jsonify(type='danger', msg=str(e))
+          elif request.json['task'] == 'mysql_upgrade':
+            try:
+              for container in docker_client.containers.list(filters={"id": container_id}):
+                sql_shell = container.exec_run(["/bin/bash"], stdin=True, socket=True, user='mysql')
+                upgrade_cmd = "/usr/bin/mysql_upgrade -uroot -p'" + os.environ['DBROOT'].replace("'", "'\\''") + "'\n"
+                sql_socket = sql_shell.output;
+                try :
+                  sql_socket.sendall(upgrade_cmd.encode('utf-8'))
+                  sql_socket.shutdown(socket.SHUT_WR)
+                except socket.error:
+                  return jsonify(type='danger', msg=str('socket error'))
+                worker_response = recv_socket_data(sql_socket)
+                matched = False
+                for line in worker_response.split("\n"):
+                  if 'is already upgraded to' in line:
+                    matched = True
+                if matched:
+                  return jsonify(type='success', msg='mysql_upgrade: already upgraded')
+                else:
+                  container.restart()
+                  return jsonify(type='warning', msg='mysql_upgrade: upgrade was applied')
+            except Exception as e:
+              return jsonify(type='danger', msg=str(e))
+
+        elif request.json['cmd'] == 'reload':
+          if request.json['task'] == 'dovecot':
+            try:
+              for container in docker_client.containers.list(filters={"id": container_id}):
+                reload_return = container.exec_run(["/bin/bash", "-c", "/usr/local/sbin/dovecot reload"])
+                return exec_run_handler('generic', reload_return)
+            except Exception as e:
+              return jsonify(type='danger', msg=str(e))
+          if request.json['task'] == 'postfix':
+            try:
+              for container in docker_client.containers.list(filters={"id": container_id}):
+                reload_return = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postfix reload"])
+                return exec_run_handler('generic', reload_return)
+            except Exception as e:
+              return jsonify(type='danger', msg=str(e))
+          if request.json['task'] == 'nginx':
+            try:
+              for container in docker_client.containers.list(filters={"id": container_id}):
+                reload_return = container.exec_run(["/bin/sh", "-c", "/usr/sbin/nginx -s reload"])
+                return exec_run_handler('generic', reload_return)
+            except Exception as e:
+              return jsonify(type='danger', msg=str(e))
+
+        elif request.json['cmd'] == 'sieve':
+          if request.json['task'] == 'list':
+            if 'username' in request.json:
+              try:
+                for container in docker_client.containers.list(filters={"id": container_id}):
+                  sieve_return = container.exec_run(["/bin/bash", "-c", "/usr/local/bin/doveadm sieve list -u '" + request.json['username'].replace("'", "'\\''") + "'"])
+                  return exec_run_handler('utf8_text_only', sieve_return)
+              except Exception as e:
+                return jsonify(type='danger', msg=str(e))
+          elif request.json['task'] == 'print':
+            if 'username' in request.json and 'script_name' in request.json:
+              try:
+                for container in docker_client.containers.list(filters={"id": container_id}):
+                  sieve_return = container.exec_run(["/bin/bash", "-c", "/usr/local/bin/doveadm sieve get -u '" + request.json['username'].replace("'", "'\\''") + "' '" + request.json['script_name'].replace("'", "'\\''") + "'"])
+                  return exec_run_handler('utf8_text_only', sieve_return)
+              except Exception as e:
+                return jsonify(type='danger', msg=str(e))
+
+        elif request.json['cmd'] == 'maildir':
+          if request.json['task'] == 'cleanup':
+            if 'maildir' in request.json:
+              try:
+                for container in docker_client.containers.list(filters={"id": container_id}):
+                  sane_name = re.sub(r'\W+', '', request.json['maildir'])
+                  maildir_cleanup = container.exec_run(["/bin/bash", "-c", "if [[ -d '/var/vmail/" + request.json['maildir'].replace("'", "'\\''") + "' ]]; then /bin/mv '/var/vmail/" + request.json['maildir'].replace("'", "'\\''") + "' '/var/vmail/_garbage/" + str(int(time.time())) + "_" + sane_name + "'; fi"], user='vmail')
+                  return exec_run_handler('generic', maildir_cleanup)
+              except Exception as e:
+                return jsonify(type='danger', msg=str(e))
+
+        elif request.json['cmd'] == 'rspamd':
+          if request.json['task'] == 'worker_password':
+            if 'raw' in request.json:
+              try:
+                for container in docker_client.containers.list(filters={"id": container_id}):
+                  worker_shell = container.exec_run(["/bin/bash"], stdin=True, socket=True, user='_rspamd')
+                  worker_cmd = "/usr/bin/rspamadm pw -e -p '" + request.json['raw'].replace("'", "'\\''") + "' 2> /dev/null\n"
+                  worker_socket = worker_shell.output;
+                  try :
+                    worker_socket.sendall(worker_cmd.encode('utf-8'))
+                    worker_socket.shutdown(socket.SHUT_WR)
+                  except socket.error:
+                    return jsonify(type='danger', msg=str('socket error'))
+                  worker_response = recv_socket_data(worker_socket)
+                  matched = False
+                  for line in worker_response.split("\n"):
+                    if '$2$' in line:
+                      matched = True
+                      hash = line.strip()
+                      hash_out = re.search('\$2\$.+$', hash).group(0)
+                      f = open("/access.inc", "w")
+                      f.write('enable_password = "' + re.sub('[^0-9a-zA-Z\$]+', '', hash_out.rstrip()) + '";\n')
+                      f.close()
+                      container.restart()
+                  if matched:
+                    return jsonify(type='success', msg='command completed successfully')
+                  else:
+                    return jsonify(type='danger', msg='command did not complete')
+              except Exception as e:
+                return jsonify(type='danger', msg=str(e))
 
         else:
           return jsonify(type='danger', msg='Unknown command')
@@ -137,11 +305,84 @@ class GracefulKiller:
     signal.signal(signal.SIGINT, self.exit_gracefully)
     signal.signal(signal.SIGTERM, self.exit_gracefully)
 
-  def exit_gracefully(self,signum, frame):
+  def exit_gracefully(self, signum, frame):
     self.kill_now = True
 
 def startFlaskAPI():
-  app.run(debug=False, host='0.0.0.0', port=8080, threaded=True)
+  create_self_signed_cert()
+  try:
+    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+    ctx.check_hostname = False
+    ctx.load_cert_chain(certfile='/cert.pem', keyfile='/key.pem')
+  except:
+    print "Cannot initialize TLS, retrying in 5s..."
+    time.sleep(5)
+  app.run(debug=False, host='0.0.0.0', port=443, threaded=True, ssl_context=ctx)
+
+def recv_socket_data(c_socket, timeout=10):
+  c_socket.setblocking(0)
+  total_data=[];
+  data='';
+  begin=time.time()
+  while True:
+    if total_data and time.time()-begin > timeout:
+      break
+    elif time.time()-begin > timeout*2:
+      break
+    try:
+      data = c_socket.recv(8192)
+      if data:
+        total_data.append(data)
+        #change the beginning time for measurement
+        begin=time.time()
+      else:
+        #sleep for sometime to indicate a gap
+        time.sleep(0.1)
+        break
+    except:
+      pass
+  return ''.join(total_data)
+
+def exec_run_handler(type, output):
+  if type == 'generic':
+    if output.exit_code == 0:
+      return jsonify(type='success', msg='command completed successfully')
+    else:
+      return jsonify(type='danger', msg='command failed: ' + output.output)
+  if type == 'utf8_text_only':
+    r = Response(response=output.output, status=200, mimetype="text/plain")
+    r.headers["Content-Type"] = "text/plain; charset=utf-8"
+    return r
+
+def create_self_signed_cert():
+  success = False
+  while not success:
+    try:
+      pkey = crypto.PKey()
+      pkey.generate_key(crypto.TYPE_RSA, 2048)
+      cert = crypto.X509()
+      cert.get_subject().O = "mailcow"
+      cert.get_subject().CN = "dockerapi"
+      cert.set_serial_number(int(uuid.uuid4()))
+      cert.gmtime_adj_notBefore(0)
+      cert.gmtime_adj_notAfter(10*365*24*60*60)
+      cert.set_issuer(cert.get_subject())
+      cert.set_pubkey(pkey)
+      cert.sign(pkey, 'sha512')
+      cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
+      pkey = crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey)
+      with os.fdopen(os.open('/cert.pem', os.O_WRONLY | os.O_CREAT, 0o644), 'w') as handle:
+        handle.write(cert)
+      with os.fdopen(os.open('/key.pem', os.O_WRONLY | os.O_CREAT, 0o600), 'w') as handle:
+        handle.write(pkey)
+      success = True
+    except:
+      time.sleep(1)
+      try:
+        os.remove('/cert.pem')
+        os.remove('/key.pem')
+      except OSError:
+        pass
 
 api.add_resource(containers_get, '/containers/json')
 api.add_resource(container_get, '/containers/<string:container_id>/json')

data/Dockerfiles/dovecot/Dockerfile

@@ -3,8 +3,8 @@ LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
 ENV LC_ALL C
-ENV DOVECOT_VERSION 2.3.2.1
+ENV DOVECOT_VERSION 2.3.5.1
-ENV PIGEONHOLE_VERSION 0.5.2
+ENV PIGEONHOLE_VERSION 0.5.5
 
 RUN apt-get update && apt-get -y --no-install-recommends install \
   automake \
@@ -14,6 +14,9 @@ RUN apt-get update && apt-get -y --no-install-recommends install \
   cpanminus \
   curl \
   default-libmysqlclient-dev \
+  dnsutils \
+  gettext \
+  jq \
   libjson-webtoken-perl \
   libcgi-pm-perl \
   libcrypt-openssl-rsa-perl \
@@ -38,6 +41,7 @@ RUN apt-get update && apt-get -y --no-install-recommends install \
   libio-socket-ssl-perl \
   libio-tee-perl \
   libipc-run-perl \
+  libldap2-dev \
   liblockfile-simple-perl \
   liblz-dev \
   liblz4-dev \
@@ -60,6 +64,10 @@ RUN apt-get update && apt-get -y --no-install-recommends install \
   libregexp-common-perl \
   liburi-perl \
   lzma-dev \
+  python-html2text \
+  python-jinja2 \
+  python-mysql.connector \
+  python-redis \
   make \
   mysql-client \
   procps \
@@ -69,11 +77,10 @@ RUN apt-get update && apt-get -y --no-install-recommends install \
   syslog-ng \
   syslog-ng-core \
   syslog-ng-mod-redis \
-  && rm -rf /var/lib/apt/lists/*
+  && rm -rf /var/lib/apt/lists/* \
-
+  && curl https://www.dovecot.org/releases/2.3/dovecot-$DOVECOT_VERSION.tar.gz | tar xvz  \
-RUN curl https://www.dovecot.org/releases/2.3/dovecot-$DOVECOT_VERSION.tar.gz | tar xvz  \
   && cd dovecot-$DOVECOT_VERSION \
-  && ./configure --with-solr --with-mysql --with-lzma --with-lz4 --with-ssl=openssl --with-notify=inotify --with-storages=mdbox,sdbox,maildir,mbox,imapc,pop3c --with-bzlib --with-zlib \
+  && ./configure --with-solr --with-mysql --with-ldap --with-lzma --with-lz4 --with-ssl=openssl --with-notify=inotify --with-storages=mdbox,sdbox,maildir,mbox,imapc,pop3c --with-bzlib --with-zlib --enable-hardening \
   && make -j3 \
   && make install \
   && make clean \
@@ -85,12 +92,18 @@ RUN curl https://www.dovecot.org/releases/2.3/dovecot-$DOVECOT_VERSION.tar.gz |
   && make install \
   && make clean \
   && cd .. \
-  && rm -rf dovecot-2.3-pigeonhole-$PIGEONHOLE_VERSION
+  && rm -rf dovecot-2.3-pigeonhole-$PIGEONHOLE_VERSION \
-
+  && cpanm Data::Uniqid Mail::IMAPClient String::Util \
-RUN cpanm Data::Uniqid Mail::IMAPClient String::Util
+  && groupadd -g 5000 vmail \
-RUN echo '* * * * *   root   /usr/local/bin/imapsync_cron.pl' > /etc/cron.d/imapsync
+  && groupadd -g 401 dovecot \
-RUN echo '30 3 * * *   vmail  /usr/local/bin/doveadm quota recalc -A' > /etc/cron.d/dovecot-sync
+  && groupadd -g 402 dovenull \
-RUN echo '* * * * *   root  /usr/local/bin/trim_logs.sh >> /dev/stdout 2>&1' > /etc/cron.d/trim_logs
+  && useradd -g vmail -u 5000 vmail -d /var/vmail \
+  && useradd -c "Dovecot unprivileged user" -d /dev/null -u 401 -g dovecot -s /bin/false dovecot \
+  && useradd -c "Dovecot login user" -d /dev/null -u 402 -g dovenull -s /bin/false dovenull \
+  && touch /etc/default/locale \
+  && apt-get purge -y build-essential automake autotools-dev default-libmysqlclient-dev libbz2-dev libcurl4-openssl-dev libexpat1-dev liblz-dev liblz4-dev liblzma-dev libpam-dev libssl-dev lzma-dev \
+  && apt-get autoremove --purge -y \
+  && rm -rf /tmp/* /var/tmp/*
 
 COPY trim_logs.sh /usr/local/bin/trim_logs.sh
 COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
@@ -101,31 +114,13 @@ COPY report-spam.sieve /usr/local/lib/dovecot/sieve/report-spam.sieve
 COPY report-ham.sieve /usr/local/lib/dovecot/sieve/report-ham.sieve
 COPY rspamd-pipe-ham /usr/local/lib/dovecot/sieve/rspamd-pipe-ham
 COPY rspamd-pipe-spam /usr/local/lib/dovecot/sieve/rspamd-pipe-spam
+COPY sa-rules.sh /usr/local/bin/sa-rules.sh
+COPY maildir_gc.sh /usr/local/bin/maildir_gc.sh
 COPY docker-entrypoint.sh /
 COPY supervisord.conf /etc/supervisor/supervisord.conf
-
+COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
-RUN chmod +x /usr/local/lib/dovecot/sieve/rspamd-pipe-ham \
+COPY quarantine_notify.py /usr/local/bin/quarantine_notify.py
-  /usr/local/lib/dovecot/sieve/rspamd-pipe-spam \
+COPY quota_notify.py /usr/local/bin/quota_notify.py
-  /usr/local/bin/imapsync_cron.pl \
-  /usr/local/bin/postlogin.sh \
-  /usr/local/bin/imapsync \
-  /usr/local/bin/trim_logs.sh
-
-RUN groupadd -g 5000 vmail \
-  && groupadd -g 401 dovecot \
-  && groupadd -g 402 dovenull \
-  && useradd -g vmail -u 5000 vmail -d /var/vmail \
-  && useradd -c "Dovecot unprivileged user" -d /dev/null -u 401 -g dovecot -s /bin/false dovecot \
-  && useradd -c "Dovecot login user" -d /dev/null -u 402 -g dovenull -s /bin/false dovenull
-
-RUN touch /etc/default/locale
-RUN apt-get purge -y build-essential automake autotools-dev default-libmysqlclient-dev libbz2-dev libcurl4-openssl-dev libexpat1-dev liblz-dev liblz4-dev liblzma-dev libpam-dev libssl-dev lzma-dev \
-  && apt-get autoremove --purge -y
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
-
-RUN rm -rf \
-  /tmp/* \
-  /var/tmp/*
-

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -2,28 +2,35 @@
 set -e
 
 # Wait for MySQL to warm-up
-while ! mysqladmin ping --host mysql -u${DBUSER} -p${DBPASS} --silent; do
+while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
   echo "Waiting for database to come up..."
   sleep 2
 done
 
-# Hard-code env vars to scripts due to cron not passing them to the perl script
+# Hard-code env vars to scripts due to cron not passing them to the scripts
-sed -i "/^\$DBUSER/c\\\$DBUSER='${DBUSER}';" /usr/local/bin/imapsync_cron.pl
+sed -i "s/__DBUSER__/${DBUSER}/g" /usr/local/bin/imapsync_cron.pl
-sed -i "/^\$DBPASS/c\\\$DBPASS='${DBPASS}';" /usr/local/bin/imapsync_cron.pl
+sed -i "s/__DBPASS__/${DBPASS}/g" /usr/local/bin/imapsync_cron.pl
-sed -i "/^\$DBNAME/c\\\$DBNAME='${DBNAME}';" /usr/local/bin/imapsync_cron.pl
+sed -i "s/__DBNAME__/${DBNAME}/g" /usr/local/bin/imapsync_cron.pl
-sed -i "s/LOG_LINES/${LOG_LINES}/g" /usr/local/bin/trim_logs.sh
+
+sed -i "s/__DBUSER__/${DBUSER}/g" /usr/local/bin/quarantine_notify.py
+sed -i "s/__DBPASS__/${DBPASS}/g" /usr/local/bin/quarantine_notify.py
+sed -i "s/__DBNAME__/${DBNAME}/g" /usr/local/bin/quarantine_notify.py
+
+sed -i "s/__LOG_LINES__/${LOG_LINES}/g" /usr/local/bin/trim_logs.sh
 
 # Create missing directories
 [[ ! -d /usr/local/etc/dovecot/sql/ ]] && mkdir -p /usr/local/etc/dovecot/sql/
+[[ ! -d /var/vmail/_garbage ]] && mkdir -p /var/vmail/_garbage
 [[ ! -d /var/vmail/sieve ]] && mkdir -p /var/vmail/sieve
 [[ ! -d /etc/sogo ]] && mkdir -p /etc/sogo
+[[ ! -d /var/volatile ]] && mkdir -p /var/volatile
 
 # Set Dovecot sql config parameters, escape " in db password
 DBPASS=$(echo ${DBPASS} | sed 's/"/\\"/g')
 
 # Create quota dict for Dovecot
 cat <<EOF > /usr/local/etc/dovecot/sql/dovecot-dict-sql-quota.conf
-connect = "host=mysql dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
 map {
   pattern = priv/quota/storage
   table = quota2
@@ -40,7 +47,7 @@ EOF
 
 # Create dict used for sieve pre and postfilters
 cat <<EOF > /usr/local/etc/dovecot/sql/dovecot-dict-sql-sieve_before.conf
-connect = "host=mysql dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
 map {
   pattern = priv/sieve/name/\$script_name
   table = sieve_before
@@ -62,7 +69,7 @@ map {
 EOF
 
 cat <<EOF > /usr/local/etc/dovecot/sql/dovecot-dict-sql-sieve_after.conf
-connect = "host=mysql dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
 map {
   pattern = priv/sieve/name/\$script_name
   table = sieve_after
@@ -83,39 +90,72 @@ map {
 }
 EOF
 
+echo -n ${ACL_ANYONE} > /usr/local/etc/dovecot/acl_anyone
+
+if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+echo -n 'quota acl zlib listescape mail_crypt mail_crypt_acl mail_log notify' > /usr/local/etc/dovecot/mail_plugins
+echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve listescape mail_crypt mail_crypt_acl notify mail_log' > /usr/local/etc/dovecot/mail_plugins_imap
+echo -n 'quota sieve acl zlib listescape mail_crypt mail_crypt_acl' > /usr/local/etc/dovecot/mail_plugins_lmtp
+else
+echo -n 'quota acl zlib listescape mail_crypt mail_crypt_acl mail_log notify fts fts_solr' > /usr/local/etc/dovecot/mail_plugins
+echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve listescape mail_crypt mail_crypt_acl notify mail_log fts fts_solr' > /usr/local/etc/dovecot/mail_plugins_imap
+echo -n 'quota sieve acl zlib listescape mail_crypt mail_crypt_acl fts fts_solr' > /usr/local/etc/dovecot/mail_plugins_lmtp
+fi
+chmod 644 /usr/local/etc/dovecot/mail_plugins /usr/local/etc/dovecot/mail_plugins_imap /usr/local/etc/dovecot/mail_plugins_lmtp /templates/quarantine.tpl
 
-# Create userdb dict for Dovecot
 cat <<EOF > /usr/local/etc/dovecot/sql/dovecot-dict-sql-userdb.conf
 driver = mysql
-connect = "host=mysql dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
-user_query = SELECT CONCAT('maildir:/var/vmail/',maildir) AS mail, 5000 AS uid, 5000 AS gid, concat('*:bytes=', quota) AS quota_rule FROM mailbox WHERE username = '%u' AND active = '1'
+user_query = SELECT CONCAT(JSON_UNQUOTE(JSON_EXTRACT(attributes, '$.mailbox_format')), mailbox_path_prefix, '%d/%n/${MAILDIR_SUB}:VOLATILEDIR=/var/volatile/%u') AS mail, 5000 AS uid, 5000 AS gid, concat('*:bytes=', quota) AS quota_rule FROM mailbox WHERE username = '%u' AND active = '1'
 iterate_query = SELECT username FROM mailbox WHERE active='1';
 EOF
 
 # Create pass dict for Dovecot
 cat <<EOF > /usr/local/etc/dovecot/sql/dovecot-dict-sql-passdb.conf
 driver = mysql
-connect = "host=mysql dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
 default_pass_scheme = SSHA256
-password_query = SELECT password FROM mailbox WHERE username = '%u' AND domain IN (SELECT domain FROM domain WHERE domain='%d' AND active='1') AND JSON_EXTRACT(attributes, '$.force_pw_update') NOT LIKE '%%1%%'
+password_query = SELECT password FROM mailbox WHERE active = '1' AND username = '%u' AND domain IN (SELECT domain FROM domain WHERE domain='%d' AND active='1') AND JSON_EXTRACT(attributes, '$.force_pw_update') NOT LIKE '%%1%%'
 EOF
 
 # Create global sieve_after script
 cat /usr/local/etc/dovecot/sieve_after > /var/vmail/sieve/global.sieve
 
-# Check permissions of vmail directory.
+# Check permissions of vmail/attachments directory.
 # Do not do this every start-up, it may take a very long time. So we use a stat check here.
 if [[ $(stat -c %U /var/vmail/) != "vmail" ]] ; then chown -R vmail:vmail /var/vmail ; fi
+if [[ $(stat -c %U /var/vmail/_garbage) != "vmail" ]] ; then chown -R vmail:vmail /var/vmail/_garbage ; fi
+if [[ $(stat -c %U /var/attachments) != "vmail" ]] ; then chown -R vmail:vmail /var/attachments ; fi
+
+# Cleanup random user maildirs
+rm -rf /var/vmail/mailcow.local/*
+
 
 # Create random master for SOGo sieve features
 RAND_USER=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 16 | head -n 1)
 RAND_PASS=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 24 | head -n 1)
 
-echo ${RAND_USER}@mailcow.local:$(doveadm pw -s SHA1 -p ${RAND_PASS}) > /usr/local/etc/dovecot/dovecot-master.passwd
+echo ${RAND_USER}@mailcow.local:{SHA1}$(echo -n ${RAND_PASS} | sha1sum | awk '{print $1}') > /usr/local/etc/dovecot/dovecot-master.passwd
+echo ${RAND_USER}@mailcow.local::5000:5000:::: > /usr/local/etc/dovecot/dovecot-master.userdb
 echo ${RAND_USER}@mailcow.local:${RAND_PASS} > /etc/sogo/sieve.creds
 
+if [[ "${ALLOW_ADMIN_EMAIL_LOGIN}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+    # Create random master Password for SOGo 'login as user' via proxy auth
+    RAND_PASS=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 32 | head -n 1)
+    echo -n ${RAND_PASS} > /etc/phpfpm/sogo-sso.pass
+    cat <<EOF > /usr/local/etc/dovecot/sogo-sso.conf
+passdb {
+  driver = static
+  args = allow_real_nets=${IPV4_NETWORK}.248/32 password={plain}${RAND_PASS}
+}
+EOF
+else
+    rm -f /usr/local/etc/dovecot/sogo-sso.pass
+    rm -f /usr/local/etc/dovecot/sogo-sso.conf
+fi
+
 # 401 is user dovecot
-if [[ ! -f /mail_crypt/ecprivkey.pem || ! -f /mail_crypt/ecpubkey.pem ]]; then
+if [[ ! -s /mail_crypt/ecprivkey.pem || ! -s /mail_crypt/ecpubkey.pem ]]; then
 	openssl ecparam -name prime256v1 -genkey | openssl pkey -out /mail_crypt/ecprivkey.pem
 	openssl pkey -in /mail_crypt/ecprivkey.pem -pubout -out /mail_crypt/ecpubkey.pem
 	chown 401 /mail_crypt/ecprivkey.pem /mail_crypt/ecpubkey.pem
@@ -129,7 +169,32 @@ sievec /usr/local/lib/dovecot/sieve/report-spam.sieve
 sievec /usr/local/lib/dovecot/sieve/report-ham.sieve
 
 # Fix permissions
+chown root:root /usr/local/etc/dovecot/sql/*.conf
+chown root:dovecot /usr/local/etc/dovecot/sql/dovecot-dict-sql-sieve* /usr/local/etc/dovecot/sql/dovecot-dict-sql-quota*
+chmod 640 /usr/local/etc/dovecot/sql/*.conf
 chown -R vmail:vmail /var/vmail/sieve
+chown -R vmail:vmail /var/volatile
+adduser vmail tty
+chmod g+rw /dev/console
+chmod +x /usr/local/lib/dovecot/sieve/rspamd-pipe-ham \
+  /usr/local/lib/dovecot/sieve/rspamd-pipe-spam \
+  /usr/local/bin/imapsync_cron.pl \
+  /usr/local/bin/postlogin.sh \
+  /usr/local/bin/imapsync \
+  /usr/local/bin/trim_logs.sh \
+  /usr/local/bin/sa-rules.sh \
+  /usr/local/bin/maildir_gc.sh \
+  /usr/local/sbin/stop-supervisor.sh \
+  /usr/local/bin/quota_notify.py
+
+# Setup cronjobs
+echo '* * * * *    root  /usr/local/bin/imapsync_cron.pl 2>&1 | /usr/bin/logger' > /etc/cron.d/imapsync
+echo '30 3 * * *   vmail /usr/local/bin/doveadm quota recalc -A' > /etc/cron.d/dovecot-sync
+echo '* * * * *    vmail /usr/local/bin/trim_logs.sh >> /dev/console 2>&1' > /etc/cron.d/trim_logs
+echo '25 * * * *   vmail /usr/local/bin/maildir_gc.sh >> /dev/console 2>&1' > /etc/cron.d/maildir_gc
+echo '30 1 * * *   root  /usr/local/bin/sa-rules.sh  >> /dev/console 2>&1' > /etc/cron.d/sa-rules
+echo '0 2 * * *    root  /usr/bin/curl http://solr:8983/solr/dovecot-fts/update?optimize=true >> /dev/console 2>&1' > /etc/cron.d/solr-optimize
+echo '*/20 * * * * vmail /usr/local/bin/quarantine_notify.py >> /dev/console 2>&1' > /etc/cron.d/quarantine_notify
 
 # Fix more than 1 hardlink issue
 touch /etc/crontab /etc/cron.*/*
@@ -139,7 +204,13 @@ touch /etc/crontab /etc/cron.*/*
 
 # Clean stopped imapsync jobs
 rm -f /tmp/imapsync_busy.lock
-IMAPSYNC_TABLE=$(mysql -h mysql-mailcow -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SHOW TABLES LIKE 'imapsync'" -Bs)
+IMAPSYNC_TABLE=$(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SHOW TABLES LIKE 'imapsync'" -Bs)
-[[ ! -z ${IMAPSYNC_TABLE} ]] && mysql -h mysql-mailcow -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "UPDATE imapsync SET is_running='0'"
+[[ ! -z ${IMAPSYNC_TABLE} ]] && mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "UPDATE imapsync SET is_running='0'"
+
+# Envsubst maildir_gc
+echo "$(envsubst < /usr/local/bin/maildir_gc.sh)" > /usr/local/bin/maildir_gc.sh
+
+# Collect SA rules once now
+/usr/local/bin/sa-rules.sh
 
 exec "$@"

data/Dockerfiles/dovecot/imapsync_cron.pl

@@ -18,18 +18,20 @@ if ($imapsync_running eq 1)
   exit;
 }
 
-sub qqw($) { split /\s+/, $_[0] }
+sub qqw($) {
-
+  my @values = split('(?=--)', $_[0]);
-$DBNAME = '';
+  foreach my $val (@values) {
-$DBUSER = '';
+    $val=trim($val);
-$DBPASS = '';
+  }
+  return @values
+}
 
 $run_dir="/tmp";
-$dsn = "DBI:mysql:database=" . $DBNAME . ";host=mysql";
+$dsn = 'DBI:mysql:database=__DBNAME__;mysql_socket=/var/run/mysqld/mysqld.sock';
 $lock_file = $run_dir . "/imapsync_busy";
 $lockmgr = LockFile::Simple->make(-autoclean => 1, -max => 1);
 $lockmgr->lock($lock_file) || die "can't lock ${lock_file}";
-$dbh = DBI->connect($dsn, $DBUSER, $DBPASS, {
+$dbh = DBI->connect($dsn, '__DBUSER__', '__DBPASS__', {
   mysql_auto_reconnect => 1,
   mysql_enable_utf8mb4 => 1
 });

data/Dockerfiles/dovecot/maildir_gc.sh

@@ -0,0 +1,2 @@
+#/bin/bash
+[ -d /var/vmail/_garbage/ ] && /usr/bin/find /var/vmail/_garbage/ -mindepth 1 -maxdepth 1 -type d -cmin +${MAILDIR_GC_TIME} -exec rm -r {} \;

data/Dockerfiles/dovecot/postlogin.sh

@@ -1,4 +1,3 @@
 #!/bin/sh
-
 export MASTER_USER=$USER
 exec "$@"

data/Dockerfiles/dovecot/quarantine_notify.py

@@ -0,0 +1,125 @@
+#!/usr/bin/python
+
+import smtplib
+import os
+import mysql.connector
+from email.MIMEMultipart import MIMEMultipart
+from email.MIMEText import MIMEText
+from email.Utils import COMMASPACE, formatdate
+import cgi
+import jinja2
+from jinja2 import Template
+import json
+import redis
+import time
+import html2text
+import socket
+
+while True:
+  try:
+    r = redis.StrictRedis(host='redis', decode_responses=True, port=6379, db=0)
+    r.ping()
+  except Exception as ex:
+    print '%s - trying again...'  % (ex)
+    time.sleep(3)
+  else:
+    break
+
+time_now = int(time.time())
+
+def query_mysql(query, headers = True, update = False):
+  while True:
+    try:
+      cnx = mysql.connector.connect(unix_socket = '/var/run/mysqld/mysqld.sock', user='__DBUSER__', passwd='__DBPASS__', database='__DBNAME__', charset="utf8")
+    except Exception as ex:
+      print '%s - trying again...'  % (ex)
+      time.sleep(3)
+    else:
+      break
+  cur = cnx.cursor()
+  cur.execute(query)
+  if not update:
+    result = []
+    columns = tuple( [d[0].decode('utf8') for d in cur.description] )
+    for row in cur:
+      if headers:
+        result.append(dict(zip(columns, row)))
+      else:
+        result.append(row)
+    cur.close()
+    cnx.close()
+    return result
+  else:
+    cnx.commit()
+    cur.close()
+    cnx.close()
+
+def notify_rcpt(rcpt, msg_count, quarantine_acl):
+  meta_query = query_mysql('SELECT SHA2(CONCAT(id, qid), 256) AS qhash, id, subject, score, sender, created FROM quarantine WHERE notified = 0 AND rcpt = "%s"' % (rcpt))
+  if r.get('Q_HTML'):
+    try:
+      template = Template(r.get('Q_HTML'))
+    except:
+      print "Error: Cannot parse quarantine template, falling back to default template."
+      with open('/templates/quarantine.tpl') as file_:
+        template = Template(file_.read())
+  else:
+    with open('/templates/quarantine.tpl') as file_:
+      template = Template(file_.read())
+  html = template.render(meta=meta_query, counter=msg_count, hostname=socket.gethostname(), quarantine_acl=quarantine_acl)
+  text = html2text.html2text(html)
+  count = 0
+  while count < 15:
+    try:
+      server = smtplib.SMTP('postfix', 590, 'quarantine')
+      server.ehlo()
+      msg = MIMEMultipart('alternative')
+      msg['From'] = r.get('Q_SENDER') or "quarantine@localhost"
+      msg['Subject'] = r.get('Q_SUBJ') or "Spam Quarantine Notification"
+      msg['Date'] = formatdate(localtime = True)
+      text_part = MIMEText(text, 'plain', 'utf-8')
+      html_part = MIMEText(html, 'html', 'utf-8')
+      msg.attach(text_part)
+      msg.attach(html_part)
+      msg['To'] = str(rcpt)
+      text = msg.as_string()
+      server.sendmail(msg['From'], msg['To'], text)
+      server.quit()
+      for res in meta_query:
+        query_mysql('UPDATE quarantine SET notified = 1 WHERE id = "%d"' % (res['id']), update = True)
+      r.hset('Q_LAST_NOTIFIED', record['rcpt'], time_now)
+      break
+    except Exception as ex:
+      print '%s'  % (ex)
+      time.sleep(3)
+
+records = query_mysql('SELECT IFNULL(user_acl.quarantine, 0) AS quarantine_acl, count(id) AS counter, rcpt FROM quarantine LEFT OUTER JOIN user_acl ON user_acl.username = rcpt WHERE notified = 0 AND rcpt in (SELECT username FROM mailbox) GROUP BY rcpt')
+
+for record in records:
+  attrs = ''
+  attrs_json = ''
+  try:
+    last_notification = int(r.hget('Q_LAST_NOTIFIED', record['rcpt']))
+    if last_notification > time_now:
+      print 'Last notification is > time now, assuming never'
+      last_notification = 0
+  except Exception as ex:
+    print 'Could not determine last notification for %s, assuming never' % (record['rcpt'])
+    last_notification = 0
+  attrs_json = query_mysql('SELECT attributes FROM mailbox WHERE username = "%s"' % (record['rcpt']))
+  attrs = json.loads(str(attrs_json[0]['attributes']))
+  if attrs['quarantine_notification'] not in ('hourly', 'daily', 'weekly', 'never'):
+    print 'Abnormal quarantine_notification value'
+    continue
+  if attrs['quarantine_notification'] == 'hourly':
+    if last_notification == 0 or (last_notification + 3600) < time_now:
+      print "Notifying %s about %d new items in quarantine" % (record['rcpt'], record['counter'])
+      notify_rcpt(record['rcpt'], record['counter'], record['quarantine_acl'])
+  elif attrs['quarantine_notification'] == 'daily':
+    if last_notification == 0 or (last_notification + 86400) < time_now:
+      print "Notifying %s about %d new items in quarantine" % (record['rcpt'], record['counter'])
+      notify_rcpt(record['rcpt'], record['counter'], record['quarantine_acl'])
+  elif attrs['quarantine_notification'] == 'weekly':
+    if last_notification == 0 or (last_notification + 604800) < time_now:
+      print "Notifying %s about %d new items in quarantine" % (record['rcpt'], record['counter'])
+      notify_rcpt(record['rcpt'], record['counter'], record['quarantine_acl'])

data/Dockerfiles/dovecot/quota_notify.py

@@ -0,0 +1,72 @@
+#!/usr/bin/python
+
+import smtplib
+import os
+from email.MIMEMultipart import MIMEMultipart
+from email.MIMEText import MIMEText
+from email.Utils import COMMASPACE, formatdate
+import jinja2
+from jinja2 import Template
+import redis
+import time
+import sys
+import html2text
+from subprocess import Popen, PIPE, STDOUT
+
+if len(sys.argv) > 2:
+  percent = int(sys.argv[1])
+  username = str(sys.argv[2])
+else:
+  print "Args missing"
+  sys.exit(1)
+
+while True:
+  try:
+    r = redis.StrictRedis(host='redis', decode_responses=True, port=6379, db=0)
+    r.ping()
+  except Exception as ex:
+    print '%s - trying again...'  % (ex)
+    time.sleep(3)
+  else:
+    break
+
+if r.get('QW_HTML'):
+  try:
+    template = Template(r.get('QW_HTML'))
+  except:
+    print "Error: Cannot parse quarantine template, falling back to default template."
+    with open('/templates/quota.tpl') as file_:
+      template = Template(file_.read())
+else:
+  with open('/templates/quota.tpl') as file_:
+    template = Template(file_.read())
+
+html = template.render(username=username, percent=percent)
+text = html2text.html2text(html)
+
+try:
+  msg = MIMEMultipart('alternative')
+  msg['From'] = r.get('QW_SENDER') or "quota-warning@localhost"
+  msg['Subject'] = r.get('QW_SUBJ') or "Quota warning"
+  msg['Date'] = formatdate(localtime = True)
+  text_part = MIMEText(text, 'plain', 'utf-8')
+  html_part = MIMEText(html, 'html', 'utf-8')
+  msg.attach(text_part)
+  msg.attach(html_part)
+  msg['To'] = username
+  p = Popen(['/usr/local/libexec/dovecot/dovecot-lda', '-d', username, '-o', '"plugin/quota=maildir:User quota:noenforcing"'], stdout=PIPE, stdin=PIPE, stderr=STDOUT)
+  p.communicate(input=msg.as_string())
+
+except Exception as ex:
+  print 'Failed to send quota notification: %s' % (ex)
+  sys.exit(1)
+
+try:
+  sys.stdout.close()
+except:
+  pass
+
+try:
+  sys.stderr.close()
+except:
+  pass

data/Dockerfiles/dovecot/rspamd-pipe-ham

@@ -3,7 +3,7 @@ FILE=/tmp/mail$$
 cat > $FILE
 trap "/bin/rm -f $FILE" 0 1 2 3 13 15
 
-cat ${FILE} | /usr/bin/curl -s --data-binary @- --unix-socket /rspamd-sock/rspamd.sock http://rspamd/learnham
+cat ${FILE} | /usr/bin/curl -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/learnham
-cat ${FILE} | /usr/bin/curl -H "Flag: 13" -s --data-binary @- --unix-socket /rspamd-sock/rspamd.sock http://rspamd/fuzzyadd
+cat ${FILE} | /usr/bin/curl -H "Flag: 13" -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/fuzzyadd
 
 exit 0

data/Dockerfiles/dovecot/rspamd-pipe-spam

@@ -3,7 +3,7 @@ FILE=/tmp/mail$$
 cat > $FILE
 trap "/bin/rm -f $FILE" 0 1 2 3 13 15
 
-cat ${FILE} | /usr/bin/curl -s --data-binary @- --unix-socket /rspamd-sock/rspamd.sock http://rspamd/learnspam
+cat ${FILE} | /usr/bin/curl -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/learnspam
-cat ${FILE} | /usr/bin/curl -H "Flag: 11" -s --data-binary @- --unix-socket /rspamd-sock/rspamd.sock http://rspamd/fuzzyadd
+cat ${FILE} | /usr/bin/curl -H "Flag: 11" -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/fuzzyadd
 
 exit 0

data/Dockerfiles/dovecot/sa-rules.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+[[ ! -d /tmp/sa-rules-heinlein ]] && mkdir -p /tmp/sa-rules-heinlein
+if [[ ! -f /etc/rspamd/custom/sa-rules-heinlein ]]; then
+  HASH_SA_RULES=0
+else
+  HASH_SA_RULES=$(cat /etc/rspamd/custom/sa-rules-heinlein | md5sum | cut -d' ' -f1)
+fi
+
+curl --connect-timeout 15 --max-time 30 http://www.spamassassin.heinlein-support.de/$(dig txt 1.4.3.spamassassin.heinlein-support.de +short | tr -d '"').tar.gz --output /tmp/sa-rules.tar.gz
+if [[ -f /tmp/sa-rules.tar.gz ]]; then
+  tar xfvz /tmp/sa-rules.tar.gz -C /tmp/sa-rules-heinlein
+  # create complete list of rules in a single file
+  cat /tmp/sa-rules-heinlein/*cf > /etc/rspamd/custom/sa-rules-heinlein
+  # Only restart rspamd-mailcow when rules changed
+  if [[ $(cat /etc/rspamd/custom/sa-rules-heinlein | md5sum | cut -d' ' -f1) != ${HASH_SA_RULES} ]]; then
+    CONTAINER_NAME=rspamd-mailcow
+    CONTAINER_ID=$(curl --silent --insecure https://dockerapi/containers/json | \
+      jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" | \
+      jq -rc "select( .name | tostring | contains(\"${CONTAINER_NAME}\")) | .id")
+    if [[ ! -z ${CONTAINER_ID} ]]; then
+      curl --silent --insecure -XPOST --connect-timeout 15 --max-time 120 https://dockerapi/containers/${CONTAINER_ID}/restart
+    fi
+  fi
+fi
+rm -rf /tmp/sa-rules-heinlein /tmp/sa-rules.tar.gz

data/Dockerfiles/dovecot/stop-supervisor.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+printf "READY\n";
+
+while read line; do
+  echo "Processing Event: $line" >&2;
+  kill -3 $(cat "/var/run/supervisord.pid")
+done < /dev/stdin

data/Dockerfiles/dovecot/supervisord.conf

@@ -1,6 +1,7 @@
 [supervisord]
 nodaemon=true
 user=root
+pidfile=/var/run/supervisord.pid
 
 [program:syslog-ng]
 command=/usr/sbin/syslog-ng --foreground --no-caps
@@ -17,3 +18,7 @@ autorestart=true
 [program:cron]
 command=/usr/sbin/cron -f
 autorestart=true
+
+[eventlistener:processes]
+command=/usr/local/sbin/stop-supervisor.sh
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL

data/Dockerfiles/dovecot/trim_logs.sh

@@ -1,8 +1,18 @@
 #!/bin/bash
+catch_non_zero() {
+  CMD=${1}
+  ${CMD} > /dev/null
+  EC=$?
+  if [ ${EC} -ne 0 ]; then
+    echo "Command ${CMD} failed to execute, exit code was ${EC}"
+  fi
+}
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM ACME_LOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM POSTFIX_MAILLOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM DOVECOT_MAILLOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM SOGO_LOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM NETFILTER_LOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM AUTODISCOVER_LOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM API_LOG 0 __LOG_LINES__"
+catch_non_zero "/usr/bin/redis-cli -h redis LTRIM RL_LOG 0 __LOG_LINES__"
 
-redis-cli -h redis LTRIM ACME_LOG 0 LOG_LINES
-redis-cli -h redis LTRIM POSTFIX_MAILLOG 0 LOG_LINES
-redis-cli -h redis LTRIM DOVECOT_MAILLOG 0 LOG_LINES
-redis-cli -h redis LTRIM SOGO_LOG 0 LOG_LINES
-redis-cli -h redis LTRIM NETFILTER_LOG 0 LOG_LINES
-redis-cli -h redis LTRIM AUTODISCOVER_LOG 0 LOG_LINES

data/Dockerfiles/netfilter/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.8
+FROM alpine:3.9
 LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
 
 ENV XTABLES_LIBDIR /usr/lib/xtables

data/Dockerfiles/netfilter/server.py

@@ -10,7 +10,6 @@ from random import randint
 from threading import Thread
 from threading import Lock
 import redis
-import time
 import json
 import iptc
 
@@ -29,10 +28,11 @@ pubsub = r.pubsub()
 RULES = {}
 RULES[1] = 'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed'
 RULES[2] = '-login: Disconnected \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),'
-RULES[3] = '-login: Aborted login \(no auth .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
+RULES[3] = '-login: Aborted login \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
-RULES[4] = '-login: Aborted login \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
+RULES[4] = 'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
-RULES[5] = 'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
+RULES[5] = 'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
-RULES[6] = 'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
+RULES[6] = '([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
+#RULES[7] = '-login: Aborted login \(no auth .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
 
 bans = {}
 log = {}

data/Dockerfiles/phpfpm/Dockerfile

@@ -1,11 +1,11 @@
-FROM php:7.2-fpm-alpine3.7
+FROM php:7.3-fpm-alpine3.8
 LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
 
-ENV APCU_PECL 5.1.11
+ENV APCU_PECL 5.1.16
 ENV IMAGICK_PECL 3.4.3
-ENV MAILPARSE_PECL 3.0.2
+#ENV MAILPARSE_PECL 3.0.2
-ENV MEMCACHED_PECL 3.0.4
+ENV MEMCACHED_PECL 3.1.3
-ENV REDIS_PECL 4.0.2
+ENV REDIS_PECL 4.2.0
 
 RUN apk add -U --no-cache autoconf \
   bash \
@@ -14,12 +14,14 @@ RUN apk add -U --no-cache autoconf \
   freetype \
   freetype-dev \
   g++ \
+  git \
   gettext-dev \
   icu-dev \
   icu-libs \
   imagemagick \
   imagemagick-dev \
   imap-dev \
+  jq \
   libjpeg-turbo \
   libjpeg-turbo-dev \
   libmemcached-dev \
@@ -32,6 +34,7 @@ RUN apk add -U --no-cache autoconf \
   libwebp-dev \
   libxml2-dev \
   libxpm-dev \
+  libzip-dev \
   make \
   mysql-client \
   openldap-dev \
@@ -41,14 +44,13 @@ RUN apk add -U --no-cache autoconf \
   samba-client \
   zlib-dev \
   tzdata \
-  && pear install channel://pear.php.net/Net_IDNA2-0.2.0 \
+  && git clone https://github.com/php/pecl-mail-mailparse \
-    channel://pear.php.net/Auth_SASL-1.1.0 \
+  && cd pecl-mail-mailparse \
-    Net_IMAP \
+  && pecl install package.xml \
-    Net_Sieve \
+  && cd .. \
-    NET_SMTP \
+  && rm -r pecl-mail-mailparse \
-    Mail_mime \
+  && pecl install redis-${REDIS_PECL} memcached-${MEMCACHED_PECL} APCu-${APCU_PECL} imagick-${IMAGICK_PECL} \
-  && pecl install redis-${REDIS_PECL} memcached-${MEMCACHED_PECL} APCu-${APCU_PECL} imagick-${IMAGICK_PECL} mailparse-${MAILPARSE_PECL} \
+  && docker-php-ext-enable apcu imagick memcached mailparse redis \
-  && docker-php-ext-enable apcu imagick mailparse memcached redis \
   && pecl clear-cache \
   && docker-php-ext-configure intl \
   && docker-php-ext-configure gd \

data/Dockerfiles/phpfpm/docker-entrypoint.sh

@@ -1,28 +1,67 @@
 #!/bin/bash
-set -e
 
 function array_by_comma { local IFS=","; echo "$*"; }
 
 # Wait for containers
-while ! mysqladmin ping --host mysql -u${DBUSER} -p${DBPASS} --silent; do
+while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
+  echo "Waiting for SQL..."
   sleep 2
 done
 
 until [[ $(redis-cli -h redis-mailcow PING) == "PONG" ]]; do
+  echo "Waiting for Redis..."
   sleep 2
 done
 
+# Set a default release format
+
+if [[ -z $(redis-cli --raw -h redis-mailcow GET Q_RELEASE_FORMAT) ]]; then
+  redis-cli --raw -h redis-mailcow SET Q_RELEASE_FORMAT raw
+fi
+
+# Check of mysql_upgrade
+
+CONTAINER_ID=
+# Todo: Better check if upgrade failed
+# This can happen due to a broken sogo_view
+[ -s /mysql_upgrade_loop ] && SQL_LOOP_C=$(cat /mysql_upgrade_loop)
+until [[ ! -z "${CONTAINER_ID}" ]] && [[ "${CONTAINER_ID}" =~ ^[[:alnum:]]*$ ]]; do
+  CONTAINER_ID=$(curl --silent --insecure https://dockerapi/containers/json | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" 2> /dev/null | jq -rc "select( .name | tostring | contains(\"mysql-mailcow\")) | .id" 2> /dev/null)
+done
+echo "MySQL @ ${CONTAINER_ID}"
+SQL_UPGRADE_RETURN=$(curl --silent --insecure -XPOST https://dockerapi/containers/${CONTAINER_ID}/exec -d '{"cmd":"system", "task":"mysql_upgrade"}' --silent -H 'Content-type: application/json' | jq -r .type)
+if [[ ${SQL_UPGRADE_RETURN} == 'warning' ]]; then
+  if [ -z ${SQL_LOOP_C} ]; then
+    echo 1 > /mysql_upgrade_loop
+    echo "MySQL applied an upgrade, restarting PHP-FPM..."
+    exit 1
+  else
+    rm /mysql_upgrade_loop
+    echo "MySQL was not applied previously, skipping. Restart php-fpm-mailcow to retry or run mysql_upgrade manually."
+    while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
+      echo "Waiting for SQL to return..."
+      sleep 2
+    done
+  fi
+else
+  echo "MySQL is up-to-date"
+fi
+
+# Trigger db init
+echo "Running DB init..."
+php -c /usr/local/etc/php -f /web/inc/init_db.inc.php
+
 # Migrate domain map
 declare -a DOMAIN_ARR
 redis-cli -h redis-mailcow DEL DOMAIN_MAP
 while read line
 do
   DOMAIN_ARR+=("$line")
-done < <(mysql -h mysql-mailcow -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT domain FROM domain" -Bs)
+done < <(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT domain FROM domain" -Bs)
 while read line
 do
   DOMAIN_ARR+=("$line")
-done < <(mysql -h mysql-mailcow -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT alias_domain FROM alias_domain" -Bs)
+done < <(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT alias_domain FROM alias_domain" -Bs)
 
 if [[ ! -z ${DOMAIN_ARR} ]]; then
 for domain in "${DOMAIN_ARR[@]}"; do
@@ -48,10 +87,9 @@ if [[ ${API_ALLOW_FROM} != "invalid" ]] && \
   done
   VALIDATED_IPS=$(array_by_comma ${VALIDATED_API_ALLOW_FROM_ARR[*]})
   if [[ ! -z ${VALIDATED_IPS} ]]; then
-    mysql --host mysql-mailcow -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
+    mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
-INSERT INTO api (username, api_key, active, allow_from)
+DELETE FROM api;
-SELECT username, "${API_KEY}", '1', "${VALIDATED_IPS}" FROM admin WHERE superadmin='1' AND active='1'
+INSERT INTO api (api_key, active, allow_from) VALUES ("${API_KEY}", "1", "${VALIDATED_IPS}");
-ON DUPLICATE KEY UPDATE active = '1', allow_from = "${VALIDATED_IPS}", api_key = "${API_KEY}";
 EOF
   fi
 fi

data/Dockerfiles/postfix/Dockerfile

@@ -48,6 +48,13 @@ COPY postfix.sh /opt/postfix.sh
 COPY rspamd-pipe-ham /usr/local/bin/rspamd-pipe-ham
 COPY rspamd-pipe-spam /usr/local/bin/rspamd-pipe-spam
 COPY whitelist_forwardinghosts.sh /usr/local/bin/whitelist_forwardinghosts.sh
+COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
+
+RUN chmod +x /opt/postfix.sh \
+  /usr/local/bin/rspamd-pipe-ham \
+  /usr/local/bin/rspamd-pipe-spam \
+  /usr/local/bin/whitelist_forwardinghosts.sh \
+  /usr/local/sbin/stop-supervisor.sh
 
 EXPOSE 588
 

data/Dockerfiles/postfix/postfix.sh

@@ -14,7 +14,7 @@ newaliases;
 cat <<EOF > /opt/postfix/conf/sql/mysql_relay_recipient_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT DISTINCT
   CASE WHEN '%d' IN (
@@ -29,10 +29,18 @@ query = SELECT DISTINCT
   END AS result;
 EOF
 
+cat <<EOF > /opt/postfix/conf/sql/mysql_tls_policy_override_maps.cf
+user = ${DBUSER}
+password = ${DBPASS}
+hosts = unix:/var/run/mysqld/mysqld.sock
+dbname = ${DBNAME}
+query = SELECT CONCAT(policy, ' ', parameters) AS tls_policy FROM tls_policy_override WHERE active = '1' AND dest = '%s'
+EOF
+
 cat <<EOF > /opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT IF(EXISTS(
   SELECT 'TLS_ACTIVE' FROM alias
@@ -49,7 +57,7 @@ EOF
 cat <<EOF > /opt/postfix/conf/sql/mysql_sender_dependent_default_transport_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT GROUP_CONCAT(transport SEPARATOR '') AS transport_maps
   FROM (
@@ -77,26 +85,49 @@ query = SELECT GROUP_CONCAT(transport SEPARATOR '') AS transport_maps
   AS transport_view;
 EOF
 
-cat <<EOF > /opt/postfix/conf/sql/mysql_sasl_passwd_maps.cf
+cat <<EOF > /opt/postfix/conf/sql/mysql_transport_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
+dbname = ${DBNAME}
+query = SELECT CONCAT('smtp_via_transport_maps:', nexthop) AS transport FROM transports
+  WHERE active = '1'
+  AND destination = '%s';
+EOF
+
+cat <<EOF > /opt/postfix/conf/sql/mysql_sasl_passwd_maps_sender_dependent.cf
+user = ${DBUSER}
+password = ${DBPASS}
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM relayhosts
   WHERE id IN (
     SELECT relayhost FROM domain
       WHERE CONCAT('@', domain) = '%s'
-      OR '%s' IN (
+      OR domain IN (
-        SELECT CONCAT('@', alias_domain) FROM alias_domain
+        SELECT target_domain FROM alias_domain WHERE CONCAT('@', alias_domain) =  '%s'
       )
   )
+  AND active = '1'
   AND username != '';
 EOF
 
+cat <<EOF > /opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf
+user = ${DBUSER}
+password = ${DBPASS}
+hosts = unix:/var/run/mysqld/mysqld.sock
+dbname = ${DBNAME}
+query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM transports
+  WHERE nexthop = '%s'
+  AND active = '1'
+  AND username != ''
+  LIMIT 1;
+EOF
+
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_alias_domain_catchall_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT goto FROM alias, alias_domain
   WHERE alias_domain.alias_domain = '%d'
@@ -107,7 +138,7 @@ EOF
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_alias_domain_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT username FROM mailbox, alias_domain
   WHERE alias_domain.alias_domain = '%d'
@@ -119,7 +150,7 @@ EOF
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_alias_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT goto FROM alias
   WHERE address='%s'
@@ -129,7 +160,7 @@ EOF
 cat <<EOF > /opt/postfix/conf/sql/mysql_recipient_bcc_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT bcc_dest FROM bcc_maps
   WHERE local_dest='%s'
@@ -140,7 +171,7 @@ EOF
 cat <<EOF > /opt/postfix/conf/sql/mysql_sender_bcc_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT bcc_dest FROM bcc_maps
   WHERE local_dest='%s'
@@ -151,7 +182,7 @@ EOF
 cat <<EOF > /opt/postfix/conf/sql/mysql_recipient_canonical_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT new_dest FROM recipient_maps
   WHERE old_dest='%s'
@@ -161,7 +192,7 @@ EOF
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_domains_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT alias_domain from alias_domain WHERE alias_domain='%s' AND active='1'
   UNION
@@ -174,15 +205,15 @@ EOF
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_mailbox_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
-query = SELECT maildir FROM mailbox WHERE username='%s' AND active = '1'
+query = SELECT CONCAT(JSON_UNQUOTE(JSON_EXTRACT(attributes, '$.mailbox_format')), mailbox_path_prefix, '%d/%u/') FROM mailbox WHERE username='%s' AND active = '1'
 EOF
 
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_relay_domain_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT domain FROM domain WHERE domain='%s' AND backupmx = '1' AND active = '1'
 EOF
@@ -190,7 +221,7 @@ EOF
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_sender_acl.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 # First select queries domain and alias_domain to determine if domains are active.
 query = SELECT goto FROM alias
@@ -231,7 +262,7 @@ EOF
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_spamalias_maps.cf
 user = ${DBUSER}
 password = ${DBPASS}
-hosts = mysql
+hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT goto FROM spamalias
   WHERE address='%s'
@@ -244,6 +275,8 @@ chmod 700 /var/lib/zeyple/keys
 chown -R 600:600 /var/lib/zeyple/keys
 
 # Fix Postfix permissions
+chown -R root:postfix /opt/postfix/conf/sql/
+chmod 640 /opt/postfix/conf/sql/*.cf
 chgrp -R postdrop /var/spool/postfix/public
 chgrp -R postdrop /var/spool/postfix/maildrop
 postfix set-permissions

data/Dockerfiles/postfix/rspamd-pipe-ham

@@ -3,7 +3,7 @@ FILE=/tmp/mail$$
 cat > $FILE
 trap "/bin/rm -f $FILE" 0 1 2 3 13 15
 
-cat ${FILE} | /usr/bin/curl -s --data-binary @- --unix-socket /rspamd-sock/rspamd.sock http://rspamd/learnham
+cat ${FILE} | /usr/bin/curl -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/learnham
-cat ${FILE} | /usr/bin/curl -H "Flag: 13" -s --data-binary @- --unix-socket /rspamd-sock/rspamd.sock http://rspamd/fuzzyadd
+cat ${FILE} | /usr/bin/curl -H "Flag: 13" -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/fuzzyadd
 
 exit 0

data/Dockerfiles/postfix/rspamd-pipe-spam

@@ -3,7 +3,7 @@ FILE=/tmp/mail$$
 cat > $FILE
 trap "/bin/rm -f $FILE" 0 1 2 3 13 15
 
-cat ${FILE} | /usr/bin/curl -s --data-binary @- --unix-socket /rspamd-sock/rspamd.sock http://rspamd/learnspam
+cat ${FILE} | /usr/bin/curl -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/learnspam
-cat ${FILE} | /usr/bin/curl -H "Flag: 11" -s --data-binary @- --unix-socket /rspamd-sock/rspamd.sock http://rspamd/fuzzyadd
+cat ${FILE} | /usr/bin/curl -H "Flag: 11" -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/fuzzyadd
 
 exit 0

data/Dockerfiles/postfix/stop-supervisor.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+printf "READY\n";
+
+while read line; do
+  echo "Processing Event: $line" >&2;
+  kill -3 $(cat "/var/run/supervisord.pid")
+done < /dev/stdin

data/Dockerfiles/postfix/supervisord.conf

@@ -13,3 +13,7 @@ autostart=true
 [program:postfix]
 command=/opt/postfix.sh
 autorestart=true
+
+[eventlistener:processes]
+command=/usr/local/sbin/stop-supervisor.sh
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL

data/Dockerfiles/rspamd/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:xenial
+FROM ubuntu:bionic
 LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
@@ -8,21 +8,21 @@ RUN apt-get update && apt-get install -y \
   tzdata \
 	ca-certificates \
 	gnupg2 \
-  gnupg-curl \
 	apt-transport-https \
 	&& apt-key adv --fetch-keys https://rspamd.com/apt/gpg.key \
-	&& echo "deb https://rspamd.com/apt-stable/ xenial main" > /etc/apt/sources.list.d/rspamd.list \
+	&& echo "deb https://rspamd.com/apt-stable/ bionic main" > /etc/apt/sources.list.d/rspamd.list \
 	&& apt-get update && apt-get install -y rspamd \
 	&& rm -rf /var/lib/apt/lists/* \
-	&& echo '.include $LOCAL_CONFDIR/local.d/rspamd.conf.local' > /etc/rspamd/rspamd.conf.local \
 	&& apt-get autoremove --purge \
 	&& apt-get clean \
 	&& mkdir -p /run/rspamd \
 	&& chown _rspamd:_rspamd /run/rspamd
 
-COPY settings.conf /etc/rspamd/modules.d/settings.conf
+COPY settings.conf /etc/rspamd/settings.conf
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
 
+STOPSIGNAL SIGTERM
+
 CMD ["/usr/bin/rspamd", "-f", "-u", "_rspamd", "-g", "_rspamd"]

data/Dockerfiles/rspamd/docker-entrypoint.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 
-chown -R _rspamd:_rspamd /var/lib/rspamd
+chown -R _rspamd:_rspamd /var/lib/rspamd /etc/rspamd/local.d /etc/rspamd/override.d /etc/rspamd/custom
+chmod 755 /var/lib/rspamd
 [[ ! -f /etc/rspamd/override.d/worker-controller-password.inc ]] && echo '# Placeholder' > /etc/rspamd/override.d/worker-controller-password.inc
+chown _rspamd:_rspamd /etc/rspamd/override.d/worker-controller-password.inc
+[[ ! -f /etc/rspamd/custom/sa-rules-heinlein ]] && echo '# to be auto-filled by dovecot-mailcow' > /etc/rspamd/custom/sa-rules-heinlein
 
 exec "$@"

data/Dockerfiles/rspamd/lua_util.lua

@@ -1,152 +0,0 @@
-local exports = {}
-local lpeg = require 'lpeg'
-
-local split_grammar = {}
-local function rspamd_str_split(s, sep)
-  local gr = split_grammar[sep]
-
-  if not gr then
-    local _sep = lpeg.P(sep)
-    local elem = lpeg.C((1 - _sep)^0)
-    local p = lpeg.Ct(elem * (_sep * elem)^0)
-    gr = p
-    split_grammar[sep] = gr
-  end
-
-  return gr:match(s)
-end
-
-exports.rspamd_str_split = rspamd_str_split
-
-local space = lpeg.S' \t\n\v\f\r'
-local nospace = 1 - space
-local ptrim = space^0 * lpeg.C((space^0 * nospace^1)^0)
-local match = lpeg.match
-exports.rspamd_str_trim = function(s)
-  return match(ptrim, s)
-end
-
--- Robert Jay Gould http://lua-users.org/wiki/SimpleRound
-exports.round = function(num, numDecimalPlaces)
-  local mult = 10^(numDecimalPlaces or 0)
-  return math.floor(num * mult) / mult
-end
-
-exports.template = function(tmpl, keys)
-  local var_lit = lpeg.P { lpeg.R("az") + lpeg.R("AZ") + lpeg.R("09") + "_" }
-  local var = lpeg.P { (lpeg.P("$") / "") * ((var_lit^1) / keys) }
-  local var_braced = lpeg.P { (lpeg.P("${") / "") * ((var_lit^1) / keys) * (lpeg.P("}") / "") }
-
-  local template_grammar = lpeg.Cs((var + var_braced + 1)^0)
-
-  return lpeg.match(template_grammar, tmpl)
-end
-
-exports.remove_email_aliases = function(email_addr)
-  local function check_gmail_user(addr)
-    -- Remove all points
-    local no_dots_user = string.gsub(addr.user, '%.', '')
-    local cap, pluses = string.match(no_dots_user, '^([^%+][^%+]*)(%+.*)$')
-    if cap then
-      return cap, rspamd_str_split(pluses, '+'), nil
-    elseif no_dots_user ~= addr.user then
-      return no_dots_user,{},nil
-    end
-
-    return nil
-  end
-
-  local function check_address(addr)
-    if addr.user then
-      local cap, pluses = string.match(addr.user, '^([^%+][^%+]*)(%+.*)$')
-      if cap then
-        return cap, rspamd_str_split(pluses, '+'), nil
-      end
-    end
-
-    return nil
-  end
-
-  local function set_addr(addr, new_user, new_domain)
-    if new_user then
-      addr.user = new_user
-    end
-    if new_domain then
-      addr.domain = new_domain
-    end
-
-    if addr.domain then
-      addr.addr = string.format('%s@%s', addr.user, addr.domain)
-    else
-      addr.addr = string.format('%s@', addr.user)
-    end
-
-    if addr.name and #addr.name > 0 then
-      addr.raw = string.format('"%s" <%s>', addr.name, addr.addr)
-    else
-      addr.raw = string.format('<%s>', addr.addr)
-    end
-  end
-
-  local function check_gmail(addr)
-    local nu, tags, nd = check_gmail_user(addr)
-
-    if nu then
-      return nu, tags, nd
-    end
-
-    return nil
-  end
-
-  local function check_googlemail(addr)
-    local nd = 'gmail.com'
-    local nu, tags = check_gmail_user(addr)
-
-    if nu then
-      return nu, tags, nd
-    end
-
-    return nil, nil, nd
-  end
-
-  local specific_domains = {
-    ['gmail.com'] = check_gmail,
-    ['googlemail.com'] = check_googlemail,
-  }
-
-  if email_addr then
-    if email_addr.domain and specific_domains[email_addr.domain] then
-      local nu, tags, nd = specific_domains[email_addr.domain](email_addr)
-      if nu or nd then
-        set_addr(email_addr, nu, nd)
-
-        return nu, tags
-      end
-    else
-      local nu, tags, nd = check_address(email_addr)
-      if nu or nd then
-        set_addr(email_addr, nu, nd)
-
-        return nu, tags
-      end
-    end
-
-    return nil
-  end
-end
-
-exports.is_rspamc_or_controller = function(task)
-  local ua = task:get_request_header('User-Agent') or ''
-  local pwd = task:get_request_header('Password')
-  local is_rspamc = false
-  if tostring(ua) == 'rspamc' or pwd then is_rspamc = true end
-
-  return is_rspamc
-end
-
-local unpack_function = table.unpack or unpack
-exports.unpack = function(t)
-  return unpack_function(t)
-end
-
-return exports

data/Dockerfiles/rspamd/metadata_exporter.lua

@@ -0,0 +1,722 @@
+--[[
+Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+Copyright (c) 2016, Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+  return
+end
+
+-- A plugin that pushes metadata (or whole messages) to external services
+
+local redis_params
+local lua_util = require "lua_util"
+local rspamd_http = require "rspamd_http"
+local rspamd_tcp = require "rspamd_tcp"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+local E = {}
+local N = 'metadata_exporter'
+
+local settings = {
+  pusher_enabled = {},
+  pusher_format = {},
+  pusher_select = {},
+  mime_type = 'text/plain',
+  defer = false,
+  mail_from = '',
+  mail_to = 'postmaster@localhost',
+  helo = 'rspamd',
+  email_template = [[From: "Rspamd" <$mail_from>
+To: $mail_to
+Subject: Spam alert
+Date: $date
+MIME-Version: 1.0
+Message-ID: <$our_message_id>
+Content-type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+Authenticated username: $user
+IP: $ip
+Queue ID: $qid
+SMTP FROM: $from
+SMTP RCPT: $rcpt
+MIME From: $header_from
+MIME To: $header_to
+MIME Date: $header_date
+Subject: $header_subject
+Message-ID: $message_id
+Action: $action
+Score: $score
+Symbols: $symbols]],
+}
+
+local function get_general_metadata(task, flatten, no_content)
+  local r = {}
+  local ip = task:get_from_ip()
+  if ip and ip:is_valid() then
+    r.ip = tostring(ip)
+  else
+    r.ip = 'unknown'
+  end
+  r.user = task:get_user() or 'unknown'
+  r.qid = task:get_queue_id() or 'unknown'
+  r.subject = task:get_subject() or 'unknown'
+  r.action = task:get_metric_action('default')
+
+  local s = task:get_metric_score('default')[1]
+  r.score = flatten and string.format('%.2f', s) or s
+
+  local rcpt = task:get_recipients('smtp')
+  if rcpt then
+    local l = {}
+    for _, a in ipairs(rcpt) do
+      table.insert(l, a['addr'])
+    end
+    if not flatten then
+      r.rcpt = l
+    else
+      r.rcpt = table.concat(l, ', ')
+    end
+  else
+    r.rcpt = 'unknown'
+  end
+  local from = task:get_from('smtp')
+  if ((from or E)[1] or E).addr then
+    r.from = from[1].addr
+  else
+    r.from = 'unknown'
+  end
+  local syminf = task:get_symbols_all()
+  if flatten then
+    local l = {}
+    for _, sym in ipairs(syminf) do
+      local txt
+      if sym.options then
+        local topt = table.concat(sym.options, ', ')
+        txt = sym.name .. '(' .. string.format('%.2f', sym.score) .. ')' .. ' [' .. topt .. ']'
+      else
+        txt = sym.name .. '(' .. string.format('%.2f', sym.score) .. ')'
+      end
+      table.insert(l, txt)
+    end
+    r.symbols = table.concat(l, '\n\t')
+  else
+    r.symbols = syminf
+  end
+  local function process_header(name)
+    local hdr = task:get_header_full(name)
+    if hdr then
+      local l = {}
+      for _, h in ipairs(hdr) do
+        table.insert(l, h.decoded)
+      end
+      if not flatten then
+        return l
+      else
+        return table.concat(l, '\n')
+      end
+    else
+      return 'unknown'
+    end
+  end
+  if not no_content then
+    r.header_from = process_header('from')
+    r.header_to = process_header('to')
+    r.header_subject = process_header('subject')
+    r.header_date = process_header('date')
+    r.message_id = task:get_message_id()
+  end
+  return r
+end
+
+local formatters = {
+  default = function(task)
+    return task:get_content()
+  end,
+  email_alert = function(task, rule, extra)
+    local meta = get_general_metadata(task, true)
+    local display_emails = {}
+    meta.mail_from = rule.mail_from or settings.mail_from
+    local mail_targets = rule.mail_to or settings.mail_to
+    if type(mail_targets) ~= 'table' then
+      table.insert(display_emails, string.format('<%s>', mail_targets))
+      mail_targets = {[mail_targets] = true}
+    else
+      for _, e in ipairs(mail_targets) do
+        table.insert(display_emails, string.format('<%s>', e))
+      end
+    end
+    if rule.email_alert_sender then
+      local x = task:get_from('smtp')
+      if x and string.len(x[1].addr) > 0 then
+        mail_targets[x] = true
+        table.insert(display_emails, string.format('<%s>', x[1].addr))
+      end
+    end
+    if rule.email_alert_user then
+      local x = task:get_user()
+      if x then
+        mail_targets[x] = true
+        table.insert(display_emails, string.format('<%s>', x))
+      end
+    end
+    if rule.email_alert_recipients then
+      local x = task:get_recipients('smtp')
+      if x then
+        for _, e in ipairs(x) do
+          if string.len(e.addr) > 0 then
+            mail_targets[e.addr] = true
+            table.insert(display_emails, string.format('<%s>', e.addr))
+          end
+        end
+      end
+    end
+    meta.mail_to = table.concat(display_emails, ', ')
+    meta.our_message_id = rspamd_util.random_hex(12) .. '@rspamd'
+    meta.date = rspamd_util.time_to_string(rspamd_util.get_time())
+    return lua_util.template(rule.email_template or settings.email_template, meta), { mail_targets = mail_targets}
+  end,
+  json = function(task)
+    return ucl.to_format(get_general_metadata(task), 'json-compact')
+  end
+}
+
+local function is_spam(action)
+  return (action == 'reject' or action == 'add header' or action == 'rewrite subject')
+end
+
+local selectors = {
+  default = function(task)
+    return true
+  end,
+  is_spam = function(task)
+    local action = task:get_metric_action('default')
+    return is_spam(action)
+  end,
+  is_spam_authed = function(task)
+    if not task:get_user() then
+      return false
+    end
+    local action = task:get_metric_action('default')
+    return is_spam(action)
+  end,
+  is_reject = function(task)
+    local action = task:get_metric_action('default')
+    return (action == 'reject')
+  end,
+  is_reject_authed = function(task)
+    if not task:get_user() then
+      return false
+    end
+    local action = task:get_metric_action('default')
+    return (action == 'reject')
+  end,
+}
+
+local function maybe_defer(task, rule)
+  if rule.defer then
+    rspamd_logger.warnx(task, 'deferring message')
+    task:set_pre_result('soft reject', 'deferred', N)
+  end
+end
+
+local pushers = {
+  redis_pubsub = function(task, formatted, rule)
+    local _,ret,upstream
+    local function redis_pub_cb(err)
+      if err then
+        rspamd_logger.errx(task, 'got error %s when publishing on server %s',
+            err, upstream:get_addr())
+        return maybe_defer(task, rule)
+      end
+      return true
+    end
+    ret,_,upstream = rspamd_redis_make_request(task,
+      redis_params, -- connect params
+      nil, -- hash key
+      true, -- is write
+      redis_pub_cb, --callback
+      'PUBLISH', -- command
+      {rule.channel, formatted} -- arguments
+    )
+    if not ret then
+      rspamd_logger.errx(task, 'error connecting to redis')
+      maybe_defer(task, rule)
+    end
+  end,
+  http = function(task, formatted, rule)
+    local function http_callback(err, code)
+      if err then
+        rspamd_logger.errx(task, 'got error %s in http callback', err)
+        return maybe_defer(task, rule)
+      end
+      if code ~= 200 then
+        rspamd_logger.errx(task, 'got unexpected http status: %s', code)
+        return maybe_defer(task, rule)
+      end
+      return true
+    end
+    local hdrs = {}
+    if rule.meta_headers then
+      local gm = get_general_metadata(task, false, true)
+      local pfx = rule.meta_header_prefix or 'X-Rspamd-'
+      for k, v in pairs(gm) do
+        if type(v) == 'table' then
+          hdrs[pfx .. k] = ucl.to_format(v, 'json-compact')
+        else
+          hdrs[pfx .. k] = v
+        end
+      end
+    end
+    rspamd_http.request({
+      task=task,
+      url=rule.url,
+      body=formatted,
+      callback=http_callback,
+      mime_type=rule.mime_type or settings.mime_type,
+      headers=hdrs,
+    })
+  end,
+  send_mail = function(task, formatted, rule, extra)
+    local function mail_cb(err, data, conn)
+      local function no_error(merr, mdata, wantcode)
+        wantcode = wantcode or '2'
+        if merr then
+          rspamd_logger.errx(task, 'got error in tcp callback: %s', merr)
+          if conn then
+            conn:close()
+          end
+          maybe_defer(task, rule)
+          return false
+        end
+        if mdata then
+          if type(mdata) ~= 'string' then
+            mdata = tostring(mdata)
+            end
+          if string.sub(mdata, 1, 1) ~= wantcode then
+            rspamd_logger.errx(task, 'got bad smtp response: %s', mdata)
+            if conn then
+              conn:close()
+            end
+            maybe_defer(task, rule)
+            return false
+            end
+        else
+            rspamd_logger.errx(task, 'no data')
+          if conn then
+            conn:close()
+          end
+          maybe_defer(task, rule)
+          return false
+        end
+        return true
+      end
+      local function all_done_cb(merr, mdata)
+        if conn then
+          conn:close()
+        end
+        return true
+      end
+      local function quit_done_cb(merr, mdata)
+        conn:add_read(all_done_cb, '\r\n')
+      end
+      local function quit_cb(merr, mdata)
+        if no_error(merr, mdata) then
+          conn:add_write(quit_done_cb, 'QUIT\r\n')
+        end
+      end
+      local function pre_quit_cb(merr, mdata)
+        if no_error(merr, '2') then
+          conn:add_read(quit_cb, '\r\n')
+        end
+      end
+      local function data_done_cb(merr, mdata)
+        if no_error(merr, mdata, '3') then
+          conn:add_write(pre_quit_cb, {formatted, '\r\n.\r\n'})
+        end
+      end
+      local function data_cb(merr, mdata)
+        if no_error(merr, '2') then
+          conn:add_read(data_done_cb, '\r\n')
+        end
+      end
+      local from_done_cb
+      local function rcpt_done_cb(merr, mdata)
+        if no_error(merr, mdata) then
+          local k = next(extra.mail_targets)
+          if not k then
+            conn:add_write(data_cb, 'DATA\r\n')
+          else
+            from_done_cb('2', '2')
+          end
+        end
+      end
+      local function rcpt_cb(merr, mdata)
+        if no_error(merr, '2') then
+          conn:add_read(rcpt_done_cb, '\r\n')
+        end
+      end
+      from_done_cb = function(merr, mdata)
+        local k
+        if extra then
+          k = next(extra.mail_targets)
+        else
+          extra = {mail_targets = {}}
+          if type(rule.mail_to) == 'string' then
+            extra = {mail_targets = {}}
+            k = rule.mail_to
+          elseif type(rule.mail_to) == 'table' then
+            for _, r in ipairs(rule.mail_to) do
+              extra.mail_targets[r] = true
+            end
+            k = next(extra.mail_targets)
+          end
+        end
+        extra.mail_targets[k] = nil
+        conn:add_write(rcpt_cb, {'RCPT TO: <', k, '>\r\n'})
+      end
+      local function from_cb(merr, mdata)
+        if no_error(merr, '2') then
+          conn:add_read(from_done_cb, '\r\n')
+        end
+      end
+        local function hello_done_cb(merr, mdata)
+        if no_error(merr, mdata) then
+          conn:add_write(from_cb, {'MAIL FROM: <', rule.mail_from or settings.mail_from, '>\r\n'})
+        end
+      end
+      local function hello_cb(merr)
+        if no_error(merr, '2') then
+          conn:add_read(hello_done_cb, '\r\n')
+        end
+      end
+      if no_error(err, data) then
+        conn:add_write(hello_cb, {'HELO ', rule.helo or settings.helo, '\r\n'})
+      end
+    end
+    rspamd_tcp.request({
+      task = task,
+      callback = mail_cb,
+      stop_pattern = '\r\n',
+      host = rule.smtp,
+      port = rule.smtp_port or settings.smtp_port or 25,
+    })
+  end,
+}
+
+local opts = rspamd_config:get_all_opt(N)
+if not opts then return end
+local process_settings = {
+  select = function(val)
+    selectors.custom = assert(load(val))()
+  end,
+  format = function(val)
+    formatters.custom = assert(load(val))()
+  end,
+  push = function(val)
+    pushers.custom = assert(load(val))()
+  end,
+  custom_push = function(val)
+    if type(val) == 'table' then
+      for k, v in pairs(val) do
+        pushers[k] = assert(load(v))()
+      end
+    end
+  end,
+  custom_select = function(val)
+    if type(val) == 'table' then
+      for k, v in pairs(val) do
+        selectors[k] = assert(load(v))()
+      end
+    end
+  end,
+  custom_format = function(val)
+    if type(val) == 'table' then
+      for k, v in pairs(val) do
+        formatters[k] = assert(load(v))()
+      end
+    end
+  end,
+  pusher_enabled = function(val)
+    if type(val) == 'string' then
+      if pushers[val] then
+        settings.pusher_enabled[val] = true
+      else
+        rspamd_logger.errx(rspamd_config, 'Pusher type: %s is invalid', val)
+      end
+    elseif type(val) == 'table' then
+      for _, v in ipairs(val) do
+        if pushers[v] then
+          settings.pusher_enabled[v] = true
+        else
+          rspamd_logger.errx(rspamd_config, 'Pusher type: %s is invalid', val)
+        end
+      end
+    end
+  end,
+}
+for k, v in pairs(opts) do
+  local f = process_settings[k]
+  if f then
+    f(opts[k])
+  else
+    settings[k] = v
+  end
+end
+if type(settings.rules) ~= 'table' then
+  -- Legacy config
+  settings.rules = {}
+  if not next(settings.pusher_enabled) then
+    if pushers.custom then
+      rspamd_logger.infox(rspamd_config, 'Custom pusher implicitly enabled')
+      settings.pusher_enabled.custom = true
+    else
+      -- Check legacy options
+      if settings.url then
+        rspamd_logger.warnx(rspamd_config, 'HTTP pusher implicitly enabled')
+        settings.pusher_enabled.http = true
+      end
+      if settings.channel then
+        rspamd_logger.warnx(rspamd_config, 'Redis Pubsub pusher implicitly enabled')
+        settings.pusher_enabled.redis_pubsub = true
+      end
+      if settings.smtp and settings.mail_to then
+        rspamd_logger.warnx(rspamd_config, 'SMTP pusher implicitly enabled')
+        settings.pusher_enabled.send_mail = true
+      end
+    end
+  end
+  if not next(settings.pusher_enabled) then
+    rspamd_logger.errx(rspamd_config, 'No push backend enabled')
+    return
+  end
+  if settings.formatter then
+    settings.format = formatters[settings.formatter]
+    if not settings.format then
+      rspamd_logger.errx(rspamd_config, 'No such formatter: %s', settings.formatter)
+      return
+    end
+  end
+  if settings.selector then
+    settings.select = selectors[settings.selector]
+    if not settings.select then
+      rspamd_logger.errx(rspamd_config, 'No such selector: %s', settings.selector)
+      return
+    end
+  end
+  for k in pairs(settings.pusher_enabled) do
+    local formatter = settings.pusher_format[k]
+    local selector = settings.pusher_select[k]
+    if not formatter then
+      settings.pusher_format[k] = settings.formatter or 'default'
+      rspamd_logger.infox(rspamd_config, 'Using default formatter for %s pusher', k)
+    else
+      if not formatters[formatter] then
+        rspamd_logger.errx(rspamd_config, 'No such formatter: %s - disabling %s', formatter, k)
+        settings.pusher_enabled.k = nil
+      end
+    end
+    if not selector then
+      settings.pusher_select[k] = settings.selector or 'default'
+      rspamd_logger.infox(rspamd_config, 'Using default selector for %s pusher', k)
+    else
+      if not selectors[selector] then
+        rspamd_logger.errx(rspamd_config, 'No such selector: %s - disabling %s', selector, k)
+        settings.pusher_enabled.k = nil
+      end
+    end
+  end
+  if settings.pusher_enabled.redis_pubsub then
+    redis_params = rspamd_parse_redis_server(N)
+    if not redis_params then
+      rspamd_logger.errx(rspamd_config, 'No redis servers are specified')
+      settings.pusher_enabled.redis_pubsub = nil
+    else
+      local r = {}
+      r.backend = 'redis_pubsub'
+      r.channel = settings.channel
+      r.defer = settings.defer
+      r.selector = settings.pusher_select.redis_pubsub
+      r.formatter = settings.pusher_format.redis_pubsub
+      settings.rules[r.backend:upper()] = r
+    end
+  end
+  if settings.pusher_enabled.http then
+    if not settings.url then
+      rspamd_logger.errx(rspamd_config, 'No URL is specified')
+      settings.pusher_enabled.http = nil
+    else
+      local r = {}
+      r.backend = 'http'
+      r.url = settings.url
+      r.mime_type = settings.mime_type
+      r.defer = settings.defer
+      r.selector = settings.pusher_select.http
+      r.formatter = settings.pusher_format.http
+      settings.rules[r.backend:upper()] = r
+    end
+  end
+  if settings.pusher_enabled.send_mail then
+    if not (settings.mail_to and settings.smtp) then
+      rspamd_logger.errx(rspamd_config, 'No mail_to and/or smtp setting is specified')
+      settings.pusher_enabled.send_mail = nil
+    else
+      local r = {}
+      r.backend = 'send_mail'
+      r.mail_to = settings.mail_to
+      r.mail_from = settings.mail_from
+      r.helo = settings.hello
+      r.smtp = settings.smtp
+      r.smtp_port = settings.smtp_port
+      r.email_template = settings.email_template
+      r.defer = settings.defer
+      r.selector = settings.pusher_select.send_mail
+      r.formatter = settings.pusher_format.send_mail
+      settings.rules[r.backend:upper()] = r
+    end
+  end
+  if not next(settings.pusher_enabled) then
+    rspamd_logger.errx(rspamd_config, 'No push backend enabled')
+    return
+  end
+elseif not next(settings.rules) then
+  lua_util.debugm(N, rspamd_config, 'No rules enabled')
+  return
+end
+if not settings.rules or not next(settings.rules) then
+  rspamd_logger.errx(rspamd_config, 'No rules enabled')
+  return
+end
+local backend_required_elements = {
+  http = {
+    'url',
+  },
+  smtp = {
+    'mail_to',
+    'smtp',
+  },
+  redis_pubsub = {
+    'channel',
+  },
+}
+local check_element = {
+  selector = function(k, v)
+    if not selectors[v] then
+      rspamd_logger.errx(rspamd_config, 'Rule %s has invalid selector %s', k, v)
+      return false
+    else
+      return true
+    end
+  end,
+  formatter = function(k, v)
+    if not formatters[v] then
+      rspamd_logger.errx(rspamd_config, 'Rule %s has invalid formatter %s', k, v)
+      return false
+    else
+      return true
+    end
+  end,
+}
+local backend_check = {
+  default = function(k, rule)
+    local reqset = backend_required_elements[rule.backend]
+    if reqset then
+      for _, e in ipairs(reqset) do
+        if not rule[e] then
+          rspamd_logger.errx(rspamd_config, 'Rule %s misses required setting %s', k, e)
+          settings.rules[k] = nil
+        end
+      end
+    end
+    for sett, v in pairs(rule) do
+      local f = check_element[sett]
+      if f then
+        if not f(sett, v) then
+          settings.rules[k] = nil
+        end
+      end
+    end
+  end,
+}
+backend_check.redis_pubsub = function(k, rule)
+  if not redis_params then
+    redis_params = rspamd_parse_redis_server(N)
+  end
+  if not redis_params then
+    rspamd_logger.errx(rspamd_config, 'No redis servers are specified')
+    settings.rules[k] = nil
+  else
+    backend_check.default(k, rule)
+  end
+end
+setmetatable(backend_check, {
+  __index = function()
+    return backend_check.default
+  end,
+})
+for k, v in pairs(settings.rules) do
+  if type(v) == 'table' then
+    local backend = v.backend
+    if not backend then
+      rspamd_logger.errx(rspamd_config, 'Rule %s has no backend', k)
+      settings.rules[k] = nil
+    elseif not pushers[backend] then
+      rspamd_logger.errx(rspamd_config, 'Rule %s has invalid backend %s', k, backend)
+      settings.rules[k] = nil
+    else
+      local f = backend_check[backend]
+      f(k, v)
+    end
+  else
+    rspamd_logger.errx(rspamd_config, 'Rule %s has bad type: %s', k, type(v))
+    settings.rules[k] = nil
+  end
+end
+
+local function gen_exporter(rule)
+  return function (task)
+    if task:has_flag('skip') then return end
+    local selector = rule.selector or 'default'
+    local selected = selectors[selector](task)
+    if selected then
+      lua_util.debugm(N, task, 'Message selected for processing')
+      local formatter = rule.formatter or 'default'
+      local formatted, extra = formatters[formatter](task, rule)
+      if formatted then
+        pushers[rule.backend](task, formatted, rule, extra)
+      else
+        lua_util.debugm(N, task, 'Formatter [%s] returned non-truthy value [%s]', formatter, formatted)
+      end
+    else
+      lua_util.debugm(N, task, 'Selector [%s] returned non-truthy value [%s]', selector, selected)
+    end
+  end
+end
+
+if not next(settings.rules) then
+  rspamd_logger.errx(rspamd_config, 'No rules enabled')
+  lua_util.disable_module(N, "config")
+end
+for k, r in pairs(settings.rules) do
+  rspamd_config:register_symbol({
+    name = 'EXPORT_METADATA_' .. k,
+    type = 'postfilter,idempotent',
+    callback = gen_exporter(r),
+    priority = 10,
+    flags = 'empty',
+  })
+end

data/Dockerfiles/rspamd/ratelimit.lua

@@ -1,674 +0,0 @@
---[[
-Copyright (c) 2011-2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
-Copyright (c) 2016-2017, Andrew Lewis <nerf@judo.za.org>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-]]--
-
-if confighelp then
-  return
-end
-
--- A plugin that implements ratelimits using redis
-
-local E = {}
-local N = 'ratelimit'
-local redis_params
--- Senders that are considered as bounce
-local settings = {
-  bounce_senders = { 'postmaster', 'mailer-daemon', '', 'null', 'fetchmail-daemon', 'mdaemon' },
--- Do not check ratelimits for these recipients
-  whitelisted_rcpts = { 'postmaster', 'mailer-daemon' },
-  prefix = 'RL',
-  ham_factor_rate = 1.01,
-  spam_factor_rate = 0.99,
-  ham_factor_burst = 1.02,
-  spam_factor_burst = 0.98,
-  max_rate_mult = 5,
-  max_bucket_mult = 10,
-  expire = 60 * 60 * 24 * 2, -- 2 days by default
-  limits = {},
-  allow_local = false,
-}
-
--- Checks bucket, updating it if needed
--- KEYS[1] - prefix to update, e.g. RL_<triplet>_<seconds>
--- KEYS[2] - current time in milliseconds
--- KEYS[3] - bucket leak rate (messages per millisecond)
--- KEYS[4] - bucket burst
--- KEYS[5] - expire for a bucket
--- return 1 if message should be ratelimited and 0 if not
--- Redis keys used:
---   l - last hit
---   b - current burst
---   dr - current dynamic rate multiplier (*10000)
---   db - current dynamic burst multiplier (*10000)
-local bucket_check_script = [[
-  local last = redis.call('HGET', KEYS[1], 'l')
-  local now = tonumber(KEYS[2])
-  local dynr, dynb = 0, 0
-  if not last then
-    -- New bucket
-    redis.call('HSET', KEYS[1], 'l', KEYS[2])
-    redis.call('HSET', KEYS[1], 'b', '0')
-    redis.call('HSET', KEYS[1], 'dr', '10000')
-    redis.call('HSET', KEYS[1], 'db', '10000')
-    redis.call('EXPIRE', KEYS[1], KEYS[5])
-    return {0, 0, 1, 1}
-  end
-
-  last = tonumber(last)
-  local burst = tonumber(redis.call('HGET', KEYS[1], 'b'))
-  -- Perform leak
-  if burst > 0 then
-   if last < tonumber(KEYS[2]) then
-    local rate = tonumber(KEYS[3])
-    dynr = tonumber(redis.call('HGET', KEYS[1], 'dr')) / 10000.0
-    rate = rate * dynr
-    local leaked = ((now - last) * rate)
-    burst = burst - leaked
-    redis.call('HINCRBYFLOAT', KEYS[1], 'b', -(leaked))
-   end
-  else
-   burst = 0
-   redis.call('HSET', KEYS[1], 'b', '0')
-  end
-
-  dynb = tonumber(redis.call('HGET', KEYS[1], 'db')) / 10000.0
-
-  if (burst + 1) * dynb > tonumber(KEYS[4]) then
-   return {1, tostring(burst), tostring(dynr), tostring(dynb)}
-  end
-
-  return {0, tostring(burst), tostring(dynr), tostring(dynb)}
-]]
-local bucket_check_id
-
-
--- Updates a bucket
--- KEYS[1] - prefix to update, e.g. RL_<triplet>_<seconds>
--- KEYS[2] - current time in milliseconds
--- KEYS[3] - dynamic rate multiplier
--- KEYS[4] - dynamic burst multiplier
--- KEYS[5] - max dyn rate (min: 1/x)
--- KEYS[6] - max burst rate (min: 1/x)
--- KEYS[7] - expire for a bucket
--- Redis keys used:
---   l - last hit
---   b - current burst
---   dr - current dynamic rate multiplier
---   db - current dynamic burst multiplier
-local bucket_update_script = [[
-  local last = redis.call('HGET', KEYS[1], 'l')
-  local now = tonumber(KEYS[2])
-  if not last then
-    -- New bucket
-    redis.call('HSET', KEYS[1], 'l', KEYS[2])
-    redis.call('HSET', KEYS[1], 'b', '1')
-    redis.call('HSET', KEYS[1], 'dr', '10000')
-    redis.call('HSET', KEYS[1], 'db', '10000')
-    redis.call('EXPIRE', KEYS[1], KEYS[7])
-    return {1, 1, 1}
-  end
-
-  local burst = tonumber(redis.call('HGET', KEYS[1], 'b'))
-  local db = tonumber(redis.call('HGET', KEYS[1], 'db')) / 10000
-  local dr = tonumber(redis.call('HGET', KEYS[1], 'dr')) / 10000
-
-  if dr < tonumber(KEYS[5]) and dr > 1.0 / tonumber(KEYS[5]) then
-    dr = dr * tonumber(KEYS[3])
-    redis.call('HSET', KEYS[1], 'dr', tostring(math.floor(dr * 10000)))
-  end
-
-  if db < tonumber(KEYS[6]) and db > 1.0 / tonumber(KEYS[6]) then
-    db = db * tonumber(KEYS[4])
-    redis.call('HSET', KEYS[1], 'db', tostring(math.floor(db * 10000)))
-  end
-
-  redis.call('HINCRBYFLOAT', KEYS[1], 'b', 1)
-  redis.call('HSET', KEYS[1], 'l', KEYS[2])
-  redis.call('EXPIRE', KEYS[1], KEYS[7])
-
-  return {tostring(burst), tostring(dr), tostring(db)}
-]]
-local bucket_update_id
-
--- message_func(task, limit_type, prefix, bucket)
-local message_func = function(_, limit_type, _, _)
-  return string.format('Ratelimit "%s" exceeded', limit_type)
-end
-
-local rspamd_logger = require "rspamd_logger"
-local rspamd_util = require "rspamd_util"
-local rspamd_lua_utils = require "lua_util"
-local lua_redis = require "lua_redis"
-local fun = require "fun"
-local lua_maps = require "lua_maps"
-local lua_util = require "lua_util"
-local rspamd_hash = require "rspamd_cryptobox_hash"
-
-
-local function load_scripts(cfg, ev_base)
-  bucket_check_id = lua_redis.add_redis_script(bucket_check_script, redis_params)
-  bucket_update_id = lua_redis.add_redis_script(bucket_update_script, redis_params)
-end
-
-local limit_parser
-local function parse_string_limit(lim, no_error)
-  local function parse_time_suffix(s)
-    if s == 's' then
-      return 1
-    elseif s == 'm' then
-      return 60
-    elseif s == 'h' then
-      return 3600
-    elseif s == 'd' then
-      return 86400
-    end
-  end
-  local function parse_num_suffix(s)
-    if s == '' then
-      return 1
-    elseif s == 'k' then
-      return 1000
-    elseif s == 'm' then
-      return 1000000
-    elseif s == 'g' then
-      return 1000000000
-    end
-  end
-  local lpeg = require "lpeg"
-
-  if not limit_parser then
-    local digit = lpeg.R("09")
-    limit_parser = {}
-    limit_parser.integer =
-    (lpeg.S("+-") ^ -1) *
-            (digit   ^  1)
-    limit_parser.fractional =
-    (lpeg.P(".")   ) *
-            (digit ^ 1)
-    limit_parser.number =
-    (limit_parser.integer *
-            (limit_parser.fractional ^ -1)) +
-            (lpeg.S("+-") * limit_parser.fractional)
-    limit_parser.time = lpeg.Cf(lpeg.Cc(1) *
-            (limit_parser.number / tonumber) *
-            ((lpeg.S("smhd") / parse_time_suffix) ^ -1),
-      function (acc, val) return acc * val end)
-    limit_parser.suffixed_number = lpeg.Cf(lpeg.Cc(1) *
-            (limit_parser.number / tonumber) *
-            ((lpeg.S("kmg") / parse_num_suffix) ^ -1),
-      function (acc, val) return acc * val end)
-    limit_parser.limit = lpeg.Ct(limit_parser.suffixed_number *
-            (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) *
-            limit_parser.time)
-  end
-  local t = lpeg.match(limit_parser.limit, lim)
-
-  if t and t[1] and t[2] and t[2] ~= 0 then
-    return t[2], t[1]
-  end
-
-  if not no_error then
-    rspamd_logger.errx(rspamd_config, 'bad limit: %s', lim)
-  end
-
-  return nil
-end
-
-local function parse_limit(name, data)
-  local buckets = {}
-  if type(data) == 'table' then
-    -- 3 cases here:
-    --  * old limit in format [burst, rate]
-    --  * vector of strings in Andrew's string format
-    --  * proper bucket table
-    if #data == 2 and tonumber(data[1]) and tonumber(data[2]) then
-      -- Old style ratelimit
-      rspamd_logger.warnx(rspamd_config, 'old style ratelimit for %s', name)
-      if tonumber(data[1]) > 0 and tonumber(data[2]) > 0 then
-        table.insert(buckets, {
-          burst = data[1],
-          rate = data[2]
-        })
-      elseif data[1] ~= 0 then
-        rspamd_logger.warnx(rspamd_config, 'invalid numbers for %s', name)
-      else
-        rspamd_logger.infox(rspamd_config, 'disable limit %s, burst is zero', name)
-      end
-    else
-      -- Recursively map parse_limit and flatten the list
-      fun.each(function(l)
-        -- Flatten list
-        for _,b in ipairs(l) do table.insert(buckets, b) end
-      end, fun.map(function(d) return parse_limit(d, name) end, data))
-    end
-  elseif type(data) == 'string' then
-    local rep_rate, burst = parse_string_limit(data)
-
-    if rep_rate and burst then
-      table.insert(buckets, {
-        burst = burst,
-        rate = 1.0 / rep_rate -- reciprocal
-      })
-    end
-  end
-
-  -- Filter valid
-  return fun.totable(fun.filter(function(val)
-    return type(val.burst) == 'number' and type(val.rate) == 'number'
-  end, buckets))
-end
-
---- Check whether this addr is bounce
-local function check_bounce(from)
-  return fun.any(function(b) return b == from end, settings.bounce_senders)
-end
-
-local keywords = {
-  ['ip'] = {
-    ['get_value'] = function(task)
-      local ip = task:get_ip()
-      if ip and ip:is_valid() then return tostring(ip) end
-      return nil
-    end,
-  },
-  ['rip'] = {
-    ['get_value'] = function(task)
-      local ip = task:get_ip()
-      if ip and ip:is_valid() and not ip:is_local() then return tostring(ip) end
-      return nil
-    end,
-  },
-  ['from'] = {
-    ['get_value'] = function(task)
-      local from = task:get_from(0)
-      if ((from or E)[1] or E).addr then
-        return string.lower(from[1]['addr'])
-      end
-      return nil
-    end,
-  },
-  ['bounce'] = {
-    ['get_value'] = function(task)
-      local from = task:get_from(0)
-      if not ((from or E)[1] or E).user then
-        return '_'
-      end
-      if check_bounce(from[1]['user']) then return '_' else return nil end
-    end,
-  },
-  ['asn'] = {
-    ['get_value'] = function(task)
-      local asn = task:get_mempool():get_variable('asn')
-      if not asn then
-        return nil
-      else
-        return asn
-      end
-    end,
-  },
-  ['user'] = {
-    ['get_value'] = function(task)
-      local auser = task:get_user()
-      if not auser then
-        return nil
-      else
-        return auser
-      end
-    end,
-  },
-  ['to'] = {
-    ['get_value'] = function(task)
-      return task:get_principal_recipient()
-    end,
-  },
-}
-
-local function gen_rate_key(task, rtype, bucket)
-  local key_t = {tostring(lua_util.round(100000.0 / bucket.burst))}
-  local key_keywords = lua_util.str_split(rtype, '_')
-  local have_user = false
-
-  for _, v in ipairs(key_keywords) do
-    local ret
-
-    if keywords[v] and type(keywords[v]['get_value']) == 'function' then
-      ret = keywords[v]['get_value'](task)
-    end
-    if not ret then return nil end
-    if v == 'user' then have_user = true end
-    if type(ret) ~= 'string' then ret = tostring(ret) end
-    table.insert(key_t, ret)
-  end
-
-  if have_user and not task:get_user() then
-    return nil
-  end
-
-  return table.concat(key_t, ":")
-end
-
-local function make_prefix(redis_key, name, bucket)
-  local hash_len = 24
-  if hash_len > #redis_key then hash_len = #redis_key end
-  local hash = settings.prefix ..
-      string.sub(rspamd_hash.create(redis_key):base32(), 1, hash_len)
-  -- Fill defaults
-  if not bucket.spam_factor_rate then
-    bucket.spam_factor_rate = settings.spam_factor_rate
-  end
-  if not bucket.ham_factor_rate then
-    bucket.ham_factor_rate = settings.ham_factor_rate
-  end
-  if not bucket.spam_factor_burst then
-    bucket.spam_factor_burst = settings.spam_factor_burst
-  end
-  if not bucket.ham_factor_burst then
-    bucket.ham_factor_burst = settings.ham_factor_burst
-  end
-
-  return {
-    bucket = bucket,
-    name = name,
-    hash = hash
-  }
-end
-
-local function limit_to_prefixes(task, k, v, prefixes)
-  local n = 0
-  for _,bucket in ipairs(v) do
-    local prefix = gen_rate_key(task, k, bucket)
-
-    if prefix then
-      prefixes[prefix] = make_prefix(prefix, k, bucket)
-      n = n + 1
-    end
-  end
-
-  return n
-end
-
-local function ratelimit_cb(task)
-  if not settings.allow_local and
-          rspamd_lua_utils.is_rspamc_or_controller(task) then return end
-
-  -- Get initial task data
-  local ip = task:get_from_ip()
-  if ip and ip:is_valid() and settings.whitelisted_ip then
-    if settings.whitelisted_ip:get_key(ip) then
-      -- Do not check whitelisted ip
-      rspamd_logger.infox(task, 'skip ratelimit for whitelisted IP')
-      return
-    end
-  end
-  -- Parse all rcpts
-  local rcpts = task:get_recipients()
-  local rcpts_user = {}
-  if rcpts then
-    fun.each(function(r)
-      fun.each(function(type) table.insert(rcpts_user, r[type]) end, {'user', 'addr'})
-    end, rcpts)
-
-    if fun.any(function(r) return settings.whitelisted_rcpts:get_key(r) end, rcpts_user) then
-      rspamd_logger.infox(task, 'skip ratelimit for whitelisted recipient')
-      return
-    end
-  end
-  -- Get user (authuser)
-  if settings.whitelisted_user then
-    local auser = task:get_user()
-    if settings.whitelisted_user:get_key(auser) then
-      rspamd_logger.infox(task, 'skip ratelimit for whitelisted user')
-      return
-    end
-  end
-  -- Now create all ratelimit prefixes
-  local prefixes = {}
-  local nprefixes = 0
-
-  for k,v in pairs(settings.limits) do
-    nprefixes = nprefixes + limit_to_prefixes(task, k, v, prefixes)
-  end
-
-  for k, hdl in pairs(settings.custom_keywords or E) do
-    local ret, redis_key, bd = pcall(hdl, task)
-
-    if ret then
-      local bucket = parse_limit(k, bd)
-      if bucket[1] then
-        prefixes[redis_key] = make_prefix(redis_key, k, bucket[1])
-      end
-      nprefixes = nprefixes + 1
-    else
-      rspamd_logger.errx(task, 'cannot call handler for %s: %s',
-          k, redis_key)
-    end
-  end
-
-  local function gen_check_cb(prefix, bucket, lim_name)
-    return function(err, data)
-      if err then
-        rspamd_logger.errx('cannot check limit %s: %s %s', prefix, err, data)
-      elseif type(data) == 'table' and data[1] and data[1] == 1 then
-        -- set symbol only and do NOT soft reject
-        if settings.symbol then
-          task:insert_result(settings.symbol, 0.0, lim_name .. "(" .. prefix .. ")")
-          rspamd_logger.infox(task,
-              'set_symbol_only: ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn)',
-              lim_name, prefix,
-              bucket.burst, bucket.rate,
-              data[2], data[3], data[4])
-          return
-        -- set INFO symbol and soft reject
-        elseif settings.info_symbol then
-          task:insert_result(settings.info_symbol, 1.0,
-              lim_name .. "(" .. prefix .. ")")
-        end
-        rspamd_logger.infox(task,
-            'ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn)',
-            lim_name, prefix,
-            bucket.burst, bucket.rate,
-            data[2], data[3], data[4])
-        task:set_pre_result('soft reject',
-                message_func(task, lim_name, prefix, bucket))
-      end
-    end
-  end
-
-  -- Don't do anything if pre-result has been already set
-  if task:has_pre_result() then return end
-
-  if nprefixes > 0 then
-    -- Save prefixes to the cache to allow update
-    task:cache_set('ratelimit_prefixes', prefixes)
-    local now = rspamd_util.get_time()
-    now = lua_util.round(now * 1000.0) -- Get milliseconds
-    -- Now call check script for all defined prefixes
-
-    for pr,value in pairs(prefixes) do
-      local bucket = value.bucket
-      local rate = (bucket.rate) / 1000.0 -- Leak rate in messages/ms
-      rspamd_logger.debugm(N, task, "check limit %s:%s -> %s (%s/%s)",
-          value.name, pr, value.hash, bucket.burst, bucket.rate)
-      lua_redis.exec_redis_script(bucket_check_id,
-              {key = value.hash, task = task, is_write = true},
-              gen_check_cb(pr, bucket, value.name),
-              {value.hash, tostring(now), tostring(rate), tostring(bucket.burst),
-                  tostring(settings.expire)})
-    end
-  end
-end
-
-local function ratelimit_update_cb(task)
-  local prefixes = task:cache_get('ratelimit_prefixes')
-
-  if prefixes then
-    if task:has_pre_result() then
-      -- Already rate limited/greylisted, do nothing
-      rspamd_logger.debugm(N, task, 'pre-action has been set, do not update')
-      return
-    end
-
-    local is_spam = not (task:get_metric_action() == 'no action')
-
-    -- Update each bucket
-    for k, v in pairs(prefixes) do
-      local bucket = v.bucket
-      local function update_bucket_cb(err, data)
-        if err then
-          rspamd_logger.errx(task, 'cannot update rate bucket %s: %s',
-                  k, err)
-        else
-          rspamd_logger.debugm(N, task,
-              "updated limit %s:%s -> %s (%s/%s), burst: %s, dyn_rate: %s, dyn_burst: %s",
-              v.name, k, v.hash,
-              bucket.burst, bucket.rate,
-              data[1], data[2], data[3])
-        end
-      end
-      local now = rspamd_util.get_time()
-      now = lua_util.round(now * 1000.0) -- Get milliseconds
-      local mult_burst = bucket.ham_factor_burst or 1.0
-      local mult_rate = bucket.ham_factor_burst or 1.0
-
-      if is_spam then
-        mult_burst = bucket.spam_factor_burst or 1.0
-        mult_rate = bucket.spam_factor_rate or 1.0
-      end
-
-      lua_redis.exec_redis_script(bucket_update_id,
-              {key = v.hash, task = task, is_write = true},
-              update_bucket_cb,
-              {v.hash, tostring(now), tostring(mult_rate), tostring(mult_burst),
-               tostring(settings.max_rate_mult), tostring(settings.max_bucket_mult),
-               tostring(settings.expire)})
-    end
-  end
-end
-
-local opts = rspamd_config:get_all_opt(N)
-if opts then
-
-  settings = lua_util.override_defaults(settings, opts)
-
-  if opts['limit'] then
-    rspamd_logger.errx(rspamd_config, 'Legacy ratelimit config format no longer supported')
-  end
-
-  if opts['rates'] and type(opts['rates']) == 'table' then
-    -- new way of setting limits
-    fun.each(function(t, lim)
-      local buckets = parse_limit(t, lim)
-
-      if buckets and #buckets > 0 then
-        settings.limits[t] = buckets
-      end
-    end, opts['rates'])
-  end
-
-  local enabled_limits = fun.totable(fun.map(function(t)
-    return t
-  end, settings.limits))
-  rspamd_logger.infox(rspamd_config,
-          'enabled rate buckets: [%1]', table.concat(enabled_limits, ','))
-
-  -- Ret, ret, ret: stupid legacy stuff:
-  -- If we have a string with commas then load it as as static map
-  -- otherwise, apply normal logic of Rspamd maps
-
-  local wrcpts = opts['whitelisted_rcpts']
-  if type(wrcpts) == 'string' then
-    if string.find(wrcpts, ',') then
-      settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(
-        lua_util.rspamd_str_split(wrcpts, ','), 'set', 'Ratelimit whitelisted rcpts')
-    else
-      settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set',
-        'Ratelimit whitelisted rcpts')
-    end
-  elseif type(opts['whitelisted_rcpts']) == 'table' then
-    settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set',
-      'Ratelimit whitelisted rcpts')
-  else
-    -- Stupid default...
-    settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(
-        settings.whitelisted_rcpts, 'set', 'Ratelimit whitelisted rcpts')
-  end
-
-  if opts['whitelisted_ip'] then
-    settings.whitelisted_ip = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_ip', 'radix',
-      'Ratelimit whitelist ip map')
-  end
-
-  if opts['whitelisted_user'] then
-    settings.whitelisted_user = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_user', 'set',
-      'Ratelimit whitelist user map')
-  end
-
-  settings.custom_keywords = {}
-  if opts['custom_keywords'] then
-    local ret, res_or_err = pcall(loadfile(opts['custom_keywords']))
-
-    if ret then
-      opts['custom_keywords'] = {}
-      if type(res_or_err) == 'table' then
-        for k,hdl in pairs(res_or_err) do
-          settings['custom_keywords'][k] = hdl
-        end
-      elseif type(res_or_err) == 'function' then
-        settings['custom_keywords']['custom'] = res_or_err
-      end
-    else
-      rspamd_logger.errx(rspamd_config, 'cannot execute %s: %s',
-          opts['custom_keywords'], res_or_err)
-      settings['custom_keywords'] = {}
-    end
-  end
-
-  if opts['message_func'] then
-    message_func = assert(load(opts['message_func']))()
-  end
-
-  redis_params = lua_redis.parse_redis_server('ratelimit')
-
-  if not redis_params then
-    rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module')
-    lua_util.disable_module(N, "redis")
-  else
-    local s = {
-      type = 'prefilter,nostat',
-      name = 'RATELIMIT_CHECK',
-      priority = 7,
-      callback = ratelimit_cb,
-      flags = 'empty',
-    }
-
-    if settings.symbol then
-      s.name = settings.symbol
-    elseif settings.info_symbol then
-      s.name = settings.info_symbol
-    end
-
-    rspamd_config:register_symbol(s)
-    rspamd_config:register_symbol {
-      type = 'idempotent',
-      name = 'RATELIMIT_UPDATE',
-      callback = ratelimit_update_cb,
-    }
-  end
-end
-
-rspamd_config:add_on_load(function(cfg, ev_base, worker)
-  load_scripts(cfg, ev_base)
-end)

data/Dockerfiles/sogo/Dockerfile

@@ -7,46 +7,50 @@ ENV GOSU_VERSION 1.9
 
 # Prerequisites
 RUN apt-get update && apt-get install -y --no-install-recommends \
-		apt-transport-https \
+  apt-transport-https \
-		ca-certificates \
+  ca-certificates \
-		cron \
+  cron \
-		gnupg \
+  gettext \
-		mysql-client \
+  gnupg \
-		supervisor \
+  mysql-client \
-		syslog-ng \
+  rsync \
-		syslog-ng-core \
+  supervisor \
-		syslog-ng-mod-redis \
+  syslog-ng \
-		dirmngr \
+  syslog-ng-core \
-		netcat \
+  syslog-ng-mod-redis \
-		psmisc \
+  dirmngr \
-		wget \
+  netcat \
-    patch \
+  psmisc \
-	&& rm -rf /var/lib/apt/lists/* \
+  wget \
-	&& dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \
+  patch \
-	&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" \
+  && rm -rf /var/lib/apt/lists/* \
-	&& chmod +x /usr/local/bin/gosu \
+  && dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \
-	&& gosu nobody true
+  && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" \
+  && chmod +x /usr/local/bin/gosu \
+  && gosu nobody true
 
 RUN mkdir /usr/share/doc/sogo \
-	&& touch /usr/share/doc/sogo/empty.sh \
+  && touch /usr/share/doc/sogo/empty.sh \
-	&& apt-key adv --keyserver keyserver.ubuntu.com --recv-key 0x810273C4 \
+  && apt-key adv --keyserver keyserver.ubuntu.com --recv-key 0x810273C4 \
-	&& echo "deb http://packages.inverse.ca/SOGo/nightly/4/debian/ stretch stretch" > /etc/apt/sources.list.d/sogo.list \
+  && echo "deb http://packages.inverse.ca/SOGo/nightly/4/debian/ stretch stretch" > /etc/apt/sources.list.d/sogo.list \
-	&& apt-get update && apt-get install -y --force-yes \
+  && apt-get update && apt-get install -y --force-yes \
-		sogo \
+    sogo \
-		sogo-activesync \
+    sogo-activesync \
-	&& rm -rf /var/lib/apt/lists/* \
+  && rm -rf /var/lib/apt/lists/* \
-	&& echo '* * * * *   sogo   /usr/sbin/sogo-ealarms-notify 2>/dev/null' > /etc/cron.d/sogo \
+  && echo '* * * * *   sogo   /usr/sbin/sogo-ealarms-notify -p /etc/sogo/sieve.creds 2>/dev/null' > /etc/cron.d/sogo \
-	&& echo '* * * * *   sogo   /usr/sbin/sogo-tool expire-sessions 60' >> /etc/cron.d/sogo \
+  && echo '* * * * *   sogo   /usr/sbin/sogo-tool expire-sessions 60' >> /etc/cron.d/sogo \
-	&& echo '0 0 * * *   sogo   /usr/sbin/sogo-tool update-autoreply -p /etc/sogo/sieve.creds' >> /etc/cron.d/sogo \
+  && echo '0 0 * * *   sogo   /usr/sbin/sogo-tool update-autoreply -p /etc/sogo/sieve.creds' >> /etc/cron.d/sogo \
-	&& touch /etc/default/locale
+  && touch /etc/default/locale
 
-COPY ./bootstrap-sogo.sh /
+COPY ./bootstrap-sogo.sh /bootstrap-sogo.sh
 COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY supervisord.conf /etc/supervisor/supervisord.conf
-COPY theme-blue.js /usr/lib/GNUstep/SOGo/WebServerResources/js/theme-blue.js
-COPY theme-blue.css /usr/lib/GNUstep/SOGo/WebServerResources/css/theme-default.css
-COPY sogo-full.svg /usr/lib/GNUstep/SOGo/WebServerResources/img/sogo-full.svg
 COPY acl.diff /acl.diff
+COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
+
+RUN chmod +x /bootstrap-sogo.sh \
+  /usr/local/sbin/stop-supervisor.sh
+
 CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
 
 RUN rm -rf /tmp/* /var/tmp/*

data/Dockerfiles/sogo/bootstrap-sogo.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Wait for MySQL to warm-up
-while ! mysqladmin ping --host mysql -u${DBUSER} -p${DBPASS} --silent; do
+while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
   echo "Waiting for database to come up..."
   sleep 2
 done
@@ -13,20 +13,31 @@ do
   sleep 3
 done
 
+# Wait for updated schema
+DBV_NOW=$(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions;" -BN)
+DBV_NEW=$(grep -oE '\$db_version = .*;' init_db.inc.php | sed 's/$db_version = //g;s/;//g' | cut -d \" -f2)
+while [[ ${DBV_NOW} != ${DBV_NEW} ]]; do
+  echo "Waiting for schema update..."
+  DBV_NOW=$(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions;" -BN)
+  DBV_NEW=$(grep -oE '\$db_version = .*;' init_db.inc.php | sed 's/$db_version = //g;s/;//g' | cut -d \" -f2)
+  sleep 5
+done
+echo "DB schema is ${DBV_NOW}"
+
 # Recreate view
 
-mysql --host mysql -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "DROP VIEW IF EXISTS sogo_view"
+mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "DROP VIEW IF EXISTS sogo_view"
 
 while [[ ${VIEW_OK} != 'OK' ]]; do
-  mysql --host mysql -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
+  mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
-CREATE VIEW sogo_view (c_uid, domain, c_name, c_password, c_cn, mail, aliases, ad_aliases, home, kind, multiple_bookings) AS
+CREATE VIEW sogo_view (c_uid, domain, c_name, c_password, c_cn, mail, aliases, ad_aliases, kind, multiple_bookings) AS
-SELECT mailbox.username, mailbox.domain, mailbox.username, if(json_extract(attributes, '$.force_pw_update') LIKE '%0%', password, 'invalid'), mailbox.name, mailbox.username, IFNULL(GROUP_CONCAT(ga.aliases SEPARATOR ' '), ''), IFNULL(gda.ad_alias, ''), CONCAT('/var/vmail/', maildir), mailbox.kind, mailbox.multiple_bookings FROM mailbox
+SELECT mailbox.username, mailbox.domain, mailbox.username, if(json_extract(attributes, '$.force_pw_update') LIKE '%0%', if(json_extract(attributes, '$.sogo_access') LIKE '%1%', password, 'invalid'), 'invalid'), mailbox.name, mailbox.username, IFNULL(GROUP_CONCAT(ga.aliases SEPARATOR ' '), ''), IFNULL(gda.ad_alias, ''), mailbox.kind, mailbox.multiple_bookings FROM mailbox
 LEFT OUTER JOIN grouped_mail_aliases ga ON ga.username REGEXP CONCAT('(^|,)', mailbox.username, '($|,)')
 LEFT OUTER JOIN grouped_domain_alias_address gda ON gda.username = mailbox.username
 WHERE mailbox.active = '1'
 GROUP BY mailbox.username;
 EOF
-  if [[ ! -z $(mysql --host mysql -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'sogo_view'") ]]; then
+  if [[ ! -z $(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'sogo_view'") ]]; then
     VIEW_OK=OK
   else
     echo "Will retry to setup SOGo view in 3s"
@@ -37,11 +48,11 @@ done
 # Wait for static view table if missing after update and update content
 
 while [[ ${STATIC_VIEW_OK} != 'OK' ]]; do
-  if [[ ! -z $(mysql --host mysql -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '_sogo_static_view'") ]]; then
+  if [[ ! -z $(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '_sogo_static_view'") ]]; then
     STATIC_VIEW_OK=OK
     echo "Updating _sogo_static_view content..."
-    mysql --host mysql -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "REPLACE INTO _sogo_static_view SELECT * from sogo_view"
+    mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "REPLACE INTO _sogo_static_view (c_uid, domain, c_name, c_password, c_cn, mail, aliases, ad_aliases, kind, multiple_bookings) SELECT c_uid, domain, c_name, c_password, c_cn, mail, aliases, ad_aliases, kind, multiple_bookings from sogo_view;"
-    mysql --host mysql -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "DELETE FROM _sogo_static_view WHERE c_uid NOT IN (SELECT username FROM mailbox WHERE active = '1')"
+    mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "DELETE FROM _sogo_static_view WHERE c_uid NOT IN (SELECT username FROM mailbox WHERE active = '1')"
   else
     echo "Waiting for database initialization..."
     sleep 3
@@ -50,10 +61,10 @@ done
 
 # Recreate password update trigger
 
-mysql --host mysql -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "DROP TRIGGER IF EXISTS sogo_update_password"
+mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "DROP TRIGGER IF EXISTS sogo_update_password"
 
 while [[ ${TRIGGER_OK} != 'OK' ]]; do
-  mysql --host mysql -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
+  mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
 DELIMITER -
 CREATE TRIGGER sogo_update_password AFTER UPDATE ON _sogo_static_view
 FOR EACH ROW
@@ -63,7 +74,7 @@ END;
 -
 DELIMITER ;
 EOF
-  if [[ ! -z $(mysql --host mysql -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TRIGGERS WHERE TRIGGER_NAME = 'sogo_update_password'") ]]; then
+  if [[ ! -z $(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TRIGGERS WHERE TRIGGER_NAME = 'sogo_update_password'") ]]; then
     TRIGGER_OK=OK
   else
     echo "Will retry to setup SOGo password update trigger in 3s"
@@ -72,28 +83,41 @@ EOF
 done
 
 
-mkdir -p /var/lib/sogo/GNUstep/Defaults/
+if [[ "${ALLOW_ADMIN_EMAIL_LOGIN}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  TRUST_PROXY="YES"
+else
+  TRUST_PROXY="NO"
+fi
+# cat /dev/urandom seems to hang here occasionally and is not recommended anyway, better use openssl
+RAND_PASS=$(openssl rand -base64 16 | tr -dc _A-Z-a-z-0-9)
 
 # Generate plist header with timezone data
+mkdir -p /var/lib/sogo/GNUstep/Defaults/
 cat <<EOF > /var/lib/sogo/GNUstep/Defaults/sogod.plist
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//GNUstep//DTD plist 0.9//EN" "http://www.gnustep.org/plist-0_9.xml">
 <plist version="0.9">
 <dict>
     <key>OCSAclURL</key>
-    <string>mysql://${DBUSER}:${DBPASS}@mysql:3306/${DBNAME}/sogo_acl</string>
+    <string>mysql://${DBUSER}:${DBPASS}@%2Fvar%2Frun%2Fmysqld%2Fmysqld.sock/${DBNAME}/sogo_acl</string>
+    <key>SOGoIMAPServer</key>
+    <string>imap://${IPV4_NETWORK}.250:143/?tls=YES</string>
+    <key>SOGoTrustProxyAuthentication</key>
+    <string>${TRUST_PROXY}</string>
+    <key>SOGoEncryptionKey</key>
+    <string>${RAND_PASS}</string>
     <key>OCSCacheFolderURL</key>
-    <string>mysql://${DBUSER}:${DBPASS}@mysql:3306/${DBNAME}/sogo_cache_folder</string>
+    <string>mysql://${DBUSER}:${DBPASS}@%2Fvar%2Frun%2Fmysqld%2Fmysqld.sock/${DBNAME}/sogo_cache_folder</string>
     <key>OCSEMailAlarmsFolderURL</key>
-    <string>mysql://${DBUSER}:${DBPASS}@mysql:3306/${DBNAME}/sogo_alarms_folder</string>
+    <string>mysql://${DBUSER}:${DBPASS}@%2Fvar%2Frun%2Fmysqld%2Fmysqld.sock/${DBNAME}/sogo_alarms_folder</string>
     <key>OCSFolderInfoURL</key>
-    <string>mysql://${DBUSER}:${DBPASS}@mysql:3306/${DBNAME}/sogo_folder_info</string>
+    <string>mysql://${DBUSER}:${DBPASS}@%2Fvar%2Frun%2Fmysqld%2Fmysqld.sock/${DBNAME}/sogo_folder_info</string>
     <key>OCSSessionsFolderURL</key>
-    <string>mysql://${DBUSER}:${DBPASS}@mysql:3306/${DBNAME}/sogo_sessions_folder</string>
+    <string>mysql://${DBUSER}:${DBPASS}@%2Fvar%2Frun%2Fmysqld%2Fmysqld.sock/${DBNAME}/sogo_sessions_folder</string>
     <key>OCSStoreURL</key>
-    <string>mysql://${DBUSER}:${DBPASS}@mysql:3306/${DBNAME}/sogo_store</string>
+    <string>mysql://${DBUSER}:${DBPASS}@%2Fvar%2Frun%2Fmysqld%2Fmysqld.sock/${DBNAME}/sogo_store</string>
     <key>SOGoProfileURL</key>
-    <string>mysql://${DBUSER}:${DBPASS}@mysql:3306/${DBNAME}/sogo_user_profile</string>
+    <string>mysql://${DBUSER}:${DBPASS}@%2Fvar%2Frun%2Fmysqld%2Fmysqld.sock/${DBNAME}/sogo_user_profile</string>
     <key>SOGoTimeZone</key>
     <string>${TZ}</string>
     <key>domains</key>
@@ -101,9 +125,9 @@ cat <<EOF > /var/lib/sogo/GNUstep/Defaults/sogod.plist
 EOF
 
 # Generate multi-domain setup
-while read line
+while read -r line gal
-        do
+  do
-        echo "        <key>${line}</key>
+  echo "        <key>${line}</key>
         <dict>
             <key>SOGoMailDomain</key>
             <string>${line}</string>
@@ -126,11 +150,11 @@ while read line
                     <key>canAuthenticate</key>
                     <string>YES</string>
                     <key>displayName</key>
-                    <string>GAL</string>
+                    <string>GAL ${line}</string>
                     <key>id</key>
                     <string>${line}</string>
                     <key>isAddressBook</key>
-                    <string>YES</string>
+                    <string>${gal}</string>
                     <key>type</key>
                     <string>sql</string>
                     <key>userPasswordAlgorithm</key>
@@ -138,11 +162,14 @@ while read line
                     <key>prependPasswordScheme</key>
                     <string>YES</string>
                     <key>viewURL</key>
-                    <string>mysql://${DBUSER}:${DBPASS}@mysql:3306/${DBNAME}/_sogo_static_view</string>
+                    <string>mysql://${DBUSER}:${DBPASS}@%2Fvar%2Frun%2Fmysqld%2Fmysqld.sock/${DBNAME}/_sogo_static_view</string>
-                </dict>
+                </dict>" >> /var/lib/sogo/GNUstep/Defaults/sogod.plist
-            </array>
+  # Generate alternative LDAP authentication dict, when SQL authentication fails
+  # This will nevertheless read attributes from LDAP
+  line=${line} envsubst < /etc/sogo/plist_ldap >> /var/lib/sogo/GNUstep/Defaults/sogod.plist
+  echo "            </array>
         </dict>" >> /var/lib/sogo/GNUstep/Defaults/sogod.plist
-done < <(mysql --host mysql -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT domain FROM domain;" -B -N)
+done < <(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT domain, CASE gal WHEN '1' THEN 'YES' ELSE 'NO' END AS gal FROM domain;" -B -N)
 
 # Generate footer
 echo '    </dict>
@@ -153,46 +180,24 @@ echo '    </dict>
 chown sogo:sogo -R /var/lib/sogo/
 chmod 600 /var/lib/sogo/GNUstep/Defaults/sogod.plist
 
-# Prevent theme switching
+# Patch ACLs
-sed -i \
+if [[ ${ACL_ANYONE} == 'allow' ]]; then
-  -e 's/eaf5e9/E3F2FD/g' \
+  #enable any or authenticated targets for ACL
-  -e 's/cbe5c8/BBDEFB/g' \
+  if patch -R -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
-  -e 's/aad6a5/90CAF9/g' \
+    patch -R /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
-  -e 's/88c781/64B5F6/g' \
+  fi
-  -e 's/66b86a/42A5F5/g' \
+else
-  -e 's/56b04c/2196F3/g' \
+  #disable any or authenticated targets for ACL
-  -e 's/4da143/1E88E5/g' \
+  if patch -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
-  -e 's/388e3c/1976D2/g' \
+    patch /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
-  -e 's/367d2e/1565C0/g' \
+  fi
-  -e 's/225e1b/0D47A1/g' \
-  -e 's/fafafa/82B1FF/g' \
-  -e 's/69f0ae/448AFF/g' \
-  -e 's/00e676/2979ff/g' \
-  -e 's/00c853/2962ff/g'  \
-  /usr/lib/GNUstep/SOGo/WebServerResources/js/Common/Common.app.js \
-  /usr/lib/GNUstep/SOGo/WebServerResources/js/Common.js
-
-sed -i \
-  -e 's/default: "900"/default: "700"/g' \
-  -e 's/default: "500"/default: "700"/g' \
-  -e 's/"hue-1": "400"/"hue-1": "500"/g' \
-  -e 's/"hue-1": "A100"/"hue-1": "500"/g' \
-  -e 's/"hue-2": "800"/"hue-2": "700"/g' \
-  -e 's/"hue-2": "300"/"hue-2": "700"/g' \
-  -e 's/"hue-3": "A700"/"hue-3": "A200"/' \
-  -e 's/default:"900"/default:"700"/g' \
-  -e 's/default:"500"/default:"700"/g' \
-  -e 's/"hue-1":"400"/"hue-1":"500"/g' \
-  -e 's/"hue-1":"A100"/"hue-1":"500"/g' \
-  -e 's/"hue-2":"800"/"hue-2":"700"/g' \
-  -e 's/"hue-2":"300"/"hue-2":"700"/g' \
-  -e 's/"hue-3":"A700"/"hue-3":"A200"/' \
-  /usr/lib/GNUstep/SOGo/WebServerResources/js/Common/Common.app.js \
-  /usr/lib/GNUstep/SOGo/WebServerResources/js/Common.js
-
-# Patch ACLs (comment this out to enable any or authenticated targets for ACL)
-if patch -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
-  patch /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
 fi
 
+# Copy logo, if any
+[[ -f /etc/sogo/sogo-full.svg ]] && cp /etc/sogo/sogo-full.svg /usr/lib/GNUstep/SOGo/WebServerResources/img/sogo-full.svg
+
+# Rsync web content
+echo "Syncing web content with named volume"
+rsync -a /usr/lib/GNUstep/SOGo/. /sogo_web/
+
 exec gosu sogo /usr/sbin/sogod

data/Dockerfiles/sogo/sogo-full.svg

@@ -1,160 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!-- Generator: Adobe Illustrator 16.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   version="1.1"
-   id="Layer_1"
-   x="0px"
-   y="0px"
-   width="640px"
-   height="350px"
-   viewBox="78.712 58.488 640 350"
-   style="enable-background:new 78.712 58.488 640 350;"
-   xml:space="preserve"
-   inkscape:version="0.91 r13725"
-   sodipodi:docname="sogo-full.svg"><metadata
-     id="metadata9"><rdf:RDF><cc:Work
-         rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" /></cc:Work></rdf:RDF></metadata><defs
-     id="defs7" /><sodipodi:namedview
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1"
-     objecttolerance="10"
-     gridtolerance="10"
-     guidetolerance="10"
-     inkscape:pageopacity="0"
-     inkscape:pageshadow="2"
-     inkscape:window-width="1721"
-     inkscape:window-height="1177"
-     id="namedview5"
-     showgrid="false"
-     inkscape:zoom="0.8396893"
-     inkscape:cx="360.23913"
-     inkscape:cy="334.02085"
-     inkscape:window-x="-8"
-     inkscape:window-y="-8"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="Layer_1" /><path
-     style="fill:#1976d2;fill-opacity:0.71428573"
-     d="M648.541,145.679c-9.947,0-17.009-7.278-17.009-17.048c0-9.777,7.062-17.057,17.009-17.057  c10.024,0,17.086,7.279,17.086,17.057C665.627,138.401,658.565,145.679,648.541,145.679z M648.511,94.893  c-19.693,0-33.679,14.4-33.679,33.738c0,19.33,13.985,33.729,33.679,33.729c19.822,0,33.808-14.4,33.808-33.729  C682.318,109.293,668.333,94.893,648.511,94.893z M648.482,179.843c-29.889,0-51.123-21.868-51.123-51.212  c0-29.353,21.234-51.209,51.123-51.209c30.082,0,51.307,21.856,51.307,51.209C699.789,157.975,678.564,179.843,648.482,179.843z   M648.442,58.488c-40.929,0-69.995,29.946-69.995,70.143c0,40.189,29.066,70.125,69.995,70.125c41.194,0,70.27-29.937,70.27-70.125  C718.712,88.434,689.637,58.488,648.442,58.488z M158.166,183.902l-21.018-5.008c-19.131-4.396-28.849-9.413-28.849-23.21  c0-15.684,15.99-21.965,30.419-21.965c14.667,0,25.382,7.329,31.693,18.737c0.02,0.048,0.051,0.097,0.09,0.157  c0.127,0.247,0.276,0.484,0.403,0.731l0.03-0.02c1.985,3.002,5.323,5.008,8.919,5.008c6.122,0,10.558-4.425,10.558-10.547  c0-2.341-0.504-4.82-1.601-6.688c-10.764-18.302-28.513-26.192-48.838-26.192c-27.594,0-54.262,13.797-54.262,44.218  c0,27.921,27.605,36.079,37.64,38.578l20.069,4.71c15.368,3.763,27.912,8.791,27.912,23.517c0,16.938-17.561,23.943-34.499,23.943  c-17.245,0-30.015-9.37-38.814-22.37h-0.01c-1.956-3-4.988-4.328-8.702-4.328c-5.984,0-10.805,5.185-10.587,11.162  c0.098,2.438,0.909,4.637,2.153,6.405c13.787,20.633,33.728,28.41,55.96,28.41c28.543,0,57.085-13.143,57.085-45.132  C193.918,203.325,178.551,188.613,158.166,183.902z M298.479,250.312c-33.866,0-55.199-25.403-55.199-58.331  c0-32.939,21.333-58.343,55.199-58.343c34.192,0,55.516,25.403,55.516,58.343C353.996,224.91,332.672,250.312,298.479,250.312z   M298.479,114.823c-45.471,0-77.777,32.93-77.777,77.158c0,44.217,32.306,77.146,77.777,77.146  c45.786,0,78.093-32.929,78.093-77.146C376.572,147.753,344.266,114.823,298.479,114.823z M518.715,234.312  c-0.771,0.74-1.549,1.472-2.399,2.175c-1.106,1.014-2.391,2.112-3.854,3.208c-8.829,6.391-19.979,10.094-33.017,10.094  c-33.876,0-55.198-25.402-55.198-58.332c0-32.939,21.322-58.342,55.198-58.342c34.183,0,55.506,25.403,55.506,58.342  C534.951,208.653,529.135,223.774,518.715,234.312z M468.097,317.938c2.528,0,5.146-0.168,7.863-0.504  c5.018-0.631,9.588-0.909,13.729-0.909c19.24,0.109,29.036,5.7,34.943,12.158c5.895,6.499,8.168,15.311,8.158,22.796  c0.01,3.586-0.555,6.795-1.177,8.721c-2.944,8.93-8.888,15.002-17.996,19.576c-9.035,4.484-21.095,6.777-33.707,6.757  c-4.514,0-9.105-0.288-13.639-0.831c-8.573-0.987-19.911-4.671-28.13-11.093c-4.138-3.199-6.458-6.991-8.858-11.485  c-2.379-4.514-2.783-9.748-2.783-16.442v-0.742c0-12.346,4.84-20.544,11.051-26.5c3.07-2.904,5.69-5.064,7.99-6.438  c0.366-0.218,0.438-0.416,0.755-0.593C452.39,316.014,459.684,317.968,468.097,317.938z M479.445,114.301  c-45.471,0-77.786,32.929-77.786,77.157c0,29.887,14.765,54.598,38.378,67.489c-0.314,0.314-0.621,0.641-0.916,0.966  c-6.104,6.687-9.226,15.25-9.236,23.913c-0.008,3.821,0.624,7.741,1.977,11.494c-3.062,1.956-6.717,4.634-10.46,8.147  c-9.026,8.408-18.734,22.541-19.021,42.097c-0.01,0.454-0.01,0.829-0.01,1.118c-0.01,10.071,2.379,19.157,6.459,26.774  c6.133,11.466,15.683,19.445,25.539,24.77c9.917,5.334,20.257,8.166,29.273,9.274c5.373,0.643,10.826,0.988,16.268,0.988  c15.151-0.02,30.261-2.578,43.409-9.019c13.085-6.34,24.333-17.253,29.192-32.562c1.443-4.553,2.212-9.719,2.231-15.428  c-0.02-11.595-3.349-25.759-13.767-37.452c-10.421-11.734-27.654-19.566-51.288-19.459c-5.138,0-10.606,0.356-16.426,1.078  c-1.877,0.227-3.596,0.334-5.166,0.334c-7.239-0.048-10.872-2.053-13.036-4.098c-2.133-2.084-3.2-4.839-3.229-8.058  c-0.01-3.28,1.284-6.727,3.467-9.078c2.231-2.332,5.008-3.91,9.846-3.97c0.436,0,0.9,0.01,1.374,0.05  c3.101,0.216,6.112,0.325,9.037,0.325c24.188,0.047,42.38-7.448,54.756-17.759c12.415-10.312,18.971-22.854,22.071-32.76l-0.04-0.01  c3.37-8.899,5.197-18.715,5.197-29.166C557.539,147.229,525.234,114.301,479.445,114.301z"
-     id="path3" /><g
-     id="g3"
-     transform="matrix(0.69327133,0,0,0.69327133,-230.59227,-153.05511)"><g
-       id="g5"><g
-         id="g7"><path
-           d="m 748.616,546.705 c -6.272,4.929 9.576,36.937 20.52,33.516 10.944,-3.42 -10.945,-41.04 -20.52,-33.516 z"
-           id="path19"
-           inkscape:connector-curvature="0"
-           style="fill:#acef29" /></g></g><g
-       id="g37"><g
-         id="g39"><g
-           id="g41" /></g><g
-         id="g45"><g
-           id="g47"><g
-             id="g49" /></g></g></g><g
-       id="g57"><g
-         id="g59"><g
-           id="g61" /></g><g
-         id="g65"><g
-           id="g67"><g
-             id="g69" /></g></g></g><g
-       id="g73"
-       style="opacity:0.38999999"><g
-         id="g75" /></g><g
-       id="g81" /><g
-       id="g85" /><g
-       id="g99"><polyline
-         points="690.928,453.92 678.401,490.532 689.894,539.76 710.135,559.116   "
-         id="polyline101"
-         style="fill:#3d5263" /><g
-         id="g103"><g
-           id="g105"><polyline
-             points="665.082,423.023 677.433,450.19 693.064,457.2 715.139,427.604     "
-             id="polyline107"
-             style="fill:#fef3df" /><g
-             id="g109"><path
-               d="m 705.288,438.868 c 0,0 -17.599,-18.89 -38.165,-13.309 0,0 4.277,25.767 36.096,32.372 l 2.164,6.413 c 0,0 -49.958,-5.925 -46.456,-49.964 0,0 36.728,-16.138 55.372,9.881"
-               id="path111"
-               inkscape:connector-curvature="0"
-               style="fill:#b58765" /><polyline
-               points="855.735,417.576 844.957,445.404 829.753,453.295 806.023,425.008      "
-               id="polyline113"
-               style="fill:#fef3df" /><path
-               d="m 816.503,435.691 c 0,0 16.491,-19.864 37.34,-15.466 0,0 -2.797,25.969 -34.187,34.38 l -1.796,6.527 c 0,0 49.537,-8.768 43.528,-52.535 0,0 -37.588,-14.015 -54.719,13.026"
-               id="path115"
-               inkscape:connector-curvature="0"
-               style="fill:#b58765" /></g></g><path
-           d="m 721.173,570.346 42.418,-1.212 15.59845,-160.5887 c -42.319,1.209 -92.18245,30.1357 -91.14645,66.4047 0.031,1.102 0.125,2.111 0.182,3.163 0.181,2.98 0.504,5.741 0.89,8.424 1.602,11.197 4.722,20.488 7.355,32.71 1.27,5.906 4.299,17.614 4.299,17.614 0.052,0.308 0.143,0.606 0.196,0.913 2.281,12.491 9.666,24.028 20.208,32.572 z"
-           id="path117"
-           inkscape:connector-curvature="0"
-           style="fill:#b58765"
-           sodipodi:nodetypes="cccccccccc" /><path
-           d="m 758.532,407.21 4.626,161.937 42.418,-1.212 c 10.038,-9.132 16.75,-21.072 18.317,-33.672 0.035,-0.31 0.107,-0.613 0.141,-0.923 0,0 2.354,-11.862 3.287,-17.831 1.932,-12.352 4.518,-21.807 5.478,-33.075 0.23,-2.702 0.393,-5.477 0.4,-8.463 0.002,-1.053 0.036,-2.066 0.005,-3.168 -1.037,-36.269 -32.35,-64.802 -74.672,-63.593 z"
-           id="path119"
-           inkscape:connector-curvature="0"
-           style="fill:#b58765" /><g
-           id="g121"><g
-             id="g123"><path
-               d="m 822.293,541.988 c 0.613,21.473 -25.496,39.648 -58.32,40.586 -32.83,0.938 -59.932,-15.717 -60.546,-37.19 -0.614,-21.477 25.494,-39.648 58.324,-40.586 32.825,-0.938 59.929,15.713 60.542,37.19 z"
-               id="path125"
-               inkscape:connector-curvature="0"
-               style="fill:#fef3df" /></g></g><g
-           id="g127"><g
-             id="g129"><g
-               id="g131"><path
-                 d="m 735.761,538.45 c 0.135,4.712 -3.578,8.644 -8.294,8.778 -4.708,0.134 -8.641,-3.579 -8.776,-8.291 -0.135,-4.718 3.58,-8.644 8.288,-8.779 4.717,-0.134 8.647,3.573 8.782,8.292 z"
-                 id="path133"
-                 inkscape:connector-curvature="0"
-                 style="fill:#5a3620" /></g></g><g
-             id="g135"><g
-               id="g137"><path
-                 d="m 806.891,536.418 c 0.135,4.712 -3.575,8.644 -8.291,8.778 -4.714,0.135 -8.646,-3.579 -8.781,-8.291 -0.135,-4.718 3.579,-8.644 8.293,-8.779 4.716,-0.134 8.644,3.573 8.779,8.292 z"
-                 id="path139"
-                 inkscape:connector-curvature="0"
-                 style="fill:#5a3620" /></g></g></g><g
-           id="g141"><path
-             d="m 831.225,475.208 c 0.201,-2.368 0.344,-4.799 0.352,-7.411 0,-0.924 0.031,-1.81 0.003,-2.778 -0.883,-30.924 -26.904,-55.418 -62.478,-55.716 l -5.561,0.163 -0.005,0 c -0.005,0.014 -19.18666,70.69971 61.71134,107.73071 0.624,-3.294 9.87079,-9.74237 10.28979,-12.41137 1.694,-10.822 -5.15313,-19.70834 -4.31213,-29.57734 z"
-             id="path143"
-             inkscape:connector-curvature="0"
-             style="fill:#87654a"
-             sodipodi:nodetypes="ccccccccc" /></g><g
-           id="g145"><g
-             id="g147"><g
-               id="g149"><g
-                 id="g151"><path
-                   d="m 807.344,471.221 c 0.151,5.28 -4.011,9.684 -9.294,9.835 -5.279,0.151 -9.686,-4.008 -9.837,-9.288 -0.151,-5.285 4.011,-9.687 9.291,-9.838 5.282,-0.151 9.689,4.006 9.84,9.291 z"
-                   id="path153"
-                   inkscape:connector-curvature="0"
-                   style="fill:#5a3620" /></g></g></g><g
-             id="g155"><g
-               id="g157"><g
-                 id="g159"><path
-                   d="m 737.68,473.211 c 0.151,5.28 -4.01,9.684 -9.289,9.835 -5.284,0.151 -9.685,-4.008 -9.835,-9.288 -0.151,-5.285 4.005,-9.687 9.289,-9.837 5.279,-0.152 9.684,4.005 9.835,9.29 z"
-                   id="path161"
-                   inkscape:connector-curvature="0"
-                   style="fill:#5a3620" /></g></g></g><g
-             id="g163"><g
-               id="g165"><path
-                 d="m 735.112,470.41 c 0.055,1.939 -1.47,3.555 -3.41,3.61 -1.939,0.055 -3.558,-1.47 -3.613,-3.41 -0.055,-1.935 1.474,-3.552 3.413,-3.607 1.94,-0.055 3.555,1.472 3.61,3.407 z"
-                 id="path167"
-                 inkscape:connector-curvature="0"
-                 style="fill:#ffffff" /></g></g><g
-             id="g169"><g
-               id="g171"><path
-                 d="m 804.125,468.439 c 0.055,1.939 -1.472,3.555 -3.409,3.61 -1.94,0.055 -3.556,-1.47 -3.611,-3.41 -0.055,-1.935 1.471,-3.552 3.411,-3.607 1.937,-0.056 3.554,1.471 3.609,3.407 z"
-                 id="path173"
-                 inkscape:connector-curvature="0"
-                 style="fill:#ffffff" /></g></g></g></g><path
-         d="m 761.738,405.962 c -50.342,1.438 -89.989,43.417 -88.55,93.759 1.438,50.342 43.417,89.987 93.758,88.549 50.343,-1.438 89.988,-43.415 88.55,-93.757 -1.438,-50.343 -43.415,-89.989 -93.758,-88.551 z m -4.396,163.807 c -40.192,1.148 -73.725,-31.172 -74.896,-72.19 -1.172,-41.017 30.461,-75.2 70.653,-76.348 40.191,-1.148 73.723,31.172 74.895,72.19 1.171,41.018 -30.462,75.2 -70.652,76.348 z"
-         id="path179"
-         inkscape:connector-curvature="0"
-         style="fill:#f1f2f2" /><g
-         id="g181" /></g></g></svg>

data/Dockerfiles/sogo/stop-supervisor.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+printf "READY\n";
+
+while read line; do
+  echo "Processing Event: $line" >&2;
+  kill -3 $(cat "/var/run/supervisord.pid")
+done < /dev/stdin

data/Dockerfiles/sogo/supervisord.conf

@@ -16,13 +16,6 @@ command=/usr/sbin/cron -f
 autorestart=true
 priority=2
 
-[program:sogo-webres]
-command=/usr/bin/python -u -m SimpleHTTPServer 9192
-directory=/usr/lib/GNUstep/SOGo/
-user=sogo
-autorestart=true
-priority=4
-
 [program:bootstrap-sogo]
 command=/bootstrap-sogo.sh
 stdout_logfile=/dev/stdout
@@ -33,3 +26,7 @@ priority=3
 startretries=10
 autorestart=true
 stopwaitsecs=120
+
+[eventlistener:processes]
+command=/usr/local/sbin/stop-supervisor.sh
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL

data/Dockerfiles/sogo/theme-blue.js

@@ -1,103 +0,0 @@
-/* -*- Mode: javascript; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-
-(function() {
-  'use strict';
-
-  angular.module('SOGo.Common')
-    .config(configure)
-
-  /**
-   * @ngInject
-   */
-  configure.$inject = ['$mdThemingProvider'];
-  function configure($mdThemingProvider) {
-
-    // Overwrite values to prevent flipping colors on login screen
-    $mdThemingProvider.definePalette('mailcow-blue', {
-      '50': 'E3F2FD',
-      '100': 'BBDEFB',
-      '200': '90CAF9',
-      '300': '64B5F6',
-      '400': '42A5F5',
-      '500': '2196F3',
-      '600': '1E88E5',
-      '700': '1976D2',
-      '800': '1565C0',
-      '900': '0D47A1',
-      '1000': '0D47A1',
-      'A100': '82B1FF',
-      'A200': '448AFF',
-      'A400': '2979ff',
-      'A700': '2962ff',
-      'contrastDefaultColor': 'dark',
-      'contrastLightColors': ['700', '800', '900'],
-      'contrastDarkColors': undefined
-    });
-
-    $mdThemingProvider.definePalette('sogo-green', {
-      '50': 'E3F2FD',
-      '100': 'BBDEFB',
-      '200': '90CAF9',
-      '300': '64B5F6',
-      '400': '42A5F5',
-      '500': '2196F3',
-      '600': '1E88E5',
-      '700': '1976D2',
-      '800': '1565C0',
-      '900': '0D47A1',
-      '1000': '0D47A1',
-      'A100': '82B1FF',
-      'A200': '448AFF',
-      'A400': '2979ff',
-      'A700': '2962ff',
-      'contrastDefaultColor': 'dark',
-      'contrastLightColors': ['700', '800', '900'],
-      'contrastDarkColors': undefined
-    });
-
-    $mdThemingProvider.definePalette('default', {
-      '50': 'E3F2FD',
-      '100': 'BBDEFB',
-      '200': '90CAF9',
-      '300': '64B5F6',
-      '400': '42A5F5',
-      '500': '2196F3',
-      '600': '1E88E5',
-      '700': '1976D2',
-      '800': '1565C0',
-      '900': '0D47A1',
-      '1000': '0D47A1',
-      'A100': '82B1FF',
-      'A200': '448AFF',
-      'A400': '2979ff',
-      'A700': '2962ff',
-      'contrastDefaultColor': 'dark',
-      'contrastLightColors': ['700', '800', '900'],
-      'contrastDarkColors': undefined
-    });
-
-    $mdThemingProvider.theme('default')
-      .primaryPalette('mailcow-blue', {
-        'default': '700',  // top toolbar
-        'hue-1': '500',
-        'hue-2': '700',    // sidebar toolbar
-        'hue-3': 'A200'
-      })
-      .accentPalette('mailcow-blue', {
-        'default': '800',  // fab buttons
-        'hue-1': '50',     // center list toolbar
-        'hue-2': '500',
-        'hue-3': 'A700'
-      })
-      .backgroundPalette('grey', {
-        'default': '50',   // center list background
-        'hue-1': '100',
-        'hue-2': '200',
-        'hue-3': '300'
-      });
-
-    $mdThemingProvider.setDefaultTheme('default');
-    $mdThemingProvider.generateThemesOnDemand(false);
-    $mdThemingProvider.alwaysWatchTheme(true);
-  }
-})();

data/Dockerfiles/solr/Dockerfile

@@ -0,0 +1,13 @@
+FROM solr:7.7-alpine
+USER root
+COPY docker-entrypoint.sh /
+COPY solr-config-7.7.0.xml /
+COPY solr-schema-7.7.0.xml /
+
+
+RUN apk --no-cache add su-exec curl tzdata \
+  && chmod +x /docker-entrypoint.sh \
+  && sync \
+  && bash /docker-entrypoint.sh --bootstrap
+
+ENTRYPOINT ["/docker-entrypoint.sh"]

data/Dockerfiles/solr/docker-entrypoint.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  echo "SKIP_SOLR=y, skipping Solr..."
+  sleep 365d
+  exit 0
+fi
+
+MEM_TOTAL=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
+
+if [[ "${1}" != "--bootstrap" ]]; then
+  if [ ${MEM_TOTAL} -lt "2097152" ]; then
+    echo "System memory less than 2 GB, skipping Solr..."
+    sleep 365d
+    exit 0
+  fi
+fi
+
+set -e
+
+# run the optional initdb
+. /opt/docker-solr/scripts/run-initdb
+
+# fixing volume permission
+[[ -d /opt/solr/server/solr/dovecot-fts/data ]] && chown -R solr:solr /opt/solr/server/solr/dovecot-fts/data
+if [[ "${1}" != "--bootstrap" ]]; then
+  sed -i '/SOLR_HEAP=/c\SOLR_HEAP="'${SOLR_HEAP:-1024}'m"' /opt/solr/bin/solr.in.sh
+else
+  sed -i '/SOLR_HEAP=/c\SOLR_HEAP="256m"' /opt/solr/bin/solr.in.sh
+fi
+
+if [[ "${1}" == "--bootstrap" ]]; then
+  echo "Creating initial configuration"
+  echo "Modifying default config set"
+  cp /solr-config-7.7.0.xml /opt/solr/server/solr/configsets/_default/conf/solrconfig.xml
+  cp /solr-schema-7.7.0.xml /opt/solr/server/solr/configsets/_default/conf/schema.xml
+  rm /opt/solr/server/solr/configsets/_default/conf/managed-schema
+
+  echo "Starting local Solr instance to setup configuration"
+  su-exec solr start-local-solr
+
+  echo "Creating core \"dovecot-fts\""
+  su-exec solr /opt/solr/bin/solr create -c "dovecot-fts"
+
+  # See https://github.com/docker-solr/docker-solr/issues/27
+  echo "Checking core"
+  while ! wget -O - 'http://localhost:8983/solr/admin/cores?action=STATUS' | grep -q instanceDir; do
+    echo "Could not find any cores, waiting..."
+    sleep 3
+  done
+
+  echo "Created core \"dovecot-fts\""
+
+  echo "Stopping local Solr"
+  su-exec solr stop-local-solr
+
+  exit 0
+fi
+
+exec su-exec solr solr-foreground
+

data/Dockerfiles/solr/solr-config-7.7.0.xml

@@ -0,0 +1,289 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<!-- This is the default config with stuff non-essential to Dovecot removed. -->
+
+<config>
+  <!-- Controls what version of Lucene various components of Solr
+       adhere to.  Generally, you want to use the latest version to
+       get all bug fixes and improvements. It is highly recommended
+       that you fully re-index after changing this setting as it can
+       affect both how text is indexed and queried.
+  -->
+  <luceneMatchVersion>7.7.0</luceneMatchVersion>
+
+  <!-- A 'dir' option by itself adds any files found in the directory
+       to the classpath, this is useful for including all jars in a
+       directory.
+
+       When a 'regex' is specified in addition to a 'dir', only the
+       files in that directory which completely match the regex
+       (anchored on both ends) will be included.
+
+       If a 'dir' option (with or without a regex) is used and nothing
+       is found that matches, a warning will be logged.
+
+       The examples below can be used to load some solr-contribs along
+       with their external dependencies.
+    -->
+  <lib dir="${solr.install.dir:../../../..}/contrib/extraction/lib" regex=".*\.jar" />
+  <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-cell-\d.*\.jar" />
+
+  <lib dir="${solr.install.dir:../../../..}/contrib/clustering/lib/" regex=".*\.jar" />
+  <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-clustering-\d.*\.jar" />
+
+  <lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" />
+  <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" />
+
+  <lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
+  <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" />
+
+  <!-- Data Directory
+
+       Used to specify an alternate directory to hold all index data
+       other than the default ./data under the Solr home.  If
+       replication is in use, this should match the replication
+       configuration.
+    -->
+  <dataDir>${solr.data.dir:}</dataDir>
+
+  <!-- The default high-performance update handler -->
+  <updateHandler class="solr.DirectUpdateHandler2">
+
+    <!-- Enables a transaction log, used for real-time get, durability, and
+         and solr cloud replica recovery.  The log can grow as big as
+         uncommitted changes to the index, so use of a hard autoCommit
+         is recommended (see below).
+         "dir" - the target directory for transaction logs, defaults to the
+                solr data directory.
+         "numVersionBuckets" - sets the number of buckets used to keep
+                track of max version values when checking for re-ordered
+                updates; increase this value to reduce the cost of
+                synchronizing access to version buckets during high-volume
+                indexing, this requires 8 bytes (long) * numVersionBuckets
+                of heap space per Solr core.
+    -->
+    <updateLog>
+      <str name="dir">${solr.ulog.dir:}</str>
+      <int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int>
+    </updateLog>
+
+    <!-- AutoCommit
+
+         Perform a hard commit automatically under certain conditions.
+         Instead of enabling autoCommit, consider using "commitWithin"
+         when adding documents.
+
+         http://wiki.apache.org/solr/UpdateXmlMessages
+
+         maxDocs - Maximum number of documents to add since the last
+                   commit before automatically triggering a new commit.
+
+         maxTime - Maximum amount of time in ms that is allowed to pass
+                   since a document was added before automatically
+                   triggering a new commit.
+         openSearcher - if false, the commit causes recent index changes
+           to be flushed to stable storage, but does not cause a new
+           searcher to be opened to make those changes visible.
+
+         If the updateLog is enabled, then it's highly recommended to
+         have some sort of hard autoCommit to limit the log size.
+      -->
+    <autoCommit>
+      <maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
+      <openSearcher>false</openSearcher>
+    </autoCommit>
+
+    <!-- softAutoCommit is like autoCommit except it causes a
+         'soft' commit which only ensures that changes are visible
+         but does not ensure that data is synced to disk.  This is
+         faster and more near-realtime friendly than a hard commit.
+      -->
+    <autoSoftCommit>
+      <maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
+    </autoSoftCommit>
+
+    <!-- Update Related Event Listeners
+
+         Various IndexWriter related events can trigger Listeners to
+         take actions.
+
+         postCommit - fired after every commit or optimize command
+         postOptimize - fired after every optimize command
+      -->
+
+  </updateHandler>
+
+  <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+       Query section - these settings control query time things like caches
+       ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
+  <query>
+    <!-- Solr Internal Query Caches
+
+         There are two implementations of cache available for Solr,
+         LRUCache, based on a synchronized LinkedHashMap, and
+         FastLRUCache, based on a ConcurrentHashMap.
+
+         FastLRUCache has faster gets and slower puts in single
+         threaded operation and thus is generally faster than LRUCache
+         when the hit ratio of the cache is high (> 75%), and may be
+         faster under other scenarios on multi-cpu systems.
+    -->
+
+    <!-- Filter Cache
+
+         Cache used by SolrIndexSearcher for filters (DocSets),
+         unordered sets of *all* documents that match a query.  When a
+         new searcher is opened, its caches may be prepopulated or
+         "autowarmed" using data from caches in the old searcher.
+         autowarmCount is the number of items to prepopulate.  For
+         LRUCache, the autowarmed items will be the most recently
+         accessed items.
+
+         Parameters:
+           class - the SolrCache implementation LRUCache or
+               (LRUCache or FastLRUCache)
+           size - the maximum number of entries in the cache
+           initialSize - the initial capacity (number of entries) of
+               the cache.  (see java.util.HashMap)
+           autowarmCount - the number of entries to prepopulate from
+               and old cache.
+           maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
+                      to occupy. Note that when this option is specified, the size
+                      and initialSize parameters are ignored.
+      -->
+    <filterCache class="solr.FastLRUCache"
+                 size="512"
+                 initialSize="512"
+                 autowarmCount="0"/>
+
+    <!-- Query Result Cache
+
+         Caches results of searches - ordered lists of document ids
+         (DocList) based on a query, a sort, and the range of documents requested.
+         Additional supported parameter by LRUCache:
+            maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
+                       to occupy
+      -->
+    <queryResultCache class="solr.LRUCache"
+                      size="512"
+                      initialSize="512"
+                      autowarmCount="0"/>
+
+    <!-- Document Cache
+
+         Caches Lucene Document objects (the stored fields for each
+         document).  Since Lucene internal document ids are transient,
+         this cache will not be autowarmed.
+      -->
+    <documentCache class="solr.LRUCache"
+                   size="512"
+                   initialSize="512"
+                   autowarmCount="0"/>
+
+    <!-- custom cache currently used by block join -->
+    <cache name="perSegFilter"
+           class="solr.search.LRUCache"
+           size="10"
+           initialSize="0"
+           autowarmCount="10"
+           regenerator="solr.NoOpRegenerator" />
+
+    <!-- Lazy Field Loading
+
+         If true, stored fields that are not requested will be loaded
+         lazily.  This can result in a significant speed improvement
+         if the usual case is to not load all stored fields,
+         especially if the skipped fields are large compressed text
+         fields.
+    -->
+    <enableLazyFieldLoading>true</enableLazyFieldLoading>
+
+    <!-- Result Window Size
+
+         An optimization for use with the queryResultCache.  When a search
+         is requested, a superset of the requested number of document ids
+         are collected.  For example, if a search for a particular query
+         requests matching documents 10 through 19, and queryWindowSize is 50,
+         then documents 0 through 49 will be collected and cached.  Any further
+         requests in that range can be satisfied via the cache.
+      -->
+    <queryResultWindowSize>20</queryResultWindowSize>
+
+    <!-- Maximum number of documents to cache for any entry in the
+         queryResultCache.
+      -->
+    <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+
+    <!-- Use Cold Searcher
+
+         If a search request comes in and there is no current
+         registered searcher, then immediately register the still
+         warming searcher and use it.  If "false" then all requests
+         will block until the first searcher is done warming.
+      -->
+    <useColdSearcher>false</useColdSearcher>
+
+  </query>
+
+
+  <!-- Request Dispatcher
+
+       This section contains instructions for how the SolrDispatchFilter
+       should behave when processing requests for this SolrCore.
+
+    -->
+  <requestDispatcher>
+    <httpCaching never304="true" />
+  </requestDispatcher>
+
+  <!-- Request Handlers
+
+       http://wiki.apache.org/solr/SolrRequestHandler
+
+       Incoming queries will be dispatched to a specific handler by name
+       based on the path specified in the request.
+
+       If a Request Handler is declared with startup="lazy", then it will
+       not be initialized until the first request that uses it.
+
+    -->
+  <!-- SearchHandler
+
+       http://wiki.apache.org/solr/SearchHandler
+
+       For processing Search Queries, the primary Request Handler
+       provided with Solr is "SearchHandler" It delegates to a sequent
+       of SearchComponents (see below) and supports distributed
+       queries across multiple shards
+    -->
+  <requestHandler name="/select" class="solr.SearchHandler">
+    <!-- default values for query parameters can be specified, these
+         will be overridden by parameters in the request
+      -->
+    <lst name="defaults">
+      <str name="echoParams">explicit</str>
+      <int name="rows">10</int>
+    </lst>
+  </requestHandler>
+
+  <initParams path="/update/**,/select">
+    <lst name="defaults">
+      <str name="df">_text_</str>
+    </lst>
+  </initParams>
+
+  <!-- Response Writers
+
+       http://wiki.apache.org/solr/QueryResponseWriter
+
+       Request responses will be written using the writer specified by
+       the 'wt' request parameter matching the name of a registered
+       writer.
+
+       The "default" writer is the default and will be used if 'wt' is
+       not specified in the request.
+    -->
+  <queryResponseWriter name="xml"
+                       default="true"
+                       class="solr.XMLResponseWriter" />
+</config>

data/Dockerfiles/solr/solr-schema-7.7.0.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<schema name="dovecot-fts" version="2.0">
+  <fieldType name="string" class="solr.StrField" omitNorms="true" sortMissingLast="true"/>
+  <fieldType name="long" class="solr.LongPointField" positionIncrementGap="0"/>
+  <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
+
+  <fieldType name="text" class="solr.TextField" autoGeneratePhraseQueries="true" positionIncrementGap="100">
+    <analyzer type="index">
+      <tokenizer class="solr.StandardTokenizerFactory"/>
+      <filter class="solr.EdgeNGramFilterFactory" minGramSize="3" maxGramSize="20"/>
+      <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
+      <filter class="solr.WordDelimiterGraphFilterFactory" catenateNumbers="1" generateNumberParts="1" splitOnCaseChange="1" generateWordParts="1" splitOnNumerics="1" catenateAll="1" catenateWords="1"/>
+      <filter class="solr.FlattenGraphFilterFactory"/>
+      <filter class="solr.LowerCaseFilterFactory"/>
+      <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+      <filter class="solr.PorterStemFilterFactory"/>
+    </analyzer>
+    <analyzer type="query">
+      <tokenizer class="solr.StandardTokenizerFactory"/>
+      <filter class="solr.SynonymGraphFilterFactory" expand="true" ignoreCase="true" synonyms="synonyms.txt"/>
+      <filter class="solr.FlattenGraphFilterFactory"/>
+      <filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
+      <filter class="solr.WordDelimiterGraphFilterFactory" catenateNumbers="1" generateNumberParts="1" splitOnCaseChange="1" generateWordParts="1" splitOnNumerics="1" catenateAll="1" catenateWords="1"/>
+      <filter class="solr.LowerCaseFilterFactory"/>
+      <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+      <filter class="solr.PorterStemFilterFactory"/>
+    </analyzer>
+  </fieldType>
+
+  <field name="id" type="string" indexed="true" required="true" stored="true"/>
+  <field name="uid" type="long" indexed="true" required="true" stored="true"/>
+  <field name="box" type="string" indexed="true" required="true" stored="true"/>
+  <field name="user" type="string" indexed="true" required="true" stored="true"/>
+
+  <field name="hdr" type="text" indexed="true" stored="false"/>
+  <field name="body" type="text" indexed="true" stored="false"/>
+
+  <field name="from" type="text" indexed="true" stored="false"/>
+  <field name="to" type="text" indexed="true" stored="false"/>
+  <field name="cc" type="text" indexed="true" stored="false"/>
+  <field name="bcc" type="text" indexed="true" stored="false"/>
+  <field name="subject" type="text" indexed="true" stored="false"/>
+
+  <!-- Used by Solr internally: -->
+  <field name="_version_" type="long" indexed="true" stored="true"/>
+
+  <uniqueKey>id</uniqueKey>
+</schema>

data/Dockerfiles/unbound/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.6
+FROM alpine:3.9
 
 LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
 
@@ -8,8 +8,10 @@ RUN apk add --update --no-cache \
 	bash \
 	openssl \
 	drill \
+	tzdata \
 	&& curl -o /etc/unbound/root.hints https://www.internic.net/domain/named.cache \
 	&& chown root:unbound /etc/unbound \
+  && adduser unbound tty \
 	&& chmod 775 /etc/unbound
 
 EXPOSE 53/udp 53/tcp

data/Dockerfiles/unbound/docker-entrypoint.sh

@@ -1,8 +1,11 @@
 #!/bin/bash
 
+echo "Setting console permissions..."
+chown root:tty /dev/console
+chmod g+rw /dev/console
 echo "Receiving anchor key..."
 /usr/sbin/unbound-anchor -a /etc/unbound/trusted-key.key
 echo "Receiving root hints..."
 curl -#o /etc/unbound/root.hints https://www.internic.net/domain/named.cache
-
+/usr/sbin/unbound-control-setup
 exec "$@"

data/Dockerfiles/watchdog/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.6
+FROM alpine:3.9
 LABEL maintainer "André Peters <andre.peters@servercow.de>"
 
 # Installation
@@ -9,6 +9,7 @@ RUN apk add --update \
   nagios-plugins-ping \
   curl \
   bash \
+  coreutils \
   jq \
   fcgi \
   nagios-plugins-mysql \

data/Dockerfiles/watchdog/watchdog.sh

@@ -5,6 +5,8 @@ trap "kill 0" EXIT
 
 # Prepare
 BACKGROUND_TASKS=()
+echo "Waiting for containers to settle..."
+sleep 10
 
 if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then
   echo -e "$(date) - USE_WATCHDOG=n, skipping watchdog..."
@@ -30,65 +32,84 @@ progress() {
   PERCENT=$(( 200 * ${CURRENT} / ${TOTAL} % 2 + 100 * ${CURRENT} / ${TOTAL} ))
   redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null
   log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}" no_redis
+  # Return 10 to indicate a dead service
+  [ ${CURRENT} -le 0 ] && return 10
 }
 
 log_msg() {
   if [[ ${2} != "no_redis" ]]; then
     redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
-      tr '%&;$"_[]{}-\r\n' ' ')\"}" > /dev/null
+      tr '\r\n%&;$"_[]{}-' ' ')\"}" > /dev/null
   fi
   echo $(date) $(printf '%s\n' "${1}")
 }
 
 function mail_error() {
   [[ -z ${1} ]] && return 1
-  [[ -z ${2} ]] && return 2
+  [[ -z ${2} ]] && BODY="Service was restarted on $(date), please check your mailcow installation." || BODY="$(date) - ${2}"
-  RCPT_DOMAIN=$(echo ${1} | awk -F @ {'print $NF'})
+  WATCHDOG_NOTIFY_EMAIL=$(echo "${WATCHDOG_NOTIFY_EMAIL}" | sed 's/"//;s|"$||')
-  RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}')
+  IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}"
-  if [[ -z ${RCPT_MX} ]]; then
+  for rcpt in "${MAIL_RCPTS[@]}"; do
-    log_msg "Cannot determine MX for ${1}, skipping email notification..."
+    RCPT_DOMAIN=
-    return 1
+    RCPT_MX=
-  fi
+    RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'})
-  ./smtp-cli --missing-modules-ok \
+    RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}')
-    --subject="Watchdog: ${2} service hit the error rate limit" \
+    if [[ -z ${RCPT_MX} ]]; then
-    --body-plain="Service was restarted, please check your mailcow installation." \
+      log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
-    --to=${1} \
+      return 1
-    --from="watchdog@${MAILCOW_HOSTNAME}" \
+    fi
-    --server="${RCPT_MX}" \
+    [ -f "/tmp/${1}" ] && ATTACH="--attach /tmp/${1}@text/plain" || ATTACH=
-    --hello-host=${MAILCOW_HOSTNAME}
+    ./smtp-cli --missing-modules-ok \
-  log_msg "Sent notification email to ${1}"
+      --subject="Watchdog: ${1} hit the error rate limit" \
+      --body-plain="${BODY}" \
+      --to=${rcpt} \
+      --from="watchdog@${MAILCOW_HOSTNAME}" \
+      --server="${RCPT_MX}" \
+      --hello-host=${MAILCOW_HOSTNAME} \
+      ${ATTACH}
+    log_msg "Sent notification email to ${rcpt}"
+  done
 }
 
-
 get_container_ip() {
   # ${1} is container
   CONTAINER_ID=()
+  CONTAINER_IPS=()
   CONTAINER_IP=
   LOOP_C=1
   until [[ ${CONTAINER_IP} =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]] || [[ ${LOOP_C} -gt 5 ]]; do
-    sleep 0.5
+    if [ ${IP_BY_DOCKER_API} -eq 0 ]; then
-    # get long container id for exact match
+      CONTAINER_IP=$(dig a "${1}" +short)
-    CONTAINER_ID=($(curl --silent http://dockerapi:8080/containers/json | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" | jq -rc "select( .name | tostring == \"${1}\") | .id"))
+    else
-    # returned id can have multiple elements (if scaled), shuffle for random test
+      sleep 0.5
-    CONTAINER_ID=($(printf "%s\n" "${CONTAINER_ID[@]}" | shuf))
+      # get long container id for exact match
-    if [[ ! -z ${CONTAINER_ID} ]]; then
+      CONTAINER_ID=($(curl --silent --insecure https://dockerapi/containers/json | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" | jq -rc "select( .name | tostring == \"${1}\") | .id"))
-      for matched_container in "${CONTAINER_ID[@]}"; do
+      # returned id can have multiple elements (if scaled), shuffle for random test
-        CONTAINER_IP=$(curl --silent http://dockerapi:8080/containers/${matched_container}/json | jq -r '.NetworkSettings.Networks[].IPAddress')
+      CONTAINER_ID=($(printf "%s\n" "${CONTAINER_ID[@]}" | shuf))
-        # grep will do nothing if one of these vars is empty
+      if [[ ! -z ${CONTAINER_ID} ]]; then
-        [[ -z ${CONTAINER_IP} ]] && continue
+        for matched_container in "${CONTAINER_ID[@]}"; do
-        [[ -z ${IPV4_NETWORK} ]] && continue
+          CONTAINER_IPS=($(curl --silent --insecure https://dockerapi/containers/${matched_container}/json | jq -r '.NetworkSettings.Networks[].IPAddress')) 
-        # only return ips that are part of our network
+          for ip_match in "${CONTAINER_IPS[@]}"; do
-        if ! grep -q ${IPV4_NETWORK} <(echo ${CONTAINER_IP}); then
+            # grep will do nothing if one of these vars is empty
-          CONTAINER_IP=
+            [[ -z ${ip_match} ]] && continue
-        fi
+            [[ -z ${IPV4_NETWORK} ]] && continue
-      done
+            # only return ips that are part of our network
+            if ! grep -q ${IPV4_NETWORK} <(echo ${ip_match}); then
+              continue
+            else
+              CONTAINER_IP=${ip_match}
+              break
+            fi
+          done
+          [[ ! -z ${CONTAINER_IP} ]] && break
+        done
+      fi
     fi
     LOOP_C=$((LOOP_C + 1))
   done
   [[ ${LOOP_C} -gt 5 ]] && echo 240.0.0.0 || echo ${CONTAINER_IP}
 }
 
-# Check functions
 nginx_checks() {
   err_count=0
   diff_c=0
@@ -96,15 +117,52 @@ nginx_checks() {
   # Reduce error count by 2 after restarting an unhealthy container
   trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
   while [ ${err_count} -lt ${THRESHOLD} ]; do
+    touch /tmp/nginx-mailcow; echo "$(tail -50 /tmp/nginx-mailcow)" > /tmp/nginx-mailcow
     host_ip=$(get_container_ip nginx-mailcow)
     err_c_cur=${err_count}
-    /usr/lib/nagios/plugins/check_ping -4 -H ${host_ip} -w 2000,10% -c 4000,100% -p2 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u / -p 8081 2>> /tmp/nginx-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
-    /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u / -p 8081 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "Nginx" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
-    diff_c=0
+    if [[ $? == 10 ]]; then
-    sleep $(( ( RANDOM % 30 )  + 10 ))
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 30 )  + 10 ))
+    fi
+  done
+  return 1
+}
+
+unbound_checks() {
+  err_count=0
+  diff_c=0
+  THRESHOLD=8
+  # Reduce error count by 2 after restarting an unhealthy container
+  trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
+  while [ ${err_count} -lt ${THRESHOLD} ]; do
+    touch /tmp/unbound-mailcow; echo "$(tail -50 /tmp/unbound-mailcow)" > /tmp/unbound-mailcow
+    host_ip=$(get_container_ip unbound-mailcow)
+    err_c_cur=${err_count}
+    /usr/lib/nagios/plugins/check_dns -s ${host_ip} -H stackoverflow.com 2>> /tmp/unbound-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
+    DNSSEC=$(dig com +dnssec | egrep 'flags:.+ad')
+    if [[ -z ${DNSSEC} ]]; then
+      echo "DNSSEC failure" 2>> /tmp/unbound-mailcow 1>&2
+      err_count=$(( ${err_count} + 1))
+    else
+      echo "DNSSEC check succeeded" 2>> /tmp/unbound-mailcow 1>&2
+    fi
+    [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
+    [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
+    progress "Unbound" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
+    if [[ $? == 10 ]]; then
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 30 )  + 10 ))
+    fi
   done
   return 1
 }
@@ -116,15 +174,21 @@ mysql_checks() {
   # Reduce error count by 2 after restarting an unhealthy container
   trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
   while [ ${err_count} -lt ${THRESHOLD} ]; do
+    touch /tmp/mysql-mailcow; echo "$(tail -50 /tmp/mysql-mailcow)" > /tmp/mysql-mailcow
     host_ip=$(get_container_ip mysql-mailcow)
     err_c_cur=${err_count}
-    /usr/lib/nagios/plugins/check_mysql -H ${host_ip} -P 3306 -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_mysql -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
-    /usr/lib/nagios/plugins/check_mysql_query -H ${host_ip} -P 3306 -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} -q "SELECT COUNT(*) FROM information_schema.tables" 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_mysql_query -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} -q "SELECT COUNT(*) FROM information_schema.tables" 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "MySQL/MariaDB" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
-    diff_c=0
+    if [[ $? == 10 ]]; then
-    sleep $(( ( RANDOM % 30 )  + 10 ))
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 30 )  + 10 ))
+    fi
   done
   return 1
 }
@@ -132,19 +196,24 @@ mysql_checks() {
 sogo_checks() {
   err_count=0
   diff_c=0
-  THRESHOLD=20
+  THRESHOLD=10
   # Reduce error count by 2 after restarting an unhealthy container
   trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
   while [ ${err_count} -lt ${THRESHOLD} ]; do
+    touch /tmp/sogo-mailcow; echo "$(tail -50 /tmp/sogo-mailcow)" > /tmp/sogo-mailcow
     host_ip=$(get_container_ip sogo-mailcow)
     err_c_cur=${err_count}
-    /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /WebServerResources/css/theme-default.css -p 9192 -R md-default-theme 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /SOGo.index/ -p 20000 -R "SOGo\.MainUI" 2>> /tmp/sogo-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
-    /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /SOGo.index/ -p 20000 -R "SOGo\.MainUI" 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "SOGo" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
-    diff_c=0
+    if [[ $? == 10 ]]; then
-    sleep $(( ( RANDOM % 30 )  + 10 ))
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 30 )  + 10 ))
+    fi
   done
   return 1
 }
@@ -152,19 +221,50 @@ sogo_checks() {
 postfix_checks() {
   err_count=0
   diff_c=0
-  THRESHOLD=16
+  THRESHOLD=8
   # Reduce error count by 2 after restarting an unhealthy container
   trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
   while [ ${err_count} -lt ${THRESHOLD} ]; do
-	host_ip=$(get_container_ip postfix-mailcow)
+    touch /tmp/postfix-mailcow; echo "$(tail -50 /tmp/postfix-mailcow)" > /tmp/postfix-mailcow
+    host_ip=$(get_container_ip postfix-mailcow)
     err_c_cur=${err_count}
-    /usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 589 -f "watchdog@invalid" -C "RCPT TO:null@localhost" -C DATA -C . -R 250 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 589 -f "watchdog@invalid" -C "RCPT TO:null@localhost" -C DATA -C . -R 250 2>> /tmp/postfix-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
-    /usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 589 -S 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 589 -S 2>> /tmp/postfix-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "Postfix" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
-    diff_c=0
+    if [[ $? == 10 ]]; then
-    sleep $(( ( RANDOM % 30 )  + 10 ))
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 30 )  + 10 ))
+    fi
+  done
+  return 1
+}
+
+clamd_checks() {
+  err_count=0
+  diff_c=0
+  THRESHOLD=15
+  # Reduce error count by 2 after restarting an unhealthy container
+  trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
+  while [ ${err_count} -lt ${THRESHOLD} ]; do
+    touch /tmp/clamd-mailcow; echo "$(tail -50 /tmp/clamd-mailcow)" > /tmp/clamd-mailcow
+    host_ip=$(get_container_ip clamd-mailcow)
+    err_c_cur=${err_count}
+    /usr/lib/nagios/plugins/check_clamd -4 -H ${host_ip} 2>> /tmp/clamd-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
+    [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
+    [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
+    progress "Clamd" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
+    if [[ $? == 10 ]]; then
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 30 )  + 30 ))
+    fi
   done
   return 1
 }
@@ -172,22 +272,28 @@ postfix_checks() {
 dovecot_checks() {
   err_count=0
   diff_c=0
-  THRESHOLD=24
+  THRESHOLD=20
   # Reduce error count by 2 after restarting an unhealthy container
   trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
   while [ ${err_count} -lt ${THRESHOLD} ]; do
+    touch /tmp/dovecot-mailcow; echo "$(tail -50 /tmp/dovecot-mailcow)" > /tmp/dovecot-mailcow
     host_ip=$(get_container_ip dovecot-mailcow)
     err_c_cur=${err_count}
-    /usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 24 -f "watchdog@invalid" -C "RCPT TO:<watchdog@invalid>" -L -R "User doesn't exist" 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_smtp -4 -H ${host_ip} -p 24 -f "watchdog@invalid" -C "RCPT TO:<watchdog@invalid>" -L -R "User doesn't exist" 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
-    /usr/lib/nagios/plugins/check_imap -4 -H ${host_ip} -p 993 -S -e "OK " 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_imap -4 -H ${host_ip} -p 993 -S -e "OK " 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
-    /usr/lib/nagios/plugins/check_imap -4 -H ${host_ip} -p 143 -e "OK " 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_imap -4 -H ${host_ip} -p 143 -e "OK " 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
-    /usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 10001 -e "VERSION" 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 10001 -e "VERSION" 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
-    /usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 4190 -e "Dovecot ready" 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 4190 -e "Dovecot ready" 2>> /tmp/dovecot-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "Dovecot" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
-    diff_c=0
+    if [[ $? == 10 ]]; then
-    sleep $(( ( RANDOM % 30 )  + 10 ))
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 30 )  + 10 ))
+    fi
   done
   return 1
 }
@@ -195,46 +301,143 @@ dovecot_checks() {
 phpfpm_checks() {
   err_count=0
   diff_c=0
-  THRESHOLD=10
+  THRESHOLD=5
   # Reduce error count by 2 after restarting an unhealthy container
   trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
   while [ ${err_count} -lt ${THRESHOLD} ]; do
+    touch /tmp/php-fpm-mailcow; echo "$(tail -50 /tmp/php-fpm-mailcow)" > /tmp/php-fpm-mailcow
     host_ip=$(get_container_ip php-fpm-mailcow)
     err_c_cur=${err_count}
-    nc -z ${host_ip} 9001 ; err_count=$(( ${err_count} + ($? * 2)))
+    /usr/lib/nagios/plugins/check_tcp -H ${host_ip} -p 9001 2>> /tmp/php-fpm-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
-    nc -z ${host_ip} 9002 ; err_count=$(( ${err_count} + ($? * 2)))
+    /usr/lib/nagios/plugins/check_tcp -H ${host_ip} -p 9002 2>> /tmp/php-fpm-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
-    /usr/lib/nagios/plugins/check_ping -4 -H ${host_ip} -w 2000,10% -c 4000,100% -p2 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "PHP-FPM" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
-    diff_c=0
+    if [[ $? == 10 ]]; then
-    sleep $(( ( RANDOM % 30 )  + 10 ))
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 30 )  + 10 ))
+    fi
   done
   return 1
 }
 
-rspamd_checks() {
+ratelimit_checks() {
   err_count=0
   diff_c=0
-  THRESHOLD=10
+  THRESHOLD=1
+  RL_LOG_STATUS=$(redis-cli -h redis LRANGE RL_LOG 0 0 | jq .qid)
   # Reduce error count by 2 after restarting an unhealthy container
   trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
   while [ ${err_count} -lt ${THRESHOLD} ]; do
+    err_c_cur=${err_count}
+    RL_LOG_STATUS_PREV=${RL_LOG_STATUS}
+    RL_LOG_STATUS=$(redis-cli -h redis LRANGE RL_LOG 0 0 | jq .qid)
+    if [[ ${RL_LOG_STATUS_PREV} != ${RL_LOG_STATUS} ]]; then
+      err_count=$(( ${err_count} + 1 ))
+    fi
+    [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
+    [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
+    progress "Ratelimit" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
+    if [[ $? == 10 ]]; then
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 30 )  + 10 ))
+    fi
+  done
+  return 1
+}
+
+acme_checks() {
+  err_count=0
+  diff_c=0
+  THRESHOLD=1
+  ACME_LOG_STATUS=$(redis-cli -h redis GET ACME_FAIL_TIME)
+  if [[ -z "${ACME_LOG_STATUS}" ]]; then
+    redis-cli -h redis SET ACME_FAIL_TIME 0
+    ACME_LOG_STATUS=0
+  fi
+  # Reduce error count by 2 after restarting an unhealthy container
+  trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
+  while [ ${err_count} -lt ${THRESHOLD} ]; do
+    err_c_cur=${err_count}
+    ACME_LOG_STATUS_PREV=${ACME_LOG_STATUS}
+    ACME_LOG_STATUS=$(redis-cli -h redis GET ACME_FAIL_TIME)
+    if [[ ${ACME_LOG_STATUS_PREV} != ${ACME_LOG_STATUS} ]]; then
+      err_count=$(( ${err_count} + 1 ))
+    fi
+    [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
+    [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
+    progress "ACME" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
+    if [[ $? == 10 ]]; then
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 30 )  + 10 ))
+    fi
+  done
+  return 1
+}
+
+ipv6nat_checks() {
+  err_count=0
+  diff_c=0
+  THRESHOLD=1
+  # Reduce error count by 2 after restarting an unhealthy container
+  trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
+  while [ ${err_count} -lt ${THRESHOLD} ]; do
+    err_c_cur=${err_count}
+    CONTAINERS=$(curl --silent --insecure https://dockerapi/containers/json)
+    IPV6NAT_CONTAINER_ID=$(echo ${CONTAINERS} | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" | jq -rc "select( .name | tostring | contains(\"ipv6nat-mailcow\")) | .id")
+    if [[ ! -z ${IPV6NAT_CONTAINER_ID} ]]; then
+      LATEST_STARTED="$(echo ${CONTAINERS} | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], StartedAt: .State.StartedAt}" | jq -rc "select( .name | tostring | contains(\"ipv6nat-mailcow\") | not)" | jq -rc .StartedAt | xargs -n1 date +%s -d | sort | tail -n1)"
+      LATEST_IPV6NAT="$(echo ${CONTAINERS} | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], StartedAt: .State.StartedAt}" | jq -rc "select( .name | tostring | contains(\"ipv6nat-mailcow\"))" | jq -rc .StartedAt | xargs -n1 date +%s -d | sort | tail -n1)"
+      DIFFERENCE_START_TIME=$(expr ${LATEST_IPV6NAT} - ${LATEST_STARTED} 2>/dev/null)
+      if [[ "${DIFFERENCE_START_TIME}" -lt 30 ]]; then
+        err_count=$(( ${err_count} + 1 ))
+      fi
+    fi
+    [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
+    [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
+    progress "IPv6 NAT" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
+    if [[ $? == 10 ]]; then
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep 300
+    fi
+  done
+  return 1
+}
+
+
+rspamd_checks() {
+  err_count=0
+  diff_c=0
+  THRESHOLD=5
+  # Reduce error count by 2 after restarting an unhealthy container
+  trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
+  while [ ${err_count} -lt ${THRESHOLD} ]; do
+    touch /tmp/rspamd-mailcow; echo "$(tail -50 /tmp/rspamd-mailcow)" > /tmp/rspamd-mailcow
     host_ip=$(get_container_ip rspamd-mailcow)
     err_c_cur=${err_count}
-    SCORE=$(/usr/bin/curl -s --data-binary @- --unix-socket /rspamd-sock/rspamd.sock http://rspamd/scan -d '
+    SCORE=$(echo 'To: null@localhost
-To: null@localhost
 From: watchdog@localhost
 
 Empty
-' | jq -rc .required_score)
+' | usr/bin/curl -s --data-binary @- --unix-socket /var/lib/rspamd/rspamd.sock http://rspamd/scan | jq -rc .required_score)
     if [[ ${SCORE} != "9999" ]]; then
-      echo "Rspamd settings check failed" 1>&2
+      echo "Rspamd settings check failed" 2>> /tmp/rspamd-mailcow 1>&2
       err_count=$(( ${err_count} + 1))
     else
-      echo "Rspamd settings check succeeded" 1>&2
+      echo "Rspamd settings check succeeded" 2>> /tmp/rspamd-mailcow 1>&2
     fi
-    /usr/lib/nagios/plugins/check_ping -4 -H ${host_ip} -w 2000,10% -c 4000,100% -p2 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "Rspamd" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
@@ -249,7 +452,6 @@ Empty
 while true; do
   if ! nginx_checks; then
     log_msg "Nginx hit error limit"
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "nginx-mailcow"
     echo nginx-mailcow > /tmp/com_pipe
   fi
 done
@@ -260,7 +462,6 @@ BACKGROUND_TASKS+=($!)
 while true; do
   if ! mysql_checks; then
     log_msg "MySQL hit error limit"
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "mysql-mailcow"
     echo mysql-mailcow > /tmp/com_pipe
   fi
 done
@@ -271,7 +472,6 @@ BACKGROUND_TASKS+=($!)
 while true; do
   if ! phpfpm_checks; then
     log_msg "PHP-FPM hit error limit"
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "php-fpm-mailcow"
     echo php-fpm-mailcow > /tmp/com_pipe
   fi
 done
@@ -282,18 +482,40 @@ BACKGROUND_TASKS+=($!)
 while true; do
   if ! sogo_checks; then
     log_msg "SOGo hit error limit"
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "sogo-mailcow"
     echo sogo-mailcow > /tmp/com_pipe
   fi
 done
 ) &
 BACKGROUND_TASKS+=($!)
 
+if [ ${CHECK_UNBOUND} -eq 1 ]; then
+(
+while true; do
+  if ! unbound_checks; then
+    log_msg "Unbound hit error limit"
+    echo unbound-mailcow > /tmp/com_pipe
+  fi
+done
+) &
+BACKGROUND_TASKS+=($!)
+fi
+
+if [[ "${SKIP_CLAMD}" =~ ^([nN][oO]|[nN])+$ ]]; then
+(
+while true; do
+  if ! clamd_checks; then
+    log_msg "Clamd hit error limit"
+    echo clamd-mailcow > /tmp/com_pipe
+  fi
+done
+) &
+BACKGROUND_TASKS+=($!)
+fi
+
 (
 while true; do
   if ! postfix_checks; then
     log_msg "Postfix hit error limit"
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "postfix-mailcow"
     echo postfix-mailcow > /tmp/com_pipe
   fi
 done
@@ -304,7 +526,6 @@ BACKGROUND_TASKS+=($!)
 while true; do
   if ! dovecot_checks; then
     log_msg "Dovecot hit error limit"
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "dovecot-mailcow"
     echo dovecot-mailcow > /tmp/com_pipe
   fi
 done
@@ -315,13 +536,42 @@ BACKGROUND_TASKS+=($!)
 while true; do
   if ! rspamd_checks; then
     log_msg "Rspamd hit error limit"
-    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "rspamd-mailcow"
     echo rspamd-mailcow > /tmp/com_pipe
   fi
 done
 ) &
 BACKGROUND_TASKS+=($!)
 
+(
+while true; do
+  if ! ratelimit_checks; then
+    log_msg "Ratelimit hit error limit"
+    echo ratelimit > /tmp/com_pipe
+  fi
+done
+) &
+BACKGROUND_TASKS+=($!)
+
+(
+while true; do
+  if ! acme_checks; then
+    log_msg "ACME client hit error limit"
+    echo acme-tiny > /tmp/com_pipe
+  fi
+done
+) &
+BACKGROUND_TASKS+=($!)
+
+(
+while true; do
+  if ! ipv6nat_checks; then
+    log_msg "IPv6 NAT warning: ipv6nat-mailcow container was not started at least 30s after siblings (not an error)"
+    echo ipv6nat-mailcow > /tmp/com_pipe
+  fi
+done
+) &
+BACKGROUND_TASKS+=($!)
+
 # Monitor watchdog agents, stop script when agents fails and wait for respawn by Docker (restart:always:n)
 (
 while true; do
@@ -338,12 +588,12 @@ done
 # Monitor dockerapi
 (
 while true; do
-  while nc -z dockerapi 8080; do
+  while nc -z dockerapi 443; do
     sleep 3
   done
   log_msg "Cannot find dockerapi-mailcow, waiting to recover..."
   kill -STOP ${BACKGROUND_TASKS[*]}
-  until nc -z dockerapi 8080; do
+  until nc -z dockerapi 443; do
     sleep 3
   done
   kill -CONT ${BACKGROUND_TASKS[*]}
@@ -354,17 +604,41 @@ done
 # Restart container when threshold limit reached
 while true; do
   CONTAINER_ID=
+  HAS_INITDB=
   read com_pipe_answer </tmp/com_pipe
-  if [[ ${com_pipe_answer} =~ .+-mailcow ]]; then
+  if [ -s "/tmp/${com_pipe_answer}" ]; then
+    cat "/tmp/${com_pipe_answer}"
+  fi
+  if [[ ${com_pipe_answer} == "ratelimit" ]]; then
+    log_msg "At least one ratelimit was applied"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please see mailcow UI logs for further information."
+  elif [[ ${com_pipe_answer} == "acme-tiny" ]]; then
+    log_msg "acme-tiny client returned non-zero exit code"
+    [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check acme-mailcow for ruther information."
+  elif [[ ${com_pipe_answer} =~ .+-mailcow ]] || [[ ${com_pipe_answer} == "ipv6nat-mailcow" ]]; then
     kill -STOP ${BACKGROUND_TASKS[*]}
     sleep 3
-    CONTAINER_ID=$(curl --silent http://dockerapi:8080/containers/json | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" | jq -rc "select( .name | tostring | contains(\"${com_pipe_answer}\")) | .id")
+    CONTAINER_ID=$(curl --silent --insecure https://dockerapi/containers/json | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" | jq -rc "select( .name | tostring | contains(\"${com_pipe_answer}\")) | .id")
     if [[ ! -z ${CONTAINER_ID} ]]; then
-      log_msg "Sending restart command to ${CONTAINER_ID}..."
+      if [[ "${com_pipe_answer}" == "php-fpm-mailcow" ]]; then
-      curl --silent -XPOST http://dockerapi:8080/containers/${CONTAINER_ID}/restart
+        HAS_INITDB=$(curl --silent --insecure -XPOST https://dockerapi/containers/${CONTAINER_ID}/top | jq '.msg.Processes[] | contains(["php -c /usr/local/etc/php -f /web/inc/init_db.inc.php"])' | grep true)
+      fi
+      S_RUNNING=$(($(date +%s) - $(curl --silent --insecure https://dockerapi/containers/${CONTAINER_ID}/json | jq .State.StartedAt | xargs -n1 date +%s -d)))
+      if [ ${S_RUNNING} -lt 120 ]; then
+        log_msg "Container is running for less than 120 seconds, skipping action..."
+      elif [[ ! -z ${HAS_INITDB} ]]; then
+        log_msg "Database is being initialized by php-fpm-mailcow, not restarting but delaying checks for a minute..."
+        sleep 60
+      else
+        log_msg "Sending restart command to ${CONTAINER_ID}..."
+        curl --silent --insecure -XPOST https://dockerapi/containers/${CONTAINER_ID}/restart
+        if [[ ${com_pipe_answer} != "ipv6nat-mailcow" ]]; then
+          [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}"
+        fi
+        log_msg "Wait for restarted container to settle and continue watching..."
+        sleep 35
+      fi
     fi
-    log_msg "Wait for restarted container to settle and continue watching..."
-    sleep 30s
     kill -CONT ${BACKGROUND_TASKS[*]}
     kill -USR1 ${BACKGROUND_TASKS[*]}
   fi

data/assets/mysql/docker-entrypoint.sh

@@ -0,0 +1,192 @@
+#!/bin/bash
+set -eo pipefail
+shopt -s nullglob
+
+openssl req -x509 -sha256 -newkey rsa:2048 -keyout /var/lib/mysql/sql.key -out /var/lib/mysql/sql.crt -days 3650 -nodes -subj '/CN=mysql'
+
+# if command starts with an option, prepend mysqld
+if [ "${1:0:1}" = '-' ]; then
+	set -- mysqld "$@"
+fi
+
+# skip setup if they want an option that stops mysqld
+wantHelp=
+for arg; do
+	case "$arg" in
+		-'?'|--help|--print-defaults|-V|--version)
+			wantHelp=1
+			break
+			;;
+	esac
+done
+
+# usage: file_env VAR [DEFAULT]
+#    ie: file_env 'XYZ_DB_PASSWORD' 'example'
+# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
+#  "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
+file_env() {
+	local var="$1"
+	local fileVar="${var}_FILE"
+	local def="${2:-}"
+	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+		echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+		exit 1
+	fi
+	local val="$def"
+	if [ "${!var:-}" ]; then
+		val="${!var}"
+	elif [ "${!fileVar:-}" ]; then
+		val="$(< "${!fileVar}")"
+	fi
+	export "$var"="$val"
+	unset "$fileVar"
+}
+
+_check_config() {
+	toRun=( "$@" --verbose --help --log-bin-index="$(mktemp -u)" )
+	if ! errors="$("${toRun[@]}" 2>&1 >/dev/null)"; then
+		cat >&2 <<-EOM
+
+			ERROR: mysqld failed while attempting to check config
+			command was: "${toRun[*]}"
+
+			$errors
+		EOM
+		exit 1
+	fi
+}
+
+# Fetch value from server config
+# We use mysqld --verbose --help instead of my_print_defaults because the
+# latter only show values present in config files, and not server defaults
+_get_config() {
+	local conf="$1"; shift
+	"$@" --verbose --help --log-bin-index="$(mktemp -u)" 2>/dev/null | awk '$1 == "'"$conf"'" { print $2; exit }'
+}
+
+# allow the container to be started with `--user`
+if [ "$1" = 'mysqld' -a -z "$wantHelp" -a "$(id -u)" = '0' ]; then
+	_check_config "$@"
+	DATADIR="$(_get_config 'datadir' "$@")"
+	mkdir -p "$DATADIR"
+	chown -R mysql:mysql "$DATADIR"
+	exec gosu mysql "$BASH_SOURCE" "$@"
+fi
+
+if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
+	# still need to check config, container may have started with --user
+	_check_config "$@"
+	# Get config
+	DATADIR="$(_get_config 'datadir' "$@")"
+
+	if [ ! -d "$DATADIR/mysql" ]; then
+		file_env 'MYSQL_ROOT_PASSWORD'
+		if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" -a -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then
+			echo >&2 'error: database is uninitialized and password option is not specified '
+			echo >&2 '  You need to specify one of MYSQL_ROOT_PASSWORD, MYSQL_ALLOW_EMPTY_PASSWORD and MYSQL_RANDOM_ROOT_PASSWORD'
+			exit 1
+		fi
+
+		mkdir -p "$DATADIR"
+
+		echo 'Initializing database'
+		# "Other options are passed to mysqld." (so we pass all "mysqld" arguments directly here)
+		mysql_install_db --datadir="$DATADIR" --rpm "${@:2}"
+		echo 'Database initialized'
+
+		SOCKET="$(_get_config 'socket' "$@")"
+		"$@" --skip-networking --socket="${SOCKET}" &
+		pid="$!"
+
+		mysql=( mysql --protocol=socket -uroot -hlocalhost --socket="${SOCKET}" )
+
+		for i in {30..0}; do
+			if echo 'SELECT 1' | "${mysql[@]}" &> /dev/null; then
+				break
+			fi
+			echo 'MySQL init process in progress...'
+			sleep 1
+		done
+		if [ "$i" = 0 ]; then
+			echo >&2 'MySQL init process failed.'
+			exit 1
+		fi
+
+		if [ -z "$MYSQL_INITDB_SKIP_TZINFO" ]; then
+			# sed is for https://bugs.mysql.com/bug.php?id=20545
+			mysql_tzinfo_to_sql /usr/share/zoneinfo | sed 's/Local time zone must be set--see zic manual page/FCTY/' | "${mysql[@]}" mysql
+		fi
+
+		if [ ! -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then
+			export MYSQL_ROOT_PASSWORD="$(pwgen -1 32)"
+			echo "GENERATED ROOT PASSWORD: $MYSQL_ROOT_PASSWORD"
+		fi
+
+		rootCreate=
+		# default root to listen for connections from anywhere
+		file_env 'MYSQL_ROOT_HOST' '%'
+		if [ ! -z "$MYSQL_ROOT_HOST" -a "$MYSQL_ROOT_HOST" != 'localhost' ]; then
+			# no, we don't care if read finds a terminating character in this heredoc
+			# https://unix.stackexchange.com/questions/265149/why-is-set-o-errexit-breaking-this-read-heredoc-expression/265151#265151
+			read -r -d '' rootCreate <<-EOSQL || true
+				CREATE USER 'root'@'${MYSQL_ROOT_HOST}' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ;
+				GRANT ALL ON *.* TO 'root'@'${MYSQL_ROOT_HOST}' WITH GRANT OPTION ;
+			EOSQL
+		fi
+
+		"${mysql[@]}" <<-EOSQL
+			-- What's done in this file shouldn't be replicated
+			--  or products like mysql-fabric won't work
+			SET @@SESSION.SQL_LOG_BIN=0;
+
+			DELETE FROM mysql.user WHERE user NOT IN ('mysql.sys', 'mysqlxsys', 'root') OR host NOT IN ('localhost') ;
+			SET PASSWORD FOR 'root'@'localhost'=PASSWORD('${MYSQL_ROOT_PASSWORD}') ;
+			GRANT ALL ON *.* TO 'root'@'localhost' WITH GRANT OPTION ;
+			${rootCreate}
+			DROP DATABASE IF EXISTS test ;
+			FLUSH PRIVILEGES ;
+		EOSQL
+
+		if [ ! -z "$MYSQL_ROOT_PASSWORD" ]; then
+			mysql+=( -p"${MYSQL_ROOT_PASSWORD}" )
+		fi
+
+		file_env 'MYSQL_DATABASE'
+		if [ "$MYSQL_DATABASE" ]; then
+			echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" | "${mysql[@]}"
+			mysql+=( "$MYSQL_DATABASE" )
+		fi
+
+		file_env 'MYSQL_USER'
+		file_env 'MYSQL_PASSWORD'
+		if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then
+			echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" | "${mysql[@]}"
+
+			if [ "$MYSQL_DATABASE" ]; then
+				echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" | "${mysql[@]}"
+			fi
+		fi
+
+		echo
+		for f in /docker-entrypoint-initdb.d/*; do
+			case "$f" in
+				*.sh)     echo "$0: running $f"; . "$f" ;;
+				*.sql)    echo "$0: running $f"; "${mysql[@]}" < "$f"; echo ;;
+				*.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${mysql[@]}"; echo ;;
+				*)        echo "$0: ignoring $f" ;;
+			esac
+			echo
+		done
+
+		if ! kill -s TERM "$pid" || ! wait "$pid"; then
+			echo >&2 'MySQL init process failed.'
+			exit 1
+		fi
+
+		echo
+		echo 'MySQL init process done. Ready for start up.'
+		echo
+	fi
+fi
+
+exec "$@"

data/assets/nextcloud/nextcloud.conf

@@ -5,11 +5,11 @@ map $http_x_forwarded_proto $client_req_scheme_nc {
 
 server {
   include /etc/nginx/conf.d/listen_ssl.active;
+  include /etc/nginx/conf.d/listen_plain.active;
   include /etc/nginx/mime.types;
   charset utf-8;
   override_charset on;
 
-  ssl on;
   ssl_certificate /etc/ssl/mail/cert.pem;
   ssl_certificate_key /etc/ssl/mail/key.pem;
   ssl_protocols TLSv1.2;
@@ -24,7 +24,8 @@ server {
   add_header X-Robots-Tag none;
   add_header X-Download-Options noopen;
   add_header X-Permitted-Cross-Domain-Policies none;
-  add_header X-Frame-Options "SAMEORIGIN";
+  #add_header X-Frame-Options "SAMEORIGIN";
+  add_header Referrer-Policy "no-referrer";
 
   server_name NC_SUBD;
 

data/assets/templates/quarantine.tpl

@@ -0,0 +1,49 @@
+<html>
+  <head>
+  <style>
+  body {
+    font-family: Helvetica, Arial, Sans-Serif;
+  }
+  table {
+    border-collapse: collapse;
+    width: 100%;
+    margin-bottom: 20px;
+  }
+  th, td {
+    padding: 8px;
+    text-align: left;
+    border-bottom: 1px solid #ddd;
+    vertical-align: top;
+  }
+  th {
+    background-color: #56B04C;
+    color: white;
+  }
+  tr:nth-child(even){background-color: #f2f2f2}
+
+  </style>
+  </head>
+  <body>
+    <p>Hi!<br>
+    {% if counter == 1 %}
+    There is 1 new message waiting in quarantine:<br>
+    {% else %}
+    There are {{counter}} new messages waiting in quarantine:<br>
+    {% endif %}
+    <table>
+    <tr><th>Subject</th><th>Sender</th><th>Score</th><th>Arrived on</th>{% if quarantine_acl == 1 %}<th>Actions</th>{% endif %}</tr>
+    {% for line in meta %}
+    <tr>
+    <td>{{ line.subject|e }}</td>
+    <td>{{ line.sender|e }}</td>
+    <td>{{ line.score }}</td>
+    <td>{{ line.created }}</td>
+    {% if quarantine_acl == 1 %}
+    <td><a href="https://{{ hostname }}/qhandler/release/{{ line.qhash }}">release</a> | <a href="https://{{ hostname }}/qhandler/delete/{{ line.qhash }}">delete</a></td>
+    {% endif %}
+    </tr>
+    {% endfor %}
+    </table>
+    </p>
+  </body>
+</html>

data/assets/templates/quota.tpl

@@ -0,0 +1,29 @@
+<html>
+  <head>
+  <style>
+  body {
+    font-family: sans-serif;
+  }
+  #progressbar {
+    background-color: #f0f0f0;
+    border-radius: 0px;
+    padding: 0px;
+    width:50%;
+  }  
+  #progressbar > div {
+    background-color: #ff9c9c;
+    width: {{percent}}%;
+    height: 20px;
+    border-radius: 0px;
+  }
+  </style>
+  </head>
+  <body>
+    <p>Hi {{username}}!<br><br>
+    Your mailbox is now {{percent}}% full, please consider deleting old messages to still be able to receive new mails in the future.<br>
+    <div id="progressbar">
+      <div></div>
+    </div>
+    </p>
+  </body>
+</html>

data/conf/clamav/clamd.conf

@@ -1,4 +1,5 @@
-LogFile /dev/console
+#Debug true
+#LogFile /dev/null
 LogTime yes
 LogClean yes
 ExtendedDetectionInfo yes
@@ -23,9 +24,9 @@ DetectPUA yes
 #IncludePUA Spy
 #IncludePUA Scanner
 #IncludePUA RAT
-AlgorithmicDetection yes
+HeuristicAlerts yes
 ScanOLE2 yes
-OLE2BlockMacros yes
+AlertOLE2Macros no
 ScanPDF yes
 ScanSWF yes
 ScanXMLDOCS yes

data/conf/clamav/freshclam.conf

@@ -1,8 +1,7 @@
-UpdateLogFile /var/log/clamav/freshclam.log
+#UpdateLogFile /dev/console
 LogTime yes
 PidFile /run/clamav/freshclam.pid
 DatabaseOwner clamav
-AllowSupplementaryGroups yes
 DNSDatabaseInfo current.cvd.clamav.net
 DatabaseMirror database.clamav.net
 MaxAttempts 4

data/conf/dovecot/dovecot.conf

@@ -1,6 +1,12 @@
 # --------------------------------------------------------------------------
 # Please create a file "extra.conf" for persistent overrides to dovecot.conf
 # --------------------------------------------------------------------------
+# LDAP example:
+#passdb {
+#  args = /usr/local/etc/dovecot/ldap/passdb.conf
+#  driver = ldap
+#}
+
 auth_mechanisms = plain login
 #mail_debug = yes
 #auth_debug = yes
@@ -14,7 +20,10 @@ disable_plaintext_auth = yes
 login_log_format_elements = "user=<%u> method=%m rip=%r lip=%l mpid=%e %c %k"
 mail_home = /var/vmail/%d/%n
 mail_location = maildir:~/
-mail_plugins = quota acl zlib listescape #mail_crypt
+mail_plugins = </usr/local/etc/dovecot/mail_plugins
+mail_attachment_fs = crypt:set_prefix=mail_crypt_global:posix:
+mail_attachment_dir = /var/attachments
+mail_attachment_min_size = 128k
 
 # Dovecot 2.2
 #ssl_protocols = !SSLv3
@@ -45,6 +54,14 @@ passdb {
 passdb {
   args = /usr/local/etc/dovecot/sql/dovecot-dict-sql-passdb.conf
   driver = sql
+  result_success = return-ok
+  result_failure = continue
+  result_internalfail = continue
+}
+passdb {
+  driver = passwd-file
+  args = /usr/local/etc/dovecot/dovecot-master.passwd
+  skip = authenticated
 }
 # Set doveadm_password=your-secret-password in data/conf/dovecot/extra.conf (create if missing)
 service doveadm {
@@ -72,6 +89,9 @@ namespace inbox {
   mailbox "Gelöschte Objekte" {
     special_use = \Trash
   }
+  mailbox "Gelöschte Elemente" {
+    special_use = \Trash
+  }
   mailbox "Papierkorb" {
     special_use = \Trash
   }
@@ -125,6 +145,9 @@ namespace inbox {
   mailbox "Gesendete Objekte" {
     special_use = \Sent
   }
+  mailbox "Gesendete Elemente" {
+    special_use = \Sent
+  }
   mailbox "Itens Enviados" {
     special_use = \Sent
   }
@@ -169,13 +192,25 @@ namespace inbox {
   mailbox "Ongewenste e-mail" {
     special_use = \Junk
   }
+  mailbox "Koncepty" {
+    special_use = \Drafts
+  }
+  mailbox "Nevyžádaná pošta" {
+    special_use = \Junk
+  }
+  mailbox "Odstraněná pošta" {
+    special_use = \Trash
+  }
+  mailbox "Odeslaná pošta" {
+    special_use = \Sent
+  }
   prefix =
 }
 namespace {
     type = shared
     separator = /
     prefix = Shared/%%u/
-    location = maildir:%%h/:CONTROL=~/Shared/%%u:INDEXPVT=~/Shared/%%u
+    location = maildir:%%h/:INDEX=~/Shared/%%u;CONTROL=~/Shared/%%u
     subscriptions = no
     list = children
 }
@@ -190,6 +225,13 @@ service dict {
 service log {
   user = dovenull
 }
+service config {
+  unix_listener config {
+    user = root
+    group = vmail
+    mode = 0660
+  }
+}
 service auth {
   inet_listener auth-inet {
     port = 10001
@@ -209,22 +251,22 @@ service managesieve-login {
   }
   service_count = 1
   process_min_avail = 2
-  vsz_limit = 256 M
+  vsz_limit = 1G
 }
 service imap-login {
   service_count = 1
-  process_limit = 500
+  process_limit = 10000
-  vsz_limit = 256 M
+  vsz_limit = 1G
   user = dovenull
 }
 service pop3-login {
   service_count = 1
-  vsz_limit = 256 M
+  vsz_limit = 1G
 }
 service imap {
   executable = imap imap-postlogin
-  user = dovenull
+  user = vmail
-  vsz_limit = 256 M
+  vsz_limit = 1G
 }
 service managesieve {
   process_limit = 256
@@ -238,17 +280,22 @@ service lmtp {
 listen = *,[::]
 ssl_cert = </etc/ssl/mail/cert.pem
 ssl_key = </etc/ssl/mail/key.pem
+userdb {
+  driver = passwd-file
+  args = /usr/local/etc/dovecot/dovecot-master.userdb
+}
 userdb {
   args = /usr/local/etc/dovecot/sql/dovecot-dict-sql-userdb.conf
   driver = sql
+  skip = found
 }
 protocol imap {
+  mail_plugins = </usr/local/etc/dovecot/mail_plugins_imap
   imap_metadata = yes
-  mail_plugins = quota imap_quota imap_acl acl zlib imap_zlib imap_sieve listescape #mail_crypt
 }
 mail_attribute_dict = file:%h/dovecot-attributes
 protocol lmtp {
-  mail_plugins = quota sieve acl zlib listescape #mail_crypt
+  mail_plugins = </usr/local/etc/dovecot/mail_plugins_lmtp
   auth_socket_path = /usr/local/var/run/dovecot/auth-master
 }
 protocol sieve {
@@ -256,9 +303,12 @@ protocol sieve {
 }
 plugin {
   # Allow "any" or "authenticated" to be used in ACLs
-  #acl_anyone = allow
+  acl_anyone = </usr/local/etc/dovecot/acl_anyone
   acl_shared_dict = file:/var/vmail/shared-mailboxes.db
   acl = vfile
+  fts = solr
+  fts_autoindex = yes
+  fts_solr = url=http://solr:8983/solr/dovecot-fts/
   quota = dict:Userquota::proxy::sqlquota
   quota_rule2 = Trash:storage=+100%%
   sieve = /var/vmail/sieve/%u.sieve
@@ -275,8 +325,11 @@ plugin {
   imapsieve_mailbox2_causes = COPY
   imapsieve_mailbox2_before = file:/usr/local/lib/dovecot/sieve/report-ham.sieve
   # END
+  quota_warning = storage=95%% quota-warning 95 %u
+  quota_warning2 = storage=80%% quota-warning 80 %u
   sieve_pipe_bin_dir = /usr/local/lib/dovecot/sieve
-  sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.execute +vacation-seconds
+  sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.execute
+  sieve_extensions = +notify +imapflags +vacation-seconds
   sieve_max_script_size = 1M
   sieve_max_redirects = 30
   sieve_quota_max_scripts = 0
@@ -288,11 +341,26 @@ plugin {
   sieve_before = dict:proxy::sieve_before;name=active;bindir=/var/vmail/sieve_before_bindir
   sieve_after = dict:proxy::sieve_after;name=active;bindir=/var/vmail/sieve_after_bindir
   sieve_after2 = /var/vmail/sieve/global.sieve
-  #mail_crypt_global_private_key = </mail_crypt/ecprivkey.pem
+
-  #mail_crypt_global_public_key = </mail_crypt/ecpubkey.pem
+  # -- Global keys
-  #mail_crypt_save_version = 2
+  mail_crypt_global_private_key = </mail_crypt/ecprivkey.pem
+  mail_crypt_global_public_key = </mail_crypt/ecpubkey.pem
+  mail_crypt_save_version = 2
+
   # Enable compression while saving, lz4 Dovecot v2.2.11+
   zlib_save = lz4
+
+  mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
+  mail_log_fields = uid box msgid size
+  mail_log_cached_only = yes
+}
+service quota-warning {
+  executable = script /usr/local/bin/quota_notify.py
+  # use some unprivileged user for executing the quota warnings
+  user = vmail
+  unix_listener quota-warning {
+    user = vmail
+  }
 }
 dict {
   sqlquota = mysql:/usr/local/etc/dovecot/sql/dovecot-dict-sql-quota.conf
@@ -315,4 +383,11 @@ service stats {
     user = vmail
   }
 }
+imap_max_line_length = 2 M
+#auth_cache_verify_password_with_worker = yes
+#auth_cache_negative_ttl = 0
+#auth_cache_ttl = 30 s
+#auth_cache_size = 2 M
 !include_try /usr/local/etc/dovecot/extra.conf
+!include_try /usr/local/etc/dovecot/sogo-sso.conf
+default_client_limit = 10400

data/conf/dovecot/ldap/passdb.conf

@@ -0,0 +1,9 @@
+#hosts = 1.2.3.4
+#dn = cn=admin,dc=example,dc=local
+#dnpass = password
+#ldap_version = 3
+#base = ou=People,dc=example,dc=local
+#auth_bind = no
+#pass_filter = (&(objectClass=posixAccount)(mail=%u))
+#pass_attrs = mail=user,userPassword=password
+#default_pass_scheme = SSHA

data/conf/mysql/my.cnf

@@ -2,9 +2,9 @@
 character-set-client-handshake = FALSE
 character-set-server           = utf8mb4
 collation-server               = utf8mb4_unicode_ci
-innodb_file_per_table          = TRUE
+#innodb_file_per_table          = TRUE
-innodb_file_format             = barracuda
+#innodb_file_format             = barracuda
-innodb_large_prefix            = TRUE
+#innodb_large_prefix            = TRUE
 #sql_mode=IGNORE_SPACE,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION
 max_allowed_packet=192M
 max-connections=1500

data/conf/nginx/site.conf

@@ -7,15 +7,6 @@ map $http_x_forwarded_proto $client_req_scheme {
      https https;
 }
 
-map $sent_http_content_type $expires {
-        default off;
-        text/html off;
-        text/css 1d;
-        application/javascript 1d;
-        application/json off;
-        image/png       1d;
-}
-
 server {
   include /etc/nginx/mime.types;
   charset utf-8;
@@ -23,33 +14,68 @@ server {
 
   ssl_certificate /etc/ssl/mail/cert.pem;
   ssl_certificate_key /etc/ssl/mail/key.pem;
-  ssl_protocols TLSv1.2;
+  ssl_protocols TLSv1.2 TLSv1.3;
   ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
   ssl_prefer_server_ciphers on;
   ssl_session_cache shared:SSL:50m;
   ssl_session_timeout 1d;
   ssl_session_tickets off;
 
-  add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";
+  add_header Strict-Transport-Security "max-age=15768000;";
   add_header X-Content-Type-Options nosniff;
   add_header X-XSS-Protection "1; mode=block";
   add_header X-Robots-Tag none;
   add_header X-Download-Options noopen;
+  add_header X-Frame-Options "SAMEORIGIN" always;
   add_header X-Permitted-Cross-Domain-Policies none;
+  add_header Referrer-Policy strict-origin;
 
   index index.php index.html;
 
   client_max_body_size 0;
 
+  listen 127.0.0.1:65510;
   include /etc/nginx/conf.d/listen_plain.active;
   include /etc/nginx/conf.d/listen_ssl.active;
   include /etc/nginx/conf.d/server_name.active;
 
+  gzip on;
+  gzip_disable "msie6";
+
+  gzip_vary on;
+  gzip_proxied off;
+  gzip_comp_level 6;
+  gzip_buffers 16 8k;
+  gzip_http_version 1.1;
+  gzip_min_length 256;
+  gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/vnd.ms-fontobject application/x-font-ttf font/opentype image/svg+xml image/x-icon;
+
+  location ~ ^/(fonts|js|css|img)/ {
+    expires max;
+    add_header Cache-Control public;
+  }
+
   error_log  /var/log/nginx/error.log;
   access_log /var/log/nginx/access.log;
   absolute_redirect off;
   root /web;
 
+  location / {
+    try_files $uri $uri/ @strip-ext;
+  }
+
+  location /qhandler {
+    rewrite ^/qhandler/(.*)/(.*) /qhandler.php?action=$1&hash=$2;
+  }
+
+  location /edit {
+    rewrite ^/edit/(.*)/(.*) /edit.php?$1=$2;
+  }
+
+  location @strip-ext {
+    rewrite ^(.*)$ $1.php last;
+  }
+
   location ~ ^/api/v1/(.*)$ {
     try_files $uri $uri/ /json_api.php?query=$1;
   }
@@ -91,7 +117,6 @@ server {
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
     proxy_set_header X-Real-IP $remote_addr;
     proxy_redirect off;
-    expires $expires;
   }
 
   location ~* ^/Autodiscover/Autodiscover.xml {
@@ -118,14 +143,26 @@ server {
     try_files /autoconfig.php =404;
   }
 
+  # auth_request endpoint if ALLOW_ADMIN_EMAIL_LOGIN is set
+  location /sogo-auth-verify {
+    internal;
+    proxy_set_header  X-Original-URI $request_uri;
+    proxy_set_header  X-Real-IP $remote_addr;
+    proxy_set_header  Host $http_host;
+    proxy_set_header  Content-Length "";
+    proxy_pass        http://127.0.0.1:65510/sogo-auth;
+    proxy_pass_request_body off;
+  }
+
   location ^~ /Microsoft-Server-ActiveSync {
+    include /etc/nginx/conf.d/sogo_proxy_auth.active;
     include /etc/nginx/conf.d/sogo_eas.active;
-    proxy_connect_timeout 1000;
+    proxy_connect_timeout 4000;
     proxy_next_upstream timeout error;
-    proxy_send_timeout 1000;
+    proxy_send_timeout 4000;
-    proxy_read_timeout 1000;
+    proxy_read_timeout 4000;
     proxy_buffer_size 8k;
-    proxy_buffers 4 32k;
+    proxy_buffers 16 64k;
     proxy_temp_file_write_size 64k;
     proxy_busy_buffers_size 64k;
     proxy_set_header X-Real-IP $remote_addr;
@@ -141,6 +178,7 @@ server {
   }
 
   location ^~ /SOGo {
+    include /etc/nginx/conf.d/sogo_proxy_auth.active;
     include /etc/nginx/conf.d/sogo.active;
     proxy_set_header X-Real-IP $remote_addr;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -156,44 +194,19 @@ server {
   }
 
   location /SOGo.woa/WebServerResources/ {
-    proxy_pass http://sogo:9192/WebServerResources/;
+    alias /usr/lib/GNUstep/SOGo/WebServerResources/;
-    proxy_set_header Host $http_host;
-    proxy_cache sogo;
-    proxy_cache_valid 200 1d;
-    proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504;
-    #alias /usr/lib/GNUstep/SOGo/WebServerResources/;
-    expires $expires;
-    allow all;
   }
 
   location /.woa/WebServerResources/ {
-    proxy_pass http://sogo:9192/WebServerResources/;
+    alias /usr/lib/GNUstep/SOGo/WebServerResources/;
-    proxy_set_header Host $http_host;
-    proxy_cache sogo;
-    proxy_cache_valid 200 1d;
-    proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504;
-    #alias /usr/lib/GNUstep/SOGo/WebServerResources/;
-    expires $expires;
-    allow all;
   }
 
   location /SOGo/WebServerResources/ {
-    proxy_pass http://sogo:9192/WebServerResources/;
+    alias /usr/lib/GNUstep/SOGo/WebServerResources/;
-    proxy_set_header Host $http_host;
-    proxy_cache sogo;
-    proxy_cache_valid 200 1d;
-    proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504;
-    #alias /usr/lib/GNUstep/SOGo/WebServerResources/;
-    allow all;
   }
 
   location (^/SOGo/so/ControlPanel/Products/[^/]*UI/Resources/.*\.(jpg|png|gif|css|js)$) {
-    proxy_pass http://sogo:9192/$1.SOGo/Resources/$2;
+    alias /usr/lib/GNUstep/SOGo/$1.SOGo/Resources/$2;
-    proxy_set_header Host $http_host;
-    proxy_cache sogo;
-    proxy_cache_valid 200 1d;
-    proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504;
-    #alias /usr/lib/GNUstep/SOGo/$1.SOGo/Resources/$2;
   }
 
   include /etc/nginx/conf.d/site.*.custom;

data/conf/nginx/templates/sogo.auth_request.template.sh

@@ -0,0 +1,10 @@
+if printf "%s\n" "${ALLOW_ADMIN_EMAIL_LOGIN}" | grep -E '^([yY][eE][sS]|[yY])+$' >/dev/null; then
+    echo 'auth_request /sogo-auth-verify;
+auth_request_set $user $upstream_http_x_user;
+auth_request_set $auth $upstream_http_x_auth;
+auth_request_set $auth_type $upstream_http_x_auth_type;
+proxy_set_header x-webobjects-remote-user "$user";
+proxy_set_header Authorization "$auth";
+proxy_set_header x-webobjects-auth-type "$auth_type";
+'
+fi

data/conf/phpfpm/php-conf.d/other.ini

@@ -1,2 +1,4 @@
 session.save_handler = redis
 session.save_path = "tcp://redis:6379"
+max_execution_time = 1200
+max_input_time = 1200

data/conf/phpfpm/php-fpm.d/pools.conf

@@ -11,8 +11,7 @@ access.log = /proc/self/fd/2
 clear_env = no
 catch_workers_output = yes
 php_admin_value[memory_limit] = 256M
-php_admin_value[max_execution_time] = 1200
+php_admin_value[disable_functions] = show_source, highlight_file, apache_child_terminate, apache_get_modules, apache_note, apache_setenv, virtual, dl, disk_total_space, posix_getpwnam, posix_getpwuid, posix_mkfifo, posix_mknod, posix_setpgid, posix_setsid, posix_setuid, posix_uname, proc_nice, openlog, syslog, pfsockopen, system, shell_exec, passthru, popen, proc_open, exec, ini_alter, pcntl_exec, proc_close, proc_get_status, proc_terminate, symlink
-php_admin_value[max_input_time] = 1200
 
 [web-worker]
 user = www-data
@@ -26,7 +25,5 @@ listen = [::]:9002
 access.log = /proc/self/fd/2
 clear_env = no
 catch_workers_output = yes
-php_admin_value[memory_limit] = 256M
+php_admin_value[memory_limit] = 512M
-php_admin_value[max_execution_time] = 1200
+php_admin_value[disable_functions] = show_source, highlight_file, apache_child_terminate, apache_get_modules, apache_note, apache_setenv, virtual, dl, disk_total_space, posix_getpwnam, posix_getpwuid, posix_mkfifo, posix_mknod, posix_setpgid, posix_setsid, posix_setuid, posix_uname, proc_nice, openlog, syslog, pfsockopen, system, shell_exec, passthru, popen, proc_open, exec, ini_alter, pcntl_exec, proc_close, proc_get_status, proc_terminate, symlink
-php_admin_value[max_input_time] = 1200
-

data/conf/postfix/allow_mailcow_local.regexp

@@ -0,0 +1 @@
+/^(.+)@mailcow.local/       OK

data/conf/postfix/anonymize_headers.pcre

@@ -0,0 +1,8 @@
+if /^\s*Received:.*Authenticated sender.*\(Postcow\)/
+/^\s*Received:.*Authenticated sender:(.+)/
+  REPLACE Received: from localhost (localhost [127.0.0.1]) (Authenticated sender:$1
+endif
+/^\s*X-Enigmail/        IGNORE
+/^\s*X-Mailer/          IGNORE
+/^\s*X-Originating-IP/  IGNORE
+/^\s*X-Forward/         IGNORE

data/conf/postfix/local_transport

@@ -0,0 +1 @@
+/localhost$/  local:

data/conf/postfix/main.cf

@@ -20,7 +20,7 @@ broken_sasl_auth_clients = yes
 disable_vrfy_command = yes
 maximal_backoff_time = 1800s
 maximal_queue_lifetime = 1d
-message_size_limit = 26214400
+message_size_limit = 104857600
 milter_default_action = accept
 milter_protocol = 6
 minimal_backoff_time = 300s
@@ -41,8 +41,11 @@ postscreen_greet_wait = 3s
 postscreen_non_smtp_command_enable = no
 postscreen_pipelining_enable = no
 proxy_read_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_virtual_sender_acl.cf,
+  proxy:mysql:/opt/postfix/conf/sql/mysql_tls_policy_override_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_sender_dependent_default_transport_maps.cf,
-  proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps.cf,
+  proxy:mysql:/opt/postfix/conf/sql/mysql_transport_maps.cf,
+  proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_sender_dependent.cf,
+  proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_sender_bcc_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_recipient_bcc_maps.cf,
@@ -91,13 +94,18 @@ smtpd_tls_dh1024_param_file = /etc/ssl/mail/dhparams.pem
 smtpd_tls_eecdh_grade = auto
 smtpd_tls_exclude_ciphers = ECDHE-RSA-RC4-SHA, RC4, aNULL, DES-CBC3-SHA, ECDHE-RSA-DES-CBC3-SHA, EDH-RSA-DES-CBC3-SHA
 smtpd_tls_loglevel = 1
-smtp_tls_mandatory_protocols = !SSLv2, !SSLv3
+
+smtp_tls_mandatory_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1
 smtp_tls_protocols = !SSLv2, !SSLv3
-lmtp_tls_mandatory_protocols = !SSLv2, !SSLv3
+
-lmtp_tls_protocols = !SSLv2, !SSLv2, !SSLv3
+lmtp_tls_mandatory_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1
-smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3
+lmtp_tls_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1
+
+smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1
 smtpd_tls_protocols = !SSLv2, !SSLv3
+
 smtpd_tls_security_level = may
+tls_preempt_cipherlist = yes
 tls_ssl_options = NO_COMPRESSION
 smtpd_tls_mandatory_ciphers = high
 virtual_alias_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_virtual_alias_maps.cf,
@@ -124,6 +132,11 @@ mydestination = localhost.localdomain, localhost
 smtp_address_preference = ipv4
 smtp_sender_dependent_authentication = yes
 smtp_sasl_auth_enable = yes
-smtp_sasl_password_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps.cf
+smtp_sasl_password_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_sender_dependent.cf
 smtp_sasl_security_options = 
 smtp_sasl_mechanism_filter = plain, login
+smtp_tls_policy_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_tls_policy_override_maps.cf
+smtp_header_checks = pcre:/opt/postfix/conf/anonymize_headers.pcre
+mail_name = Postcow
+transport_maps = pcre:/opt/postfix/conf/local_transport, proxy:mysql:/opt/postfix/conf/sql/mysql_transport_maps.cf
+smtp_sasl_auth_soft_bounce = no

data/conf/postfix/master.cf

@@ -1,17 +1,23 @@
 smtp       inet  n       -       n       -       1       postscreen
 smtpd      pass  -       -       n       -       -       smtpd
   -o smtpd_helo_restrictions=permit_mynetworks,reject_non_fqdn_helo_hostname
+  -o smtpd_sasl_auth_enable=no
+  -o smtpd_sender_restrictions=permit_mynetworks,reject_unlisted_sender,reject_unknown_sender_domain
 smtps    inet  n       -       n       -       -       smtpd
   -o smtpd_tls_wrappermode=yes
   -o smtpd_client_restrictions=permit_mynetworks,permit_sasl_authenticated,reject
+  -o smtpd_tls_mandatory_protocols=!SSLv2,!SSLv3
+  -o tls_preempt_cipherlist=yes
 submission inet n       -       n       -       -       smtpd
   -o smtpd_client_restrictions=permit_mynetworks,permit_sasl_authenticated,reject
   -o smtpd_enforce_tls=yes
   -o smtpd_tls_security_level=encrypt
+  -o smtpd_tls_mandatory_protocols=!SSLv2,!SSLv3
   -o tls_preempt_cipherlist=yes
 588 inet n      -       n       -       -       smtpd
   -o smtpd_client_restrictions=permit_mynetworks,permit_sasl_authenticated,reject
   -o smtpd_tls_auth_only=no
+  -o smtpd_sender_restrictions=check_sasl_access,regexp:/opt/postfix/conf/allow_mailcow_local.regexp,reject_authenticated_sender_login_mismatch,permit_mynetworks,permit_sasl_authenticated,reject_unlisted_sender,reject_unknown_sender_domain
 590 inet n      -       n       -       -       smtpd
   -o smtpd_client_restrictions=permit_mynetworks,reject
   -o smtpd_tls_auth_only=no
@@ -21,6 +27,8 @@ smtp_enforced_tls      unix  -       -       n       -       -       smtp
   -o smtp_tls_security_level=encrypt
   -o syslog_name=enforced-tls-smtp
   -o smtp_delivery_status_filter=pcre:/opt/postfix/conf/smtp_dsn_filter
+smtp_via_transport_maps      unix  -       -       n       -       -       smtp
+  -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf
 
 tlsproxy   unix  -       -       n       -       0       tlsproxy
 dnsblog    unix  -       -       n       -       0       dnsblog

data/conf/rspamd/custom/bad_asn.map

@@ -1,11 +1,12 @@
 # High spam networks, disabled by default
+# ASN:SCORE DESC
+# Remove comment to enable score
 #201942:5 #Soltia Consulting SL - ipinfo.io
-#16276:5 #OVH
+#16276:2 #OVH
-#12876:5 #ONLINE S.A.S
+#12876:2 #ONLINE S.A.S
-#31034:5
+#31034:5 #ARUBA-ASN, IT
-#12874:5
+#12874:5 #FASTWEB, IT
-#30823:5
+#30823:3 #PKV spam
-#197071:5
 #42831:5 #UK Dedicated Servers Ltd
 #29119:5 #Aire Networks del Mediterraneo S.L.U.
 #13335:5 #Cloudflare
@@ -17,7 +18,7 @@
 #14061:4 #Digitalocean
 #55293:4 #A2 Hosting
 #63018:4 #US Dedicated
-#197518:2
+#197518:2 #RACKMARKT
 #44493:2
 #46606:2
 #49505:2
@@ -25,3 +26,5 @@
 #197695:2
 #198068:2
 #43146:2
+#49100:4
+#39364:4

data/conf/rspamd/custom/global_mime_from_blacklist.map

@@ -0,0 +1 @@
+# /.+example\.com/i

data/conf/rspamd/custom/global_mime_from_whitelist.map

@@ -0,0 +1 @@
+# /.+example\.com/i

data/conf/rspamd/custom/global_rcpt_blacklist.map

@@ -0,0 +1 @@
+# /.+example\.com/i

data/conf/rspamd/custom/global_rcpt_whitelist.map

@@ -0,0 +1 @@
+# /.+example\.com/i

data/conf/rspamd/custom/global_smtp_from_blacklist.map

@@ -0,0 +1 @@
+# /.+example\.com/i

data/conf/rspamd/custom/global_smtp_from_whitelist.map

@@ -0,0 +1 @@
+# /.+example\.com/i

data/conf/rspamd/dynmaps/settings.php

@@ -6,10 +6,13 @@ then any of these will trigger the rule. If a rule is triggered then no more rul
 */
 header('Content-Type: text/plain');
 require_once "vars.inc.php";
+// Getting headers sent by the client.
+//$headers = apache_request_headers();
 
 ini_set('error_reporting', 0);
 
-$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
+//$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
 $opt = [
     PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
     PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
@@ -24,14 +27,50 @@ catch (PDOException $e) {
   exit;
 }
 
+// Check if db changed and return header
+/*$stmt = $pdo->prepare("SELECT UNIX_TIMESTAMP(UPDATE_TIME) AS `db_update_time` FROM information_schema.tables
+  WHERE `TABLE_NAME` = 'filterconf'
+    AND TABLE_SCHEMA = :dbname;");
+$stmt->execute(array(
+  ':dbname' => $database_name
+));
+$db_update_time = $stmt->fetch(PDO::FETCH_ASSOC)['db_update_time'];
+
+if (isset($headers['If-Modified-Since']) && (strtotime($headers['If-Modified-Since']) == $db_update_time)) {
+  header('Last-Modified: '.gmdate('D, d M Y H:i:s', $db_update_time).' GMT', true, 304);
+  exit;
+} else {
+  header('Last-Modified: '.gmdate('D, d M Y H:i:s', $db_update_time).' GMT', true, 200);
+}
+*/
+
 function parse_email($email) {
-  if(!filter_var($email, FILTER_VALIDATE_EMAIL)) return false;
+  if (!filter_var($email, FILTER_VALIDATE_EMAIL)) return false;
   $a = strrpos($email, '@');
   return array('local' => substr($email, 0, $a), 'domain' => substr($email, $a));
 }
 
+function wl_by_sogo() {
+  global $pdo;
+  $rcpt = array();
+  $stmt = $pdo->query("SELECT DISTINCT(`sogo_folder_info`.`c_path2`) AS `user`, GROUP_CONCAT(`sogo_quick_contact`.`c_mail`) AS `contacts` FROM `sogo_folder_info`
+    INNER JOIN `sogo_quick_contact` ON `sogo_quick_contact`.`c_folder_id` = `sogo_folder_info`.`c_folder_id`
+      GROUP BY `c_path2`");
+  $sogo_contacts = $stmt->fetchAll(PDO::FETCH_ASSOC);
+  while ($row = array_shift($sogo_contacts)) {
+    foreach (explode(',', $row['contacts']) as $contact) {
+      if (!filter_var($contact, FILTER_VALIDATE_EMAIL)) {
+        continue;
+      }
+      $rcpt[$row['user']][] = '/^' . str_replace('/', '\/', $contact) . '$/i';
+    }
+  }
+  return $rcpt;
+}
+
 function ucl_rcpts($object, $type) {
   global $pdo;
+  $rcpt = array();
   if ($type == 'mailbox') {
     // Standard aliases
     $stmt = $pdo->prepare("SELECT `address` FROM `alias`
@@ -81,17 +120,14 @@ function ucl_rcpts($object, $type) {
       $rcpt[] = '/.*@' . $row['alias_domain'] . '/i';
     }
   }
-  if (!empty($rcpt)) {
+  return $rcpt;
-    return $rcpt;
-  }
-  return false;
 }
 ?>
 settings {
   watchdog {
     priority = 10;
-    rcpt = "/null@localhost/i";
+    rcpt_mime = "/null@localhost/i";
-    from = "/watchdog@localhost/i";
+    from_mime = "/watchdog@localhost/i";
     apply "default" {
       actions {
         reject = 9999.0;
@@ -137,6 +173,40 @@ while ($row = array_shift($rows)) {
 <?php
 }
 
+/*
+// Start SOGo contacts whitelist
+// Priority 4, lower than a domain whitelist (5) and lower than a mailbox whitelist (6)
+*/
+
+foreach (wl_by_sogo() as $user => $contacts) {
+  $username_sane = preg_replace("/[^a-zA-Z0-9]+/", "", $user);
+?>
+  whitelist_sogo_<?=$username_sane;?> {
+<?php
+  foreach ($contacts as $contact) {
+?>
+    from = <?=json_encode($contact, JSON_UNESCAPED_SLASHES);?>;
+<?php
+  }
+?>
+    priority = 4;
+<?php
+    foreach (ucl_rcpts($user, 'mailbox') as $rcpt) {
+?>
+    rcpt = <?=json_encode($rcpt, JSON_UNESCAPED_SLASHES);?>;
+<?php
+    }
+?>
+    apply "default" {
+      SOGO_CONTACT = -99.0;
+    }
+    symbols [
+      "SOGO_CONTACT"
+    ]
+  }
+<?php
+}
+
 /*
 // Start whitelist
 */
@@ -148,15 +218,17 @@ while ($row = array_shift($rows)) {
 ?>
   whitelist_<?=$username_sane;?> {
 <?php
-  $stmt = $pdo->prepare("SELECT GROUP_CONCAT(REPLACE(CONCAT('^', `value`, '$'), '*', '.*') SEPARATOR '|') AS `value` FROM `filterconf`
+  $list_items = array();
+  $stmt = $pdo->prepare("SELECT `value` FROM `filterconf`
     WHERE `object`= :object
       AND `option` = 'whitelist_from'");
   $stmt->execute(array(':object' => $row['object']));
-  $grouped_lists = $stmt->fetchAll(PDO::FETCH_COLUMN);
+  $list_items = $stmt->fetchAll(PDO::FETCH_ASSOC);
-  $value_sane = preg_replace("/\.\./", ".", (preg_replace("/\*/", ".*", $grouped_lists[0])));
+  foreach ($list_items as $item) {
 ?>
-    from = "/(<?=$value_sane;?>)/i";
+    from = "/<?='^' . str_replace('\*', '.*', preg_quote($item['value'], '/')) . '$' ;?>/i";
 <?php
+  }
   if (!filter_var(trim($row['object']), FILTER_VALIDATE_EMAIL)) {
 ?>
     priority = 5;
@@ -185,19 +257,13 @@ while ($row = array_shift($rows)) {
       "MAILCOW_WHITE"
     ]
   }
-  whitelist_header_<?=$username_sane;?> {
+  whitelist_mime_<?=$username_sane;?> {
 <?php
-  $stmt = $pdo->prepare("SELECT GROUP_CONCAT(REPLACE(CONCAT('\<', `value`, '\>'), '*', '.*') SEPARATOR '|') AS `value` FROM `filterconf`
+  foreach ($list_items as $item) {
-    WHERE `object`= :object
-      AND `option` = 'whitelist_from'");
-  $stmt->execute(array(':object' => $row['object']));
-  $grouped_lists = $stmt->fetchAll(PDO::FETCH_COLUMN);
-  $value_sane = preg_replace("/\.\./", ".", (preg_replace("/\*/", ".*", $grouped_lists[0])));
 ?>
-    header = {
+    from_mime = "/<?='^' . str_replace('\*', '.*', preg_quote($item['value'], '/')) . '$' ;?>/i";
-      "From" = "/(<?=$value_sane;?>)/i";
-    }
 <?php
+  }
   if (!filter_var(trim($row['object']), FILTER_VALIDATE_EMAIL)) {
 ?>
     priority = 5;
@@ -240,15 +306,17 @@ while ($row = array_shift($rows)) {
 ?>
   blacklist_<?=$username_sane;?> {
 <?php
-  $stmt = $pdo->prepare("SELECT GROUP_CONCAT(REPLACE(CONCAT('^', `value`, '$'), '*', '.*') SEPARATOR '|') AS `value` FROM `filterconf`
+  $list_items = array();
+  $stmt = $pdo->prepare("SELECT `value` FROM `filterconf`
     WHERE `object`= :object
       AND `option` = 'blacklist_from'");
   $stmt->execute(array(':object' => $row['object']));
-  $grouped_lists = $stmt->fetchAll(PDO::FETCH_COLUMN);
+  $list_items = $stmt->fetchAll(PDO::FETCH_ASSOC);
-  $value_sane = preg_replace("/\.\./", ".", (preg_replace("/\*/", ".*", $grouped_lists[0])));
+  foreach ($list_items as $item) {
 ?>
-    from = "/(<?=$value_sane;?>)/i";
+    from = "/<?='^' . str_replace('\*', '.*', preg_quote($item['value'], '/')) . '$' ;?>/i";
 <?php
+  }
   if (!filter_var(trim($row['object']), FILTER_VALIDATE_EMAIL)) {
 ?>
     priority = 5;
@@ -279,17 +347,11 @@ while ($row = array_shift($rows)) {
   }
   blacklist_header_<?=$username_sane;?> {
 <?php
-  $stmt = $pdo->prepare("SELECT GROUP_CONCAT(REPLACE(CONCAT('\<', `value`, '\>'), '*', '.*') SEPARATOR '|') AS `value` FROM `filterconf`
+  foreach ($list_items as $item) {
-    WHERE `object`= :object
-      AND `option` = 'blacklist_from'");
-  $stmt->execute(array(':object' => $row['object']));
-  $grouped_lists = $stmt->fetchAll(PDO::FETCH_COLUMN);
-  $value_sane = preg_replace("/\.\./", ".", (preg_replace("/\*/", ".*", $grouped_lists[0])));
 ?>
-    header = {
+    from_mime = "/<?='^' . str_replace('\*', '.*', preg_quote($item['value'], '/')) . '$' ;?>/i";
-      "From" = "/(<?=$value_sane;?>)/i";
-    }
 <?php
+  }
   if (!filter_var(trim($row['object']), FILTER_VALIDATE_EMAIL)) {
 ?>
     priority = 5;
@@ -342,7 +404,6 @@ while ($row = array_shift($rows)) {
     priority = 9;
     want_spam = yes;
   }
-
 <?php
 // Start additional content
 
@@ -357,6 +418,9 @@ while ($row = array_shift($rows)) {
     foreach ($content as $line) {
       echo '    ' . $line . PHP_EOL;
     }
+?>
+  }
+<?php
 }
 ?>
 }

data/conf/rspamd/local.d/antivirus.conf

@@ -1,7 +1,11 @@
 clamav {
-  attachments_only = true;
+  # Scan whole message
+  scan_mime_parts = false;
+  #scan_text_mime = true;
+  #scan_image_mime = true;
   symbol = "CLAM_VIRUS";
   type = "clamav";
   log_clean = true;
   servers = "clamd:3310";
+  max_size = 20971520;
 }

data/conf/rspamd/local.d/composites.conf

@@ -1,5 +1,5 @@
 MX_IMPLICIT {
-  expression = "MX_GOOD and MX_MISSING";
+  expression = "MX_GOOD & MX_MISSING";
   score = -0.01;
 }
 VIRUS_FOUND {
@@ -10,3 +10,9 @@ SPF_FAIL_NO_DKIM {
   expression = "R_SPF_FAIL & R_DKIM_NA & !MAILCOW_WHITE";
   score = 10;
 }
+SOGO_CONTACT_EXCLUDE_FWD_HOST {
+  expression = "WHITELISTED_FWD_HOST & ~SOGO_CONTACT";
+}
+SOGO_CONTACT_SPOOFED {
+  expression = "(R_SPF_PERMFAIL | R_SPF_SOFTFAIL | R_SPF_FAIL) & ~SOGO_CONTACT";
+}

data/conf/rspamd/local.d/fuzzy_group.conf

@@ -3,9 +3,9 @@ symbols = {
         weight = 2.0;
     }
     "LOCAL_FUZZY_DENIED" {
-        weight = 10.0;
+        weight = 15.0;
     }
     "LOCAL_FUZZY_WHITE" {
-        weight = -3.4;
+        weight = -10.0;
     }
 }

data/conf/rspamd/local.d/history_redis.conf

@@ -0,0 +1 @@
+nrows = 1000;

data/conf/rspamd/local.d/metadata_exporter.conf

@@ -1,10 +1,26 @@
 rules {
-        QUARANTINE {
+  QUARANTINE {
-                backend = "http";
+    backend = "http";
-                url = "http://nginx:9081/pipe.php";
+    url = "http://nginx:9081/pipe.php";
-                selector = "is_reject";
+    selector = "is_reject";
-                formatter = "default";
+    formatter = "default";
-                meta_headers = true;
+    meta_headers = true;
-        }
+  }
+	RLINFO {
+		backend = "http";
+		url = "http://nginx:9081/pipe_rl.php";
+		selector = "ratelimited";
+		formatter = "json";
+	}
+}
+custom_select {
+  ratelimited = <<EOD
+return function(task)
+  local ratelimited = task:get_symbol("RATELIMITED")
+  if ratelimited then
+    return true
+  end
+  return
+end
+EOD;
 }
-

data/conf/rspamd/local.d/metrics.conf

@@ -1,7 +1,7 @@
 actions {
 	reject = 15;
-	add_header = 5;
+	add_header = 8;
-	greylist = 4;
+	greylist = 7;
 }
 
 symbol "MAILCOW_AUTH" {
@@ -13,25 +13,19 @@ group "MX" {
 	symbol "MX_INVALID" {
 	  score = 0.5;
 	  description = "No connectable MX";
-	  one_shot = "true";
+	  one_shot = true;
 	}
 	symbol "MX_MISSING" {
 	  score = 2.0;
 	  description = "No MX record";
-	  one_shot = "true";
+	  one_shot = true;
 	}
 	symbol "MX_GOOD" {
 	  score = -0.01;
 	  description = "MX was ok";
-	  one_shot = "true";
+	  one_shot = true;
 	}
 }
-
-symbol "SPOOFED_SENDER" {
-	description = "Sender is not authenticated but part of mailcow managed domains";
-	score = 1.0;
-}
-
 symbol "CTYPE_MIXED_BOGUS" {
   score = 0.0;
 }

data/conf/rspamd/local.d/milter_headers.conf

@@ -7,6 +7,9 @@ routines {
     value = "YES";
     remove = 1;
   }
+  fuzzy-hashes {
+    header = "X-Rspamd-Fuzzy";
+  }
   authentication-results {
     header = "Authentication-Results";
     remove = 1;

data/conf/rspamd/local.d/multimap.conf

@@ -25,13 +25,6 @@ WHITELISTED_FWD_HOST {
   symbols_set = ["WHITELISTED_FWD_HOST"];
 }
 
-KEEP_SPAM {
-  type = "ip";
-  map = "redis://KEEP_SPAM";
-  action = "accept";
-  symbols_set = ["KEEP_SPAM"];
-}
-
 LOCAL_BL_ASN {
   require_symbols = "!MAILCOW_WHITE";
   type = "asn";
@@ -41,10 +34,52 @@ LOCAL_BL_ASN {
   symbols_set = ["LOCAL_BL_ASN"];
 }
 
-#SPOOFED_SENDER {
+GLOBAL_SMTP_FROM_WL {
-#  type = "rcpt";
+  type = "from";
-#  filter = "email:domain:tld";
+  map = "$LOCAL_CONFDIR/custom/global_smtp_from_whitelist.map";
-#  map = "redis://DOMAIN_MAP";
+  regexp = true;
-#  require_symbols = "AUTH_NA | !RCVD_VIA_SMTP_AUTH";
+  prefilter = true;
-#  symbols_set = ["SPOOFED_SENDER"];
+  action = "accept";
-#}
+}
+
+GLOBAL_SMTP_FROM_BL {
+  type = "from";
+  map = "$LOCAL_CONFDIR/custom/global_smtp_from_blacklist.map";
+  regexp = true;
+  prefilter = true;
+  action = "reject";
+}
+
+GLOBAL_MIME_FROM_WL {
+  type = "header";
+  header = "from";
+  map = "$LOCAL_CONFDIR/custom/global_mime_from_whitelist.map";
+  regexp = true;
+  prefilter = true;
+  action = "accept";
+}
+
+GLOBAL_MIME_FROM_BL {
+  type = "header";
+  header = "from";
+  map = "$LOCAL_CONFDIR/custom/global_mime_from_blacklist.map";
+  regexp = true;
+  prefilter = true;
+  action = "reject";
+}
+
+GLOBAL_RCPT_WL {
+  type = "rcpt";
+  map = "$LOCAL_CONFDIR/custom/global_rcpt_whitelist.map";
+  regexp = true;
+  prefilter = true;
+  action = "accept";
+}
+
+GLOBAL_RCPT_BL {
+  type = "rcpt";
+  map = "$LOCAL_CONFDIR/custom/global_rcpt_blacklist.map";
+  regexp = true;
+  prefilter = true;
+  action = "reject";
+}

data/conf/rspamd/local.d/options.inc

@@ -3,7 +3,7 @@ dns {
 }
 map_watch_interval = 30s;
 dns {
-  timeout = 15s;
+  timeout = 4s;
-  retransmits = 5;
+  retransmits = 2;
 }
 disable_monitoring = true;

data/conf/rspamd/local.d/policies_group.conf

@@ -3,6 +3,15 @@ symbols = {
         score = 0.0;
     }
     "R_SPF_FAIL" {
-        score = 4.0;
+        score = 10.0;
+    }
+    "R_SPF_PERMFAIL" {
+        score = 10.0;
+    }
+    "R_DKIM_REJECT" {
+        score = 10.0;
+    }
+    "R_DKIM_PERMFAIL" {
+        score = 10.0;
     }
 }

data/conf/rspamd/local.d/rspamd.conf.local

@@ -1,16 +0,0 @@
-# rspamd.conf.local
-
-worker "fuzzy" {
-  # Socket to listen on (UDP and TCP from rspamd 1.3)
-  bind_socket = "*:11445";
-  allow_update = ["127.0.0.1", "::1"];
-  # Number of processes to serve this storage (useful for read scaling)
-  count = 2;
-  # Backend ("sqlite" or "redis" - default "sqlite")
-  backend = "redis";
-  # Hashes storage time (3 months)
-  expire = 90d;
-  # Synchronize updates to the storage each minute
-  sync = 1min;
-}
-

data/conf/rspamd/lua/rspamd.local.lua

@@ -7,6 +7,68 @@ rspamd_config.MAILCOW_AUTH = {
 	end
 }
 
+rspamd_config:register_symbol({
+  name = 'KEEP_SPAM',
+  type = 'prefilter',
+  callback = function(task)
+    local util = require("rspamd_util")
+    local rspamd_logger = require "rspamd_logger"
+    local rspamd_ip = require 'rspamd_ip'
+    local uname = task:get_user()
+
+    if uname then
+      return false
+    end
+
+    local redis_params = rspamd_parse_redis_server('keep_spam')
+    local ip = task:get_from_ip()
+
+    if not ip:is_valid() then
+      return false
+    end
+
+    local from_ip_string = tostring(ip)
+    ip_check_table = {from_ip_string}
+
+    local maxbits = 128
+    local minbits = 32
+    if ip:get_version() == 4 then
+        maxbits = 32
+        minbits = 8
+    end
+    for i=maxbits,minbits,-1 do
+      local nip = ip:apply_mask(i):to_string() .. "/" .. i
+      table.insert(ip_check_table, nip)
+    end
+    local function keep_spam_cb(err, data)
+      if err then
+        rspamd_logger.infox(rspamd_config, "keep_spam query request for ip %s returned invalid or empty data (\"%s\") or error (\"%s\")", ip, data, err)
+        return false
+      else
+        for k,v in pairs(data) do
+          if (v and v ~= userdata and v == '1') then
+            rspamd_logger.infox(rspamd_config, "found ip in keep_spam map, setting pre-result", v)
+            task:set_pre_result('accept', 'IP matched with forward hosts')
+          end
+        end
+      end
+    end
+    table.insert(ip_check_table, 1, 'KEEP_SPAM')
+    local redis_ret_user = rspamd_redis_make_request(task,
+      redis_params, -- connect params
+      'KEEP_SPAM', -- hash key
+      false, -- is write
+      keep_spam_cb, --callback
+      'HMGET', -- command
+      ip_check_table -- arguments
+    )
+    if not redis_ret_user then
+      rspamd_logger.infox(rspamd_config, "cannot check keep_spam redis map")
+    end
+  end,
+  priority = 19
+})
+
 rspamd_config:register_symbol({
   name = 'TAG_MOO',
   type = 'postfilter',

data/conf/rspamd/meta_exporter/pipe.php

@@ -6,7 +6,8 @@ require_once "vars.inc.php";
 // Do not show errors, we log to using error_log
 ini_set('error_reporting', 0);
 // Init database
-$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
+//$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
 $opt = [
     PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
     PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
@@ -16,6 +17,7 @@ try {
   $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
 }
 catch (PDOException $e) {
+  error_log("QUARANTINE: " . $e);
   http_response_code(501);
   exit;
 }
@@ -49,6 +51,7 @@ $raw_data = mb_convert_encoding($raw_data_content, 'HTML-ENTITIES', "UTF-8");
 $headers = getallheaders();
 
 $qid      = $headers['X-Rspamd-Qid'];
+$subject  = $headers['X-Rspamd-Subject'];
 $score    = $headers['X-Rspamd-Score'];
 $rcpts    = $headers['X-Rspamd-Rcpt'];
 $user     = $headers['X-Rspamd-User'];
@@ -60,12 +63,11 @@ $symbols  = $headers['X-Rspamd-Symbols'];
 $raw_size = (int)$_SERVER['CONTENT_LENGTH'];
 
 try {
-  if ($max_size = $redis->Get('Q_MAX_SIZE')) {
+  $max_size = (int)$redis->Get('Q_MAX_SIZE');
-    if (!empty($max_size) && ($max_size * 1048576) < $raw_size) {
+  if (($max_size * 1048576) < $raw_size) {
-      error_log(sprintf("Message too large: %d exceeds %d", $raw_size, ($max_size * 1048576)));
+    error_log(sprintf("QUARANTINE: Message too large: %d b exceeds %d b", $raw_size, ($max_size * 1048576)));
-      http_response_code(505);
+    http_response_code(505);
-      exit;
+    exit;
-    }
   }
   if ($exclude_domains = $redis->Get('Q_EXCLUDE_DOMAINS')) {
     $exclude_domains = json_decode($exclude_domains, true);
@@ -73,7 +75,7 @@ try {
   $retention_size = (int)$redis->Get('Q_RETENTION_SIZE');
 }
 catch (RedisException $e) {
-  error_log($e);
+  error_log("QUARANTINE: " . $e);
   http_response_code(504);
   exit;
 }
@@ -82,6 +84,9 @@ $rcpt_final_mailboxes = array();
 
 // Loop through all rcpts
 foreach (json_decode($rcpts, true) as $rcpt) {
+  // Remove tag
+  $rcpt = preg_replace('/^(.*?)\+.*(@.*)$/', '$1$2', $rcpt);
+  
   // Break rcpt into local part and domain part
   $parsed_rcpt = parse_email($rcpt);
   
@@ -92,14 +97,14 @@ foreach (json_decode($rcpts, true) as $rcpt) {
     }
   }
   catch (RedisException $e) {
-    error_log($e);
+    error_log("QUARANTINE: " . $e);
     http_response_code(504);
     exit;
   }
 
   // Skip if domain is excluded
   if (in_array($parsed_rcpt['domain'], $exclude_domains)) {
-    error_log(sprintf("Skipped domain %s", $parsed_rcpt['domain']));
+    error_log(sprintf("QUARANTINE: Skipped domain %s", $parsed_rcpt['domain']));
     continue;
   }
 
@@ -134,12 +139,12 @@ foreach (json_decode($rcpts, true) as $rcpt) {
 
       // Loop through all found gotos
       foreach ($gotos_array as $index => &$goto) {
-        error_log("quarantine pipe: query " . $goto . " as username from mailbox");
+        error_log("QUARANTINE: quarantine pipe: query " . $goto . " as username from mailbox");
         $stmt = $pdo->prepare("SELECT `username` FROM `mailbox` WHERE `username` = :goto AND `active`= '1';");
         $stmt->execute(array(':goto' => $goto));
         $username = $stmt->fetch(PDO::FETCH_ASSOC)['username'];
         if (!empty($username)) {
-          error_log("quarantine pipe: mailbox found: " . $username);
+          error_log("QUARANTINE: quarantine pipe: mailbox found: " . $username);
           // Current goto is a mailbox, save to rcpt_final_mailboxes if not a duplicate
           if (!in_array($username, $rcpt_final_mailboxes)) {
             $rcpt_final_mailboxes[] = $username;
@@ -148,13 +153,13 @@ foreach (json_decode($rcpts, true) as $rcpt) {
         else {
           $parsed_goto = parse_email($goto);
           if (!$redis->hGet('DOMAIN_MAP', $parsed_goto['domain'])) {
-            error_log($goto . " is not a mailcow handled mailbox or alias address");
+            error_log("QUARANTINE:" . $goto . " is not a mailcow handled mailbox or alias address");
           }
           else {
             $stmt = $pdo->prepare("SELECT `goto` FROM `alias` WHERE `address` = :goto AND `active` = '1'");
             $stmt->execute(array(':goto' => $goto));
             $goto_branch = $stmt->fetch(PDO::FETCH_ASSOC)['goto'];
-            error_log("quarantine pipe: goto address " . $goto . " is a alias branch for " . $goto_branch);
+            error_log("QUARANTINE: quarantine pipe: goto address " . $goto . " is a alias branch for " . $goto_branch);
             $goto_branch_array = explode(',', $goto_branch);
           }
         }
@@ -174,23 +179,24 @@ foreach (json_decode($rcpts, true) as $rcpt) {
       // Force exit if loop cannot be solved
       // Postfix does not allow for alias loops, so this should never happen.
       $loop_c++;
-      error_log("quarantine pipe: goto array count on loop #". $loop_c . " is " . count($gotos_array));
+      error_log("QUARANTINE: quarantine pipe: goto array count on loop #". $loop_c . " is " . count($gotos_array));
     }
   }
   catch (PDOException $e) {
-    error_log($e->getMessage());
+    error_log("QUARANTINE: " . $e->getMessage());
     http_response_code(502);
     exit;
   }
 }
 
 foreach ($rcpt_final_mailboxes as $rcpt) {
-  error_log("quarantine pipe: processing quarantine message for rcpt " . $rcpt);
+  error_log("QUARANTINE: quarantine pipe: processing quarantine message for rcpt " . $rcpt);
   try {
-    $stmt = $pdo->prepare("INSERT INTO `quarantine` (`qid`, `score`, `sender`, `rcpt`, `symbols`, `user`, `ip`, `msg`, `action`)
+    $stmt = $pdo->prepare("INSERT INTO `quarantine` (`qid`, `subject`, `score`, `sender`, `rcpt`, `symbols`, `user`, `ip`, `msg`, `action`)
-      VALUES (:qid, :score, :sender, :rcpt, :symbols, :user, :ip, :msg, :action)");
+      VALUES (:qid, :subject, :score, :sender, :rcpt, :symbols, :user, :ip, :msg, :action)");
     $stmt->execute(array(
       ':qid' => $qid,
+      ':subject' => $subject,
       ':score' => $score,
       ':sender' => $sender,
       ':rcpt' => $rcpt,
@@ -217,7 +223,7 @@ foreach ($rcpt_final_mailboxes as $rcpt) {
     ));
   }
   catch (PDOException $e) {
-    error_log($e->getMessage());
+    error_log("QUARANTINE: " . $e->getMessage());
     http_response_code(503);
     exit;
   }

data/conf/rspamd/meta_exporter/pipe_rl.php

@@ -0,0 +1,38 @@
+<?php
+// File size is limited by Nginx site to 10M
+// To speed things up, we do not include prerequisites
+header('Content-Type: text/plain');
+require_once "vars.inc.php";
+// Do not show errors, we log to using error_log
+ini_set('error_reporting', 0);
+// Init Redis
+$redis = new Redis();
+$redis->connect('redis-mailcow', 6379);
+
+$raw_data_content = file_get_contents('php://input');
+$raw_data_decoded = json_decode($raw_data_content, true);
+
+$data['time'] = time();
+$data['rcpt'] = implode(', ', $raw_data_decoded['rcpt']);
+$data['from'] = $raw_data_decoded['from'];
+$data['user'] = $raw_data_decoded['user'];
+$symbol_rl_key = array_search('RATELIMITED', array_column($raw_data_decoded['symbols'], 'name'));
+$data['rl_info'] = implode($raw_data_decoded['symbols'][$symbol_rl_key]['options']);
+preg_match('/(.+)\((.+)\)/i', $data['rl_info'], $rl_matches);
+if (!empty($rl_matches[1]) && !empty($rl_matches[2])) {
+  $data['rl_name'] = $rl_matches[1];
+  $data['rl_hash'] = $rl_matches[2];
+}
+else {
+  $data['rl_name'] = 'err';
+  $data['rl_hash'] = 'err';
+}
+$data['qid'] = $raw_data_decoded['qid'];
+$data['ip'] = $raw_data_decoded['ip'];
+$data['message_id'] = $raw_data_decoded['message_id'];
+$data['header_subject'] = implode(' ', $raw_data_decoded['header_subject']);
+$data['header_from'] = implode(', ', $raw_data_decoded['header_from']);
+
+$redis->lpush('RL_LOG', json_encode($data));
+exit;
+

data/conf/rspamd/override.d/logging.inc

@@ -1,4 +1,5 @@
 type = "console";
 systemd = false;
+level = "silent";
 .include "$CONFDIR/logging.inc"
 .include(try=true; priority=20) "$CONFDIR/override.d/logging.custom.inc"

data/conf/rspamd/override.d/ratelimit.conf

@@ -1,11 +1,12 @@
 rates {
     # Format: "1 / 1h" or "20 / 1m" etc. - global ratelimits are disabled by default
-    to = "100 / 1s";
+    to = "45 / 1m";
-    to_ip = "100 / 1s";
+    to_ip = "360 / 1m";
-    to_ip_from = "100 / 1s";
+    to_ip_from = "180 / 1m";
     bounce_to = "100 / 1s";
     bounce_to_ip = "100 / 1s";
 }
 whitelisted_rcpts = "postmaster,mailer-daemon";
 max_rcpt = 5;
 custom_keywords = "/etc/rspamd/lua/ratelimit.lua";
+info_symbol = "RATELIMITED";

data/conf/rspamd/override.d/worker-controller.inc

@@ -1,7 +1,7 @@
 bind_socket = "*:11334";
-count = 2;
+count = 1;
 secure_ip = "127.0.0.1";
 secure_ip = "::1";
-bind_socket = "/rspamd-sock/rspamd.sock mode=0666 owner=nobody";
+bind_socket = "/var/lib/rspamd/rspamd.sock mode=0666 owner=nobody";
 .include(try=true; priority=10) "$CONFDIR/override.d/worker-controller-password.inc"
 .include(try=true; priority=20) "$CONFDIR/override.d/worker-controller.custom.inc"