From 223e73e8f77c3282c1f3e06bc0543fb3fb415530 Mon Sep 17 00:00:00 2001 From: andryyy Date: Mon, 10 Dec 2018 23:23:46 +0100 Subject: [PATCH 01/50] [Compose] Update SOGo and Dovecot images --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c45137d6..5970ca8d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -138,7 +138,7 @@ services: - phpfpm sogo-mailcow: - image: mailcow/sogo:1.44 + image: mailcow/sogo:1.45 build: ./data/Dockerfiles/sogo environment: - DBNAME=${DBNAME} @@ -162,7 +162,7 @@ services: - sogo dovecot-mailcow: - image: mailcow/dovecot:1.48 + image: mailcow/dovecot:1.49 build: ./data/Dockerfiles/dovecot cap_add: - NET_BIND_SERVICE From 51536235393664db9effb913a768ab09a0f2ff26 Mon Sep 17 00:00:00 2001 From: andryyy Date: Mon, 10 Dec 2018 23:23:56 +0100 Subject: [PATCH 02/50] [Dovecot] Add master user to userdb (to be used in SOGo) [SOGo] Use sieve.creds to authenticate against Dovecot and send email reminders --- data/Dockerfiles/dovecot/docker-entrypoint.sh | 2 +- data/Dockerfiles/sogo/bootstrap-sogo.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/data/Dockerfiles/dovecot/docker-entrypoint.sh b/data/Dockerfiles/dovecot/docker-entrypoint.sh index 86b0db77..ec4b63a5 100755 --- a/data/Dockerfiles/dovecot/docker-entrypoint.sh +++ b/data/Dockerfiles/dovecot/docker-entrypoint.sh @@ -116,7 +116,7 @@ RAND_USER=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 16 | head -n 1) RAND_PASS=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 24 | head -n 1) echo ${RAND_USER}@mailcow.local:{SHA1}$(echo -n ${RAND_PASS} | sha1sum | awk '{print $1}') > /usr/local/etc/dovecot/dovecot-master.passwd - +echo ${RAND_USER}@mailcow.local::::::: > /usr/local/etc/dovecot/dovecot-master.userdb echo ${RAND_USER}@mailcow.local:${RAND_PASS} > /etc/sogo/sieve.creds # 401 is user dovecot diff --git a/data/Dockerfiles/sogo/bootstrap-sogo.sh b/data/Dockerfiles/sogo/bootstrap-sogo.sh index 9fc8b502..7fa0668d 100755 --- a/data/Dockerfiles/sogo/bootstrap-sogo.sh +++ b/data/Dockerfiles/sogo/bootstrap-sogo.sh @@ -167,6 +167,9 @@ echo ' chown sogo:sogo -R /var/lib/sogo/ chmod 600 /var/lib/sogo/GNUstep/Defaults/sogod.plist +# Add credentials to alarms +sed -i 's/\/usr\/sbin\/sogo-ealarms-notify/\/usr\/sbin\/sogo-ealarms-notify -p \/etc\/sogo\/sieve.creds/g' /etc/cron.d/sogo + # Prevent theme switching sed -i \ -e 's/eaf5e9/E3F2FD/g' \ From fa3525e2ddb349e8f9938e8226d0811fc583dbf3 Mon Sep 17 00:00:00 2001 From: andryyy Date: Mon, 10 Dec 2018 23:24:49 +0100 Subject: [PATCH 03/50] [SOGo] Enable EMailAlarms --- data/conf/sogo/sogo.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/conf/sogo/sogo.conf b/data/conf/sogo/sogo.conf index de030ae9..5bb7963c 100644 --- a/data/conf/sogo/sogo.conf +++ b/data/conf/sogo/sogo.conf @@ -11,7 +11,7 @@ SOGoDraftsFolderName = "Drafts"; SOGoJunkFolderName= "Junk"; SOGoMailDomain = "sogo.local"; - SOGoEnableEMailAlarms = NO; + SOGoEnableEMailAlarms = YES; SOGoFoldersSendEMailNotifications = YES; SOGoForwardEnabled = YES; SOGoUIAdditionalJSFiles = (js/theme-blue.js, js/custom-sogo.js); From 9b720bb07a19d301573917b01b06172baf4a602d Mon Sep 17 00:00:00 2001 From: andryyy Date: Mon, 10 Dec 2018 23:25:37 +0100 Subject: [PATCH 04/50] [Dovecot] Add master user to userdb (to be used in SOGo) --- data/conf/dovecot/dovecot.conf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/data/conf/dovecot/dovecot.conf b/data/conf/dovecot/dovecot.conf index 230defb5..8da9c06e 100644 --- a/data/conf/dovecot/dovecot.conf +++ b/data/conf/dovecot/dovecot.conf @@ -58,6 +58,11 @@ passdb { result_failure = continue result_internalfail = continue } +passdb { + driver = passwd-file + args = /usr/local/etc/dovecot/dovecot-master.passwd + skip = authenticated +} # Set doveadm_password=your-secret-password in data/conf/dovecot/extra.conf (create if missing) service doveadm { inet_listener { @@ -257,9 +262,14 @@ service lmtp { listen = *,[::] ssl_cert = Date: Mon, 10 Dec 2018 23:26:28 +0100 Subject: [PATCH 05/50] [Git] Add allow_mailcow_local.regexp and dovecot-master.userdb --- .gitignore | 2 ++ data/conf/postfix/master.cf | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 91233788..f858422d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ rebuild-images.sh data/conf/sogo/sieve.creds data/conf/dovecot/dovecot-master.passwd +data/conf/dovecot/dovecot-master.userdb mailcow.conf mailcow.conf_backup data/conf/nginx/*.active data/conf/postfix/sql +data/conf/postfix/allow_mailcow_local.regexp data/conf/dovecot/sql data/conf/nextcloud-*.bak data/web/inc/vars.local.inc.php diff --git a/data/conf/postfix/master.cf b/data/conf/postfix/master.cf index 5de97ee0..07d0d853 100644 --- a/data/conf/postfix/master.cf +++ b/data/conf/postfix/master.cf @@ -13,6 +13,8 @@ submission inet n - n - - smtpd 588 inet n - n - - smtpd -o smtpd_client_restrictions=permit_mynetworks,permit_sasl_authenticated,reject -o smtpd_tls_auth_only=no + -o smtpd_sender_restrictions=check_sasl_access,regexp:/opt/postfix/conf/allow_mailcow_local.regexp,reject_authenticated_sender_login_mismatch,permit_mynetworks,permit_sasl_authenticated,reject_unlisted_sender,reject_unknown_sender_domain + 590 inet n - n - - smtpd -o smtpd_client_restrictions=permit_mynetworks,reject -o smtpd_tls_auth_only=no From 497b6a39dedbe106ca59080da8ba5bc91ac3d3d5 Mon Sep 17 00:00:00 2001 From: andryyy Date: Tue, 11 Dec 2018 17:16:53 +0100 Subject: [PATCH 06/50] [Postfix] Add missing regexp map, fixes #2083 --- data/conf/postfix/allow_mailcow_local.regexp | 1 + 1 file changed, 1 insertion(+) create mode 100644 data/conf/postfix/allow_mailcow_local.regexp diff --git a/data/conf/postfix/allow_mailcow_local.regexp b/data/conf/postfix/allow_mailcow_local.regexp new file mode 100644 index 00000000..0da4593c --- /dev/null +++ b/data/conf/postfix/allow_mailcow_local.regexp @@ -0,0 +1 @@ +/^(.+)@mailcow.local/ OK From 92b9b2413e2c6469d3a89cfb7f7b2b1476b07810 Mon Sep 17 00:00:00 2001 From: Geitenijs <40541903+Geitenijs@users.noreply.github.com> Date: Wed, 12 Dec 2018 10:00:23 +0100 Subject: [PATCH 07/50] Update lang.nl.php --- data/web/lang/lang.nl.php | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/data/web/lang/lang.nl.php b/data/web/lang/lang.nl.php index 6193db1e..240fcab5 100644 --- a/data/web/lang/lang.nl.php +++ b/data/web/lang/lang.nl.php @@ -411,8 +411,10 @@ $lang['edit']['delete1'] = 'Verwijder van oorsprong wanneer voltooid'; $lang['edit']['delete2'] = 'Verwijder berichten die zich niet in de oorsprong bevinden'; $lang['add']['custom_params'] = 'Aangepaste parameters'; $lang['add']['subscribeall'] = 'Abonneer op alle mappen'; -$lang['add']['timeout1'] = 'Time-out voor verbinding met externe host'; -$lang['add']['timeout2'] = 'Time-out voor verbinding met lokale host'; +$lang['add']['timeout1'] = 'Time-out voor verbinding met externe hosts'; +$lang['add']['timeout2'] = 'Time-out voor verbinding met lokale hosts'; +$lang['edit']['timeout1'] = 'Time-out voor verbinding met externe hosts'; +$lang['edit']['timeout2'] = 'Time-out voor verbinding met lokale hosts'; $lang['add']['domain_matches_hostname'] = 'Domein %s komt overeen met hostname'; $lang['add']['domain'] = 'Domein'; @@ -521,7 +523,7 @@ $lang['admin']['dkim_add_key'] = 'Voeg DKIM-sleutel toe'; $lang['admin']['dkim_keys'] = 'DKIM-sleutels'; $lang['admin']['dkim_private_key'] = 'Privésleutel'; $lang['admin']['dkim_domains_wo_keys'] = "Selecteer domeinen met ontbrekende sleutels"; -$lang['admin']['dkim_domains_selector'] = "Onderdeel"; +$lang['admin']['dkim_domains_selector'] = "Noemer"; $lang['admin']['add'] = 'Toevoegen'; $lang['add']['add_domain_restart'] = 'Voeg domein toe en herstart SOGo'; $lang['add']['add_domain_only'] = 'Voeg enkel domein toe'; @@ -553,7 +555,7 @@ $lang['admin']['flush_queue'] = 'Leeg wachtrij'; $lang['admin']['delete_queue'] = 'Verwijder alles'; $lang['admin']['queue_deliver_mail'] = 'Lever af'; $lang['admin']['queue_hold_mail'] = 'Houd vast'; -$lang['admin']['queue_unhold_mail'] = 'Vrijgeven'; +$lang['admin']['queue_unhold_mail'] = 'Geef vrij'; $lang['admin']['username'] = 'Gebruikersnaam'; $lang['admin']['edit'] = 'Wijzig'; $lang['admin']['remove'] = 'Verwijder'; @@ -608,6 +610,9 @@ $lang['admin']['quarantine'] = "Quarantaine"; $lang['admin']['quarantine_retention_size'] = "Maximale retenties per postvak
Gebruik 0 om deze functionaliteit uit te zetten."; $lang['admin']['quarantine_max_size'] = "Maximale grootte in MiB (mail die de limiet overschrijdt zal worden verwijderd)
0 betekent niet onbeperkt!"; $lang['admin']['quarantine_exclude_domains'] = "Sluit domeinen en aliasdomeinen uit:"; +$lang['admin']['quarantine_release_format'] = "Vrijgegeven items worden verstuurd als:"; +$lang['admin']['quarantine_release_format_raw'] = "Origineel"; +$lang['admin']['quarantine_release_format_att'] = "Bijlage"; $lang['admin']['ui_texts'] = "UI-labels en teksten"; $lang['admin']['help_text'] = "Pas hulpteksten onder inlogvenster aan (HTML toegestaan)"; @@ -648,7 +653,7 @@ $lang['danger']['imagick_exception'] = "Error: Er is een probleem opgetreden met $lang['quarantine']['quarantine'] = "Quarantaine"; $lang['quarantine']['learn_spam_delete'] = "Onthoud als spam en verwijder"; $lang['quarantine']['qinfo'] = 'Het quarantainesysteem slaat geweigerde e-mail op, terwijl het voor de afzender lijkt alsof deze niet ontvangen is.
"' . $lang['quarantine']['learn_spam_delete'] . '" traint het systeem om soortgelijke e-mails in de toekomst direct als spam te markeren.
Wees er van bewust dat wanneer er meerdere berichten worden onderzocht, dit mogelijk enige tijd kan duren.'; -$lang['quarantine']['release'] = "Vrijgeven"; +$lang['quarantine']['release'] = "Geef vrij"; $lang['quarantine']['empty'] = 'Geen resultaten'; $lang['quarantine']['toggle_all'] = 'Selecteer alles'; $lang['quarantine']['quick_actions'] = 'Handelingen'; From 3fad851278a44754b3a31042bacf03cb517da737 Mon Sep 17 00:00:00 2001 From: andryyy Date: Wed, 12 Dec 2018 20:24:18 +0100 Subject: [PATCH 08/50] [Nextcloud] Use db 10 for Redis cache --- helper-scripts/nextcloud.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/helper-scripts/nextcloud.sh b/helper-scripts/nextcloud.sh index 8dd8c0fd..1bba17e1 100755 --- a/helper-scripts/nextcloud.sh +++ b/helper-scripts/nextcloud.sh @@ -32,7 +32,11 @@ if [[ ${NC_PURGE} == "y" ]]; then docker exec -it $(docker ps -f name=mysql-mailcow -q) mysql -uroot -p${DBROOT} -e \ "$(docker exec -it $(docker ps -f name=mysql-mailcow -q) mysql -uroot -p${DBROOT} -e "SELECT GROUP_CONCAT('DROP TABLE ', TABLE_SCHEMA, '.', TABLE_NAME SEPARATOR ';') FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME LIKE 'nc_%' AND TABLE_SCHEMA = '${DBNAME}';" -BN)" - docker exec -it $(docker ps -f name=redis-mailcow -q) /bin/sh -c 'redis-cli KEYS "*nextcloud*" | xargs redis-cli DEL' + docker exec -it $(docker ps -f name=redis-mailcow -q) /bin/sh -c ' cat < Date: Wed, 12 Dec 2018 20:31:19 +0100 Subject: [PATCH 09/50] [Nextcloud] Fix headers --- data/assets/nextcloud/nextcloud.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/assets/nextcloud/nextcloud.conf b/data/assets/nextcloud/nextcloud.conf index 72f30240..31bd088a 100644 --- a/data/assets/nextcloud/nextcloud.conf +++ b/data/assets/nextcloud/nextcloud.conf @@ -24,7 +24,8 @@ server { add_header X-Robots-Tag none; add_header X-Download-Options noopen; add_header X-Permitted-Cross-Domain-Policies none; - add_header X-Frame-Options "SAMEORIGIN"; + #add_header X-Frame-Options "SAMEORIGIN"; + add_header Referrer-Policy "no-referrer"; server_name NC_SUBD; From d8906e3d6ce2700cc72e2fad65addf1ce58ab729 Mon Sep 17 00:00:00 2001 From: andryyy Date: Wed, 12 Dec 2018 22:51:55 +0100 Subject: [PATCH 10/50] [Dovecot] Trim more logs --- data/Dockerfiles/dovecot/trim_logs.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/Dockerfiles/dovecot/trim_logs.sh b/data/Dockerfiles/dovecot/trim_logs.sh index 5a4259ba..7630b0ae 100755 --- a/data/Dockerfiles/dovecot/trim_logs.sh +++ b/data/Dockerfiles/dovecot/trim_logs.sh @@ -13,3 +13,6 @@ catch_non_zero "/usr/bin/redis-cli -h redis LTRIM DOVECOT_MAILLOG 0 LOG_LINES" catch_non_zero "/usr/bin/redis-cli -h redis LTRIM SOGO_LOG 0 LOG_LINES" catch_non_zero "/usr/bin/redis-cli -h redis LTRIM NETFILTER_LOG 0 LOG_LINES" catch_non_zero "/usr/bin/redis-cli -h redis LTRIM AUTODISCOVER_LOG 0 LOG_LINES" +catch_non_zero "/usr/bin/redis-cli -h redis LTRIM API_LOG 0 LOG_LINES" +catch_non_zero "/usr/bin/redis-cli -h redis LTRIM RL_LOG 0 LOG_LINES" + From b7c9af5e7530a7ee6246b6e0d585f30b76e2eef7 Mon Sep 17 00:00:00 2001 From: andryyy Date: Thu, 13 Dec 2018 19:52:44 +0100 Subject: [PATCH 11/50] [Dovecot] Give master user a uid and gid, fixes #2093 --- data/Dockerfiles/dovecot/docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/Dockerfiles/dovecot/docker-entrypoint.sh b/data/Dockerfiles/dovecot/docker-entrypoint.sh index ec4b63a5..44cc64cc 100755 --- a/data/Dockerfiles/dovecot/docker-entrypoint.sh +++ b/data/Dockerfiles/dovecot/docker-entrypoint.sh @@ -116,7 +116,7 @@ RAND_USER=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 16 | head -n 1) RAND_PASS=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 24 | head -n 1) echo ${RAND_USER}@mailcow.local:{SHA1}$(echo -n ${RAND_PASS} | sha1sum | awk '{print $1}') > /usr/local/etc/dovecot/dovecot-master.passwd -echo ${RAND_USER}@mailcow.local::::::: > /usr/local/etc/dovecot/dovecot-master.userdb +echo ${RAND_USER}@mailcow.local::5000:5000:::: > /usr/local/etc/dovecot/dovecot-master.userdb echo ${RAND_USER}@mailcow.local:${RAND_PASS} > /etc/sogo/sieve.creds # 401 is user dovecot From 558ba23b93d5d7779769afd6e12ac593312fd8b0 Mon Sep 17 00:00:00 2001 From: andryyy Date: Thu, 13 Dec 2018 19:53:19 +0100 Subject: [PATCH 12/50] [Compose] Update Dovecot image --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5970ca8d..2edbd8d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -162,7 +162,7 @@ services: - sogo dovecot-mailcow: - image: mailcow/dovecot:1.49 + image: mailcow/dovecot:1.51 build: ./data/Dockerfiles/dovecot cap_add: - NET_BIND_SERVICE From c4446b85f1bea5ea8139e119a11f98747dd99aea Mon Sep 17 00:00:00 2001 From: andryyy Date: Sat, 15 Dec 2018 21:19:35 +0100 Subject: [PATCH 13/50] [Rspamd] Add ratelimit.lua (to be removed from Dockerfile with next Rspamd release) --- data/Dockerfiles/rspamd/Dockerfile | 1 + data/Dockerfiles/rspamd/ratelimit.lua | 864 ++++++++++++++++++++++++++ 2 files changed, 865 insertions(+) create mode 100644 data/Dockerfiles/rspamd/ratelimit.lua diff --git a/data/Dockerfiles/rspamd/Dockerfile b/data/Dockerfiles/rspamd/Dockerfile index f51656d9..3b90015b 100644 --- a/data/Dockerfiles/rspamd/Dockerfile +++ b/data/Dockerfiles/rspamd/Dockerfile @@ -21,6 +21,7 @@ RUN apt-get update && apt-get install -y \ COPY settings.conf /etc/rspamd/settings.conf COPY docker-entrypoint.sh /docker-entrypoint.sh +COPY ratelimit.lua /usr/share/rspamd/lua/ratelimit.lua ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/data/Dockerfiles/rspamd/ratelimit.lua b/data/Dockerfiles/rspamd/ratelimit.lua new file mode 100644 index 00000000..f2358a48 --- /dev/null +++ b/data/Dockerfiles/rspamd/ratelimit.lua @@ -0,0 +1,864 @@ +--[[ +Copyright (c) 2011-2017, Vsevolod Stakhov +Copyright (c) 2016-2017, Andrew Lewis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]]-- + +if confighelp then + return +end + +local rspamd_logger = require "rspamd_logger" +local rspamd_util = require "rspamd_util" +local rspamd_lua_utils = require "lua_util" +local lua_redis = require "lua_redis" +local fun = require "fun" +local lua_maps = require "lua_maps" +local lua_util = require "lua_util" +local rspamd_hash = require "rspamd_cryptobox_hash" +local lua_selectors = require "lua_selectors" +local ts = require("tableshape").types + +-- A plugin that implements ratelimits using redis + +local E = {} +local N = 'ratelimit' +local redis_params +-- Senders that are considered as bounce +local settings = { + bounce_senders = { 'postmaster', 'mailer-daemon', '', 'null', 'fetchmail-daemon', 'mdaemon' }, +-- Do not check ratelimits for these recipients + whitelisted_rcpts = { 'postmaster', 'mailer-daemon' }, + prefix = 'RL', + ham_factor_rate = 1.01, + spam_factor_rate = 0.99, + ham_factor_burst = 1.02, + spam_factor_burst = 0.98, + max_rate_mult = 5, + max_bucket_mult = 10, + expire = 60 * 60 * 24 * 2, -- 2 days by default + limits = {}, + allow_local = false, +} + +-- Checks bucket, updating it if needed +-- KEYS[1] - prefix to update, e.g. RL__ +-- KEYS[2] - current time in milliseconds +-- KEYS[3] - bucket leak rate (messages per millisecond) +-- KEYS[4] - bucket burst +-- KEYS[5] - expire for a bucket +-- return 1 if message should be ratelimited and 0 if not +-- Redis keys used: +-- l - last hit +-- b - current burst +-- dr - current dynamic rate multiplier (*10000) +-- db - current dynamic burst multiplier (*10000) +local bucket_check_script = [[ + local last = redis.call('HGET', KEYS[1], 'l') + local now = tonumber(KEYS[2]) + local dynr, dynb, leaked = 0, 0, 0 + if not last then + -- New bucket + redis.call('HSET', KEYS[1], 'l', KEYS[2]) + redis.call('HSET', KEYS[1], 'b', '0') + redis.call('HSET', KEYS[1], 'dr', '10000') + redis.call('HSET', KEYS[1], 'db', '10000') + redis.call('EXPIRE', KEYS[1], KEYS[5]) + return {0, '0', '1', '1', '0'} + end + + last = tonumber(last) + local burst = tonumber(redis.call('HGET', KEYS[1], 'b')) + -- Perform leak + if burst > 0 then + if last < tonumber(KEYS[2]) then + local rate = tonumber(KEYS[3]) + dynr = tonumber(redis.call('HGET', KEYS[1], 'dr')) / 10000.0 + if dynr == 0 then dynr = 0.0001 end + rate = rate * dynr + leaked = ((now - last) * rate) + if leaked > burst then leaked = burst end + burst = burst - leaked + redis.call('HINCRBYFLOAT', KEYS[1], 'b', -(leaked)) + redis.call('HSET', KEYS[1], 'l', KEYS[2]) + end + + dynb = tonumber(redis.call('HGET', KEYS[1], 'db')) / 10000.0 + if dynb == 0 then dynb = 0.0001 end + + if burst > 0 and (burst + 1) > tonumber(KEYS[4]) * dynb then + return {1, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked)} + end + else + burst = 0 + redis.call('HSET', KEYS[1], 'b', '0') + end + + return {0, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked)} +]] +local bucket_check_id + + +-- Updates a bucket +-- KEYS[1] - prefix to update, e.g. RL__ +-- KEYS[2] - current time in milliseconds +-- KEYS[3] - dynamic rate multiplier +-- KEYS[4] - dynamic burst multiplier +-- KEYS[5] - max dyn rate (min: 1/x) +-- KEYS[6] - max burst rate (min: 1/x) +-- KEYS[7] - expire for a bucket +-- Redis keys used: +-- l - last hit +-- b - current burst +-- dr - current dynamic rate multiplier +-- db - current dynamic burst multiplier +local bucket_update_script = [[ + local last = redis.call('HGET', KEYS[1], 'l') + local now = tonumber(KEYS[2]) + if not last then + -- New bucket + redis.call('HSET', KEYS[1], 'l', KEYS[2]) + redis.call('HSET', KEYS[1], 'b', '1') + redis.call('HSET', KEYS[1], 'dr', '10000') + redis.call('HSET', KEYS[1], 'db', '10000') + redis.call('EXPIRE', KEYS[1], KEYS[7]) + return {1, 1, 1} + end + + local dr, db = 1.0, 1.0 + + if tonumber(KEYS[5]) > 1 then + local rate_mult = tonumber(KEYS[3]) + local rate_limit = tonumber(KEYS[5]) + dr = tonumber(redis.call('HGET', KEYS[1], 'dr')) / 10000 + + if rate_mult > 1.0 and dr < rate_limit then + dr = dr * rate_mult + if dr > 0.0001 then + redis.call('HSET', KEYS[1], 'dr', tostring(math.floor(dr * 10000))) + else + redis.call('HSET', KEYS[1], 'dr', '1') + end + elseif rate_mult < 1.0 and dr > (1.0 / rate_limit) then + dr = dr * rate_mult + if dr > 0.0001 then + redis.call('HSET', KEYS[1], 'dr', tostring(math.floor(dr * 10000))) + else + redis.call('HSET', KEYS[1], 'dr', '1') + end + end + end + + if tonumber(KEYS[6]) > 1 then + local rate_mult = tonumber(KEYS[4]) + local rate_limit = tonumber(KEYS[6]) + db = tonumber(redis.call('HGET', KEYS[1], 'db')) / 10000 + + if rate_mult > 1.0 and db < rate_limit then + db = db * rate_mult + if db > 0.0001 then + redis.call('HSET', KEYS[1], 'db', tostring(math.floor(db * 10000))) + else + redis.call('HSET', KEYS[1], 'db', '1') + end + elseif rate_mult < 1.0 and db > (1.0 / rate_limit) then + db = db * rate_mult + if db > 0.0001 then + redis.call('HSET', KEYS[1], 'db', tostring(math.floor(db * 10000))) + else + redis.call('HSET', KEYS[1], 'db', '1') + end + end + end + + local burst = tonumber(redis.call('HGET', KEYS[1], 'b')) + if burst < 0 then burst = 0 end + + redis.call('HINCRBYFLOAT', KEYS[1], 'b', 1) + redis.call('HSET', KEYS[1], 'l', KEYS[2]) + redis.call('EXPIRE', KEYS[1], KEYS[7]) + + return {tostring(burst), tostring(dr), tostring(db)} +]] +local bucket_update_id + +-- message_func(task, limit_type, prefix, bucket, limit_key) +local message_func = function(_, limit_type, _, _, _) + return string.format('Ratelimit "%s" exceeded', limit_type) +end + + +local function load_scripts(cfg, ev_base) + bucket_check_id = lua_redis.add_redis_script(bucket_check_script, redis_params) + bucket_update_id = lua_redis.add_redis_script(bucket_update_script, redis_params) +end + +local limit_parser +local function parse_string_limit(lim, no_error) + local function parse_time_suffix(s) + if s == 's' then + return 1 + elseif s == 'm' then + return 60 + elseif s == 'h' then + return 3600 + elseif s == 'd' then + return 86400 + end + end + local function parse_num_suffix(s) + if s == '' then + return 1 + elseif s == 'k' then + return 1000 + elseif s == 'm' then + return 1000000 + elseif s == 'g' then + return 1000000000 + end + end + local lpeg = require "lpeg" + + if not limit_parser then + local digit = lpeg.R("09") + limit_parser = {} + limit_parser.integer = + (lpeg.S("+-") ^ -1) * + (digit ^ 1) + limit_parser.fractional = + (lpeg.P(".") ) * + (digit ^ 1) + limit_parser.number = + (limit_parser.integer * + (limit_parser.fractional ^ -1)) + + (lpeg.S("+-") * limit_parser.fractional) + limit_parser.time = lpeg.Cf(lpeg.Cc(1) * + (limit_parser.number / tonumber) * + ((lpeg.S("smhd") / parse_time_suffix) ^ -1), + function (acc, val) return acc * val end) + limit_parser.suffixed_number = lpeg.Cf(lpeg.Cc(1) * + (limit_parser.number / tonumber) * + ((lpeg.S("kmg") / parse_num_suffix) ^ -1), + function (acc, val) return acc * val end) + limit_parser.limit = lpeg.Ct(limit_parser.suffixed_number * + (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) * + limit_parser.time) + end + local t = lpeg.match(limit_parser.limit, lim) + + if t and t[1] and t[2] and t[2] ~= 0 then + return t[2], t[1] + end + + if not no_error then + rspamd_logger.errx(rspamd_config, 'bad limit: %s', lim) + end + + return nil +end + +local function str_to_rate(str) + local divider,divisor = parse_string_limit(str, false) + + if not divisor then + rspamd_logger.errx(rspamd_config, 'bad rate string: %s', str) + + return nil + end + + return divisor / divider +end + +local bucket_schema = ts.shape{ + burst = ts.number + ts.string / lua_util.dehumanize_number, + rate = ts.number + ts.string / str_to_rate +} + +local function parse_limit(name, data) + if type(data) == 'table' then + -- 2 cases here: + -- * old limit in format [burst, rate] + -- * vector of strings in Andrew's string format (removed from 1.8.2) + -- * proper bucket table + if #data == 2 and tonumber(data[1]) and tonumber(data[2]) then + -- Old style ratelimit + rspamd_logger.warnx(rspamd_config, 'old style ratelimit for %s', name) + if tonumber(data[1]) > 0 and tonumber(data[2]) > 0 then + return { + burst = data[1], + rate = data[2] + } + elseif data[1] ~= 0 then + rspamd_logger.warnx(rspamd_config, 'invalid numbers for %s', name) + else + rspamd_logger.infox(rspamd_config, 'disable limit %s, burst is zero', name) + end + + return nil + else + local parsed_bucket,err = bucket_schema:transform(data) + + if not parsed_bucket or err then + rspamd_logger.errx(rspamd_config, 'cannot parse bucket for %s: %s; original value: %s', + name, err, data) + else + return parsed_bucket + end + end + elseif type(data) == 'string' then + local rep_rate, burst = parse_string_limit(data) + rspamd_logger.warnx(rspamd_config, 'old style rate bucket config detected for %s: %s', + name, data) + if rep_rate and burst then + return { + burst = burst, + rate = burst / rep_rate -- reciprocal + } + end + end + + return nil +end + +--- Check whether this addr is bounce +local function check_bounce(from) + return fun.any(function(b) return b == from end, settings.bounce_senders) +end + +local keywords = { + ['ip'] = { + ['get_value'] = function(task) + local ip = task:get_ip() + if ip and ip:is_valid() then return tostring(ip) end + return nil + end, + }, + ['rip'] = { + ['get_value'] = function(task) + local ip = task:get_ip() + if ip and ip:is_valid() and not ip:is_local() then return tostring(ip) end + return nil + end, + }, + ['from'] = { + ['get_value'] = function(task) + local from = task:get_from(0) + if ((from or E)[1] or E).addr then + return string.lower(from[1]['addr']) + end + return nil + end, + }, + ['bounce'] = { + ['get_value'] = function(task) + local from = task:get_from(0) + if not ((from or E)[1] or E).user then + return '_' + end + if check_bounce(from[1]['user']) then return '_' else return nil end + end, + }, + ['asn'] = { + ['get_value'] = function(task) + local asn = task:get_mempool():get_variable('asn') + if not asn then + return nil + else + return asn + end + end, + }, + ['user'] = { + ['get_value'] = function(task) + local auser = task:get_user() + if not auser then + return nil + else + return auser + end + end, + }, + ['to'] = { + ['get_value'] = function(task) + return task:get_principal_recipient() + end, + }, + ['digest'] = { + ['get_value'] = function(task) + return task:get_digest() + end, + }, + ['attachments'] = { + ['get_value'] = function(task) + local parts = task:get_parts() or E + local digests = {} + + for _,p in ipairs(parts) do + if p:get_filename() then + table.insert(digests, p:get_digest()) + end + end + + if #digests > 0 then + return table.concat(digests, '') + end + + return nil + end, + }, + ['files'] = { + ['get_value'] = function(task) + local parts = task:get_parts() or E + local files = {} + + for _,p in ipairs(parts) do + local fname = p:get_filename() + if fname then + table.insert(files, fname) + end + end + + if #files > 0 then + return table.concat(files, ':') + end + + return nil + end, + }, +} + +local function gen_rate_key(task, rtype, bucket) + local key_t = {tostring(lua_util.round(100000.0 / bucket.burst))} + local key_keywords = lua_util.str_split(rtype, '_') + local have_user = false + + for _, v in ipairs(key_keywords) do + local ret + + if keywords[v] and type(keywords[v]['get_value']) == 'function' then + ret = keywords[v]['get_value'](task) + end + if not ret then return nil end + if v == 'user' then have_user = true end + if type(ret) ~= 'string' then ret = tostring(ret) end + table.insert(key_t, ret) + end + + if have_user and not task:get_user() then + return nil + end + + return table.concat(key_t, ":") +end + +local function make_prefix(redis_key, name, bucket) + local hash_len = 24 + if hash_len > #redis_key then hash_len = #redis_key end + local hash = settings.prefix .. + string.sub(rspamd_hash.create(redis_key):base32(), 1, hash_len) + -- Fill defaults + if not bucket.spam_factor_rate then + bucket.spam_factor_rate = settings.spam_factor_rate + end + if not bucket.ham_factor_rate then + bucket.ham_factor_rate = settings.ham_factor_rate + end + if not bucket.spam_factor_burst then + bucket.spam_factor_burst = settings.spam_factor_burst + end + if not bucket.ham_factor_burst then + bucket.ham_factor_burst = settings.ham_factor_burst + end + + return { + bucket = bucket, + name = name, + hash = hash + } +end + +local function limit_to_prefixes(task, k, v, prefixes) + local n = 0 + for _,bucket in ipairs(v.buckets) do + if v.selector then + local selectors = lua_selectors.process_selectors(task, v.selector) + if selectors then + local combined = lua_selectors.combine_selectors(task, selectors, ':') + if type(combined) == 'string' then + prefixes[combined] = make_prefix(combined, k, bucket) + n = n + 1 + else + fun.each(function(p) + prefixes[p] = make_prefix(p, k, bucket) + n = n + 1 + end, combined) + end + end + else + local prefix = gen_rate_key(task, k, bucket) + if prefix then + if type(prefix) == 'string' then + prefixes[prefix] = make_prefix(prefix, k, bucket) + n = n + 1 + else + fun.each(function(p) + prefixes[p] = make_prefix(p, k, bucket) + n = n + 1 + end, prefix) + end + end + end + end + + return n +end + +local function ratelimit_cb(task) + if not settings.allow_local and + rspamd_lua_utils.is_rspamc_or_controller(task) then return end + + -- Get initial task data + local ip = task:get_from_ip() + if ip and ip:is_valid() and settings.whitelisted_ip then + if settings.whitelisted_ip:get_key(ip) then + -- Do not check whitelisted ip + rspamd_logger.infox(task, 'skip ratelimit for whitelisted IP') + return + end + end + -- Parse all rcpts + local rcpts = task:get_recipients() + local rcpts_user = {} + if rcpts then + fun.each(function(r) + fun.each(function(type) table.insert(rcpts_user, r[type]) end, {'user', 'addr'}) + end, rcpts) + + if fun.any(function(r) return settings.whitelisted_rcpts:get_key(r) end, rcpts_user) then + rspamd_logger.infox(task, 'skip ratelimit for whitelisted recipient') + return + end + end + -- Get user (authuser) + if settings.whitelisted_user then + local auser = task:get_user() + if settings.whitelisted_user:get_key(auser) then + rspamd_logger.infox(task, 'skip ratelimit for whitelisted user') + return + end + end + -- Now create all ratelimit prefixes + local prefixes = {} + local nprefixes = 0 + + for k,v in pairs(settings.limits) do + nprefixes = nprefixes + limit_to_prefixes(task, k, v, prefixes) + end + + for k, hdl in pairs(settings.custom_keywords or E) do + local ret, redis_key, bd = pcall(hdl, task) + + if ret then + local bucket = parse_limit(k, bd) + if bucket then + prefixes[redis_key] = make_prefix(redis_key, k, bucket) + end + nprefixes = nprefixes + 1 + else + rspamd_logger.errx(task, 'cannot call handler for %s: %s', + k, redis_key) + end + end + + local function gen_check_cb(prefix, bucket, lim_name, lim_key) + return function(err, data) + if err then + rspamd_logger.errx('cannot check limit %s: %s %s', prefix, err, data) + elseif type(data) == 'table' and data[1] then + lua_util.debugm(N, task, + "got reply for limit %s (%s / %s); %s burst, %s:%s dyn, %s leaked", + prefix, bucket.burst, bucket.rate, + data[2], data[3], data[4], data[5]) + + if data[1] == 1 then + -- set symbol only and do NOT soft reject + if settings.symbol then + task:insert_result(settings.symbol, 0.0, + string.format('%s(%s)', lim_name, lim_key)) + rspamd_logger.infox(task, + 'set_symbol_only: ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn); redis key: %s', + lim_name, prefix, + bucket.burst, bucket.rate, + data[2], data[3], data[4], lim_key) + return + -- set INFO symbol and soft reject + elseif settings.info_symbol then + task:insert_result(settings.info_symbol, 1.0, + string.format('%s(%s)', lim_name, lim_key)) + end + rspamd_logger.infox(task, + 'ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn); redis key: %s', + lim_name, prefix, + bucket.burst, bucket.rate, + data[2], data[3], data[4], lim_key) + task:set_pre_result('soft reject', + message_func(task, lim_name, prefix, bucket, lim_key), N) + end + end + end + end + + -- Don't do anything if pre-result has been already set + if task:has_pre_result() then return end + + if nprefixes > 0 then + -- Save prefixes to the cache to allow update + task:cache_set('ratelimit_prefixes', prefixes) + local now = rspamd_util.get_time() + now = lua_util.round(now * 1000.0) -- Get milliseconds + -- Now call check script for all defined prefixes + + for pr,value in pairs(prefixes) do + local bucket = value.bucket + local rate = (bucket.rate) / 1000.0 -- Leak rate in messages/ms + lua_util.debugm(N, task, "check limit %s:%s -> %s (%s/%s)", + value.name, pr, value.hash, bucket.burst, bucket.rate) + lua_redis.exec_redis_script(bucket_check_id, + {key = value.hash, task = task, is_write = true}, + gen_check_cb(pr, bucket, value.name, value.hash), + {value.hash, tostring(now), tostring(rate), tostring(bucket.burst), + tostring(settings.expire)}) + end + end +end + +local function ratelimit_update_cb(task) + if task:has_flag('skip') then return end + if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then return end + local prefixes = task:cache_get('ratelimit_prefixes') + + if prefixes then + if task:has_pre_result() then + -- Already rate limited/greylisted, do nothing + lua_util.debugm(N, task, 'pre-action has been set, do not update') + return + end + + local verdict = lua_util.get_task_verdict(task) + + -- Update each bucket + for k, v in pairs(prefixes) do + local bucket = v.bucket + local function update_bucket_cb(err, data) + if err then + rspamd_logger.errx(task, 'cannot update rate bucket %s: %s', + k, err) + else + lua_util.debugm(N, task, + "updated limit %s:%s -> %s (%s/%s), burst: %s, dyn_rate: %s, dyn_burst: %s", + v.name, k, v.hash, + bucket.burst, bucket.rate, + data[1], data[2], data[3]) + end + end + local now = rspamd_util.get_time() + now = lua_util.round(now * 1000.0) -- Get milliseconds + local mult_burst = 1.0 + local mult_rate = 1.0 + + if verdict == 'spam' or verdict == 'junk' then + mult_burst = bucket.spam_factor_burst or 1.0 + mult_rate = bucket.spam_factor_rate or 1.0 + elseif verdict == 'ham' then + mult_burst = bucket.ham_factor_burst or 1.0 + mult_rate = bucket.ham_factor_rate or 1.0 + end + + lua_redis.exec_redis_script(bucket_update_id, + {key = v.hash, task = task, is_write = true}, + update_bucket_cb, + {v.hash, tostring(now), tostring(mult_rate), tostring(mult_burst), + tostring(settings.max_rate_mult), tostring(settings.max_bucket_mult), + tostring(settings.expire)}) + end + end +end + +local opts = rspamd_config:get_all_opt(N) +if opts then + + settings = lua_util.override_defaults(settings, opts) + + if opts['limit'] then + rspamd_logger.errx(rspamd_config, 'Legacy ratelimit config format no longer supported') + end + + if opts['rates'] and type(opts['rates']) == 'table' then + -- new way of setting limits + fun.each(function(t, lim) + local buckets = {} + + if type(lim) == 'table' and lim.bucket then + + if lim.bucket[1] then + for _,bucket in ipairs(lim.bucket) do + local b = parse_limit(t, bucket) + + if not b then + rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"', + t, b) + return + end + + table.insert(buckets, b) + end + else + local bucket = parse_limit(t, lim.bucket) + + if not bucket then + rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"', + t, lim.bucket) + return + end + + buckets = {bucket} + end + + settings.limits[t] = { + buckets = buckets + } + + if lim.selector then + local selector = lua_selectors.parse_selector(rspamd_config, lim.selector) + if not selector then + rspamd_logger.errx(rspamd_config, 'bad ratelimit selector for %s: "%s"', + t, lim.selector) + settings.limits[t] = nil + return + end + + settings.limits[t].selector = selector + end + else + rspamd_logger.warnx(rspamd_config, 'old syntax for ratelimits: %s', lim) + buckets = parse_limit(t, lim) + if buckets then + settings.limits[t] = { + buckets = {buckets} + } + end + end + end, opts['rates']) + end + + -- Display what's enabled + fun.each(function(s) + rspamd_logger.infox(rspamd_config, 'enabled ratelimit: %s', s) + end, fun.map(function(n,d) + return string.format('%s [%s]', n, + table.concat(fun.totable(fun.map(function(v) + return string.format('%s msgs burst, %s msgs/sec rate', + v.burst, v.rate) + end, d.buckets)), '; ') + ) + end, settings.limits)) + + -- Ret, ret, ret: stupid legacy stuff: + -- If we have a string with commas then load it as as static map + -- otherwise, apply normal logic of Rspamd maps + + local wrcpts = opts['whitelisted_rcpts'] + if type(wrcpts) == 'string' then + if string.find(wrcpts, ',') then + settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl( + lua_util.rspamd_str_split(wrcpts, ','), 'set', 'Ratelimit whitelisted rcpts') + else + settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set', + 'Ratelimit whitelisted rcpts') + end + elseif type(opts['whitelisted_rcpts']) == 'table' then + settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set', + 'Ratelimit whitelisted rcpts') + else + -- Stupid default... + settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl( + settings.whitelisted_rcpts, 'set', 'Ratelimit whitelisted rcpts') + end + + if opts['whitelisted_ip'] then + settings.whitelisted_ip = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_ip', 'radix', + 'Ratelimit whitelist ip map') + end + + if opts['whitelisted_user'] then + settings.whitelisted_user = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_user', 'set', + 'Ratelimit whitelist user map') + end + + settings.custom_keywords = {} + if opts['custom_keywords'] then + local ret, res_or_err = pcall(loadfile(opts['custom_keywords'])) + + if ret then + opts['custom_keywords'] = {} + if type(res_or_err) == 'table' then + for k,hdl in pairs(res_or_err) do + settings['custom_keywords'][k] = hdl + end + elseif type(res_or_err) == 'function' then + settings['custom_keywords']['custom'] = res_or_err + end + else + rspamd_logger.errx(rspamd_config, 'cannot execute %s: %s', + opts['custom_keywords'], res_or_err) + settings['custom_keywords'] = {} + end + end + + if opts['message_func'] then + message_func = assert(load(opts['message_func']))() + end + + redis_params = lua_redis.parse_redis_server('ratelimit') + + if not redis_params then + rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module') + lua_util.disable_module(N, "redis") + else + local s = { + type = 'prefilter,nostat', + name = 'RATELIMIT_CHECK', + priority = 7, + callback = ratelimit_cb, + flags = 'empty', + } + + if settings.symbol then + s.name = settings.symbol + elseif settings.info_symbol then + s.name = settings.info_symbol + end + + rspamd_config:register_symbol(s) + rspamd_config:register_symbol { + type = 'idempotent', + name = 'RATELIMIT_UPDATE', + callback = ratelimit_update_cb, + } + end +end + +rspamd_config:add_on_load(function(cfg, ev_base, worker) + load_scripts(cfg, ev_base) +end) From 468e3dbe12884bac567834cae7b44e44210ea61b Mon Sep 17 00:00:00 2001 From: andryyy Date: Sat, 15 Dec 2018 21:20:21 +0100 Subject: [PATCH 14/50] [PHP-FPM] Try SQL once, prevent loops (todo: fix view before upgrade) --- data/Dockerfiles/phpfpm/docker-entrypoint.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/data/Dockerfiles/phpfpm/docker-entrypoint.sh b/data/Dockerfiles/phpfpm/docker-entrypoint.sh index 481e097a..c54da457 100755 --- a/data/Dockerfiles/phpfpm/docker-entrypoint.sh +++ b/data/Dockerfiles/phpfpm/docker-entrypoint.sh @@ -23,12 +23,25 @@ fi # Check of mysql_upgrade CONTAINER_ID= +# Todo: Better check if upgrade failed +# This can happen due to a broken sogo_view +[ -s /mysql_upgrade_loop ] && SQL_LOOP_C=$(cat /mysql_upgrade_loop) CONTAINER_ID=$(curl --silent --insecure https://dockerapi/containers/json | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" | jq -rc "select( .name | tostring | contains(\"mysql-mailcow\")) | .id") if [[ ! -z ${CONTAINER_ID} ]]; then SQL_UPGRADE_RETURN=$(curl --silent --insecure -XPOST https://dockerapi/containers/${CONTAINER_ID}/exec -d '{"cmd":"system", "task":"mysql_upgrade"}' --silent -H 'Content-type: application/json' | jq -r .type) if [[ ${SQL_UPGRADE_RETURN} == 'warning' ]]; then - echo "MySQL applied an upgrade, restarting PHP-FPM..." - exit 1 + if [ -z ${SQL_LOOP_C} ]; then + echo 1 > /mysql_upgrade_loop + echo "MySQL applied an upgrade, restarting PHP-FPM..." + exit 1 + else + rm /mysql_upgrade_loop + echo "MySQL was not applied previously, skipping. Restart php-fpm-mailcow to retry or run mysql_upgrade manually." + while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do + echo "Waiting for SQL to return..." + sleep 2 + done + fi fi fi From 6f1ec5acbf0a2dd19fdaf2c4749b3e0932b74469 Mon Sep 17 00:00:00 2001 From: andryyy Date: Sat, 15 Dec 2018 21:21:22 +0100 Subject: [PATCH 15/50] [Watchdog] Alert when ratelimit log changed (does NOT send one mail per triggered ratelimit) --- data/Dockerfiles/watchdog/watchdog.sh | 42 ++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/data/Dockerfiles/watchdog/watchdog.sh b/data/Dockerfiles/watchdog/watchdog.sh index 0b903354..15bf6e2a 100755 --- a/data/Dockerfiles/watchdog/watchdog.sh +++ b/data/Dockerfiles/watchdog/watchdog.sh @@ -322,6 +322,34 @@ phpfpm_checks() { return 1 } +ratelimit_checks() { + err_count=0 + diff_c=0 + THRESHOLD=1 + RL_LOG_STATUS=$(redis-cli -h redis LRANGE RL_LOG 0 0 | jq .qid) + # Reduce error count by 2 after restarting an unhealthy container + trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1 + while [ ${err_count} -lt ${THRESHOLD} ]; do + err_c_cur=${err_count} + RL_LOG_STATUS_PREV=${RL_LOG_STATUS} + RL_LOG_STATUS=$(redis-cli -h redis LRANGE RL_LOG 0 0 | jq .qid) + if [[ ${RL_LOG_STATUS_PREV} != ${RL_LOG_STATUS} ]]; then + err_count=$(( ${err_count} + 1 )) + fi + [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 + [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) + progress "Ratelimit" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} + if [[ $? == 10 ]]; then + diff_c=0 + sleep 1 + else + diff_c=0 + sleep $(( ( RANDOM % 30 ) + 10 )) + fi + done + return 1 +} + rspamd_checks() { err_count=0 diff_c=0 @@ -448,6 +476,15 @@ done ) & BACKGROUND_TASKS+=($!) +( +while true; do + if ! ratelimit_checks; then + log_msg "Ratelimit hit error limit" + echo ratelimit > /tmp/com_pipe + fi +done +) & +BACKGROUND_TASKS+=($!) # Monitor watchdog agents, stop script when agents fails and wait for respawn by Docker (restart:always:n) ( while true; do @@ -482,7 +519,10 @@ while true; do CONTAINER_ID= HAS_INITDB= read com_pipe_answer Date: Sat, 15 Dec 2018 21:21:41 +0100 Subject: [PATCH 16/50] [Compose] Update Watchdog, Dovecot, PHP, Rspamd images --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2edbd8d4..e0d6ed6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,7 +72,7 @@ services: - clamd rspamd-mailcow: - image: mailcow/rspamd:1.32 + image: mailcow/rspamd:1.33 build: ./data/Dockerfiles/rspamd stop_grace_period: 30s depends_on: @@ -95,7 +95,7 @@ services: - rspamd php-fpm-mailcow: - image: mailcow/phpfpm:1.26 + image: mailcow/phpfpm:1.27 build: ./data/Dockerfiles/phpfpm command: "php-fpm -d date.timezone=${TZ} -d expose_php=0" depends_on: @@ -162,7 +162,7 @@ services: - sogo dovecot-mailcow: - image: mailcow/dovecot:1.51 + image: mailcow/dovecot:1.52 build: ./data/Dockerfiles/dovecot cap_add: - NET_BIND_SERVICE @@ -342,7 +342,7 @@ services: - /lib/modules:/lib/modules:ro watchdog-mailcow: - image: mailcow/watchdog:1.29 + image: mailcow/watchdog:1.30 # Debug #command: /watchdog.sh build: ./data/Dockerfiles/watchdog From e7427eddf379e1742829d39a4f45dfa6a77675b0 Mon Sep 17 00:00:00 2001 From: andryyy Date: Sat, 15 Dec 2018 21:22:59 +0100 Subject: [PATCH 17/50] [Rspamd] Updated values of default ratelimit settings, add info_symbol --- data/conf/rspamd/override.d/ratelimit.conf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/data/conf/rspamd/override.d/ratelimit.conf b/data/conf/rspamd/override.d/ratelimit.conf index 3150d0e5..ccd083d4 100644 --- a/data/conf/rspamd/override.d/ratelimit.conf +++ b/data/conf/rspamd/override.d/ratelimit.conf @@ -1,11 +1,12 @@ rates { # Format: "1 / 1h" or "20 / 1m" etc. - global ratelimits are disabled by default - to = "100 / 1s"; - to_ip = "100 / 1s"; - to_ip_from = "100 / 1s"; + to = "45 / 1m"; + to_ip = "360 / 1m"; + to_ip_from = "180 / 1m"; bounce_to = "100 / 1s"; bounce_to_ip = "100 / 1s"; } whitelisted_rcpts = "postmaster,mailer-daemon"; max_rcpt = 5; custom_keywords = "/etc/rspamd/lua/ratelimit.lua"; +info_symbol = "RATELIMITED"; From ed763cd668b395821c04e116fc4d7ef10fa4a6d1 Mon Sep 17 00:00:00 2001 From: andryyy Date: Sat, 15 Dec 2018 21:23:42 +0100 Subject: [PATCH 18/50] [Rspamd] Use meta exporter to pipe meta data of ratelimited msg to Redis --- .../rspamd/local.d/metadata_exporter.conf | 32 ++++++++++++---- data/conf/rspamd/meta_exporter/pipe_rl.php | 38 +++++++++++++++++++ 2 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 data/conf/rspamd/meta_exporter/pipe_rl.php diff --git a/data/conf/rspamd/local.d/metadata_exporter.conf b/data/conf/rspamd/local.d/metadata_exporter.conf index f1600708..afe5c7e1 100644 --- a/data/conf/rspamd/local.d/metadata_exporter.conf +++ b/data/conf/rspamd/local.d/metadata_exporter.conf @@ -1,10 +1,26 @@ rules { - QUARANTINE { - backend = "http"; - url = "http://nginx:9081/pipe.php"; - selector = "is_reject"; - formatter = "default"; - meta_headers = true; - } + QUARANTINE { + backend = "http"; + url = "http://nginx:9081/pipe.php"; + selector = "is_reject"; + formatter = "default"; + meta_headers = true; + } + RLINFO { + backend = "http"; + url = "http://nginx:9081/pipe_rl.php"; + selector = "ratelimited"; + formatter = "json"; + } +} +custom_select { + ratelimited = <connect('redis-mailcow', 6379); + +$raw_data_content = file_get_contents('php://input'); +$raw_data_decoded = json_decode($raw_data_content, true); + +$data['time'] = time(); +$data['rcpt'] = implode(', ', $raw_data_decoded['rcpt']); +$data['from'] = $raw_data_decoded['from']; +$data['user'] = $raw_data_decoded['user']; +$symbol_rl_key = array_search('RATELIMITED', array_column($raw_data_decoded['symbols'], 'name')); +$data['rl_info'] = implode($raw_data_decoded['symbols'][$symbol_rl_key]['options']); +preg_match('/(.+)\((.+)\)/i', $data['rl_info'], $rl_matches); +if (!empty($rl_matches[1]) && !empty($rl_matches[2])) { + $data['rl_name'] = $rl_matches[1]; + $data['rl_hash'] = $rl_matches[2]; +} +else { + $data['rl_name'] = 'err'; + $data['rl_hash'] = 'err'; +} +$data['qid'] = $raw_data_decoded['qid']; +$data['ip'] = $raw_data_decoded['ip']; +$data['message_id'] = $raw_data_decoded['message_id']; +$data['header_subject'] = implode(' ', $raw_data_decoded['header_subject']); +$data['header_from'] = implode(', ', $raw_data_decoded['header_from']); + +$redis->lpush('RL_LOG', json_encode($data)); +exit; + From 5b5976ba230b6c31ed8811062446deb660d99eef Mon Sep 17 00:00:00 2001 From: andryyy Date: Sat, 15 Dec 2018 21:24:39 +0100 Subject: [PATCH 19/50] [Web] Show ratelimited messages, allow to delete Redis hash to reset status of a bucket --- data/web/debug.php | 19 ++++++++ data/web/inc/functions.inc.php | 36 ++++++++++----- data/web/inc/functions.ratelimit.inc.php | 39 ++++++++++++++++ data/web/js/admin.js | 2 +- data/web/js/debug.js | 58 ++++++++++++++++++++++++ data/web/json_api.php | 14 ++++++ data/web/lang/lang.de.php | 8 ++++ data/web/lang/lang.en.php | 8 ++++ 8 files changed, 172 insertions(+), 12 deletions(-) diff --git a/data/web/debug.php b/data/web/debug.php index 68b1a5a0..d99213cc 100644 --- a/data/web/debug.php +++ b/data/web/debug.php @@ -22,6 +22,7 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
  • Watchdog
  • ACME
  • API
  • +
  • Ratelimits
  • Rspamd
  • @@ -272,6 +273,24 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI']; +
    +
    +
    Ratelimits +
    + + + +
    +
    +
    +

    +
    +
    +
    +
    +
    +
    + diff --git a/data/web/inc/functions.inc.php b/data/web/inc/functions.inc.php index 61104308..89ccd037 100644 --- a/data/web/inc/functions.inc.php +++ b/data/web/inc/functions.inc.php @@ -1263,7 +1263,7 @@ function get_u2f_registrations($username) { $sel->execute(array($username)); return $sel->fetchAll(PDO::FETCH_OBJ); } -function get_logs($container, $lines = false) { +function get_logs($application, $lines = false) { if ($lines === false) { $lines = $GLOBALS['LOG_LINES'] - 1; } @@ -1283,7 +1283,7 @@ function get_logs($container, $lines = false) { return false; } // SQL - if ($container == "mailcow-ui") { + if ($application == "mailcow-ui") { if (isset($from) && isset($to)) { $stmt = $pdo->prepare("SELECT * FROM `logs` ORDER BY `id` DESC LIMIT :from, :to"); $stmt->execute(array( @@ -1304,7 +1304,7 @@ function get_logs($container, $lines = false) { } } // Redis - if ($container == "dovecot-mailcow") { + if ($application == "dovecot-mailcow") { if (isset($from) && isset($to)) { $data = $redis->lRange('DOVECOT_MAILLOG', $from - 1, $to - 1); } @@ -1318,7 +1318,7 @@ function get_logs($container, $lines = false) { return $data_array; } } - if ($container == "postfix-mailcow") { + if ($application == "postfix-mailcow") { if (isset($from) && isset($to)) { $data = $redis->lRange('POSTFIX_MAILLOG', $from - 1, $to - 1); } @@ -1332,7 +1332,7 @@ function get_logs($container, $lines = false) { return $data_array; } } - if ($container == "sogo-mailcow") { + if ($application == "sogo-mailcow") { if (isset($from) && isset($to)) { $data = $redis->lRange('SOGO_LOG', $from - 1, $to - 1); } @@ -1346,7 +1346,7 @@ function get_logs($container, $lines = false) { return $data_array; } } - if ($container == "watchdog-mailcow") { + if ($application == "watchdog-mailcow") { if (isset($from) && isset($to)) { $data = $redis->lRange('WATCHDOG_LOG', $from - 1, $to - 1); } @@ -1360,7 +1360,7 @@ function get_logs($container, $lines = false) { return $data_array; } } - if ($container == "acme-mailcow") { + if ($application == "acme-mailcow") { if (isset($from) && isset($to)) { $data = $redis->lRange('ACME_LOG', $from - 1, $to - 1); } @@ -1374,7 +1374,21 @@ function get_logs($container, $lines = false) { return $data_array; } } - if ($container == "api-mailcow") { + if ($application == "ratelimited") { + if (isset($from) && isset($to)) { + $data = $redis->lRange('RL_LOG', $from - 1, $to - 1); + } + else { + $data = $redis->lRange('RL_LOG', 0, $lines); + } + if ($data) { + foreach ($data as $json_line) { + $data_array[] = json_decode($json_line, true); + } + return $data_array; + } + } + if ($application == "api-mailcow") { if (isset($from) && isset($to)) { $data = $redis->lRange('API_LOG', $from - 1, $to - 1); } @@ -1388,7 +1402,7 @@ function get_logs($container, $lines = false) { return $data_array; } } - if ($container == "netfilter-mailcow") { + if ($application == "netfilter-mailcow") { if (isset($from) && isset($to)) { $data = $redis->lRange('NETFILTER_LOG', $from - 1, $to - 1); } @@ -1402,7 +1416,7 @@ function get_logs($container, $lines = false) { return $data_array; } } - if ($container == "autodiscover-mailcow") { + if ($application == "autodiscover-mailcow") { if (isset($from) && isset($to)) { $data = $redis->lRange('AUTODISCOVER_LOG', $from - 1, $to - 1); } @@ -1416,7 +1430,7 @@ function get_logs($container, $lines = false) { return $data_array; } } - if ($container == "rspamd-history") { + if ($application == "rspamd-history") { $curl = curl_init(); curl_setopt($curl, CURLOPT_UNIX_SOCKET_PATH, '/var/lib/rspamd/rspamd.sock'); if (!is_numeric($lines)) { diff --git a/data/web/inc/functions.ratelimit.inc.php b/data/web/inc/functions.ratelimit.inc.php index 7fac01e4..efde5ec5 100644 --- a/data/web/inc/functions.ratelimit.inc.php +++ b/data/web/inc/functions.ratelimit.inc.php @@ -192,5 +192,44 @@ function ratelimit($_action, $_scope, $_data = null) { break; } break; + case 'delete': + $data['hash'] = $_data; + if ($_SESSION['mailcow_cc_role'] != 'admin' || !preg_match('/^RL[0-9A-Za-z=]+$/i', trim($data['hash']))) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), + 'msg' => 'access_denied' + ); + return false; + } + try { + if ($redis->exists($data['hash'])) { + $redis->delete($data['hash']); + $_SESSION['return'][] = array( + 'type' => 'success', + 'log' => array(__FUNCTION__, $_action, $_scope, $_data_log), + 'msg' => 'hash_deleted' + ); + return true; + } + else { + $_SESSION['return'][] = array( + 'type' => 'warning', + 'log' => array(__FUNCTION__, $_action, $_scope, $_data_log), + 'msg' => 'hash_not_found' + ); + return false; + } + } + catch (RedisException $e) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_scope, $_data_log), + 'msg' => array('redis_error', $e) + ); + return false; + } + return false; + break; } } \ No newline at end of file diff --git a/data/web/js/admin.js b/data/web/js/admin.js index 9573baf1..07bf27b6 100644 --- a/data/web/js/admin.js +++ b/data/web/js/admin.js @@ -8,7 +8,7 @@ jQuery(function($){ $("#rspamd_preset_1").on('click', function(e) { e.preventDefault(); $("form[data-id=rsetting]").find("#adminRspamdSettingsDesc").val(lang.rsettings_preset_1); - $("form[data-id=rsetting]").find("#adminRspamdSettingsContent").val('priority = 10;\nauthenticated = yes;\napply "default" {\n symbols_enabled = ["DKIM_SIGNED", "RATELIMIT_UPDATE", "RATELIMIT_CHECK", "DYN_RL_CHECK", "HISTORY_SAVE", "MILTER_HEADERS", "ARC_SIGNED"];\n}'); + $("form[data-id=rsetting]").find("#adminRspamdSettingsContent").val('priority = 10;\nauthenticated = yes;\napply "default" {\n symbols_enabled = ["DKIM_SIGNED", "RATELIMITED", "RATELIMIT_UPDATE", "RATELIMIT_CHECK", "DYN_RL_CHECK", "HISTORY_SAVE", "MILTER_HEADERS", "ARC_SIGNED"];\n}'); }); $("#rspamd_preset_2").on('click', function(e) { e.preventDefault(); diff --git a/data/web/js/debug.js b/data/web/js/debug.js index dfd88d86..b294931c 100644 --- a/data/web/js/debug.js +++ b/data/web/js/debug.js @@ -8,6 +8,8 @@ jQuery(function($){ var entityMap={"&":"&","<":"<",">":">",'"':""","'":"'","/":"/","`":"`","=":"="}; function escapeHtml(n){return String(n).replace(/[&<>"'`=\/]/g,function(n){return entityMap[n]})} function humanFileSize(i){if(Math.abs(i)<1024)return i+" B";var B=["KiB","MiB","GiB","TiB","PiB","EiB","ZiB","YiB"],e=-1;do{i/=1024,++e}while(Math.abs(i)>=1024&&e'; } }); + } else if (table == 'rllog') { + $.each(data, function (i, item) { + if (item.user == null) { + item.user = "none"; + } + if (item.rl_hash == null) { + item.rl_hash = "err"; + } + item.indicator = '  '; + if (item.rl_hash != 'err') { + item.action = ' ' + lang.reset_limit + ''; + } + }); } return data }; @@ -575,6 +632,7 @@ jQuery(function($){ draw_watchdog_logs(); draw_acme_logs(); draw_api_logs(); + draw_rl_logs(); draw_ui_logs(); draw_netfilter_logs(); draw_rspamd_history(); diff --git a/data/web/json_api.php b/data/web/json_api.php index 26ccbb25..c9db5eb6 100644 --- a/data/web/json_api.php +++ b/data/web/json_api.php @@ -387,6 +387,17 @@ if (isset($_SESSION['mailcow_cc_role']) || isset($_SESSION['pending_mailcow_cc_u } echo (isset($logs) && !empty($logs)) ? json_encode($logs, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT) : '{}'; break; + case "ratelimited": + // 0 is first record, so empty is fine + if (isset($extra)) { + $extra = preg_replace('/[^\d\-]/i', '', $extra); + $logs = get_logs('ratelimited', $extra); + } + else { + $logs = get_logs('ratelimited'); + } + echo (isset($logs) && !empty($logs)) ? json_encode($logs, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT) : '{}'; + break; case "netfilter": // 0 is first record, so empty is fine if (isset($extra)) { @@ -1043,6 +1054,9 @@ if (isset($_SESSION['mailcow_cc_role']) || isset($_SESSION['pending_mailcow_cc_u case "admin": process_delete_return(admin('delete', array('username' => $items))); break; + case "rlhash": + echo ratelimit('delete', null, implode($items)); + break; } break; case "edit": diff --git a/data/web/lang/lang.de.php b/data/web/lang/lang.de.php index 1ff443b7..ba031d8e 100644 --- a/data/web/lang/lang.de.php +++ b/data/web/lang/lang.de.php @@ -556,10 +556,18 @@ $lang['admin']['no_record'] = 'Kein Eintrag'; $lang['admin']['filter_table'] = 'Tabelle Filtern'; $lang['admin']['empty'] = 'Keine Einträge vorhanden'; $lang['admin']['time'] = 'Zeit'; +$lang['admin']['last_applied'] = 'Zuletzt angewendet'; +$lang['admin']['reset_limit'] = 'Hash entfernen'; +$lang['admin']['hash_remove_info'] = 'Das Entfernen eines Ratelimit Hashes - sofern noch existent - bewirkt den Reset gezählter Nachrichten dieses Elements.
    + Jeder Hash wird durch eine eindeutige Farbe gekennzeichnet.'; +$lang['warning']['hash_not_found'] = 'Hash nicht gefunden'; +$lang['success']['hash_deleted'] = 'Hash wurde gelöscht'; +$lang['admin']['authed_user'] = 'Auth. Benutzer'; $lang['admin']['priority'] = 'Gewichtung'; $lang['admin']['refresh'] = 'Neu laden'; $lang['admin']['to_top'] = 'Nach oben'; $lang['admin']['in_use_by'] = 'Verwendet von'; +$lang['admin']['rate_name'] = 'Rate name'; $lang['admin']['message'] = 'Nachricht'; $lang['admin']['forwarding_hosts'] = 'Weiterleitungs-Hosts'; $lang['admin']['forwarding_hosts_hint'] = 'Eingehende Nachrichten werden von den hier gelisteten Hosts bedingungslos akzeptiert. Diese Hosts werden dann nicht mit DNSBLs abgeglichen oder Greylisting unterworfen. Von ihnen empfangener Spam wird nie abgelehnt, optional kann er aber in den Spam-Ordner einsortiert werden. Die übliche Verwendung für diese Funktion ist, um Mailserver anzugeben, auf denen eine Weiterleitung zu Ihrem mailcow-Server eingerichtet wurde.'; diff --git a/data/web/lang/lang.en.php b/data/web/lang/lang.en.php index dc773150..4191446a 100644 --- a/data/web/lang/lang.en.php +++ b/data/web/lang/lang.en.php @@ -580,8 +580,16 @@ $lang['admin']['no_record'] = 'No record'; $lang['admin']['filter_table'] = 'Filter table'; $lang['admin']['empty'] = 'No results'; $lang['admin']['time'] = 'Time'; +$lang['admin']['last_applied'] = 'Last applied'; +$lang['admin']['reset_limit'] = 'Remove hash'; +$lang['admin']['hash_remove_info'] = 'Removing a ratelimit hash (if still existing) will reset its counter completely.
    + Each hash is indicated by an individual color.'; +$lang['warning']['hash_not_found'] = 'Hash not found'; +$lang['success']['hash_deleted'] = 'Hash deleted'; +$lang['admin']['authed_user'] = 'Auth. user'; $lang['admin']['priority'] = 'Priority'; $lang['admin']['message'] = 'Message'; +$lang['admin']['rate_name'] = 'Rate name'; $lang['admin']['refresh'] = 'Refresh'; $lang['admin']['to_top'] = 'Back to top'; $lang['admin']['in_use_by'] = 'In use by'; From 5dad2dded1a0e6bc14867451f2393b3338ee04fd Mon Sep 17 00:00:00 2001 From: Patrik Kernstock Date: Tue, 18 Dec 2018 20:50:24 +0100 Subject: [PATCH 20/50] [web] Duplicating DKIM key corrupts private key Missing base64_decode() corrupted private key when duplicating, as `$from_domain_dkim['privkey']` returns the public key base64-encoded. --- data/web/inc/functions.dkim.inc.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/web/inc/functions.dkim.inc.php b/data/web/inc/functions.dkim.inc.php index 819afd25..f4bfd997 100644 --- a/data/web/inc/functions.dkim.inc.php +++ b/data/web/inc/functions.dkim.inc.php @@ -123,7 +123,7 @@ function dkim($_action, $_data = null) { try { $redis->hSet('DKIM_PUB_KEYS', $to_domain, $from_domain_dkim['pubkey']); $redis->hSet('DKIM_SELECTORS', $to_domain, $from_domain_dkim['dkim_selector']); - $redis->hSet('DKIM_PRIV_KEYS', $from_domain_dkim['dkim_selector'] . '.' . $to_domain, trim($from_domain_dkim['privkey'])); + $redis->hSet('DKIM_PRIV_KEYS', $from_domain_dkim['dkim_selector'] . '.' . $to_domain, base64_decode(trim($from_domain_dkim['privkey']))); } catch (RedisException $e) { $_SESSION['return'][] = array( @@ -307,4 +307,4 @@ function dkim($_action, $_data = null) { } break; } -} \ No newline at end of file +} From 59301decab15f39dc8ffc77df663fcd33d34e2e8 Mon Sep 17 00:00:00 2001 From: Patrik Kernstock Date: Tue, 18 Dec 2018 21:00:16 +0100 Subject: [PATCH 21/50] [Web] Add hint to DKIM key import for RSA PKCS#8 Adding hint to explicitly provide a RSA Private key in the newer PKCS#8 format, as the webinterface denies the key with a cryptic error message otherwise: `Private key error: error:0EFFF06C:configuration file routines:CRYPTO_internal:no value`. To prevent frustrated users I'd add a simple notice which format is expected. PKCS#8 is also the default format when generating keys directly in the webinterface. Some interesting resources: https://stackoverflow.com/questions/20065304/differences-between-begin-rsa-private-key-and-begin-private-key https://stackoverflow.com/questions/17733536/how-to-convert-a-private-key-to-an-rsa-private-key --- data/web/admin.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/web/admin.php b/data/web/admin.php index 2241da86..3cd1f0d3 100644 --- a/data/web/admin.php +++ b/data/web/admin.php @@ -333,7 +333,7 @@ $tfa_data = get_tfa();
    - +
    From 29512fa4e1b8a3dd76da236415e65a7fd16a356d Mon Sep 17 00:00:00 2001 From: andryyy Date: Wed, 19 Dec 2018 09:33:59 +0100 Subject: [PATCH 22/50] [SOGo] Build stable SOGo versions [SOGo] Remove custom colors, there were various broken styles especially for indicators of freebusy states --- data/Dockerfiles/sogo/Dockerfile | 113 +++++++++++++++++++----- data/Dockerfiles/sogo/bootstrap-sogo.sh | 45 +--------- 2 files changed, 97 insertions(+), 61 deletions(-) diff --git a/data/Dockerfiles/sogo/Dockerfile b/data/Dockerfiles/sogo/Dockerfile index dad9682a..2e2ea104 100644 --- a/data/Dockerfiles/sogo/Dockerfile +++ b/data/Dockerfiles/sogo/Dockerfile @@ -4,6 +4,7 @@ LABEL maintainer "Andre Peters " ARG DEBIAN_FRONTEND=noninteractive ENV LC_ALL C ENV GOSU_VERSION 1.9 +ENV SOGO_VERSION 4.0.4 # Prerequisites RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -11,7 +12,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ cron \ gettext \ - gnupg \ mysql-client \ supervisor \ syslog-ng \ @@ -22,31 +22,106 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ psmisc \ wget \ patch \ + git \ + devscripts \ + debhelper \ + build-essential \ + gnustep-make \ + gnustep-base-runtime \ + libgnustep-base-dev \ + libgnustep-base1.24 \ + gobjc \ + libxml2-dev \ + libldap2-dev \ + libssl-dev \ + zlib1g-dev \ + libpq-dev \ + default-libmysqlclient-dev \ + liblasso3-dev \ + libmemcached-dev \ + mysql-client \ + libcurl4-openssl-dev \ && rm -rf /var/lib/apt/lists/* \ && dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \ && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" \ && chmod +x /usr/local/bin/gosu \ - && gosu nobody true - -RUN mkdir /usr/share/doc/sogo \ - && touch /usr/share/doc/sogo/empty.sh \ - && apt-key adv --keyserver keyserver.ubuntu.com --recv-key 0x810273C4 \ - && echo "deb http://packages.inverse.ca/SOGo/nightly/4/debian/ stretch stretch" > /etc/apt/sources.list.d/sogo.list \ - && apt-get update && apt-get install -y --force-yes \ - sogo \ - sogo-activesync \ + && gosu nobody true \ + && mkdir /tmp/sogo_build \ + && cd /tmp/sogo_build \ + && git clone -b SOPE-${SOGO_VERSION} https://github.com/inverse-inc/sope.git \ + && cd sope \ + && ./configure --enable-xml --enable-mysql --enable-openldap --with-gnustep \ + && make -j4 \ + && make install \ + && cd /tmp/sogo_build \ + && git clone -b SOGo-${SOGO_VERSION} https://github.com/inverse-inc/sogo.git \ + && cd sogo \ + && ./configure --enable-saml2 \ + && make -j4 \ + && make install \ + && groupadd -g 6000 sogo \ + && useradd -g sogo -u 6000 sogo -d /var/lib/sogo \ + && mkdir -p /usr/local/share/doc/sogo \ + && touch /usr/local/share/doc/sogo/empty.sh \ && rm -rf /var/lib/apt/lists/* \ - && echo '* * * * * sogo /usr/sbin/sogo-ealarms-notify 2>/dev/null' > /etc/cron.d/sogo \ - && echo '* * * * * sogo /usr/sbin/sogo-tool expire-sessions 60' >> /etc/cron.d/sogo \ - && echo '0 0 * * * sogo /usr/sbin/sogo-tool update-autoreply -p /etc/sogo/sieve.creds' >> /etc/cron.d/sogo \ - && touch /etc/default/locale + && echo '* * * * * sogo /usr/local/sbin/sogo-ealarms-notify 2>/dev/null' > /etc/cron.d/sogo \ + && echo '* * * * * sogo /usr/local/sbin/sogo-tool expire-sessions 60' >> /etc/cron.d/sogo \ + && echo '0 0 * * * sogo /usr/local/sbin/sogo-tool update-autoreply -p /etc/sogo/sieve.creds' >> /etc/cron.d/sogo \ + && touch /etc/default/locale \ + && echo '/usr/local/lib/sogo' > /etc/ld.so.conf.d/sogo.conf \ + && ldconfig \ + && mkdir -p /var/run/sogo /var/spool/sogo \ + && chown sogo:sogo /var/run/sogo /var/spool/sogo \ + && rm -rf /tmp/* /var/tmp/* /usr/bin/mysql_embedded /usr/bin/mariabackup /tmp/sogo_build \ + && apt-get purge -y libxml2-dev \ + libldap2-dev \ + libssl-dev \ + zlib1g-dev \ + libpq-dev \ + default-libmysqlclient-dev \ + liblasso3-dev \ + libmemcached-dev \ + libgnustep-base-dev \ + libcurl4-openssl-dev \ + devscripts \ + debhelper \ + build-essential \ + autoconf \ + automake \ + autopoint \ + autotools-dev \ + binutils \ + cpp \ + cpp-6 \ + dh-python \ + dpkg-dev \ + g++-6 \ + gcc-6 \ + gobjc-6 \ + groff-base \ + icu-devtools \ + libc-dev-bin \ + libc6-dev \ + libgcc-6-dev \ + libgcrypt20-dev \ + libgmp-dev \ + libgpg-error-dev \ + libhashkit-dev \ + libicu-dev \ + libidn11-dev \ + libnspr4-dev \ + libnss3-dev \ + libobjc-6-dev \ + libp11-kit-dev \ + libpcre3-dev \ + libsasl2-dev \ + libtasn1-6-dev \ + linux-libc-dev \ + nettle-dev COPY ./bootstrap-sogo.sh /bootstrap-sogo.sh COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf COPY supervisord.conf /etc/supervisor/supervisord.conf -COPY theme-blue.js /usr/lib/GNUstep/SOGo/WebServerResources/js/theme-blue.js -COPY theme-blue.css /usr/lib/GNUstep/SOGo/WebServerResources/css/theme-default.css -COPY sogo-full.svg /usr/lib/GNUstep/SOGo/WebServerResources/img/sogo-full.svg COPY acl.diff /acl.diff COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh @@ -55,6 +130,4 @@ RUN chmod +x /bootstrap-sogo.sh \ CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf -VOLUME /usr/lib/GNUstep/SOGo/ - -RUN rm -rf /tmp/* /var/tmp/* +VOLUME /usr/local/lib/GNUstep/SOGo/ diff --git a/data/Dockerfiles/sogo/bootstrap-sogo.sh b/data/Dockerfiles/sogo/bootstrap-sogo.sh index 7fa0668d..539314df 100755 --- a/data/Dockerfiles/sogo/bootstrap-sogo.sh +++ b/data/Dockerfiles/sogo/bootstrap-sogo.sh @@ -168,48 +168,11 @@ chown sogo:sogo -R /var/lib/sogo/ chmod 600 /var/lib/sogo/GNUstep/Defaults/sogod.plist # Add credentials to alarms -sed -i 's/\/usr\/sbin\/sogo-ealarms-notify/\/usr\/sbin\/sogo-ealarms-notify -p \/etc\/sogo\/sieve.creds/g' /etc/cron.d/sogo - -# Prevent theme switching -sed -i \ - -e 's/eaf5e9/E3F2FD/g' \ - -e 's/cbe5c8/BBDEFB/g' \ - -e 's/aad6a5/90CAF9/g' \ - -e 's/88c781/64B5F6/g' \ - -e 's/66b86a/42A5F5/g' \ - -e 's/56b04c/2196F3/g' \ - -e 's/4da143/1E88E5/g' \ - -e 's/388e3c/1976D2/g' \ - -e 's/367d2e/1565C0/g' \ - -e 's/225e1b/0D47A1/g' \ - -e 's/fafafa/82B1FF/g' \ - -e 's/69f0ae/448AFF/g' \ - -e 's/00e676/2979ff/g' \ - -e 's/00c853/2962ff/g' \ - /usr/lib/GNUstep/SOGo/WebServerResources/js/Common/Common.app.js \ - /usr/lib/GNUstep/SOGo/WebServerResources/js/Common.js - -sed -i \ - -e 's/default: "900"/default: "700"/g' \ - -e 's/default: "500"/default: "700"/g' \ - -e 's/"hue-1": "400"/"hue-1": "500"/g' \ - -e 's/"hue-1": "A100"/"hue-1": "500"/g' \ - -e 's/"hue-2": "800"/"hue-2": "700"/g' \ - -e 's/"hue-2": "300"/"hue-2": "700"/g' \ - -e 's/"hue-3": "A700"/"hue-3": "A200"/' \ - -e 's/default:"900"/default:"700"/g' \ - -e 's/default:"500"/default:"700"/g' \ - -e 's/"hue-1":"400"/"hue-1":"500"/g' \ - -e 's/"hue-1":"A100"/"hue-1":"500"/g' \ - -e 's/"hue-2":"800"/"hue-2":"700"/g' \ - -e 's/"hue-2":"300"/"hue-2":"700"/g' \ - -e 's/"hue-3":"A700"/"hue-3":"A200"/' \ - /usr/lib/GNUstep/SOGo/WebServerResources/js/Common/Common.app.js \ - /usr/lib/GNUstep/SOGo/WebServerResources/js/Common.js +sed -i 's/\/usr\/local\/sbin\/sogo-ealarms-notify/\/usr\/local\/sbin\/sogo-ealarms-notify -p \/etc\/sogo\/sieve.creds/g' /etc/cron.d/sogo # Patch ACLs (comment this out to enable any or authenticated targets for ACL) -if patch -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then - patch /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff; +if patch -sfN --dry-run /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then + patch /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff; fi -exec gosu sogo /usr/sbin/sogod +exec gosu sogo /usr/local/sbin/sogod From 084c44f9c51e1caaadfff2f6446991c1b174b74f Mon Sep 17 00:00:00 2001 From: andryyy Date: Wed, 19 Dec 2018 09:36:35 +0100 Subject: [PATCH 23/50] [Compose] Update Postfix and SOGo images --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e0d6ed6f..99881953 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -138,7 +138,7 @@ services: - phpfpm sogo-mailcow: - image: mailcow/sogo:1.45 + image: mailcow/sogo:1.46 build: ./data/Dockerfiles/sogo environment: - DBNAME=${DBNAME} @@ -206,7 +206,7 @@ services: - dovecot postfix-mailcow: - image: mailcow/postfix:1.27 + image: mailcow/postfix:1.28 build: ./data/Dockerfiles/postfix volumes: - ./data/conf/postfix:/opt/postfix/conf From 534e83a218adf7048df7db1c95b8835e8886d6f9 Mon Sep 17 00:00:00 2001 From: andryyy Date: Wed, 19 Dec 2018 09:37:07 +0100 Subject: [PATCH 24/50] [Nginx] New WebServerResources path --- data/conf/nginx/site.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/conf/nginx/site.conf b/data/conf/nginx/site.conf index 67d6f1a1..006cdea4 100644 --- a/data/conf/nginx/site.conf +++ b/data/conf/nginx/site.conf @@ -170,19 +170,19 @@ server { } location /SOGo.woa/WebServerResources/ { - alias /usr/lib/GNUstep/SOGo/WebServerResources/; + alias /usr/local/lib/GNUstep/SOGo/WebServerResources/; } location /.woa/WebServerResources/ { - alias /usr/lib/GNUstep/SOGo/WebServerResources/; + alias /usr/local/lib/GNUstep/SOGo/WebServerResources/; } location /SOGo/WebServerResources/ { - alias /usr/lib/GNUstep/SOGo/WebServerResources/; + alias /usr/local/lib/GNUstep/SOGo/WebServerResources/; } location (^/SOGo/so/ControlPanel/Products/[^/]*UI/Resources/.*\.(jpg|png|gif|css|js)$) { - alias /usr/lib/GNUstep/SOGo/$1.SOGo/Resources/$2; + alias /usr/local/lib/GNUstep/SOGo/$1.SOGo/Resources/$2; } include /etc/nginx/conf.d/site.*.custom; From 8f686c154327e2748e5aeaa372eff9fcf870e04c Mon Sep 17 00:00:00 2001 From: andryyy Date: Wed, 19 Dec 2018 09:38:56 +0100 Subject: [PATCH 25/50] [Postfix] Split sasl passwd maps to not lookup sender_dependent_default_transport_maps auth info when querying for transport_maps --- data/Dockerfiles/postfix/postfix.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/data/Dockerfiles/postfix/postfix.sh b/data/Dockerfiles/postfix/postfix.sh index 7a69258f..425d65ff 100755 --- a/data/Dockerfiles/postfix/postfix.sh +++ b/data/Dockerfiles/postfix/postfix.sh @@ -85,7 +85,17 @@ query = SELECT GROUP_CONCAT(transport SEPARATOR '') AS transport_maps AS transport_view; EOF -cat < /opt/postfix/conf/sql/mysql_sasl_passwd_maps.cf +cat < /opt/postfix/conf/sql/mysql_transport_maps.cf +user = ${DBUSER} +password = ${DBPASS} +hosts = unix:/var/run/mysqld/mysqld.sock +dbname = ${DBNAME} +query = SELECT CONCAT('smtp_via_transport_maps:', nexthop) AS transport FROM transports + WHERE active = '1' + AND destination = '%s'; +EOF + +cat < /opt/postfix/conf/sql/mysql_sasl_passwd_maps_sender_dependent.cf user = ${DBUSER} password = ${DBPASS} hosts = unix:/var/run/mysqld/mysqld.sock @@ -98,6 +108,18 @@ query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM relayhosts SELECT CONCAT('@', alias_domain) FROM alias_domain ) ) + AND active = '1' + AND username != ''; +EOF + +cat < /opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf +user = ${DBUSER} +password = ${DBPASS} +hosts = unix:/var/run/mysqld/mysqld.sock +dbname = ${DBNAME} +query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM transports + WHERE nexthop = '%s' + AND active = '1' AND username != ''; EOF From cd72a4e18b7c2a00c7bef939ca210df75b9dec86 Mon Sep 17 00:00:00 2001 From: andryyy Date: Wed, 19 Dec 2018 09:39:35 +0100 Subject: [PATCH 26/50] [Postfix] Split SASL passwd maps [Postfix] create new smtp service to skip sender-dependent SASL map [Postfix] Hard-bounce on SASL errors --- data/conf/postfix/main.cf | 8 ++++++-- data/conf/postfix/master.cf | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/data/conf/postfix/main.cf b/data/conf/postfix/main.cf index 47cbc791..83a252d8 100644 --- a/data/conf/postfix/main.cf +++ b/data/conf/postfix/main.cf @@ -43,7 +43,9 @@ postscreen_pipelining_enable = no proxy_read_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_virtual_sender_acl.cf, proxy:mysql:/opt/postfix/conf/sql/mysql_tls_policy_override_maps.cf, proxy:mysql:/opt/postfix/conf/sql/mysql_sender_dependent_default_transport_maps.cf, - proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps.cf, + proxy:mysql:/opt/postfix/conf/sql/mysql_transport_maps.cf, + proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_sender_dependent.cf, + proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf, proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf, proxy:mysql:/opt/postfix/conf/sql/mysql_sender_bcc_maps.cf, proxy:mysql:/opt/postfix/conf/sql/mysql_recipient_bcc_maps.cf, @@ -126,9 +128,11 @@ mydestination = localhost.localdomain, localhost smtp_address_preference = ipv4 smtp_sender_dependent_authentication = yes smtp_sasl_auth_enable = yes -smtp_sasl_password_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps.cf +smtp_sasl_password_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_sender_dependent.cf smtp_sasl_security_options = smtp_sasl_mechanism_filter = plain, login smtp_tls_policy_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_tls_policy_override_maps.cf smtp_header_checks = pcre:/opt/postfix/conf/anonymize_headers.pcre mail_name = Postcow +transport_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_transport_maps.cf +smtp_sasl_auth_soft_bounce = no diff --git a/data/conf/postfix/master.cf b/data/conf/postfix/master.cf index 07d0d853..40527db8 100644 --- a/data/conf/postfix/master.cf +++ b/data/conf/postfix/master.cf @@ -14,7 +14,6 @@ submission inet n - n - - smtpd -o smtpd_client_restrictions=permit_mynetworks,permit_sasl_authenticated,reject -o smtpd_tls_auth_only=no -o smtpd_sender_restrictions=check_sasl_access,regexp:/opt/postfix/conf/allow_mailcow_local.regexp,reject_authenticated_sender_login_mismatch,permit_mynetworks,permit_sasl_authenticated,reject_unlisted_sender,reject_unknown_sender_domain - 590 inet n - n - - smtpd -o smtpd_client_restrictions=permit_mynetworks,reject -o smtpd_tls_auth_only=no @@ -24,6 +23,8 @@ smtp_enforced_tls unix - - n - - smtp -o smtp_tls_security_level=encrypt -o syslog_name=enforced-tls-smtp -o smtp_delivery_status_filter=pcre:/opt/postfix/conf/smtp_dsn_filter +smtp_via_transport_maps unix - - n - - smtp -v + -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf tlsproxy unix - - n - 0 tlsproxy dnsblog unix - - n - 0 dnsblog From bcd6e436653a038ffb54817ba06c0d27de5a9aac Mon Sep 17 00:00:00 2001 From: andryyy Date: Wed, 19 Dec 2018 12:16:36 +0100 Subject: [PATCH 27/50] [Postfix] Remove verbose flag from smtp service --- data/conf/postfix/master.cf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/conf/postfix/master.cf b/data/conf/postfix/master.cf index 40527db8..224c74ab 100644 --- a/data/conf/postfix/master.cf +++ b/data/conf/postfix/master.cf @@ -23,7 +23,7 @@ smtp_enforced_tls unix - - n - - smtp -o smtp_tls_security_level=encrypt -o syslog_name=enforced-tls-smtp -o smtp_delivery_status_filter=pcre:/opt/postfix/conf/smtp_dsn_filter -smtp_via_transport_maps unix - - n - - smtp -v +smtp_via_transport_maps unix - - n - - smtp -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf tlsproxy unix - - n - 0 tlsproxy From 8e5d2fe4f99f3fbc74f198a9ea9b0acfec0545a1 Mon Sep 17 00:00:00 2001 From: andryyy Date: Thu, 20 Dec 2018 11:23:13 +0100 Subject: [PATCH 28/50] [Compose] Fix custom-sogo.js mount --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 99881953..0bdea302 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -150,7 +150,7 @@ services: volumes: - ./data/conf/sogo/:/etc/sogo/ - ./data/web/inc/init_db.inc.php:/init_db.inc.php - - ./data/conf/sogo/custom-sogo.js:/usr/lib/GNUstep/SOGo/WebServerResources/js/custom-sogo.js + - ./data/conf/sogo/custom-sogo.js:/usr/local/lib/GNUstep/SOGo/WebServerResources/js/custom-sogo.js - mysql-socket-vol-1:/var/run/mysqld/ restart: always dns: From b99820d0117e364dec337915dde7d697116ee532 Mon Sep 17 00:00:00 2001 From: andryyy Date: Thu, 20 Dec 2018 11:23:35 +0100 Subject: [PATCH 29/50] [Web] Allow to set transport maps, rename relayhosts to sender-dependent transports --- data/web/admin.php | 137 +++++-- data/web/edit.php | 53 +++ .../{relay_check.php => transport_check.php} | 40 +- data/web/inc/functions.relayhost.inc.php | 174 -------- data/web/inc/functions.transports.inc.php | 385 ++++++++++++++++++ data/web/inc/init_db.inc.php | 22 +- data/web/inc/prerequisites.inc.php | 2 +- data/web/js/admin.js | 74 +++- data/web/js/debug.js | 2 +- data/web/js/mailbox.js | 62 +++ data/web/json_api.php | 36 ++ data/web/lang/lang.de.php | 35 +- data/web/lang/lang.en.php | 32 +- data/web/modals/admin.php | 17 +- 14 files changed, 820 insertions(+), 251 deletions(-) rename data/web/inc/ajax/{relay_check.php => transport_check.php} (65%) delete mode 100644 data/web/inc/functions.relayhost.inc.php create mode 100644 data/web/inc/functions.transports.inc.php diff --git a/data/web/admin.php b/data/web/admin.php index 2241da86..07c0f7e6 100644 --- a/data/web/admin.php +++ b/data/web/admin.php @@ -10,6 +10,7 @@ $tfa_data = get_tfa(); @@ -178,6 +179,101 @@ $tfa_data = get_tfa(); +
    +
    +
    +
    +

    +
    +
    +
    +
    +
    + + + +
    +
    + +

    +
    +
    +
    +
    + + +
    +
    + + +
    +
    + + +
    + +
    +
    +
    +
    +
    + +
    +
    +
    +

    +
    +
    +
    +
    +
    + + + +
    +
    + +

    +
    +
    +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    +

    + +
    +
    +
    +
    +
    +
    + + +
    - -
    -
    Relayhosts
    -
    -

    -
    -
    -
    -
    -
    - - - -
    -
    - -

    -
    -
    - - -
    -
    - - -
    -
    - - -
    - -
    -
    -
    -
    diff --git a/data/web/edit.php b/data/web/edit.php index 188f2cb9..fed7e2c9 100644 --- a/data/web/edit.php +++ b/data/web/edit.php @@ -693,6 +693,59 @@ if (isset($_SESSION['mailcow_cc_role'])) { +

    +
    + +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    +
    +
    + +
    +
    +
    +
    +
    + +
    +
    +
    + + + '; + } + else { + echo 'No MX records for ' . $hostname . ' were found in DNS, skipping and using hostname as next-hop.
    '; + } + } // Use port 25 if no port was given $port = (empty($port)) ? 25 : $port; - $username = $relayhost_details['username']; - $password = $relayhost_details['password']; + $username = $transport_details['username']; + $password = $transport_details['password']; $mail = new PHPMailer; $mail->Timeout = 10; @@ -73,7 +99,7 @@ if (isset($_SESSION['mailcow_cc_role']) && $_SESSION['mailcow_cc_role'] == "admi $mail->send(); } else { - echo "Unknown relayhost."; + echo "Unknown transport."; } } else { diff --git a/data/web/inc/functions.relayhost.inc.php b/data/web/inc/functions.relayhost.inc.php deleted file mode 100644 index a3e1ffda..00000000 --- a/data/web/inc/functions.relayhost.inc.php +++ /dev/null @@ -1,174 +0,0 @@ - 'danger', - 'log' => array(__FUNCTION__, $_action, $_data_log), - 'msg' => 'access_denied' - ); - return false; - } - $hostname = trim($_data['hostname']); - $username = str_replace(':', '\:', trim($_data['username'])); - $password = str_replace(':', '\:', trim($_data['password'])); - if (empty($hostname)) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_data_log), - 'msg' => array('invalid_host', htmlspecialchars($host)) - ); - return false; - } - try { - $stmt = $pdo->prepare("INSERT INTO `relayhosts` (`hostname`, `username` ,`password`, `active`) - VALUES (:hostname, :username, :password, :active)"); - $stmt->execute(array( - ':hostname' => $hostname, - ':username' => $username, - ':password' => str_replace(':', '\:', $password), - ':active' => '1' - )); - } - catch (PDOException $e) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_data_log), - 'msg' => array('mysql_error', $e) - ); - return false; - } - $_SESSION['return'][] = array( - 'type' => 'success', - 'log' => array(__FUNCTION__, $_action, $_data_log), - 'msg' => array('relayhost_added', htmlspecialchars(implode(', ', $hosts))) - ); - break; - case 'edit': - if ($_SESSION['mailcow_cc_role'] != "admin") { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_data_log), - 'msg' => 'access_denied' - ); - return false; - } - $ids = (array)$_data['id']; - foreach ($ids as $id) { - $is_now = relayhost('details', $id); - if (!empty($is_now)) { - $hostname = (!empty($_data['hostname'])) ? trim($_data['hostname']) : $is_now['hostname']; - $username = (isset($_data['username'])) ? trim($_data['username']) : $is_now['username']; - $password = (isset($_data['password'])) ? trim($_data['password']) : $is_now['password']; - $active = (isset($_data['active'])) ? intval($_data['active']) : $is_now['active_int']; - } - else { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_data_log), - 'msg' => array('relayhost_invalid', $id) - ); - continue; - } - try { - $stmt = $pdo->prepare("UPDATE `relayhosts` SET - `hostname` = :hostname, - `username` = :username, - `password` = :password, - `active` = :active - WHERE `id` = :id"); - $stmt->execute(array( - ':id' => $id, - ':hostname' => $hostname, - ':username' => $username, - ':password' => $password, - ':active' => $active - )); - } - catch (PDOException $e) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_data_log), - 'msg' => array('mysql_error', $e) - ); - continue; - } - $_SESSION['return'][] = array( - 'type' => 'success', - 'log' => array(__FUNCTION__, $_action, $_data_log), - 'msg' => array('object_modified', htmlspecialchars(implode(', ', $hostnames))) - ); - } - break; - case 'delete': - if ($_SESSION['mailcow_cc_role'] != "admin") { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_data_log), - 'msg' => 'access_denied' - ); - return false; - } - $ids = (array)$_data['id']; - foreach ($ids as $id) { - try { - $stmt = $pdo->prepare("DELETE FROM `relayhosts` WHERE `id`= :id"); - $stmt->execute(array(':id' => $id)); - $stmt = $pdo->prepare("UPDATE `domain` SET `relayhost` = '0' WHERE `relayhost`= :id"); - $stmt->execute(array(':id' => $id)); - } - catch (PDOException $e) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_data_log), - 'msg' => array('mysql_error', $e) - ); - continue; - } - $_SESSION['return'][] = array( - 'type' => 'success', - 'log' => array(__FUNCTION__, $_action, $_data_log), - 'msg' => array('relayhost_removed', htmlspecialchars($id)) - ); - } - break; - case 'get': - if ($_SESSION['mailcow_cc_role'] != "admin") { - return false; - } - $relayhosts = array(); - $stmt = $pdo->query("SELECT `id`, `hostname`, `username` FROM `relayhosts`"); - $relayhosts = $stmt->fetchAll(PDO::FETCH_ASSOC); - return $relayhosts; - break; - case 'details': - if ($_SESSION['mailcow_cc_role'] != "admin" || !isset($_data)) { - return false; - } - $relayhostdata = array(); - $stmt = $pdo->prepare("SELECT `id`, - `hostname`, - `username`, - `password`, - `active` AS `active_int`, - CONCAT(LEFT(`password`, 3), '...') AS `password_short`, - CASE `active` WHEN 1 THEN '".$lang['mailbox']['yes']."' ELSE '".$lang['mailbox']['no']."' END AS `active` - FROM `relayhosts` - WHERE `id` = :id"); - $stmt->execute(array(':id' => $_data)); - $relayhostdata = $stmt->fetch(PDO::FETCH_ASSOC); - if (!empty($relayhostdata)) { - $stmt = $pdo->prepare("SELECT GROUP_CONCAT(`domain` SEPARATOR ', ') AS `used_by_domains` FROM `domain` WHERE `relayhost` = :id"); - $stmt->execute(array(':id' => $_data)); - $used_by_domains = $stmt->fetch(PDO::FETCH_ASSOC)['used_by_domains']; - $used_by_domains = (empty($used_by_domains)) ? '' : $used_by_domains; - $relayhostdata['used_by_domains'] = $used_by_domains; - } - return $relayhostdata; - break; - } -} \ No newline at end of file diff --git a/data/web/inc/functions.transports.inc.php b/data/web/inc/functions.transports.inc.php new file mode 100644 index 00000000..4f30645f --- /dev/null +++ b/data/web/inc/functions.transports.inc.php @@ -0,0 +1,385 @@ + 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => 'access_denied' + ); + return false; + } + $hostname = trim($_data['hostname']); + $username = str_replace(':', '\:', trim($_data['username'])); + $password = str_replace(':', '\:', trim($_data['password'])); + if (empty($hostname)) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('invalid_host', htmlspecialchars($host)) + ); + return false; + } + try { + $stmt = $pdo->prepare("INSERT INTO `relayhosts` (`hostname`, `username` ,`password`, `active`) + VALUES (:hostname, :username, :password, :active)"); + $stmt->execute(array( + ':hostname' => $hostname, + ':username' => $username, + ':password' => str_replace(':', '\:', $password), + ':active' => '1' + )); + } + catch (PDOException $e) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('mysql_error', $e) + ); + return false; + } + $_SESSION['return'][] = array( + 'type' => 'success', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('relayhost_added', htmlspecialchars(implode(', ', $hosts))) + ); + break; + case 'edit': + if ($_SESSION['mailcow_cc_role'] != "admin") { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => 'access_denied' + ); + return false; + } + $ids = (array)$_data['id']; + foreach ($ids as $id) { + $is_now = relayhost('details', $id); + if (!empty($is_now)) { + $hostname = (!empty($_data['hostname'])) ? trim($_data['hostname']) : $is_now['hostname']; + $username = (isset($_data['username'])) ? trim($_data['username']) : $is_now['username']; + $password = (isset($_data['password'])) ? trim($_data['password']) : $is_now['password']; + $active = (isset($_data['active'])) ? intval($_data['active']) : $is_now['active_int']; + } + else { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('relayhost_invalid', $id) + ); + continue; + } + try { + $stmt = $pdo->prepare("UPDATE `relayhosts` SET + `hostname` = :hostname, + `username` = :username, + `password` = :password, + `active` = :active + WHERE `id` = :id"); + $stmt->execute(array( + ':id' => $id, + ':hostname' => $hostname, + ':username' => $username, + ':password' => $password, + ':active' => $active + )); + } + catch (PDOException $e) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('mysql_error', $e) + ); + continue; + } + $_SESSION['return'][] = array( + 'type' => 'success', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('object_modified', htmlspecialchars(implode(', ', $hostnames))) + ); + } + break; + case 'delete': + if ($_SESSION['mailcow_cc_role'] != "admin") { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => 'access_denied' + ); + return false; + } + $ids = (array)$_data['id']; + foreach ($ids as $id) { + try { + $stmt = $pdo->prepare("DELETE FROM `relayhosts` WHERE `id`= :id"); + $stmt->execute(array(':id' => $id)); + $stmt = $pdo->prepare("UPDATE `domain` SET `relayhost` = '0' WHERE `relayhost`= :id"); + $stmt->execute(array(':id' => $id)); + } + catch (PDOException $e) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('mysql_error', $e) + ); + continue; + } + $_SESSION['return'][] = array( + 'type' => 'success', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('relayhost_removed', htmlspecialchars($id)) + ); + } + break; + case 'get': + if ($_SESSION['mailcow_cc_role'] != "admin") { + return false; + } + $relayhosts = array(); + $stmt = $pdo->query("SELECT `id`, `hostname`, `username` FROM `relayhosts`"); + $relayhosts = $stmt->fetchAll(PDO::FETCH_ASSOC); + return $relayhosts; + break; + case 'details': + if ($_SESSION['mailcow_cc_role'] != "admin" || !isset($_data)) { + return false; + } + $relayhostdata = array(); + $stmt = $pdo->prepare("SELECT `id`, + `hostname`, + `username`, + `password`, + `active` AS `active_int`, + CONCAT(LEFT(`password`, 3), '...') AS `password_short`, + CASE `active` WHEN 1 THEN '".$lang['mailbox']['yes']."' ELSE '".$lang['mailbox']['no']."' END AS `active` + FROM `relayhosts` + WHERE `id` = :id"); + $stmt->execute(array(':id' => $_data)); + $relayhostdata = $stmt->fetch(PDO::FETCH_ASSOC); + if (!empty($relayhostdata)) { + $stmt = $pdo->prepare("SELECT GROUP_CONCAT(`domain` SEPARATOR ', ') AS `used_by_domains` FROM `domain` WHERE `relayhost` = :id"); + $stmt->execute(array(':id' => $_data)); + $used_by_domains = $stmt->fetch(PDO::FETCH_ASSOC)['used_by_domains']; + $used_by_domains = (empty($used_by_domains)) ? '' : $used_by_domains; + $relayhostdata['used_by_domains'] = $used_by_domains; + } + return $relayhostdata; + break; + } +} +function transport($_action, $_data = null) { + global $pdo; + global $lang; + $_data_log = $_data; + switch ($_action) { + case 'add': + if ($_SESSION['mailcow_cc_role'] != "admin") { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => 'access_denied' + ); + return false; + } + $destination = trim($_data['destination']); + $nexthop = trim($_data['nexthop']); + $username = str_replace(':', '\:', trim($_data['username'])); + $password = str_replace(':', '\:', trim($_data['password'])); + if (empty($destination) || (is_valid_domain_name(preg_replace('/^' . preg_quote('.', '/') . '/', '', $destination)) === false && $destination != '*')) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => 'invalid_destination' + ); + return false; + } + if (empty($nexthop)) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('invalid_nexthop') + ); + return false; + } + if (!empty($username)) { + $transports = transport('get'); + if (!empty($transports)) { + foreach ($transports as $transport) { + if (transport('details', $transport['id'])['nexthop'] == $nexthop && !empty(transport('details', $transport['id'])['username'])) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => 'invalid_nexthop_authenticated' + ); + return false; + } + } + } + } + try { + $stmt = $pdo->prepare("INSERT INTO `transports` (`nexthop`, `destination`, `username` ,`password`, `active`) + VALUES (:nexthop, :destination, :username, :password, :active)"); + $stmt->execute(array( + ':nexthop' => $nexthop, + ':destination' => $destination, + ':username' => $username, + ':password' => str_replace(':', '\:', $password), + ':active' => '1' + )); + $stmt = $pdo->prepare("UPDATE `transports` SET + `username` = :username, + `password` = :password + WHERE `nexthop` = :nexthop"); + $stmt->execute(array( + ':nexthop' => $nexthop, + ':username' => $username, + ':password' => $password + )); + } + catch (PDOException $e) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('mysql_error', $e) + ); + return false; + } + $_SESSION['return'][] = array( + 'type' => 'success', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('relayhost_added', htmlspecialchars(implode(', ', $hosts))) + ); + break; + case 'edit': + if ($_SESSION['mailcow_cc_role'] != "admin") { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => 'access_denied' + ); + return false; + } + $ids = (array)$_data['id']; + foreach ($ids as $id) { + $is_now = transport('details', $id); + if (!empty($is_now)) { + $destination = (!empty($_data['destination'])) ? trim($_data['destination']) : $is_now['destination']; + $nexthop = (!empty($_data['nexthop'])) ? trim($_data['nexthop']) : $is_now['nexthop']; + $username = (isset($_data['username'])) ? trim($_data['username']) : $is_now['username']; + $password = (isset($_data['password'])) ? trim($_data['password']) : $is_now['password']; + $active = (isset($_data['active'])) ? intval($_data['active']) : $is_now['active_int']; + } + else { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('relayhost_invalid', $id) + ); + continue; + } + try { + $stmt = $pdo->prepare("UPDATE `transports` SET + `destination` = :destination, + `nexthop` = :nexthop, + `username` = :username, + `password` = :password, + `active` = :active + WHERE `id` = :id"); + $stmt->execute(array( + ':id' => $id, + ':destination' => $destination, + ':nexthop' => $nexthop, + ':username' => $username, + ':password' => $password, + ':active' => $active + )); + $stmt = $pdo->prepare("UPDATE `transports` SET + `username` = :username, + `password` = :password + WHERE `nexthop` = :nexthop"); + $stmt->execute(array( + ':nexthop' => $nexthop, + ':username' => $username, + ':password' => $password + )); + } + catch (PDOException $e) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('mysql_error', $e) + ); + continue; + } + $_SESSION['return'][] = array( + 'type' => 'success', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('object_modified', htmlspecialchars(implode(', ', $hostnames))) + ); + } + break; + case 'delete': + if ($_SESSION['mailcow_cc_role'] != "admin") { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => 'access_denied' + ); + return false; + } + $ids = (array)$_data['id']; + foreach ($ids as $id) { + try { + $stmt = $pdo->prepare("DELETE FROM `transports` WHERE `id`= :id"); + $stmt->execute(array(':id' => $id)); + } + catch (PDOException $e) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('mysql_error', $e) + ); + continue; + } + $_SESSION['return'][] = array( + 'type' => 'success', + 'log' => array(__FUNCTION__, $_action, $_data_log), + 'msg' => array('relayhost_removed', htmlspecialchars($id)) + ); + } + break; + case 'get': + if ($_SESSION['mailcow_cc_role'] != "admin") { + return false; + } + $transports = array(); + $stmt = $pdo->query("SELECT `id`, `destination`, `nexthop`, `username` FROM `transports`"); + $transports = $stmt->fetchAll(PDO::FETCH_ASSOC); + return $transports; + break; + case 'details': + if ($_SESSION['mailcow_cc_role'] != "admin" || !isset($_data)) { + return false; + } + $transportdata = array(); + $stmt = $pdo->prepare("SELECT `id`, + `destination`, + `nexthop`, + `username`, + `password`, + `active` AS `active_int`, + CONCAT(LEFT(`password`, 3), '...') AS `password_short`, + CASE `active` WHEN 1 THEN '".$lang['mailbox']['yes']."' ELSE '".$lang['mailbox']['no']."' END AS `active` + FROM `transports` + WHERE `id` = :id"); + $stmt->execute(array(':id' => $_data)); + $transportdata = $stmt->fetch(PDO::FETCH_ASSOC); + return $transportdata; + break; + } +} \ No newline at end of file diff --git a/data/web/inc/init_db.inc.php b/data/web/inc/init_db.inc.php index 3b17f145..052c1b0e 100644 --- a/data/web/inc/init_db.inc.php +++ b/data/web/inc/init_db.inc.php @@ -3,7 +3,7 @@ function init_db_schema() { try { global $pdo; - $db_version = "14112018_0717"; + $db_version = "15122018_0717"; $stmt = $pdo->query("SHOW TABLES LIKE 'versions'"); $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC)); @@ -109,6 +109,26 @@ function init_db_schema() { ), "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" ), + "transports" => array( + "cols" => array( + "id" => "INT NOT NULL AUTO_INCREMENT", + "destination" => "VARCHAR(255) NOT NULL", + "nexthop" => "VARCHAR(255) NOT NULL", + "username" => "VARCHAR(255) NOT NULL", + "password" => "VARCHAR(255) NOT NULL", + "active" => "TINYINT(1) NOT NULL DEFAULT '1'" + ), + "keys" => array( + "primary" => array( + "" => array("id") + ), + "key" => array( + "destination" => array("destination"), + "nexthop" => array("nexthop"), + ) + ), + "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" + ), "alias" => array( "cols" => array( "id" => "INT NOT NULL AUTO_INCREMENT", diff --git a/data/web/inc/prerequisites.inc.php b/data/web/inc/prerequisites.inc.php index 9eca849a..5ed34478 100644 --- a/data/web/inc/prerequisites.inc.php +++ b/data/web/inc/prerequisites.inc.php @@ -147,7 +147,7 @@ require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.dkim.inc.php'; require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.fwdhost.inc.php'; require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.mailq.inc.php'; require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.ratelimit.inc.php'; -require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.relayhost.inc.php'; +require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.transports.inc.php'; require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.rsettings.inc.php'; require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.tls_policy_maps.inc.php'; require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.fail2ban.inc.php'; diff --git a/data/web/js/admin.js b/data/web/js/admin.js index 07bf27b6..c53f47d5 100644 --- a/data/web/js/admin.js +++ b/data/web/js/admin.js @@ -5,6 +5,8 @@ jQuery(function($){ var entityMap={"&":"&","<":"<",">":">",'"':""","'":"'","/":"/","`":"`","=":"="}; function escapeHtml(n){return String(n).replace(/[&<>"'`=\/]/g,function(n){return entityMap[n]})} function humanFileSize(i){if(Math.abs(i)<1024)return i+" B";var B=["KiB","MiB","GiB","TiB","PiB","EiB","ZiB","YiB"],e=-1;do{i/=1024,++e}while(Math.abs(i)>=1024&&e Test' + + ' Test' + ' ' + lang.edit + '' + - ' ' + lang.remove + '' + + ' ' + lang.remove + '' + '
    '; item.chkbox = ''; }); + } else if (table == 'transportstable') { + $.each(data, function (i, item) { + if (item.username) { + item.username = '' + item.username + ''; + } + item.action = ''; + item.chkbox = ''; + }); } else if (table == 'queuetable') { $.each(data, function (i, item) { item.chkbox = ''; @@ -264,6 +305,7 @@ jQuery(function($){ draw_admins(); draw_fwd_hosts(); draw_relayhosts(); + draw_transport_maps(); draw_queue(); // Relayhost $('#testRelayhostModal').on('show.bs.modal', function (e) { @@ -290,6 +332,32 @@ jQuery(function($){ } }); }) + // Transport + $('#testTransportModal').on('show.bs.modal', function (e) { + $('#test_transport_result').text("-"); + button = $(e.relatedTarget) + if (button != null) { + $('#transport_id').val(button.data('transport-id')); + $('#transport_type').val(button.data('transport-type')); + } + }) + $('#test_transport').on('click', function (e) { + e.preventDefault(); + prev = $('#test_transport').text(); + $(this).prop("disabled",true); + $(this).html(' '); + $.ajax({ + type: 'GET', + url: 'inc/ajax/transport_check.php', + dataType: 'text', + data: $('#test_transport_form').serialize(), + complete: function (data) { + $('#test_transport_result').html(data.responseText); + $('#test_transport').prop("disabled",false); + $('#test_transport').text(prev); + } + }); + }) // DKIM private key modal $('#showDKIMprivKey').on('show.bs.modal', function (e) { $('#priv_key_pre').text("-"); diff --git a/data/web/js/debug.js b/data/web/js/debug.js index b294931c..4c5119cb 100644 --- a/data/web/js/debug.js +++ b/data/web/js/debug.js @@ -592,7 +592,7 @@ jQuery(function($){ if (item.rl_hash == null) { item.rl_hash = "err"; } - item.indicator = '  '; + item.indicator = ' '; if (item.rl_hash != 'err') { item.action = ' ' + lang.reset_limit + ''; } diff --git a/data/web/js/mailbox.js b/data/web/js/mailbox.js index 1f2b4bd2..3551afa2 100644 --- a/data/web/js/mailbox.js +++ b/data/web/js/mailbox.js @@ -646,6 +646,67 @@ jQuery(function($){ } }); } + function draw_transport_maps_table() { + ft_transport_maps_table = FooTable.init('#transport_maps_table', { + "columns": [ + {"name":"chkbox","title":"","style":{"maxWidth":"60px","width":"60px"},"filterable": false,"sortable": false,"type":"html"}, + {"sorted": true,"name":"id","title":"ID","style":{"maxWidth":"60px","width":"60px","text-align":"center"}}, + {"name":"dest","title":lang.tls_map_dest}, + {"name":"parameters","title":lang.tls_map_parameters}, + {"name":"active","filterable": false,"style":{"maxWidth":"80px","width":"80px"},"title":lang.active}, + {"name":"action","filterable": false,"sortable": false,"style":{"text-align":"right","maxWidth":"180px","width":"180px"},"type":"html","title":(role == "admin" ? lang.action : ""),"breakpoints":"xs sm"} + ], + "empty": lang.empty, + "rows": $.ajax({ + dataType: 'json', + url: '/api/v1/get/transport-map/all', + jsonp: false, + error: function () { + console.log('Cannot draw transport map table'); + }, + success: function (data) { + if (role == "admin") { + $.each(data, function (i, item) { + item.dest = escapeHtml(item.dest); + if (item.parameters == '') { + item.parameters = '-'; + } else { + item.parameters = '' + escapeHtml(item.parameters) + ''; + } + item.action = ''; + item.chkbox = ''; + }); + } + } + }), + "paging": { + "enabled": true, + "limit": 5, + "size": pagination_size + }, + "filtering": { + "enabled": true, + "delay": 100, + "position": "left", + "connectors": false, + "placeholder": lang.filter_table + }, + "sorting": { + "enabled": true + }, + "on": { + "ready.ft.table": function(e, ft){ + table_mailbox_ready(ft, 'transport_maps_table'); + }, + "after.ft.paging": function(e, ft){ + paging_mailbox_after(ft, 'transport_maps_table'); + } + } + }); + } function draw_alias_table() { ft_alias_table = FooTable.init('#alias_table', { "columns": [ @@ -925,5 +986,6 @@ jQuery(function($){ draw_bcc_table(); draw_recipient_map_table(); draw_tls_policy_table(); + draw_transport_maps_table(); }); diff --git a/data/web/json_api.php b/data/web/json_api.php index c9db5eb6..6a876f8f 100644 --- a/data/web/json_api.php +++ b/data/web/json_api.php @@ -102,6 +102,9 @@ if (isset($_SESSION['mailcow_cc_role']) || isset($_SESSION['pending_mailcow_cc_u case "relayhost": process_add_return(relayhost('add', $attr)); break; + case "transport": + process_add_return(transport('add', $attr)); + break; case "rsetting": process_add_return(rsettings('add', $attr)); break; @@ -320,6 +323,33 @@ if (isset($_SESSION['mailcow_cc_role']) || isset($_SESSION['pending_mailcow_cc_u } break; + case "transport": + switch ($object) { + case "all": + $transports = transport('get'); + if (!empty($transports)) { + foreach ($transports as $transport) { + if ($details = transport('details', $transport['id'])) { + $data[] = $details; + } + else { + continue; + } + } + process_get_return($data); + } + else { + echo '{}'; + } + break; + + default: + $data = transport('details', $object); + process_get_return($data); + break; + } + break; + case "rsetting": switch ($object) { case "all": @@ -990,6 +1020,9 @@ if (isset($_SESSION['mailcow_cc_role']) || isset($_SESSION['pending_mailcow_cc_u case "relayhost": process_delete_return(relayhost('delete', array('id' => $items))); break; + case "transport": + process_delete_return(transport('delete', array('id' => $items))); + break; case "rsetting": process_delete_return(rsettings('delete', array('id' => $items))); break; @@ -1107,6 +1140,9 @@ if (isset($_SESSION['mailcow_cc_role']) || isset($_SESSION['pending_mailcow_cc_u case "relayhost": process_edit_return(relayhost('edit', array_merge(array('id' => $items), $attr))); break; + case "transport": + process_edit_return(transport('edit', array_merge(array('id' => $items), $attr))); + break; case "rsetting": process_edit_return(rsettings('edit', array_merge(array('id' => $items), $attr))); break; diff --git a/data/web/lang/lang.de.php b/data/web/lang/lang.de.php index ba031d8e..8a763cb6 100644 --- a/data/web/lang/lang.de.php +++ b/data/web/lang/lang.de.php @@ -29,6 +29,7 @@ $lang['success']['verified_u2f_login'] = "U2F Anmeldung verifiziert"; $lang['success']['verified_yotp_login'] = "Yubico OTP Anmeldung verifiziert"; $lang['danger']['yotp_verification_failed'] = "Yubico OTP Verifizierung fehlgeschlagen: %s"; $lang['danger']['ip_list_empty'] = "Liste erlaubter IPs darf nicht leer sein"; +$lang['danger']['invalid_destination'] = "Ziel-Format ist ungültig"; $lang['danger']['rspamd_ui_pw_length'] = "Rspamd UI Passwort muss mindestens 6 Zeichen lang sein"; $lang['success']['rspamd_ui_pw_set'] = "Rspamd UI Passwort wurde gesetzt"; $lang['success']['queue_command_success'] = "Queue-Aufgabe erfolgreich ausgeführt"; @@ -65,7 +66,7 @@ $lang['success']['settings_map_added'] = "Regel wurde gespeichert"; $lang['danger']['settings_map_invalid'] = "Regel ID %s ist ungültig"; $lang['success']['settings_map_removed'] = "Regeln wurden entfernt: %s"; $lang['danger']['invalid_host'] = "Ungültiger Host: %s"; -$lang['danger']['relayhost_invalid'] = "Relayhost %s ist ungültig"; +$lang['danger']['relayhost_invalid'] = "Mapeintrag %s ist ungültig"; $lang['success']['saved_settings'] = "Regel wurde gespeichert"; $lang['danger']['dkim_domain_or_sel_invalid'] = 'DKIM-Domain oder Selektor nicht korrekt: %s'; @@ -392,7 +393,10 @@ $lang['acl']['prohibited'] = 'Untersagt durch Richtlinie'; $lang['add']['generate'] = 'generieren'; $lang['add']['syncjob'] = 'Syncjob hinzufügen'; $lang['add']['syncjob_hint'] = 'Passwörter werden unverschlüsselt abgelegt!'; -$lang['add']['hostname'] = 'Servername'; +$lang['add']['hostname'] = 'Host'; +$lang['add']['destination'] = 'Ziel'; +$lang['add']['nexthop'] = 'Next hop'; +$lang['edit']['nexthop'] = 'Next hop'; $lang['add']['port'] = 'Port'; $lang['add']['username'] = 'Benutzername'; $lang['add']['enc_method'] = 'Verschlüsselung'; @@ -572,12 +576,29 @@ $lang['admin']['message'] = 'Nachricht'; $lang['admin']['forwarding_hosts'] = 'Weiterleitungs-Hosts'; $lang['admin']['forwarding_hosts_hint'] = 'Eingehende Nachrichten werden von den hier gelisteten Hosts bedingungslos akzeptiert. Diese Hosts werden dann nicht mit DNSBLs abgeglichen oder Greylisting unterworfen. Von ihnen empfangener Spam wird nie abgelehnt, optional kann er aber in den Spam-Ordner einsortiert werden. Die übliche Verwendung für diese Funktion ist, um Mailserver anzugeben, auf denen eine Weiterleitung zu Ihrem mailcow-Server eingerichtet wurde.'; $lang['admin']['forwarding_hosts_add_hint'] = 'Sie können entweder IPv4/IPv6-Adressen, Netzwerke in CIDR-Notation, Hostnamen (die zu IP-Adressen aufgelöst werden), oder Domainnamen (die zu IP-Adressen aufgelöst werden, indem ihr SPF-Record abgefragt wird oder, in dessen Abwesenheit, ihre MX-Records) angeben.'; -$lang['admin']['relayhosts_hint'] = 'Erstellen Sie Relayhosts, um diese im Einstellungsdialog einer Domain auszuwählen.'; -$lang['admin']['add_relayhost_add_hint'] = 'Bitte beachten Sie, dass Relayhost Anmeldedaten im Klartext gespeichert werden.'; +$lang['admin']['relayhosts_hint'] = 'Erstellen Sie senderabhängige Transporte, um diese im Einstellungsdialog einer Domain auszuwählen.
    + Der Transporttyp lautet immer "smtp:". Benutzereinstellungen bezüglich Verschlüsselungsrichtlinie werden beim Transport berücksichtigt.'; +$lang['admin']['transports_hint'] = 'Transport Maps überwiegen senderabhängige Transport Maps und ignorieren die individuellen Einstellungen eines Benutzers bezüglich Verschlüsselungsrichtlinie, da der Absender bei Ermittlung der Transportregel nicht berücksichtigt wird.
    + Der Transport erfolgt immer via "smtp:".
    + Ein Eintrag in der TLS Policy Map kann eine Verschlüsselung erzwingen.
    + Die Authentifizierung wird anhand des Host Parameters ermittelt.'; +$lang['admin']['add_relayhost_hint'] = 'Bitte beachten Sie, dass Anmeldedaten klartext gespeichert werden.
    + Angelegte Transporte dieser Art sind senderabhängig und müssen erst einer Domain zugewiesen werden, bevor sie als Transport verwendet werden.
    + Diese Einstellungen entsprechen demach nicht dem "relayhost" Parameter in Postfix.'; +$lang['admin']['add_transports_hint'] = 'Bitte beachten Sie, dass Anmeldedaten klartext gespeichert werden.'; $lang['admin']['host'] = 'Host'; $lang['admin']['source'] = 'Quelle'; $lang['admin']['add_forwarding_host'] = 'Weiterleitungs-Host hinzufügen'; -$lang['admin']['add_relayhost'] = 'Relayhost hinzufügen'; +$lang['admin']['add_relayhost'] = 'Senderabhängigen Transport hinzufügen'; +$lang['admin']['add_transport'] = 'Transport hinzufügen'; +$lang['admin']['relayhosts'] = 'Senderabhängige Transport Maps'; +$lang['admin']['transport_maps'] = 'Transport Maps'; +$lang['admin']['routing'] = 'Routing'; +$lang['admin']['credentials_transport_warning'] = 'Warnung: Das Hinzufügen einer neuen Regel bewirkt die Aktualisierung der Authentifizierungsdaten aller vorhandenen Einträge mit identischem Host.'; + +$lang['admin']['destination'] = 'Ziel'; +$lang['admin']['nexthop'] = 'Next hop'; + $lang['admin']['api_allow_from'] = "IP-Adressen für Zugriff"; $lang['admin']['api_key'] = "API-Key"; $lang['admin']['activate_api'] = "API aktivieren"; @@ -594,8 +615,8 @@ $lang['admin']['quarantine_exclude_domains'] = "Domains und Alias-Domains aussch $lang['success']['forwarding_host_removed'] = "Weiterleitungs-Host %s wurde entfernt"; $lang['success']['forwarding_host_added'] = "Weiterleitungs-Host %s wurde hinzugefügt"; -$lang['success']['relayhost_removed'] = "Relayhost %s wurde entfernt"; -$lang['success']['relayhost_added'] = "Relayhost %s wurde hinzugefügt"; +$lang['success']['relayhost_removed'] = "Mapeintrag %s wurde entfernt"; +$lang['success']['relayhost_added'] = "Mapeintrag %s wurde hinzugefügt"; $lang['diagnostics']['dns_records'] = 'DNS-Einträge'; $lang['diagnostics']['dns_records_24hours'] = 'Bitte beachten Sie, dass es bis zu 24 Stunden dauern kann, bis Änderungen an Ihren DNS-Einträgen als aktueller Status auf dieser Seite dargestellt werden. Diese Seite ist nur als Hilfsmittel gedacht, um die korrekten Werte für DNS-Einträge anzuzeigen und zu überprüfen, ob die Daten im DNS hinterlegt sind.'; $lang['diagnostics']['dns_records_name'] = 'Name'; diff --git a/data/web/lang/lang.en.php b/data/web/lang/lang.en.php index 4191446a..887385c7 100644 --- a/data/web/lang/lang.en.php +++ b/data/web/lang/lang.en.php @@ -30,6 +30,7 @@ $lang['success']['verified_u2f_login'] = "Verified U2F login"; $lang['success']['verified_yotp_login'] = "Verified Yubico OTP login"; $lang['danger']['yotp_verification_failed'] = "Yubico OTP verification failed: %s"; $lang['danger']['ip_list_empty'] = "List of allowed IPs cannot be empty"; +$lang['danger']['invalid_destination'] = "Destination format is invalid"; $lang['danger']['rspamd_ui_pw_length'] = "Rspamd UI password should be at least 6 chars long"; $lang['success']['rspamd_ui_pw_set'] = "Rspamd UI password successfully set"; $lang['success']['queue_command_success'] = "Queue command completed successfully"; @@ -66,7 +67,7 @@ $lang['success']['settings_map_added'] = "Added settings map entry"; $lang['danger']['settings_map_invalid'] = "Settings map ID %s invalid"; $lang['success']['settings_map_removed'] = "Removed settings map ID %s"; $lang['danger']['invalid_host'] = "Invalid host specified: %s"; -$lang['danger']['relayhost_invalid'] = "Relayhost %s is invalid"; +$lang['danger']['relayhost_invalid'] = "Map entry %s is invalid"; $lang['success']['saved_settings'] = "Saved settings"; $lang['success']['db_init_complete'] = "Database initialization completed"; @@ -405,7 +406,10 @@ $lang['acl']['prohibited'] = 'Prohibited by ACL'; $lang['add']['generate'] = 'generate'; $lang['add']['syncjob'] = 'Add sync job'; $lang['add']['syncjob_hint'] = 'Be aware that passwords need to be saved plain-text!'; -$lang['add']['hostname'] = 'Hostname'; +$lang['add']['hostname'] = 'Host'; +$lang['add']['destination'] = 'Ziel'; +$lang['add']['nexthop'] = 'Next hop'; +$lang['edit']['nexthop'] = 'Next hop'; $lang['add']['port'] = 'Port'; $lang['add']['username'] = 'Username'; $lang['add']['enc_method'] = 'Encryption method'; @@ -596,16 +600,28 @@ $lang['admin']['in_use_by'] = 'In use by'; $lang['admin']['forwarding_hosts'] = 'Forwarding Hosts'; $lang['admin']['forwarding_hosts_hint'] = 'Incoming messages are unconditionally accepted from any hosts listed here. These hosts are then not checked against DNSBLs or subjected to greylisting. Spam received from them is never rejected, but optionally it can be filed into the Junk folder. The most common use for this is to specify mail servers on which you have set up a rule that forwards incoming emails to your mailcow server.'; $lang['admin']['forwarding_hosts_add_hint'] = 'You can either specify IPv4/IPv6 addresses, networks in CIDR notation, host names (which will be resolved to IP addresses), or domain names (which will be resolved to IP addresses by querying SPF records or, in their absence, MX records).'; -$lang['admin']['relayhosts_hint'] = 'Define relayhosts here to be able to select them in a domains configuration dialog.'; -$lang['admin']['add_relayhost_add_hint'] = 'Please be aware that relayhost authentication data will be stored as plain text.'; +$lang['admin']['relayhosts_hint'] = 'Define sender-dependent transports to be able to select them in a domains configuration dialog.
    + The transport service is always "smtp:". A users individual outbound TLS policy setting is taken into account.'; +$lang['admin']['transports_hint'] = 'A transport map entry overrules a sender-dependent transport map.'; +$lang['admin']['add_relayhost_hint'] = 'Please be aware that authentication data, if any, will be stored as plain text.'; +$lang['admin']['add_transports_hint'] = 'Please be aware that authentication data, if any, will be stored as plain text.'; $lang['admin']['host'] = 'Host'; $lang['admin']['source'] = 'Source'; -$lang['admin']['add_forwarding_host'] = 'Add Forwarding Host'; -$lang['admin']['add_relayhost'] = 'Add Relayhost'; +$lang['admin']['add_forwarding_host'] = 'Add forwarding host'; +$lang['admin']['add_relayhost'] = 'Add sender-dependent transport'; +$lang['admin']['add_transport'] = 'Add transport'; +$lang['admin']['relayhosts'] = 'Sender-dependent transports'; +$lang['admin']['transport_maps'] = 'Transport Maps'; +$lang['admin']['routing'] = 'Routing'; +$lang['admin']['credentials_transport_warning'] = 'Warning: Adding a new transport map entry will update the credentials for all entries with a matching nexthop column.'; + +$lang['admin']['destination'] = 'Destination'; +$lang['admin']['nexthop'] = 'Next hop'; + $lang['success']['forwarding_host_removed'] = "Forwarding host %s has been removed"; $lang['success']['forwarding_host_added'] = "Forwarding host %s has been added"; -$lang['success']['relayhost_removed'] = "Relayhost %s has been removed"; -$lang['success']['relayhost_added'] = "Relayhost %s has been added"; +$lang['success']['relayhost_removed'] = "Map entry %s has been removed"; +$lang['success']['relayhost_added'] = "Map entry %s has been added"; $lang['diagnostics']['dns_records'] = 'DNS Records'; $lang['diagnostics']['dns_records_24hours'] = 'Please note that changes made to DNS may take up to 24 hours to correctly have their current state reflected on this page. It is intended as a way for you to easily see how to configure your DNS records and to check whether all your records are correctly stored in DNS.'; $lang['diagnostics']['dns_records_name'] = 'Name'; diff --git a/data/web/modals/admin.php b/data/web/modals/admin.php index 5a995a76..2303519e 100644 --- a/data/web/modals/admin.php +++ b/data/web/modals/admin.php @@ -151,17 +151,18 @@ if (!isset($_SESSION['mailcow_cc_role'])) {
    - -