From 7181ee4658926aea4b746d815627e54ce228534f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9?= Date: Thu, 26 Apr 2018 13:51:55 +0200 Subject: [PATCH] [Rspamd] Apply ratelimit against authenticated user instead of envelope from [PHP-FPM] Create PHP-FPM listeners 9001 (system) and 9002 (web), drop 9000 [Rspamd] Parse quarantine messages as utf8 [Rspamd] Use new schema for Rspamd bayes hashes and expire them in Redis [SOGo] Change default logo [SOGo] Use different keyserver by default in Dockerfile [Rspamd] Add bad ASN list (disabled by default) [Watchdog] Change the way we check PHP-FPM, change SOGo check [Nginx] Change ports according to new PHP-FPM listeners [Update] Fix PHP-FPM ports for existing non-mailcow Nginx sites --- data/Dockerfiles/phpfpm/Dockerfile | 122 ++++++++-------- data/Dockerfiles/sogo/Dockerfile | 2 +- data/Dockerfiles/sogo/sogo-full.svg | 154 +++----------------- data/Dockerfiles/sogo/theme-blue.js | 24 +-- data/Dockerfiles/watchdog/watchdog.sh | 6 +- data/assets/nextcloud/nextcloud.conf | 7 +- data/assets/nextcloud/site.nextcloud.custom | 7 +- data/conf/nginx/meta_exporter.conf | 2 +- data/conf/nginx/site.conf | 22 ++- data/conf/phpfpm/php-fpm.d/pools.conf | 25 ++++ data/conf/rspamd/custom/bad_asn.map | 27 ++++ data/conf/rspamd/local.d/multimap.conf | 8 + data/conf/rspamd/local.d/statistic.conf | 2 + data/conf/rspamd/lua/rspamd.local.lua | 12 +- data/conf/rspamd/meta_exporter/pipe.php | 3 +- 15 files changed, 193 insertions(+), 230 deletions(-) create mode 100644 data/conf/phpfpm/php-fpm.d/pools.conf create mode 100644 data/conf/rspamd/custom/bad_asn.map diff --git a/data/Dockerfiles/phpfpm/Dockerfile b/data/Dockerfiles/phpfpm/Dockerfile index 00681ec2..e1f121a9 100644 --- a/data/Dockerfiles/phpfpm/Dockerfile +++ b/data/Dockerfiles/phpfpm/Dockerfile @@ -1,84 +1,78 @@ -FROM php:7.1-fpm-alpine +FROM php:7.2-fpm-alpine3.7 LABEL maintainer "Andre Peters " -ENV REDIS_PECL 3.1.6 -ENV MEMCACHED_PECL 3.0.4 -ENV APCU_PECL 5.1.9 +ENV APCU_PECL 5.1.10 ENV IMAGICK_PECL 3.4.3 ENV MAILPARSE_PECL 3.0.2 +ENV MEMCACHED_PECL 3.0.4 +ENV REDIS_PECL 3.1.6 -RUN apk add -U --no-cache libxml2-dev \ - icu-dev \ - imap-dev \ - libmemcached-dev \ - cyrus-sasl-dev \ - pcre-dev \ - icu-libs \ - redis \ - mysql-client \ - bash \ - autoconf \ - g++ \ - make \ - openssl \ - openssl-dev \ - samba-client \ - libpng \ - libpng-dev \ - libjpeg-turbo-dev \ - libwebp-dev \ - zlib-dev \ - libxpm-dev \ - c-client \ - imagemagick-dev \ - imagemagick \ - libtool \ +RUN apk add -U --no-cache autoconf \ + bash \ + c-client \ + cyrus-sasl-dev \ freetype \ - libpng \ - libjpeg-turbo \ freetype-dev \ - libpng-dev \ - libjpeg-turbo-dev\ + g++ \ gettext-dev \ - openldap-dev \ + icu-dev \ + icu-libs \ + imagemagick \ + imagemagick-dev \ + imap-dev \ + libjpeg-turbo \ + libjpeg-turbo-dev \ + libmemcached-dev \ + libpng \ + libpng-dev \ + libressl \ + libressl-dev \ librsvg \ + libtool \ + libwebp-dev \ + libxml2-dev \ + libxpm-dev \ + make \ + mysql-client \ + openldap-dev \ + pcre-dev \ + redis \ + samba-client \ + zlib-dev \ && pear install channel://pear.php.net/Net_IDNA2-0.2.0 \ channel://pear.php.net/Auth_SASL-1.1.0 \ Net_IMAP \ Net_Sieve \ NET_SMTP \ Mail_mime \ - && pecl install redis-${REDIS_PECL} memcached-${MEMCACHED_PECL} APCu-${APCU_PECL} imagick-${IMAGICK_PECL} mailparse-${MAILPARSE_PECL} \ - && docker-php-ext-enable redis apcu memcached imagick mailparse \ - && pecl clear-cache \ - && docker-php-ext-configure intl \ + && pecl install redis-${REDIS_PECL} memcached-${MEMCACHED_PECL} APCu-${APCU_PECL} imagick-${IMAGICK_PECL} mailparse-${MAILPARSE_PECL} \ + && docker-php-ext-enable apcu imagick mailparse memcached redis \ + && pecl clear-cache \ + && docker-php-ext-configure intl \ && docker-php-ext-configure gd \ - --with-gd \ - --enable-gd-native-ttf \ - --with-freetype-dir=/usr/include/ \ - --with-png-dir=/usr/include/ \ - --with-jpeg-dir=/usr/include/ \ - && docker-php-ext-install -j 4 intl gettext ldap sockets soap pdo pdo_mysql xmlrpc gd zip pcntl opcache \ + --with-gd \ + --enable-gd-native-ttf \ + --with-freetype-dir=/usr/include/ \ + --with-png-dir=/usr/include/ \ + --with-jpeg-dir=/usr/include/ \ + && docker-php-ext-install -j 4 gd gettext intl ldap opcache pcntl pdo pdo_mysql soap sockets xmlrpc zip \ && docker-php-ext-configure imap --with-imap --with-imap-ssl \ - && docker-php-ext-install -j 4 imap \ - && apk del --purge autoconf \ - g++ \ - make \ - libxml2-dev \ - icu-dev \ - imap-dev \ - openssl-dev \ - cyrus-sasl-dev \ - pcre-dev \ - libpng-dev \ - libpng-dev \ - libjpeg-turbo-dev \ - libwebp-dev \ - zlib-dev \ - imagemagick-dev \ - freetype-dev \ - libpng-dev \ - libjpeg-turbo-dev + && docker-php-ext-install -j 4 imap \ + && apk del --purge autoconf \ + cyrus-sasl-dev \ + freetype-dev \ + g++ \ + icu-dev \ + imagemagick-dev \ + imap-dev \ + libjpeg-turbo-dev \ + libpng-dev \ + libressl-dev \ + libwebp-dev \ + libxml2-dev \ + make \ + pcre-dev \ + zlib-dev COPY ./docker-entrypoint.sh / diff --git a/data/Dockerfiles/sogo/Dockerfile b/data/Dockerfiles/sogo/Dockerfile index 5cc94b9b..fd49dd8b 100644 --- a/data/Dockerfiles/sogo/Dockerfile +++ b/data/Dockerfiles/sogo/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN mkdir /usr/share/doc/sogo \ && touch /usr/share/doc/sogo/empty.sh \ - && apt-key adv --keyserver keys.gnupg.net --recv-key 0x810273C4 \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-key 0x810273C4 \ && echo "deb http://packages.inverse.ca/SOGo/nightly/4/debian/ stretch stretch" > /etc/apt/sources.list.d/sogo.list \ && apt-get update && apt-get install -y --force-yes \ sogo \ diff --git a/data/Dockerfiles/sogo/sogo-full.svg b/data/Dockerfiles/sogo/sogo-full.svg index 72c17214..c22e3010 100644 --- a/data/Dockerfiles/sogo/sogo-full.svg +++ b/data/Dockerfiles/sogo/sogo-full.svg @@ -1,5 +1,5 @@ - + image/svg+xml \ No newline at end of file + inkscape:zoom="0.59375" + inkscape:cx="320" + inkscape:cy="175" + inkscape:window-x="-8" + inkscape:window-y="-8" + inkscape:window-maximized="1" + inkscape:current-layer="Layer_1" /> \ No newline at end of file diff --git a/data/Dockerfiles/sogo/theme-blue.js b/data/Dockerfiles/sogo/theme-blue.js index 4efbf824..8992bba2 100644 --- a/data/Dockerfiles/sogo/theme-blue.js +++ b/data/Dockerfiles/sogo/theme-blue.js @@ -16,16 +16,16 @@ * Define the Alternative theme */ $mdThemingProvider.theme('mailcow') - .primaryPalette('indigo', { + .primaryPalette('blue', { 'default': '700', // top toolbar - 'hue-1': '400', - 'hue-2': '600', // sidebar toolbar + 'hue-1': '500', + 'hue-2': '700', // sidebar toolbar 'hue-3': 'A700' }) - .accentPalette('indigo', { - 'default': '500', // fab buttons + .accentPalette('blue', { + 'default': '700', // fab buttons 'hue-1': '50', // center list toolbar - 'hue-2': '400', + 'hue-2': '600', 'hue-3': 'A700' }) .backgroundPalette('grey', { @@ -35,16 +35,16 @@ 'hue-3': '300' }); $mdThemingProvider.theme('default') - .primaryPalette('indigo', { + .primaryPalette('blue', { 'default': '700', // top toolbar - 'hue-1': '400', - 'hue-2': '600', // sidebar toolbar + 'hue-1': '500', + 'hue-2': '700', // sidebar toolbar 'hue-3': 'A700' }) - .accentPalette('indigo', { - 'default': '500', // fab buttons + .accentPalette('blue', { + 'default': '700', // fab buttons 'hue-1': '50', // center list toolbar - 'hue-2': '400', + 'hue-2': '600', 'hue-3': 'A700' }) .backgroundPalette('grey', { diff --git a/data/Dockerfiles/watchdog/watchdog.sh b/data/Dockerfiles/watchdog/watchdog.sh index 39760ec0..c5a274e9 100755 --- a/data/Dockerfiles/watchdog/watchdog.sh +++ b/data/Dockerfiles/watchdog/watchdog.sh @@ -128,7 +128,7 @@ sogo_checks() { host_ip=$(get_container_ip sogo-mailcow) err_c_cur=${err_count} /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /WebServerResources/css/theme-default.css -p 9192 -R md-default-theme 1>&2; err_count=$(( ${err_count} + $? )) - /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /SOGo.index/ -p 20000 -R "SOGo\sGroupware" 1>&2; err_count=$(( ${err_count} + $? )) + /usr/lib/nagios/plugins/check_http -4 -H ${host_ip} -u /SOGo.index/ -p 20000 -R "SOGo\.MainUI" 1>&2; err_count=$(( ${err_count} + $? )) [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) progress "SOGo" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c} @@ -190,8 +190,8 @@ phpfpm_checks() { while [ ${err_count} -lt ${THRESHOLD} ]; do host_ip=$(get_container_ip php-fpm-mailcow) err_c_cur=${err_count} - cgi-fcgi -bind -connect ${host_ip}:9000 | grep "Content-type" 1>&2; err_count=$(( ${err_count} + ($? * 2))) - cgi-fcgi -bind -connect ${host_ip}:9001 | grep "Content-type" 1>&2; err_count=$(( ${err_count} + ($? * 2))) + nc -z ${host_ip} 9001 ; err_count=$(( ${err_count} + ($? * 2))) + nc -z ${host_ip} 9002 ; err_count=$(( ${err_count} + ($? * 2))) /usr/lib/nagios/plugins/check_ping -4 -H ${host_ip} -w 2000,10% -c 4000,100% -p2 1>&2; err_count=$(( ${err_count} + $? )) [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1 [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} )) diff --git a/data/assets/nextcloud/nextcloud.conf b/data/assets/nextcloud/nextcloud.conf index 4759bff2..74ea6a0c 100644 --- a/data/assets/nextcloud/nextcloud.conf +++ b/data/assets/nextcloud/nextcloud.conf @@ -82,10 +82,13 @@ server { #Avoid sending the security headers twice fastcgi_param modHeadersAvailable true; fastcgi_param front_controller_active true; - fastcgi_pass phpfpm:9000; + fastcgi_pass phpfpm:9002; fastcgi_intercept_errors on; fastcgi_request_buffering off; - client_max_body_size 10G; + client_max_body_size 0; + fastcgi_param PHP_VALUE "max_execution_time = 1200 + max_input_time = 1200"; + fastcgi_read_timeout 1200; } location ~ ^/(?:updater|ocs-provider)(?:$|/) { diff --git a/data/assets/nextcloud/site.nextcloud.custom b/data/assets/nextcloud/site.nextcloud.custom index f7d6dae0..41f7348f 100644 --- a/data/assets/nextcloud/site.nextcloud.custom +++ b/data/assets/nextcloud/site.nextcloud.custom @@ -16,10 +16,13 @@ fastcgi_param HTTPS on; fastcgi_param modHeadersAvailable true; fastcgi_param front_controller_active true; - fastcgi_pass phpfpm:9000; + fastcgi_pass phpfpm:9002; fastcgi_intercept_errors on; fastcgi_request_buffering off; - client_max_body_size 10G; + client_max_body_size 0; + fastcgi_param PHP_VALUE "max_execution_time = 1200 + max_input_time = 1200"; + fastcgi_read_timeout 1200; } location ~ ^/nextcloud/(?:updater|ocs-provider)(?:$|/) { try_files $uri/ =404; diff --git a/data/conf/nginx/meta_exporter.conf b/data/conf/nginx/meta_exporter.conf index 30369a32..74291b14 100644 --- a/data/conf/nginx/meta_exporter.conf +++ b/data/conf/nginx/meta_exporter.conf @@ -10,7 +10,7 @@ server { client_max_body_size 10M; try_files $uri =404; fastcgi_split_path_info ^(.+\.php)(/.+)$; - fastcgi_pass phpfpm:9000; + fastcgi_pass phpfpm:9001; fastcgi_index pipe.php; include fastcgi_params; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; diff --git a/data/conf/nginx/site.conf b/data/conf/nginx/site.conf index 83f913f6..53a48ad6 100644 --- a/data/conf/nginx/site.conf +++ b/data/conf/nginx/site.conf @@ -35,7 +35,6 @@ server { add_header X-XSS-Protection "1; mode=block"; add_header X-Robots-Tag none; add_header X-Download-Options noopen; - add_header X-Frame-Options "SAMEORIGIN"; add_header X-Permitted-Cross-Domain-Policies none; index index.php index.html; @@ -76,7 +75,7 @@ server { location ~ \.php$ { try_files $uri =404; fastcgi_split_path_info ^(.+\.php)(/.+)$; - fastcgi_pass phpfpm:9000; + fastcgi_pass phpfpm:9002; fastcgi_index index.php; include /etc/nginx/fastcgi_params; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; @@ -96,9 +95,22 @@ server { expires $expires; } + location /lists/ { + proxy_pass http://mm-web:8000/; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_redirect off; + expires $expires; + } + + location /mm_static { + alias /opt/mm_web-data/static; + } + location ~* ^/Autodiscover/Autodiscover.xml { fastcgi_split_path_info ^(.+\.php)(/.+)$; - fastcgi_pass phpfpm:9000; + fastcgi_pass phpfpm:9002; include /etc/nginx/fastcgi_params; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; try_files /autodiscover.php =404; @@ -106,7 +118,7 @@ server { location ~* ^/Autodiscover/Autodiscover.json { fastcgi_split_path_info ^(.+\.php)(/.+)$; - fastcgi_pass phpfpm:9000; + fastcgi_pass phpfpm:9002; include /etc/nginx/fastcgi_params; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; try_files /autodiscover-json.php =404; @@ -114,7 +126,7 @@ server { location ~ /(?:m|M)ail/(?:c|C)onfig-v1.1.xml { fastcgi_split_path_info ^(.+\.php)(/.+)$; - fastcgi_pass phpfpm:9000; + fastcgi_pass phpfpm:9002; include /etc/nginx/fastcgi_params; fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; try_files /autoconfig.php =404; diff --git a/data/conf/phpfpm/php-fpm.d/pools.conf b/data/conf/phpfpm/php-fpm.d/pools.conf new file mode 100644 index 00000000..9b7a9646 --- /dev/null +++ b/data/conf/phpfpm/php-fpm.d/pools.conf @@ -0,0 +1,25 @@ +[system-worker] +user = www-data +group = www-data +pm = dynamic +pm.max_children = 15 +pm.start_servers = 2 +pm.min_spare_servers = 2 +pm.max_spare_servers = 4 +listen = [::]:9001 +access.log = /proc/self/fd/2 +clear_env = no +catch_workers_output = yes + +[web-worker] +user = www-data +group = www-data +pm = dynamic +pm.max_children = 50 +pm.start_servers = 2 +pm.min_spare_servers = 2 +pm.max_spare_servers = 4 +listen = [::]:9002 +access.log = /proc/self/fd/2 +clear_env = no +catch_workers_output = yes diff --git a/data/conf/rspamd/custom/bad_asn.map b/data/conf/rspamd/custom/bad_asn.map new file mode 100644 index 00000000..8b7a4407 --- /dev/null +++ b/data/conf/rspamd/custom/bad_asn.map @@ -0,0 +1,27 @@ +# High spam networks, disabled by default +#201942:5 #Soltia Consulting SL - ipinfo.io +#16276:5 #OVH +#12876:5 #ONLINE S.A.S +#31034:5 +#12874:5 +#30823:5 +#197071:5 +#42831:5 #UK Dedicated Servers Ltd +#29119:5 #Aire Networks del Mediterraneo S.L.U. +#13335:5 #Cloudflare +#28753:5 #Leaseweb +#61272:5 #Informacines sistemos ir technologijos +#53755:5 #Input Output Flood LLC +#29422:5 #FICIX Helsinki +#62255:4 #Asmunda New Media Ltd +#14061:4 #Digitalocean +#55293:4 #A2 Hosting +#63018:4 #US Dedicated +#197518:2 +#44493:2 +#46606:2 +#49505:2 +#21100:2 +#197695:2 +#198068:2 +#43146:2 diff --git a/data/conf/rspamd/local.d/multimap.conf b/data/conf/rspamd/local.d/multimap.conf index 6237f007..d96f6bf8 100644 --- a/data/conf/rspamd/local.d/multimap.conf +++ b/data/conf/rspamd/local.d/multimap.conf @@ -27,6 +27,14 @@ KEEP_SPAM { action = "accept"; } +LOCAL_BL_ASN { + require_symbols = "!MAILCOW_WHITE"; + type = "asn"; + map = "$LOCAL_CONFDIR/custom/bad_asn.map"; + score = 5; + description = "Sender's ASN is on the local blacklist"; +} + #SPOOFED_SENDER { # type = "rcpt"; # filter = "email:domain:tld"; diff --git a/data/conf/rspamd/local.d/statistic.conf b/data/conf/rspamd/local.d/statistic.conf index b66f5018..c8c2cf42 100644 --- a/data/conf/rspamd/local.d/statistic.conf +++ b/data/conf/rspamd/local.d/statistic.conf @@ -8,6 +8,8 @@ classifier "bayes" { min_tokens = 11; min_learns = 20; autolearn = [-20, 50]; + new_schema = true; + expiry = 50d; per_user = <