From a37a8e3b2ace4f13b8832436c9e16089e0ce64f8 Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 14 Feb 2018 13:27:30 +0200 Subject: [PATCH 1/6] Fix for large Mailboxes to avoid timeouts ** I agree to the code of conduct and the contributory guidelines ** Tested with 80+GB single inbox via imap ``` --nofoldersizes --skipsize --fast ``` file and folder sizes are only used for statistics, which are completely useless. Before the actual data is transfered the message and folder sizes need to be calculated on a 200 000+ message inbox, this will almost never complete. ``` --buffersize 8192000 ``` sets the io buffer to 8mb, the default buffer is 4kbyte, this is speeds up syncs. ``` --skipheader 'X-*' ``` X headers vary wildly from system to system and do not ensure message uniqueness, they are not needed. ``` --split1 3000 --split2 3000 ``` split the requests in several parts on the server, 3000 is the number of messages handled per request. ``` --fastio1 --fastio2 ``` use fastio --- data/Dockerfiles/dovecot/imapsync_cron.pl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/data/Dockerfiles/dovecot/imapsync_cron.pl b/data/Dockerfiles/dovecot/imapsync_cron.pl index 8afc3b20..c441d7f9 100755 --- a/data/Dockerfiles/dovecot/imapsync_cron.pl +++ b/data/Dockerfiles/dovecot/imapsync_cron.pl @@ -74,6 +74,14 @@ while ($row = $sth->fetchrow_arrayref()) { "--timeout1", "10", "--tmpdir", "/tmp", "--subscribeall", + "--nofoldersizes", + "--skipsize", + "--buffersize 8192000", + "--skipheader 'X-*", + "--split1 3000", + "--split2 3000", + "--fastio1", + "--fastio2", ($exclude eq "" ? () : ("--exclude", $exclude)), ($subfolder2 eq "" ? () : ('--subfolder2', $subfolder2)), ($maxage eq "0" ? () : ('--maxage', $maxage)), From 8eb05d1450445658a1870f6ff0bc66c82f559e4a Mon Sep 17 00:00:00 2001 From: eXtremeSHOK Date: Wed, 14 Feb 2018 14:02:56 +0200 Subject: [PATCH 2/6] Fix for large Mailboxes to avoid timeouts 2 Missing ' (sorry my editor keeps correcting the '" to " ) --- data/Dockerfiles/dovecot/imapsync_cron.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/Dockerfiles/dovecot/imapsync_cron.pl b/data/Dockerfiles/dovecot/imapsync_cron.pl index c441d7f9..47f9bbf7 100755 --- a/data/Dockerfiles/dovecot/imapsync_cron.pl +++ b/data/Dockerfiles/dovecot/imapsync_cron.pl @@ -77,7 +77,7 @@ while ($row = $sth->fetchrow_arrayref()) { "--nofoldersizes", "--skipsize", "--buffersize 8192000", - "--skipheader 'X-*", + "--skipheader 'X-*'", "--split1 3000", "--split2 3000", "--fastio1", From 9da6aebfb9599f4ccfad63610503f71ef2e45541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Peters?= Date: Fri, 16 Feb 2018 10:35:22 +0100 Subject: [PATCH 3/6] [SOGo] Fix timezone bug in previous nightly --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0cd2c175..9439a345 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -128,7 +128,7 @@ services: - phpfpm sogo-mailcow: - image: mailcow/sogo:1.15 + image: mailcow/sogo:1.16 build: ./data/Dockerfiles/sogo environment: - DBNAME=${DBNAME} From 03031516e98916e93575c2f3dd359fccd98e6922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Peters?= Date: Fri, 16 Feb 2018 22:39:33 +0100 Subject: [PATCH 4/6] [Web] Fixes #1055 and changes location.reload to window = xy in footer script --- data/web/inc/ajax/destroy_tfa_auth.php | 6 ++++++ data/web/inc/footer.inc.php | 13 ++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 data/web/inc/ajax/destroy_tfa_auth.php diff --git a/data/web/inc/ajax/destroy_tfa_auth.php b/data/web/inc/ajax/destroy_tfa_auth.php new file mode 100644 index 00000000..72c7f1e3 --- /dev/null +++ b/data/web/inc/ajax/destroy_tfa_auth.php @@ -0,0 +1,6 @@ + diff --git a/data/web/inc/footer.inc.php b/data/web/inc/footer.inc.php index 3c92c28e..a4974604 100644 --- a/data/web/inc/footer.inc.php +++ b/data/web/inc/footer.inc.php @@ -79,6 +79,17 @@ $(document).ready(function() { }); } }); + $('#ConfirmTFAModal').on('hidden.bs.modal', function(){ + $.ajax({ + type: "GET", + cache: false, + dataType: 'script', + url: '/inc/ajax/destroy_tfa_auth.php', + complete: function(data){ + window.location = window.location.href.split("#")[0]; + } + }); + }); // Set TFA modals @@ -205,7 +216,7 @@ $(document).ready(function() { $('#triggerRestartContainer').html(' '); $('#statusTriggerRestartContainer2').append(data); $('#triggerRestartContainer').html(' '); - location.reload(); + window.location = window.location.href.split("#")[0]; } }); }); From 2865c892a684fe1d614cf709173ed5ddb0be6ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Peters?= Date: Fri, 16 Feb 2018 22:40:51 +0100 Subject: [PATCH 5/6] [Multi] Fixes #1058 by including a 'force password update' option and also introduces a attributes json object to be used for further mailbox configurations in the future --- data/Dockerfiles/dovecot/docker-entrypoint.sh | 2 +- data/Dockerfiles/sogo/bootstrap-sogo.sh | 3 +-- data/web/edit.php | 8 ++++++++ data/web/inc/functions.inc.php | 2 +- data/web/inc/functions.mailbox.inc.php | 20 ++++++------------- data/web/inc/init_db.inc.php | 4 ++-- data/web/lang/lang.de.php | 3 +++ data/web/lang/lang.en.php | 3 +++ data/web/user.php | 3 +++ docker-compose.yml | 4 ++-- 10 files changed, 30 insertions(+), 22 deletions(-) diff --git a/data/Dockerfiles/dovecot/docker-entrypoint.sh b/data/Dockerfiles/dovecot/docker-entrypoint.sh index 9f8f5313..469edacc 100755 --- a/data/Dockerfiles/dovecot/docker-entrypoint.sh +++ b/data/Dockerfiles/dovecot/docker-entrypoint.sh @@ -82,7 +82,7 @@ cat < /usr/local/etc/dovecot/sql/dovecot-dict-sql-passdb.conf driver = mysql connect = "host=mysql dbname=${DBNAME} user=${DBUSER} password=${DBPASS}" default_pass_scheme = SSHA256 -password_query = SELECT password FROM mailbox WHERE username = '%u' AND domain IN (SELECT domain FROM domain WHERE domain='%d' AND active='1') +password_query = SELECT password FROM mailbox WHERE username = '%u' AND domain IN (SELECT domain FROM domain WHERE domain='%d' AND active='1') AND JSON_EXTRACT(attributes, "$.force_pw_update") != 1 user_query = SELECT CONCAT('maildir:/var/vmail/',maildir) AS mail, 5000 AS uid, 5000 AS gid, concat('*:bytes=', quota) AS quota_rule FROM mailbox WHERE username = '%u' AND active = '1' iterate_query = SELECT username FROM mailbox WHERE active='1'; EOF diff --git a/data/Dockerfiles/sogo/bootstrap-sogo.sh b/data/Dockerfiles/sogo/bootstrap-sogo.sh index c360e30e..07365981 100755 --- a/data/Dockerfiles/sogo/bootstrap-sogo.sh +++ b/data/Dockerfiles/sogo/bootstrap-sogo.sh @@ -19,14 +19,13 @@ mysql --host mysql -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "DROP VIEW IF EXISTS so mysql --host mysql -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF CREATE VIEW sogo_view (c_uid, domain, c_name, c_password, c_cn, mail, aliases, ad_aliases, home, kind, multiple_bookings) AS -SELECT mailbox.username, mailbox.domain, mailbox.username, mailbox.password, mailbox.name, mailbox.username, IFNULL(GROUP_CONCAT(ga.aliases SEPARATOR ' '), ''), IFNULL(gda.ad_alias, ''), CONCAT('/var/vmail/', maildir), mailbox.kind, mailbox.multiple_bookings FROM mailbox +SELECT mailbox.username, mailbox.domain, mailbox.username, if(json_extract(attributes, '$.force_pw_update') = '0', password, 'invalid'), mailbox.name, mailbox.username, IFNULL(GROUP_CONCAT(ga.aliases SEPARATOR ' '), ''), IFNULL(gda.ad_alias, ''), CONCAT('/var/vmail/', maildir), mailbox.kind, mailbox.multiple_bookings FROM mailbox LEFT OUTER JOIN grouped_mail_aliases ga ON ga.username REGEXP CONCAT('(^|,)', mailbox.username, '($|,)') LEFT OUTER JOIN grouped_domain_alias_address gda ON gda.username = mailbox.username WHERE mailbox.active = '1' GROUP BY mailbox.username; EOF - mkdir -p /var/lib/sogo/GNUstep/Defaults/ # Generate plist header with timezone data diff --git a/data/web/edit.php b/data/web/edit.php index 8374dc76..8a12c665 100644 --- a/data/web/edit.php +++ b/data/web/edit.php @@ -390,6 +390,7 @@ if (isset($_SESSION['mailcow_cc_role'])) {
+
@@ -476,6 +477,13 @@ if (isset($_SESSION['mailcow_cc_role'])) {
+
+
+
+ +
+
+
diff --git a/data/web/inc/functions.inc.php b/data/web/inc/functions.inc.php index f7f08403..25d88d78 100644 --- a/data/web/inc/functions.inc.php +++ b/data/web/inc/functions.inc.php @@ -414,7 +414,7 @@ function edit_user_account($postarray) { } $password_hashed = hash_password($password_new); try { - $stmt = $pdo->prepare("UPDATE `mailbox` SET `password` = :password_hashed WHERE `username` = :username"); + $stmt = $pdo->prepare("UPDATE `mailbox` SET `password` = :password_hashed, `attributes` = JSON_SET(`attributes`, '$.force_pw_update', '0') WHERE `username` = :username"); $stmt->execute(array( ':password_hashed' => $password_hashed, ':username' => $username diff --git a/data/web/inc/functions.mailbox.inc.php b/data/web/inc/functions.mailbox.inc.php index 5bd9ac58..89e02166 100644 --- a/data/web/inc/functions.mailbox.inc.php +++ b/data/web/inc/functions.mailbox.inc.php @@ -1954,6 +1954,7 @@ function mailbox($_action, $_type, $_data = null, $attr = null) { $is_now = mailbox('get', 'mailbox_details', $username); if (!empty($is_now)) { $active = (isset($_data['active'])) ? intval($_data['active']) : $is_now['active_int']; + (int)$force_pw_update = (isset($_data['force_pw_update'])) ? intval($_data['force_pw_update']) : intval($is_now['attributes']['force_pw_update']); $name = (!empty($_data['name'])) ? $_data['name'] : $is_now['name']; $domain = $is_now['domain']; $quota_m = (!empty($_data['quota'])) ? $_data['quota'] : ($is_now['quota'] / 1048576); @@ -2113,24 +2114,11 @@ function mailbox($_action, $_type, $_data = null, $attr = null) { } $password_hashed = hash_password($password); try { - $stmt = $pdo->prepare("UPDATE `alias` SET - `active` = :active - WHERE `address` = :address"); - $stmt->execute(array( - ':address' => $username, - ':active' => $active - )); $stmt = $pdo->prepare("UPDATE `mailbox` SET - `active` = :active, `password` = :password_hashed, - `name`= :name, - `quota` = :quota_b WHERE `username` = :username"); $stmt->execute(array( ':password_hashed' => $password_hashed, - ':active' => $active, - ':name' => $name, - ':quota_b' => $quota_b, ':username' => $username )); } @@ -2153,12 +2141,14 @@ function mailbox($_action, $_type, $_data = null, $attr = null) { $stmt = $pdo->prepare("UPDATE `mailbox` SET `active` = :active, `name`= :name, - `quota` = :quota_b + `quota` = :quota_b, + `attributes` = JSON_SET(`attributes`, '$.force_pw_update', :force_pw_update) WHERE `username` = :username"); $stmt->execute(array( ':active' => $active, ':name' => $name, ':quota_b' => $quota_b, + ':force_pw_update' => $force_pw_update, ':username' => $username )); } @@ -3070,6 +3060,7 @@ function mailbox($_action, $_type, $_data = null, $attr = null) { `mailbox`.`domain`, `mailbox`.`quota`, `quota2`.`bytes`, + `attributes`, `quota2`.`messages` FROM `mailbox`, `quota2`, `domain` WHERE `mailbox`.`kind` NOT REGEXP 'location|thing|group' AND `mailbox`.`username` = `quota2`.`username` AND `domain`.`domain` = `mailbox`.`domain` AND `mailbox`.`username` = :mailbox"); @@ -3097,6 +3088,7 @@ function mailbox($_action, $_type, $_data = null, $attr = null) { $mailboxdata['active_int'] = $row['active_int']; $mailboxdata['domain'] = $row['domain']; $mailboxdata['quota'] = $row['quota']; + $mailboxdata['attributes'] = json_decode($row['attributes'], true); $mailboxdata['quota_used'] = intval($row['bytes']); $mailboxdata['percent_in_use'] = round((intval($row['bytes']) / intval($row['quota'])) * 100); $mailboxdata['messages'] = $row['messages']; diff --git a/data/web/inc/init_db.inc.php b/data/web/inc/init_db.inc.php index 24729c83..58091721 100644 --- a/data/web/inc/init_db.inc.php +++ b/data/web/inc/init_db.inc.php @@ -3,7 +3,7 @@ function init_db_schema() { try { global $pdo; - $db_version = "08022018_1219"; + $db_version = "16022018_1419"; $stmt = $pdo->query("SHOW TABLES LIKE 'versions'"); $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC)); @@ -191,9 +191,9 @@ function init_db_schema() { "domain" => "VARCHAR(255) NOT NULL", "tls_enforce_in" => "TINYINT(1) NOT NULL DEFAULT '0'", "tls_enforce_out" => "TINYINT(1) NOT NULL DEFAULT '0'", + "attributes" => "JSON DEFAULT '{}'", "kind" => "VARCHAR(100) NOT NULL DEFAULT ''", "multiple_bookings" => "TINYINT(1) NOT NULL DEFAULT '0'", - "wants_tagged_subject" => "TINYINT(1) NOT NULL DEFAULT '0'", "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)", "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP", "active" => "TINYINT(1) NOT NULL DEFAULT '1'" diff --git a/data/web/lang/lang.de.php b/data/web/lang/lang.de.php index a874eea1..8f4733b9 100644 --- a/data/web/lang/lang.de.php +++ b/data/web/lang/lang.de.php @@ -100,6 +100,7 @@ $lang['warning']['spam_alias_temp_error'] = 'Kann zur Zeit keinen Spam-Alias ers $lang['danger']['spam_alias_max_exceeded'] = 'Maximale Anzahl an Spam-Alias-Adressen erreicht'; $lang['danger']['validity_missing'] = 'Bitte geben Sie eine Gültigkeitsdauer an'; $lang['user']['loading'] = "Lade..."; +$lang['user']['force_pw_update'] = 'Das Passwort für diesen Benutzer muss geändert werden, damit die Zugriffssperre auf die Groupwarekomponenten wieder freigeschaltet wird.'; $lang['user']['active_sieve'] = "Aktiver Filter"; $lang['user']['show_sieve_filters'] = "Zeige aktiven Filter des Benutzers"; $lang['user']['no_active_filter'] = "Kein aktiver Filter vorhanden"; @@ -320,6 +321,8 @@ $lang['edit']['max_mailboxes'] = 'Max. Mailboxanzahl:'; $lang['edit']['title'] = 'Objekt bearbeiten'; $lang['edit']['target_address'] = 'Ziel-Adresse(n) (getrennt durch Komma):'; $lang['edit']['active'] = 'Aktiv'; +$lang['edit']['force_pw_update'] = 'Erzwinge Passwortänderung bei nächstem Login'; +$lang['edit']['force_pw_update_info'] = 'Dem Benutzer wird lediglich der Zugang zur mailcow UI ermöglicht.'; $lang['edit']['target_domain'] = 'Ziel-Domain:'; $lang['edit']['password'] = 'Passwort:'; $lang['edit']['ratelimit'] = 'Limit ausgehender Nachrichten/Stunde:'; diff --git a/data/web/lang/lang.en.php b/data/web/lang/lang.en.php index 3fc99872..474ad773 100644 --- a/data/web/lang/lang.en.php +++ b/data/web/lang/lang.en.php @@ -100,6 +100,7 @@ $lang['warning']['spam_alias_temp_error'] = "Temporary error: Cannot add spam al $lang['danger']['spam_alias_max_exceeded'] = "Max. allowed spam alias addresses exceeded"; $lang['danger']['validity_missing'] = 'Please assign a period of validity'; $lang['user']['loading'] = "Loading..."; +$lang['user']['force_pw_update'] = 'You must set a new password to be able to access groupware related services.'; $lang['user']['active_sieve'] = "Active filter"; $lang['user']['show_sieve_filters'] = "Show active user sieve filter"; $lang['user']['no_active_filter'] = "No active filter available"; @@ -321,6 +322,8 @@ $lang['edit']['max_mailboxes'] = 'Max. possible mailboxes'; $lang['edit']['title'] = 'Edit object'; $lang['edit']['target_address'] = 'Goto address/es (comma-separated)'; $lang['edit']['active'] = 'Active'; +$lang['edit']['force_pw_update'] = 'Force password update at next login'; +$lang['edit']['force_pw_update_info'] = 'This user will only be able to login to mailcow UI.'; $lang['edit']['target_domain'] = 'Target domain'; $lang['edit']['password'] = 'Password'; $lang['edit']['ratelimit'] = 'Outgoing rate limit/h'; diff --git a/data/web/user.php b/data/web/user.php index 0fb0875f..6c9069e5 100644 --- a/data/web/user.php +++ b/data/web/user.php @@ -89,6 +89,9 @@ elseif (isset($_SESSION['mailcow_cc_role']) && $_SESSION['mailcow_cc_role'] == '
+ +
+

[]

[]

diff --git a/docker-compose.yml b/docker-compose.yml index 9439a345..e4d29209 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -128,7 +128,7 @@ services: - phpfpm sogo-mailcow: - image: mailcow/sogo:1.16 + image: mailcow/sogo:1.17 build: ./data/Dockerfiles/sogo environment: - DBNAME=${DBNAME} @@ -149,7 +149,7 @@ services: - sogo dovecot-mailcow: - image: mailcow/dovecot:1.20 + image: mailcow/dovecot:1.22 build: ./data/Dockerfiles/dovecot cap_add: - NET_BIND_SERVICE From 2a3040de12e3b333c3d22b1684426e9ea5299168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Peters?= Date: Fri, 16 Feb 2018 22:42:28 +0100 Subject: [PATCH 6/6] [Web] Add missing info block to force pw update function in edit --- data/web/edit.php | 1 + 1 file changed, 1 insertion(+) diff --git a/data/web/edit.php b/data/web/edit.php index 8a12c665..912570cd 100644 --- a/data/web/edit.php +++ b/data/web/edit.php @@ -481,6 +481,7 @@ if (isset($_SESSION['mailcow_cc_role'])) {
+