diff --git a/data/Dockerfiles/dovecot/Dockerfile b/data/Dockerfiles/dovecot/Dockerfile index 1320df1f..bc4fca5c 100644 --- a/data/Dockerfiles/dovecot/Dockerfile +++ b/data/Dockerfiles/dovecot/Dockerfile @@ -3,8 +3,8 @@ LABEL maintainer "Andre Peters " ARG DEBIAN_FRONTEND=noninteractive ENV LC_ALL C -ENV DOVECOT_VERSION 2.3.4 -ENV PIGEONHOLE_VERSION 0.5.4 +ENV DOVECOT_VERSION 2.3.5 +ENV PIGEONHOLE_VERSION 0.5.5 RUN apt-get update && apt-get -y --no-install-recommends install \ automake \ diff --git a/data/Dockerfiles/dovecot/docker-entrypoint.sh b/data/Dockerfiles/dovecot/docker-entrypoint.sh index 28b31420..11fcef14 100755 --- a/data/Dockerfiles/dovecot/docker-entrypoint.sh +++ b/data/Dockerfiles/dovecot/docker-entrypoint.sh @@ -189,7 +189,7 @@ echo '30 3 * * * vmail /usr/local/bin/doveadm quota recalc -A' > /etc/cron.d/d echo '* * * * * vmail /usr/local/bin/trim_logs.sh >> /dev/console 2>&1' > /etc/cron.d/trim_logs echo '25 * * * * vmail /usr/local/bin/maildir_gc.sh >> /dev/console 2>&1' > /etc/cron.d/maildir_gc echo '30 1 * * * root /usr/local/bin/sa-rules.sh >> /dev/console 2>&1' > /etc/cron.d/sa-rules -echo '0 2 * * * root /usr/bin/curl http://solr:8983/solr/dovecot/update?optimize=true >> /dev/console 2>&1' > /etc/cron.d/solr-optimize +echo '0 2 * * * root /usr/bin/curl http://solr:8983/solr/dovecot-fts/update?optimize=true >> /dev/console 2>&1' > /etc/cron.d/solr-optimize echo '*/20 * * * * vmail /usr/local/bin/quarantine_notify.py >> /dev/console 2>&1' > /etc/cron.d/quarantine_notify # Fix more than 1 hardlink issue diff --git a/data/Dockerfiles/solr/Dockerfile b/data/Dockerfiles/solr/Dockerfile index 67cd3384..25235664 100644 --- a/data/Dockerfiles/solr/Dockerfile +++ b/data/Dockerfiles/solr/Dockerfile @@ -1,9 +1,8 @@ -FROM solr:7-alpine +FROM solr:7.7-alpine USER root COPY docker-entrypoint.sh / RUN apk --no-cache add su-exec curl tzdata \ - && chmod +x /docker-entrypoint.sh \ - && /docker-entrypoint.sh --bootstrap + && chmod +x /docker-entrypoint.sh ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/data/Dockerfiles/solr/docker-entrypoint.sh b/data/Dockerfiles/solr/docker-entrypoint.sh index 108f8b5a..0634874f 100755 --- a/data/Dockerfiles/solr/docker-entrypoint.sh +++ b/data/Dockerfiles/solr/docker-entrypoint.sh @@ -26,395 +26,51 @@ fi # run the optional initdb . /opt/docker-solr/scripts/run-initdb -function solr_config() { - curl -XPOST http://localhost:8983/solr/dovecot/schema -H 'Content-type:application/json' -d '{ - "add-field-type":{ - "name":"long", - "class":"solr.TrieLongField" - }, - "add-field-type":{ - "name":"dovecot_text", - "class":"solr.TextField", - "autoGeneratePhraseQueries":true, - "positionIncrementGap":100, - "indexAnalyser":{ - "charFilter":{ - "class":"solr.MappingCharFilterFactory", - "mapping":"mapping-FoldToASCII.txt" - }, - "charFilter":{ - "class":"solr.MappingCharFilterFactory", - "mapping":"mapping-ISOLatin1Accent.txt" - }, - "charFilter":{ - "class":"solr.HTMLStripCharFilterFactory" - }, - "tokenizer":{ - "class":"solr.StandardTokenizerFactory" - }, - "filter":{ - "class":"solr.StopFilterFactory", - "words":"stopwords.txt", - "ignoreCase":true - }, - "filter":{ - "class":"solr.WordDelimiterGraphFilterFactory", - "generateWordParts":1, - "generateNumberParts":1, - "splitOnCaseChange":1, - "splitOnNumerics":1, - "catenateWords":1, - "catenateNumbers":1, - "catenateAll":1 - }, - "filter":{ - "class":"solr.FlattenGraphFilterFactory" - }, - "filter":{ - "class":"solr.LowerCaseFilterFactory" - }, - "filter":{ - "class":"solr.KeywordMarkerFilterFactory", - "protected":"protwords.txt" - }, - "filter":{ - "class":"solr.PorterStemFilterFactory" - } - }, - "queryAnalyzer":{ - "tokenizer":{ - "class":"solr.StandardTokenizerFactory" - }, - "filter":{ - "class":"solr.SynonymGraphFilterFactory", - "expand":true, - "ignoreCase":true, - "synonyms":synonyms.txt - }, - "filter":{ - "class":"solr.FlattenGraphFilterFactory" - }, - "filter":{ - "class":"solr.StopFilterFactory", - "words":"stopwords.txt", - "ignoreCase":true - }, - "filter":{ - "class":"solr.WordDelimiterGraphFilterFactory", - "generateWordParts":1, - "generateNumberParts":1, - "splitOnCaseChange":1, - "splitOnNumerics":1, - "catenateWords":1, - "catenateNumbers":1, - "catenateAll":1 - }, - "filter":{ - "class":"solr.LowerCaseFilterFactory" - }, - "filter":{ - "class":"solr.KeywordMarkerFilterFactory", - "protected":"protwords.txt" - }, - "filter":{ - "class":"solr.PorterStemFilterFactory" - } - } - }, - "add-field":{ - "name":"uid", - "type":"long", - "indexed":true, - "stored":true, - "required":true - }, - "add-field":{ - "name":"box", - "type":"string", - "indexed":true, - "stored":true, - "required":true - }, - "add-field":{ - "name":"user", - "type":"string", - "indexed":true, - "stored":true, - "required":true - }, - "add-field":{ - "name":"hdr", - "type":"dovecot_text", - "indexed":true, - "stored":false - - }, - "add-field":{ - "name":"body", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "add-field":{ - "name":"from", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "add-field":{ - "name":"to", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "add-field":{ - "name":"cc", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "add-field":{ - "name":"bcc", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "add-field":{ - "name":"subject", - "type":"dovecot_text", - "indexed":true, - "stored":false - } - }' - - curl -XPOST http://localhost:8983/solr/dovecot/schema -H 'Content-type:application/json' -d '{ - "replace-field-type":{ - "name":"long", - "class":"solr.TrieLongField" - }, - "replace-field-type":{ - "name":"dovecot_text", - "class":"solr.TextField", - "autoGeneratePhraseQueries":true, - "positionIncrementGap":100, - "indexAnalyser":{ - "charFilter":{ - "class":"solr.MappingCharFilterFactory", - "mapping":"mapping-FoldToASCII.txt" - }, - "charFilter":{ - "class":"solr.MappingCharFilterFactory", - "mapping":"mapping-ISOLatin1Accent.txt" - }, - "charFilter":{ - "class":"solr.HTMLStripCharFilterFactory" - }, - "tokenizer":{ - "class":"solr.StandardTokenizerFactory" - }, - "filter":{ - "class":"solr.StopFilterFactory", - "words":"stopwords.txt", - "ignoreCase":true - }, - "filter":{ - "class":"solr.WordDelimiterGraphFilterFactory", - "generateWordParts":1, - "generateNumberParts":1, - "splitOnCaseChange":1, - "splitOnNumerics":1, - "catenateWords":1, - "catenateNumbers":1, - "catenateAll":1 - }, - "filter":{ - "class":"solr.FlattenGraphFilterFactory" - }, - "filter":{ - "class":"solr.LowerCaseFilterFactory" - }, - "filter":{ - "class":"solr.KeywordMarkerFilterFactory", - "protected":"protwords.txt" - }, - "filter":{ - "class":"solr.PorterStemFilterFactory" - } - }, - "queryAnalyzer":{ - "tokenizer":{ - "class":"solr.StandardTokenizerFactory" - }, - "filter":{ - "class":"solr.SynonymGraphFilterFactory", - "expand":true, - "ignoreCase":true, - "synonyms":synonyms.txt - }, - "filter":{ - "class":"solr.FlattenGraphFilterFactory" - }, - "filter":{ - "class":"solr.StopFilterFactory", - "words":"stopwords.txt", - "ignoreCase":true - }, - "filter":{ - "class":"solr.WordDelimiterGraphFilterFactory", - "generateWordParts":1, - "generateNumberParts":1, - "splitOnCaseChange":1, - "splitOnNumerics":1, - "catenateWords":1, - "catenateNumbers":1, - "catenateAll":1 - }, - "filter":{ - "class":"solr.LowerCaseFilterFactory" - }, - "filter":{ - "class":"solr.KeywordMarkerFilterFactory", - "protected":"protwords.txt" - }, - "filter":{ - "class":"solr.PorterStemFilterFactory" - } - } - }, - "replace-field":{ - "name":"uid", - "type":"long", - "indexed":true, - "stored":true, - "required":true - }, - "replace-field":{ - "name":"box", - "type":"string", - "indexed":true, - "stored":true, - "required":true - }, - "replace-field":{ - "name":"user", - "type":"string", - "indexed":true, - "stored":true, - "required":true - }, - "replace-field":{ - "name":"hdr", - "type":"dovecot_text", - "indexed":true, - "stored":false - - }, - "replace-field":{ - "name":"body", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "replace-field":{ - "name":"from", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "replace-field":{ - "name":"to", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "replace-field":{ - "name":"cc", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "replace-field":{ - "name":"bcc", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "replace-field":{ - "name":"subject", - "type":"dovecot_text", - "indexed":true, - "stored":false - } - }' - - curl -XPOST http://localhost:8983/solr/dovecot/config -H 'Content-type:application/json' -d '{ - "update-requesthandler":{ - "name":"/select", - "class":"solr.SearchHandler", - "defaults":{ - "wt":"xml" - } - } - }' - - curl -XPOST http://localhost:8983/solr/dovecot/config/updateHandler -d '{ - "set-property": { - "updateHandler.autoSoftCommit.maxDocs":500, - "updateHandler.autoSoftCommit.maxTime":120000, - "updateHandler.autoCommit.maxDocs":200, - "updateHandler.autoCommit.maxTime":1800000, - "updateHandler.autoCommit.openSearcher":false - } - }' -} - # fixing volume permission -[[ -d /opt/solr/server/solr/dovecot/data ]] && chown -R solr:solr /opt/solr/server/solr/dovecot/data +[[ -d /opt/solr/server/solr/dovecot-fts/data ]] && chown -R solr:solr /opt/solr/server/solr/dovecot-fts/data if [[ "${1}" != "--bootstrap" ]]; then sed -i '/SOLR_HEAP=/c\SOLR_HEAP="'${SOLR_HEAP:-1024}'m"' /opt/solr/bin/solr.in.sh else sed -i '/SOLR_HEAP=/c\SOLR_HEAP="256m"' /opt/solr/bin/solr.in.sh fi -# start a Solr so we can use the Schema API, but only on localhost, -# so that clients don't see Solr until we have configured it. - -echo "Starting local Solr instance to setup configuration" -su-exec solr start-local-solr - # keep a sentinel file so we don't try to create the core a second time # for example when we restart a container. +# todo: check if a core exists without sentinel file -SENTINEL=/opt/docker-solr/core_created +SENTINEL=/opt/docker-solr/fts_core_created if [[ -f ${SENTINEL} ]]; then echo "skipping core creation" else - echo "Creating core \"dovecot\"" - su-exec solr /opt/solr/bin/solr create -c "dovecot" + echo "Starting local Solr instance to setup configuration" + su-exec solr start-local-solr + + echo "Creating core \"dovecot-fts\"" + su-exec solr /opt/solr/bin/solr create -c "dovecot-fts" # See https://github.com/docker-solr/docker-solr/issues/27 echo "Checking core" while ! wget -O - 'http://localhost:8983/solr/admin/cores?action=STATUS' | grep -q instanceDir; do echo "Could not find any cores, waiting..." - sleep 5 + sleep 3 done - echo "Created core \"dovecot\"" + + echo "Created core \"dovecot-fts\"" touch ${SENTINEL} + + echo "Stopping local Solr" + su-exec solr stop-local-solr fi -echo "Starting configuration" -while ! wget -O - 'http://localhost:8983/solr/admin/cores?action=STATUS' | grep -q instanceDir; do - echo "Waiting for Solr..." - sleep 5 -done -solr_config -echo "Stopping local Solr" -su-exec solr stop-local-solr +rm -f /opt/solr/server/solr/dovecot-fts/conf/schema.xml +rm -f /opt/solr/server/solr/dovecot-fts/conf/managed-schema +rm -f /opt/solr/server/solr/dovecot-fts/conf/solrconfig.xml -if [[ "${1}" == "--bootstrap" ]]; then - exit 0 -else - exec su-exec solr solr-foreground -fi +cp /etc/solr/solr-config-7.7.0.xml /opt/solr/server/solr/dovecot-fts/conf/solrconfig.xml +cp /etc/solr/solr-schema-7.7.0.xml /opt/solr/server/solr/dovecot-fts/conf/schema.xml + +chown -R solr:solr /opt/solr/server/solr/dovecot-fts/conf/{schema.xml,solrconfig.xml} + +exec su-exec solr solr-foreground diff --git a/data/conf/dovecot/dovecot.conf b/data/conf/dovecot/dovecot.conf index 4255741d..1ad40f64 100644 --- a/data/conf/dovecot/dovecot.conf +++ b/data/conf/dovecot/dovecot.conf @@ -308,7 +308,7 @@ plugin { acl = vfile fts = solr fts_autoindex = yes - fts_solr = url=http://solr:8983/solr/dovecot/ + fts_solr = url=http://solr:8983/solr/dovecot-fts/ quota = dict:Userquota::proxy::sqlquota quota_rule2 = Trash:storage=+100%% sieve = /var/vmail/sieve/%u.sieve diff --git a/data/conf/solr/solr-config-7.7.0.xml b/data/conf/solr/solr-config-7.7.0.xml new file mode 100644 index 00000000..3661874d --- /dev/null +++ b/data/conf/solr/solr-config-7.7.0.xml @@ -0,0 +1,289 @@ + + + + + + + 7.7.0 + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + + + + + + + + + + + true + + + 20 + + + 200 + + + false + + + + + + + + + + + + + + + explicit + 10 + + + + + + _text_ + + + + + + diff --git a/data/conf/solr/solr-schema-7.7.0.xml b/data/conf/solr/solr-schema-7.7.0.xml new file mode 100644 index 00000000..a41cbb47 --- /dev/null +++ b/data/conf/solr/solr-schema-7.7.0.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + diff --git a/data/web/inc/functions.inc.php b/data/web/inc/functions.inc.php index 34872aef..c869c122 100644 --- a/data/web/inc/functions.inc.php +++ b/data/web/inc/functions.inc.php @@ -1477,7 +1477,7 @@ function solr_status() { $endpoint = 'http://solr:8983/solr/admin/cores'; $params = array( 'action' => 'STATUS', - 'core' => 'dovecot', + 'core' => 'dovecot-fts', 'indexInfo' => 'true' ); $url = $endpoint . '?' . http_build_query($params); @@ -1494,7 +1494,7 @@ function solr_status() { else { curl_close($curl); $status = json_decode($response, true); - return (!empty($status['status']['dovecot'])) ? $status['status']['dovecot'] : false; + return (!empty($status['status']['dovecot-fts'])) ? $status['status']['dovecot-fts'] : false; } return false; } diff --git a/data/web/inc/functions.mailbox.inc.php b/data/web/inc/functions.mailbox.inc.php index 753e1702..82517492 100644 --- a/data/web/inc/functions.mailbox.inc.php +++ b/data/web/inc/functions.mailbox.inc.php @@ -3525,7 +3525,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { } if (strtolower(getenv('SKIP_SOLR')) == 'n') { $curl = curl_init(); - curl_setopt($curl, CURLOPT_URL, 'http://solr:8983/solr/dovecot/update?commit=true'); + curl_setopt($curl, CURLOPT_URL, 'http://solr:8983/solr/dovecot-fts/update?commit=true'); curl_setopt($curl, CURLOPT_HTTPHEADER,array('Content-Type: text/xml')); curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); curl_setopt($curl, CURLOPT_POST, 1); diff --git a/docker-compose.yml b/docker-compose.yml index d67f603b..52a1ecc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,7 @@ services: - clamd rspamd-mailcow: - image: mailcow/rspamd:1.36 + image: mailcow/rspamd:1.37 build: ./data/Dockerfiles/rspamd stop_grace_period: 30s depends_on: @@ -405,11 +405,12 @@ services: - dockerapi solr-mailcow: - image: mailcow/solr:1.2 + image: mailcow/solr:1.3 build: ./data/Dockerfiles/solr restart: always volumes: - solr-vol-1:/opt/solr/server/solr/dovecot/data + - ./data/conf/solr:/etc/solr dns: - ${IPV4_NETWORK:-172.22.1}.254 environment: