From c7c115d63a02a7142838a37831d2defaeee7be59 Mon Sep 17 00:00:00 2001 From: andryyy Date: Sun, 10 Mar 2019 09:36:33 +0100 Subject: [PATCH] [Solr] Use fixed, recommended schema but add EdgeNGramFilterFactory --- data/Dockerfiles/solr/Dockerfile | 5 +- data/Dockerfiles/solr/docker-entrypoint.sh | 390 ++------------------- data/conf/solr/solr-config-7.7.0.xml | 289 +++++++++++++++ data/conf/solr/solr-schema-7.7.0.xml | 50 +++ data/web/inc/functions.inc.php | 4 +- 5 files changed, 366 insertions(+), 372 deletions(-) create mode 100644 data/conf/solr/solr-config-7.7.0.xml create mode 100644 data/conf/solr/solr-schema-7.7.0.xml diff --git a/data/Dockerfiles/solr/Dockerfile b/data/Dockerfiles/solr/Dockerfile index 67cd3384..25235664 100644 --- a/data/Dockerfiles/solr/Dockerfile +++ b/data/Dockerfiles/solr/Dockerfile @@ -1,9 +1,8 @@ -FROM solr:7-alpine +FROM solr:7.7-alpine USER root COPY docker-entrypoint.sh / RUN apk --no-cache add su-exec curl tzdata \ - && chmod +x /docker-entrypoint.sh \ - && /docker-entrypoint.sh --bootstrap + && chmod +x /docker-entrypoint.sh ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/data/Dockerfiles/solr/docker-entrypoint.sh b/data/Dockerfiles/solr/docker-entrypoint.sh index 108f8b5a..0634874f 100755 --- a/data/Dockerfiles/solr/docker-entrypoint.sh +++ b/data/Dockerfiles/solr/docker-entrypoint.sh @@ -26,395 +26,51 @@ fi # run the optional initdb . /opt/docker-solr/scripts/run-initdb -function solr_config() { - curl -XPOST http://localhost:8983/solr/dovecot/schema -H 'Content-type:application/json' -d '{ - "add-field-type":{ - "name":"long", - "class":"solr.TrieLongField" - }, - "add-field-type":{ - "name":"dovecot_text", - "class":"solr.TextField", - "autoGeneratePhraseQueries":true, - "positionIncrementGap":100, - "indexAnalyser":{ - "charFilter":{ - "class":"solr.MappingCharFilterFactory", - "mapping":"mapping-FoldToASCII.txt" - }, - "charFilter":{ - "class":"solr.MappingCharFilterFactory", - "mapping":"mapping-ISOLatin1Accent.txt" - }, - "charFilter":{ - "class":"solr.HTMLStripCharFilterFactory" - }, - "tokenizer":{ - "class":"solr.StandardTokenizerFactory" - }, - "filter":{ - "class":"solr.StopFilterFactory", - "words":"stopwords.txt", - "ignoreCase":true - }, - "filter":{ - "class":"solr.WordDelimiterGraphFilterFactory", - "generateWordParts":1, - "generateNumberParts":1, - "splitOnCaseChange":1, - "splitOnNumerics":1, - "catenateWords":1, - "catenateNumbers":1, - "catenateAll":1 - }, - "filter":{ - "class":"solr.FlattenGraphFilterFactory" - }, - "filter":{ - "class":"solr.LowerCaseFilterFactory" - }, - "filter":{ - "class":"solr.KeywordMarkerFilterFactory", - "protected":"protwords.txt" - }, - "filter":{ - "class":"solr.PorterStemFilterFactory" - } - }, - "queryAnalyzer":{ - "tokenizer":{ - "class":"solr.StandardTokenizerFactory" - }, - "filter":{ - "class":"solr.SynonymGraphFilterFactory", - "expand":true, - "ignoreCase":true, - "synonyms":synonyms.txt - }, - "filter":{ - "class":"solr.FlattenGraphFilterFactory" - }, - "filter":{ - "class":"solr.StopFilterFactory", - "words":"stopwords.txt", - "ignoreCase":true - }, - "filter":{ - "class":"solr.WordDelimiterGraphFilterFactory", - "generateWordParts":1, - "generateNumberParts":1, - "splitOnCaseChange":1, - "splitOnNumerics":1, - "catenateWords":1, - "catenateNumbers":1, - "catenateAll":1 - }, - "filter":{ - "class":"solr.LowerCaseFilterFactory" - }, - "filter":{ - "class":"solr.KeywordMarkerFilterFactory", - "protected":"protwords.txt" - }, - "filter":{ - "class":"solr.PorterStemFilterFactory" - } - } - }, - "add-field":{ - "name":"uid", - "type":"long", - "indexed":true, - "stored":true, - "required":true - }, - "add-field":{ - "name":"box", - "type":"string", - "indexed":true, - "stored":true, - "required":true - }, - "add-field":{ - "name":"user", - "type":"string", - "indexed":true, - "stored":true, - "required":true - }, - "add-field":{ - "name":"hdr", - "type":"dovecot_text", - "indexed":true, - "stored":false - - }, - "add-field":{ - "name":"body", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "add-field":{ - "name":"from", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "add-field":{ - "name":"to", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "add-field":{ - "name":"cc", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "add-field":{ - "name":"bcc", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "add-field":{ - "name":"subject", - "type":"dovecot_text", - "indexed":true, - "stored":false - } - }' - - curl -XPOST http://localhost:8983/solr/dovecot/schema -H 'Content-type:application/json' -d '{ - "replace-field-type":{ - "name":"long", - "class":"solr.TrieLongField" - }, - "replace-field-type":{ - "name":"dovecot_text", - "class":"solr.TextField", - "autoGeneratePhraseQueries":true, - "positionIncrementGap":100, - "indexAnalyser":{ - "charFilter":{ - "class":"solr.MappingCharFilterFactory", - "mapping":"mapping-FoldToASCII.txt" - }, - "charFilter":{ - "class":"solr.MappingCharFilterFactory", - "mapping":"mapping-ISOLatin1Accent.txt" - }, - "charFilter":{ - "class":"solr.HTMLStripCharFilterFactory" - }, - "tokenizer":{ - "class":"solr.StandardTokenizerFactory" - }, - "filter":{ - "class":"solr.StopFilterFactory", - "words":"stopwords.txt", - "ignoreCase":true - }, - "filter":{ - "class":"solr.WordDelimiterGraphFilterFactory", - "generateWordParts":1, - "generateNumberParts":1, - "splitOnCaseChange":1, - "splitOnNumerics":1, - "catenateWords":1, - "catenateNumbers":1, - "catenateAll":1 - }, - "filter":{ - "class":"solr.FlattenGraphFilterFactory" - }, - "filter":{ - "class":"solr.LowerCaseFilterFactory" - }, - "filter":{ - "class":"solr.KeywordMarkerFilterFactory", - "protected":"protwords.txt" - }, - "filter":{ - "class":"solr.PorterStemFilterFactory" - } - }, - "queryAnalyzer":{ - "tokenizer":{ - "class":"solr.StandardTokenizerFactory" - }, - "filter":{ - "class":"solr.SynonymGraphFilterFactory", - "expand":true, - "ignoreCase":true, - "synonyms":synonyms.txt - }, - "filter":{ - "class":"solr.FlattenGraphFilterFactory" - }, - "filter":{ - "class":"solr.StopFilterFactory", - "words":"stopwords.txt", - "ignoreCase":true - }, - "filter":{ - "class":"solr.WordDelimiterGraphFilterFactory", - "generateWordParts":1, - "generateNumberParts":1, - "splitOnCaseChange":1, - "splitOnNumerics":1, - "catenateWords":1, - "catenateNumbers":1, - "catenateAll":1 - }, - "filter":{ - "class":"solr.LowerCaseFilterFactory" - }, - "filter":{ - "class":"solr.KeywordMarkerFilterFactory", - "protected":"protwords.txt" - }, - "filter":{ - "class":"solr.PorterStemFilterFactory" - } - } - }, - "replace-field":{ - "name":"uid", - "type":"long", - "indexed":true, - "stored":true, - "required":true - }, - "replace-field":{ - "name":"box", - "type":"string", - "indexed":true, - "stored":true, - "required":true - }, - "replace-field":{ - "name":"user", - "type":"string", - "indexed":true, - "stored":true, - "required":true - }, - "replace-field":{ - "name":"hdr", - "type":"dovecot_text", - "indexed":true, - "stored":false - - }, - "replace-field":{ - "name":"body", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "replace-field":{ - "name":"from", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "replace-field":{ - "name":"to", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "replace-field":{ - "name":"cc", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "replace-field":{ - "name":"bcc", - "type":"dovecot_text", - "indexed":true, - "stored":false - }, - "replace-field":{ - "name":"subject", - "type":"dovecot_text", - "indexed":true, - "stored":false - } - }' - - curl -XPOST http://localhost:8983/solr/dovecot/config -H 'Content-type:application/json' -d '{ - "update-requesthandler":{ - "name":"/select", - "class":"solr.SearchHandler", - "defaults":{ - "wt":"xml" - } - } - }' - - curl -XPOST http://localhost:8983/solr/dovecot/config/updateHandler -d '{ - "set-property": { - "updateHandler.autoSoftCommit.maxDocs":500, - "updateHandler.autoSoftCommit.maxTime":120000, - "updateHandler.autoCommit.maxDocs":200, - "updateHandler.autoCommit.maxTime":1800000, - "updateHandler.autoCommit.openSearcher":false - } - }' -} - # fixing volume permission -[[ -d /opt/solr/server/solr/dovecot/data ]] && chown -R solr:solr /opt/solr/server/solr/dovecot/data +[[ -d /opt/solr/server/solr/dovecot-fts/data ]] && chown -R solr:solr /opt/solr/server/solr/dovecot-fts/data if [[ "${1}" != "--bootstrap" ]]; then sed -i '/SOLR_HEAP=/c\SOLR_HEAP="'${SOLR_HEAP:-1024}'m"' /opt/solr/bin/solr.in.sh else sed -i '/SOLR_HEAP=/c\SOLR_HEAP="256m"' /opt/solr/bin/solr.in.sh fi -# start a Solr so we can use the Schema API, but only on localhost, -# so that clients don't see Solr until we have configured it. - -echo "Starting local Solr instance to setup configuration" -su-exec solr start-local-solr - # keep a sentinel file so we don't try to create the core a second time # for example when we restart a container. +# todo: check if a core exists without sentinel file -SENTINEL=/opt/docker-solr/core_created +SENTINEL=/opt/docker-solr/fts_core_created if [[ -f ${SENTINEL} ]]; then echo "skipping core creation" else - echo "Creating core \"dovecot\"" - su-exec solr /opt/solr/bin/solr create -c "dovecot" + echo "Starting local Solr instance to setup configuration" + su-exec solr start-local-solr + + echo "Creating core \"dovecot-fts\"" + su-exec solr /opt/solr/bin/solr create -c "dovecot-fts" # See https://github.com/docker-solr/docker-solr/issues/27 echo "Checking core" while ! wget -O - 'http://localhost:8983/solr/admin/cores?action=STATUS' | grep -q instanceDir; do echo "Could not find any cores, waiting..." - sleep 5 + sleep 3 done - echo "Created core \"dovecot\"" + + echo "Created core \"dovecot-fts\"" touch ${SENTINEL} + + echo "Stopping local Solr" + su-exec solr stop-local-solr fi -echo "Starting configuration" -while ! wget -O - 'http://localhost:8983/solr/admin/cores?action=STATUS' | grep -q instanceDir; do - echo "Waiting for Solr..." - sleep 5 -done -solr_config -echo "Stopping local Solr" -su-exec solr stop-local-solr +rm -f /opt/solr/server/solr/dovecot-fts/conf/schema.xml +rm -f /opt/solr/server/solr/dovecot-fts/conf/managed-schema +rm -f /opt/solr/server/solr/dovecot-fts/conf/solrconfig.xml -if [[ "${1}" == "--bootstrap" ]]; then - exit 0 -else - exec su-exec solr solr-foreground -fi +cp /etc/solr/solr-config-7.7.0.xml /opt/solr/server/solr/dovecot-fts/conf/solrconfig.xml +cp /etc/solr/solr-schema-7.7.0.xml /opt/solr/server/solr/dovecot-fts/conf/schema.xml + +chown -R solr:solr /opt/solr/server/solr/dovecot-fts/conf/{schema.xml,solrconfig.xml} + +exec su-exec solr solr-foreground diff --git a/data/conf/solr/solr-config-7.7.0.xml b/data/conf/solr/solr-config-7.7.0.xml new file mode 100644 index 00000000..3661874d --- /dev/null +++ b/data/conf/solr/solr-config-7.7.0.xml @@ -0,0 +1,289 @@ + + + + + + + 7.7.0 + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + + + + + + + + + + + true + + + 20 + + + 200 + + + false + + + + + + + + + + + + + + + explicit + 10 + + + + + + _text_ + + + + + + diff --git a/data/conf/solr/solr-schema-7.7.0.xml b/data/conf/solr/solr-schema-7.7.0.xml new file mode 100644 index 00000000..a41cbb47 --- /dev/null +++ b/data/conf/solr/solr-schema-7.7.0.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + diff --git a/data/web/inc/functions.inc.php b/data/web/inc/functions.inc.php index 34872aef..c869c122 100644 --- a/data/web/inc/functions.inc.php +++ b/data/web/inc/functions.inc.php @@ -1477,7 +1477,7 @@ function solr_status() { $endpoint = 'http://solr:8983/solr/admin/cores'; $params = array( 'action' => 'STATUS', - 'core' => 'dovecot', + 'core' => 'dovecot-fts', 'indexInfo' => 'true' ); $url = $endpoint . '?' . http_build_query($params); @@ -1494,7 +1494,7 @@ function solr_status() { else { curl_close($curl); $status = json_decode($response, true); - return (!empty($status['status']['dovecot'])) ? $status['status']['dovecot'] : false; + return (!empty($status['status']['dovecot-fts'])) ? $status['status']['dovecot-fts'] : false; } return false; }