[mailcow] Use ofelia instead of cron daemons (wip: remove init systems)

2021-04-16 20:38:09 +02:00 · 2021-04-16 20:38:09 +02:00 · 971434ddd3
parent 2b2e139d7a
commit 971434ddd3
11 changed files with 85 additions and 69 deletions
--- a/data/Dockerfiles/dovecot/Dockerfile
+++ b/data/Dockerfiles/dovecot/Dockerfile
@ -4,6 +4,7 @@ LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
 ARG DEBIAN_FRONTEND=noninteractive
 ARG DOVECOT=2.3.14
 ENV LC_ALL C
+ENV GOSU_VERSION 1.12

 # Add groups and users before installing Dovecot to not break compatibility
 RUN groupadd -g 5000 vmail \
@ -20,7 +21,6 @@ RUN groupadd -g 5000 vmail \
  apt-transport-https \
  ca-certificates \
  cpanminus \
-  cron \
  curl \
  dnsutils \
  dirmngr \
@ -82,6 +82,11 @@ RUN groupadd -g 5000 vmail \
  syslog-ng \
  syslog-ng-core \
  syslog-ng-mod-redis \
+  wget \
+  && dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \
+  && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" \
+  && chmod +x /usr/local/bin/gosu \
+  && gosu nobody true \
  && apt-key adv --fetch-keys https://repo.dovecot.org/DOVECOT-REPO-GPG \
  && echo "deb https://repo.dovecot.org/ce-${DOVECOT}/debian/buster buster main" > /etc/apt/sources.list.d/dovecot.list \
  && apt-get update \
@ -100,7 +105,7 @@ RUN groupadd -g 5000 vmail \
  && apt-get autoremove --purge -y \
  && apt-get autoclean \
  && rm -rf /var/lib/apt/lists/* \
-  && rm -rf /tmp/* /var/tmp/* /etc/cron.daily/* /root/.cache/
+  && rm -rf /tmp/* /var/tmp/* /root/.cache/

 COPY trim_logs.sh /usr/local/bin/trim_logs.sh
 COPY clean_q_aged.sh /usr/local/bin/clean_q_aged.sh
@ -108,7 +113,7 @@ COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
 COPY imapsync /usr/local/bin/imapsync
 COPY postlogin.sh /usr/local/bin/postlogin.sh
-COPY imapsync_cron.pl /usr/local/bin/imapsync_cron.pl
+COPY imapsync_runner.pl /usr/local/bin/imapsync_runner.pl
 COPY report-spam.sieve /usr/lib/dovecot/sieve/report-spam.sieve
 COPY report-ham.sieve /usr/lib/dovecot/sieve/report-ham.sieve
 COPY rspamd-pipe-ham /usr/lib/dovecot/sieve/rspamd-pipe-ham
--- a/data/Dockerfiles/dovecot/docker-entrypoint.sh
+++ b/data/Dockerfiles/dovecot/docker-entrypoint.sh
@ -185,6 +185,12 @@ function script_deinit()
 end
 EOF

+# Replace patterns in app-passdb.lua
+sed -i "s/__DBUSER__/${DBUSER}/g" /etc/dovecot/lua/app-passdb.lua
+sed -i "s/__DBPASS__/${DBPASS}/g" /etc/dovecot/lua/app-passdb.lua
+sed -i "s/__DBNAME__/${DBNAME}/g" /etc/dovecot/lua/app-passdb.lua
+
+
 # Migrate old sieve_after file
 [[ -f /etc/dovecot/sieve_after ]] && mv /etc/dovecot/sieve_after /etc/dovecot/global_sieve_after
 # Create global sieve scripts
@ -269,15 +275,10 @@ else
    rm -f /etc/dovecot/sogo-sso.conf
 fi

-# Hard-code env vars to scripts due to cron not passing them to the scripts
-sed -i "s/__DBUSER__/${DBUSER}/g" /usr/local/bin/imapsync_cron.pl /usr/local/bin/quarantine_notify.py /usr/local/bin/clean_q_aged.sh /etc/dovecot/lua/app-passdb.lua
-sed -i "s/__DBPASS__/${DBPASS}/g" /usr/local/bin/imapsync_cron.pl /usr/local/bin/quarantine_notify.py /usr/local/bin/clean_q_aged.sh /etc/dovecot/lua/app-passdb.lua
-sed -i "s/__DBNAME__/${DBNAME}/g" /usr/local/bin/imapsync_cron.pl /usr/local/bin/quarantine_notify.py /usr/local/bin/clean_q_aged.sh /etc/dovecot/lua/app-passdb.lua
-sed -i "s/__MAILCOW_HOSTNAME__/${MAILCOW_HOSTNAME}/g" /usr/local/bin/quarantine_notify.py
-sed -i "s/__LOG_LINES__/${LOG_LINES}/g" /usr/local/bin/trim_logs.sh
+
 if [[ "${MASTER}" =~ ^([nN][oO]|[nN])+$ ]]; then
-# Toggling MASTER will result in a rebuild of containers, so the quota script will be recreated
-cat <<'EOF' > /usr/local/bin/quota_notify.py
+  # Toggling MASTER will result in a rebuild of containers, so the quota script will be recreated
+  cat <<'EOF' > /usr/local/bin/quota_notify.py
 #!/usr/bin/python3
 import sys
 sys.exit()
@ -311,7 +312,7 @@ chmod g+rw /dev/console
 chown root:tty /dev/console
 chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
  /usr/lib/dovecot/sieve/rspamd-pipe-spam \
-  /usr/local/bin/imapsync_cron.pl \
+  /usr/local/bin/imapsync_runner.pl \
  /usr/local/bin/postlogin.sh \
  /usr/local/bin/imapsync \
  /usr/local/bin/trim_logs.sh \
@ -322,27 +323,6 @@ chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
  /usr/local/bin/quota_notify.py \
  /usr/local/bin/repl_health.sh

-if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-# Setup cronjobs
-echo '* * * * *    nobody  /usr/local/bin/imapsync_cron.pl 2>&1 | /usr/bin/logger' > /etc/cron.d/imapsync
-#echo '30 3 * * *   vmail /usr/local/bin/doveadm quota recalc -A' > /etc/cron.d/dovecot-sync
-echo '* * * * *    vmail /usr/local/bin/trim_logs.sh >> /dev/console 2>&1' > /etc/cron.d/trim_logs
-echo '25 * * * *   vmail /usr/local/bin/maildir_gc.sh >> /dev/console 2>&1' > /etc/cron.d/maildir_gc
-echo '30 1 * * *   root  /usr/local/bin/sa-rules.sh  >> /dev/console 2>&1' > /etc/cron.d/sa-rules
-echo '0 2 * * *    root  /usr/bin/curl http://solr:8983/solr/dovecot-fts/update?optimize=true >> /dev/console 2>&1' > /etc/cron.d/solr-optimize
-echo '*/20 * * * * vmail /usr/local/bin/quarantine_notify.py >> /dev/console 2>&1' > /etc/cron.d/quarantine_notify
-echo '15 4 * * * vmail /usr/local/bin/clean_q_aged.sh >> /dev/console 2>&1' > /etc/cron.d/clean_q_aged
-echo '*/5 * * * *  vmail /usr/local/bin/repl_health.sh >> /dev/console 2>&1' > /etc/cron.d/repl_health
-else
-echo '25 * * * *   vmail /usr/local/bin/maildir_gc.sh >> /dev/console 2>&1' > /etc/cron.d/maildir_gc
-echo '30 1 * * *   root  /usr/local/bin/sa-rules.sh  >> /dev/console 2>&1' > /etc/cron.d/sa-rules
-echo '0 2 * * *    root  /usr/bin/curl http://solr:8983/solr/dovecot-fts/update?optimize=true >> /dev/console 2>&1' > /etc/cron.d/solr-optimize
-echo '*/5 * * * *  vmail /usr/local/bin/repl_health.sh >> /dev/console 2>&1' > /etc/cron.d/repl_health
-fi
-
-# Fix more than 1 hardlink issue
-touch /etc/crontab /etc/cron.*/*
-
 # Prepare environment file for cronjobs
 printenv | sed 's/^\(.*\)$/export \1/g' > /source_env.sh

--- a/data/Dockerfiles/dovecot/imapsync_runner.pl
+++ b/data/Dockerfiles/dovecot/imapsync_runner.pl
@ -36,11 +36,11 @@ sub qqw($) {
 }

 $run_dir="/tmp";
-$dsn = 'DBI:mysql:database=__DBNAME__;mysql_socket=/var/run/mysqld/mysqld.sock';
+$dsn = 'DBI:mysql:database=' . $ENV{'DBNAME'} . ';mysql_socket=/var/run/mysqld/mysqld.sock';
 $lock_file = $run_dir . "/imapsync_busy";
 $lockmgr = LockFile::Simple->make(-autoclean => 1, -max => 1);
 $lockmgr->lock($lock_file) || die "can't lock ${lock_file}";
-$dbh = DBI->connect($dsn, '__DBUSER__', '__DBPASS__', {
+$dbh = DBI->connect($dsn, $ENV{'DBUSER'}, $ENV{'DBPASS'}, {
  mysql_auto_reconnect => 1,
  mysql_enable_utf8mb4 => 1
 });
--- a/data/Dockerfiles/dovecot/maildir_gc.sh
+++ b/data/Dockerfiles/dovecot/maildir_gc.sh
@ -1,2 +1,2 @@
-#/bin/bash
+#!/bin/bash
 [ -d /var/vmail/_garbage/ ] && /usr/bin/find /var/vmail/_garbage/ -mindepth 1 -maxdepth 1 -type d -cmin +${MAILDIR_GC_TIME} -exec rm -r {} \;
--- a/data/Dockerfiles/dovecot/quarantine_notify.py
+++ b/data/Dockerfiles/dovecot/quarantine_notify.py
@ -41,7 +41,7 @@ try:
      break

  time_now = int(time.time())
-  mailcow_hostname = '__MAILCOW_HOSTNAME__'
+  mailcow_hostname = os.environ.get('MAILCOW_HOSTNAME')

  max_score = float(r.get('Q_MAX_SCORE') or "9999.0")
  if max_score == "":
@ -50,7 +50,7 @@ try:
  def query_mysql(query, headers = True, update = False):
    while True:
      try:
-        cnx = mysql.connector.connect(unix_socket = '/var/run/mysqld/mysqld.sock', user='__DBUSER__', passwd='__DBPASS__', database='__DBNAME__', charset="utf8")
+        cnx = mysql.connector.connect(unix_socket = '/var/run/mysqld/mysqld.sock', user=os.environ.get('DBUSER'), passwd=os.environ.get('DBPASS'), database=os.environ.get('DBNAME'), charset="utf8")
      except Exception as ex:
        print('%s - trying again...'  % (ex))
        time.sleep(3)
--- a/data/Dockerfiles/dovecot/supervisord.conf
+++ b/data/Dockerfiles/dovecot/supervisord.conf
@ -15,10 +15,6 @@ autostart=true
 command=/usr/sbin/dovecot -F
 autorestart=true

-[program:cron]
-command=/usr/sbin/cron -f
-autorestart=true
-
 [eventlistener:processes]
 command=/usr/local/sbin/stop-supervisor.sh
 events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
--- a/data/Dockerfiles/dovecot/trim_logs.sh
+++ b/data/Dockerfiles/dovecot/trim_logs.sh
@ -14,12 +14,12 @@ if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
 else
  REDIS_CMDLINE="redis-cli -h redis -p 6379"
 fi
-catch_non_zero "${REDIS_CMDLINE} LTRIM ACME_LOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM POSTFIX_MAILLOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM DOVECOT_MAILLOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM SOGO_LOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM NETFILTER_LOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM AUTODISCOVER_LOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM API_LOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM RL_LOG 0 __LOG_LINES__"
-catch_non_zero "${REDIS_CMDLINE} LTRIM WATCHDOG_LOG 0 __LOG_LINES__"
+catch_non_zero "${REDIS_CMDLINE} LTRIM ACME_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM POSTFIX_MAILLOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM DOVECOT_MAILLOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM SOGO_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM NETFILTER_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM AUTODISCOVER_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM API_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM RL_LOG 0 ${LOG_LINES}"
+catch_non_zero "${REDIS_CMDLINE} LTRIM WATCHDOG_LOG 0 ${LOG_LINES}"
--- a/data/Dockerfiles/sogo/Dockerfile
+++ b/data/Dockerfiles/sogo/Dockerfile
@ -4,14 +4,13 @@ LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
 ARG DEBIAN_FRONTEND=noninteractive
 ARG SOGO_DEBIAN_REPOSITORY=http://packages.inverse.ca/SOGo/nightly/5/debian/
 ENV LC_ALL C
-ENV GOSU_VERSION 1.11
+ENV GOSU_VERSION 1.12

 # Prerequisites
 RUN echo "Building from repository $SOGO_DEBIAN_REPOSITORY" \
  && apt-get update && apt-get install -y --no-install-recommends \
  apt-transport-https \
  ca-certificates \
-  cron \
  gettext \
  gnupg \
  mariadb-client \
--- a/data/Dockerfiles/sogo/bootstrap-sogo.sh
+++ b/data/Dockerfiles/sogo/bootstrap-sogo.sh
@ -249,14 +249,4 @@ rsync -a /usr/lib/GNUstep/SOGo/. /sogo_web/
 # Chown backup path
 chown -R sogo:sogo /sogo_backup

-# Creating cronjobs
-if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-  echo "* * * * *   sogo   /usr/sbin/sogo-ealarms-notify -p /etc/sogo/sieve.creds 2>/dev/null" > /etc/cron.d/sogo
-  echo "* * * * *   sogo   /usr/sbin/sogo-tool expire-sessions ${SOGO_EXPIRE_SESSION}" >> /etc/cron.d/sogo
-  echo "0 0 * * *   sogo   /usr/sbin/sogo-tool update-autoreply -p /etc/sogo/sieve.creds" >> /etc/cron.d/sogo
-  echo "0 2 * * *   sogo   /usr/sbin/sogo-tool backup /sogo_backup ALL" >> /etc/cron.d/sogo
-else
-  rm /etc/cron.d/sogo
-fi
-
 exec gosu sogo /usr/sbin/sogod
--- a/data/Dockerfiles/sogo/supervisord.conf
+++ b/data/Dockerfiles/sogo/supervisord.conf
@ -11,18 +11,13 @@ stderr_logfile_maxbytes=0
 autostart=true
 priority=1

-[program:cron]
-command=/usr/sbin/cron -f
-autorestart=true
-priority=2
-
 [program:bootstrap-sogo]
 command=/bootstrap-sogo.sh
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
-priority=3
+priority=2
 startretries=10
 autorestart=true
 stopwaitsecs=120
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -192,6 +192,16 @@ services:
        - mysql-socket-vol-1:/var/run/mysqld/:z
        - sogo-web-vol-1:/sogo_web:z
        - sogo-userdata-backup-vol-1:/sogo_backup:Z
+      labels:
+        ofelia.enabled: "true"
+        ofelia.job-exec.sogo_sessions.schedule: "@every 1m"
+        ofelia.job-exec.sogo_sessions.command: "/bin/bash -c \"[[ $${MASTER} == y ]] && /usr/local/bin/gosu sogo /usr/sbin/sogo-tool expire-sessions $${SOGO_EXPIRE_SESSION} || exit 0\""
+        ofelia.job-exec.sogo_ealarms.schedule: "@every 1m"
+        ofelia.job-exec.sogo_ealarms.command: "/bin/bash -c \"[[ $${MASTER} == y ]] && /usr/local/bin/gosu sogo /usr/sbin/sogo-ealarms-notify -p /etc/sogo/sieve.creds || exit 0\""
+        ofelia.job-exec.sogo_eautoreply.schedule: "@every 1d"
+        ofelia.job-exec.sogo_eautoreply.command: "/bin/bash -c \"[[ $${MASTER} == y ]] && /usr/local/bin/gosu sogo /usr/sbin/sogo-tool update-autoreply -p /etc/sogo/sieve.creds || exit 0\""
+        ofelia.job-exec.sogo_backup.schedule: "@every 1d"
+        ofelia.job-exec.sogo_backup.command: "/bin/bash -c \"[[ $${MASTER} == y ]] && /usr/local/bin/gosu sogo /usr/sbin/sogo-tool backup /sogo_backup ALL || exit 0\""
      restart: always
      networks:
        mailcow-network:
@ -249,6 +259,25 @@ services:
        - "${SIEVE_PORT:-4190}:4190"
      restart: always
      tty: true
+      labels:
+        ofelia.enabled: "true"
+        ofelia.job-exec.dovecot_imapsync_runner.schedule: "@every 1m"
+        ofelia.job-exec.dovecot_imapsync_runner.no-overlap: "true"
+        ofelia.job-exec.dovecot_imapsync_runner.command: "/bin/bash -c \"[[ $${MASTER} == y ]] && /usr/local/bin/gosu nobody /usr/local/bin/imapsync_runner.pl || exit 0\""
+        ofelia.job-exec.dovecot_trim_logs.schedule: "@every 1m"
+        ofelia.job-exec.dovecot_trim_logs.command: "/bin/bash -c \"[[ $${MASTER} == y ]] && /usr/local/bin/gosu vmail /usr/local/bin/trim_logs.sh || exit 0\""
+        ofelia.job-exec.dovecot_quarantine.schedule: "@every 20m"
+        ofelia.job-exec.dovecot_quarantine.command: "/bin/bash -c \"[[ $${MASTER} == y ]] && /usr/local/bin/gosu vmail /usr/local/bin/quarantine_notify.py || exit 0\""
+        ofelia.job-exec.dovecot_clean_q_aged.schedule: "@every 1d"
+        ofelia.job-exec.dovecot_clean_q_aged.command: "/bin/bash -c \"[[ $${MASTER} == y ]] && /usr/local/bin/gosu vmail /usr/local/bin/clean_q_aged.sh || exit 0\""
+        ofelia.job-exec.dovecot_maildir_gc.schedule: "@every 30m"
+        ofelia.job-exec.dovecot_maildir_gc.command: "/bin/bash -c \"source /source_env.sh ; /usr/local/bin/gosu vmail /usr/local/bin/maildir_gc.sh\""
+        ofelia.job-exec.dovecot_sarules.schedule: "@every 1d"
+        ofelia.job-exec.dovecot_sarules.command: "/bin/bash -c \"/usr/local/bin/sa-rules.sh\""
+        ofelia.job-exec.dovecot_fts.schedule: "@every 1d"
+        ofelia.job-exec.dovecot_fts.command: "/usr/bin/curl http://solr:8983/solr/dovecot-fts/update?optimize=true"
+        ofelia.job-exec.dovecot_repl_health.schedule: "@every 5m"
+        ofelia.job-exec.dovecot_repl_health.command: "/bin/bash -c \"/usr/local/bin/gosu vmail /usr/local/bin/repl_health.sh\""
      ulimits:
        nproc: 65535
        nofile:
@ -542,6 +571,10 @@ services:
      dns:
        - ${IPV4_NETWORK:-172.22.1}.254
      hostname: ejabberd.mailcow.local
+      labels:
+        ofelia.enabled: "true"
+        ofelia.job-exec.ejabberd_certs.schedule: "@every 14d"
+        ofelia.job-exec.ejabberd_certs.command: "/sbin/su-exec ejabberd /home/ejabberd/bin/ejabberdctl --node ejabberd@$${MAILCOW_HOSTNAME} request-certificate all"
      extra_hosts:
        - "${MAILCOW_HOSTNAME}:127.0.0.1"
      environment:
@ -562,6 +595,24 @@ services:
          aliases:
            - ejabberd

+    ofelia-mailcow:
+      image: mcuadros/ofelia:latest
+      restart: always
+      command: daemon --docker
+        - TZ=${TZ}
+      depends_on:
+        - sogo-mailcow
+        - dovecot-mailcow
+        - ejabberd-mailcow
+      labels:
+        ofelia.enabled: "true"
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock:ro
+      networks:
+        mailcow-network:
+          aliases:
+            - ofelia
+
    ipv6nat-mailcow:
      depends_on:
        - unbound-mailcow