paxton
/
mailcow
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Various changes...

master
andre.peters 2017-12-09 13:15:24 +01:00
parent 873222d5f8
commit 2519738094
6 changed files with 84 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
data/Dockerfiles/acme/Dockerfile
View File

@ -3,20 +3,21 @@ FROM alpine:3.6
LABEL maintainer "Andre Peters <andre.peters@servercow.de>"
RUN apk add --update --no-cache \
bash \
curl \
openssl \
bind-tools \
jq \
bash \
curl \
openssl \
bind-tools \
jq \
libressl-dev \
libbsd-dev \
libseccomp-dev \
mariadb-client \
mariadb-client \
tini \
make \
gcc \
libressl \
libc-dev \
redis \
linux-headers \
ca-certificates \
&& curl -s https://kristaps.bsd.lv/acme-client/snapshots/acme-client-portable.tgz | tar xfvz - \

49
data/Dockerfiles/acme/docker-entrypoint.sh
View File

@ -2,16 +2,30 @@
set -o pipefail
exec 5>&1
log_f() {
if [[ ${2} == "no_nl" ]]; then
echo -n "$(date) - ${1}"
elif [[ ${2} == "no_date" ]]; then
echo "${1}"
elif [[ ${2} != "redis_only" ]]; then
echo "$(date) - ${1}"
fi
redis-cli -h redis LPUSH ACME_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
tr '%&;$"_[]{}-\r\n' ' ')\"}" > /dev/null
redis-cli -h redis LTRIM ACME_LOG 0 9999 > /dev/null
}
if [[ "${SKIP_LETS_ENCRYPT}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
log_f "SKIP_LETS_ENCRYPT=y, skipping Let's Encrypt..."
sleep 365d
exec $(readlink -f "$0")
fi
echo "Waiting for Docker API..."
log_f "Waiting for Docker API..." no_nl
until ping dockerapi -c1 > /dev/null; do
sleep 1
done
log_f "Found Docker API" no_date
ACME_BASE=/var/lib/acme
SSL_EXAMPLE=/var/lib/ssl-example
@ -20,21 +34,12 @@ mkdir -p ${ACME_BASE}/acme/private
restart_containers(){
for container in $*; do
echo "Restarting ${container}..."
curl -X POST http://dockerapi:8080/containers/${container}/restart
log_f "Restarting ${container}..." no_nl
C_REST_OUT=$(curl -X POST http://dockerapi:8080/containers/${container}/restart | jq -r '.msg')
log_f "${C_REST_OUT}" no_date
done
}
log_f() {
if [[ ${2} == "no_nl" ]]; then
echo -n "$(date) - ${1}"
elif [[ ${2} == "no_date" ]]; then
echo "${1}"
else
echo "$(date) - ${1}"
fi
}
array_diff() {
# https://stackoverflow.com/questions/2312762, Alex Offshore
eval local ARR1=\(\"\${$2[@]}\"\)
@ -123,16 +128,23 @@ while true; do
declare -a VALIDATED_CONFIG_DOMAINS
declare -a ADDITIONAL_VALIDATED_SAN
IFS=',' read -r -a ADDITIONAL_SAN_ARR <<< "${ADDITIONAL_SAN}"
IPV4=$(get_ipv4)
until [[ ${IPV4} == ${EXTERNAL_IPV4} ]]; do
IPV4=$(get_ipv4)
if [[ ${IPV4} != ${EXTERNAL_IPV4} ]]; then
echo "Waiting for correct source ip..."
sleep 30s
fi
done
# Container ids may have changed
CONTAINERS_RESTART=($(curl --silent http://dockerapi:8080/containers/json | jq -r '.[] | {name: .Config.Labels["com.docker.compose.service"], id: .Id}' | jq -rc 'select( .name | tostring | contains("nginx-mailcow") or contains("postfix-mailcow") or contains("dovecot-mailcow")) | .id' | tr "\n" " "))
log_f "Waiting for domain tables... " no_nl
log_f "Waiting for domain table... " no_nl
while [[ -z ${DOMAIN_TABLE} ]]; do
curl --silent http://nginx/ >/dev/null 2>&1
DOMAIN_TABLE=$(mysql -h mysql-mailcow -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SHOW TABLES LIKE 'domain'" -Bs)
[[ -z ${DOMAIN_TABLE} ]] && sleep 10
done
log_f "OK" no_date
log_f "Found domain tables." no_date
while read domains; do
SQL_DOMAIN_ARR+=("${domains}")
@ -226,6 +238,7 @@ while true; do
case "$?" in
0) # new certs
log_f "${ACME_RESPONSE}" redis_only
# cp the new certificates and keys
cp ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem
cp ${ACME_BASE}/acme/private/privkey.pem ${ACME_BASE}/key.pem
@ -239,6 +252,7 @@ while true; do
restart_containers ${CONTAINERS_RESTART[*]}
;;
1) # failure
log_f "${ACME_RESPONSE}" redis_only
if [[ $ACME_RESPONSE =~ "No registration exists" ]]; then
log_f "Registration keys are invalid, deleting old keys and restarting..."
rm ${ACME_BASE}/acme/private/account.key
@ -268,6 +282,7 @@ while true; do
exec $(readlink -f "$0")
;;
2) # no change
log_f "${ACME_RESPONSE}" redis_only
if ! diff ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem; then
log_f "Certificate was not changed, but active certificate does not match the verified certificate, fixing and restarting containers..."
cp ${ACME_BASE}/acme/fullchain.pem ${ACME_BASE}/cert.pem
@ -280,9 +295,11 @@ while true; do
cp ${ACME_BASE}/acme/private/privkey.pem ${ACME_BASE}/key.pem
TRIGGER_RESTART=1
fi
log_f "Certificate was not changed"
[[ ${TRIGGER_RESTART} == 1 ]] && restart_containers ${CONTAINERS_RESTART[*]}
;;
*) # unspecified
log_f "${ACME_RESPONSE}" redis_only
if [[ -f ${ACME_BASE}/acme/private/${DATE}.bak/fullchain.pem ]] && [[ -f ${ACME_BASE}/acme/private/${DATE}.bak/privkey.pem ]]; then
log_f "Error requesting certificate, restoring previous certificate from backup and restarting containers...."
cp ${ACME_BASE}/acme/private/${DATE}.bak/fullchain.pem ${ACME_BASE}/cert.pem

41
data/Dockerfiles/dockerapi/server.py
View File

@ -6,6 +6,9 @@ from threading import Thread
import docker
import signal
import time
import os
import re
import sys
docker_client = docker.DockerClient(base_url='unix://var/run/docker.sock', version='auto')
app = Flask(__name__)
@ -15,7 +18,7 @@ class containers_get(Resource):
def get(self):
containers = {}
try:
for container in docker_client.containers.list(all=True):
for container in docker_client.containers.list(all=True, filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME']}):
containers.update({container.attrs['Id']: container.attrs})
return containers
except Exception as e:
@ -25,19 +28,30 @@ class container_get(Resource):
def get(self, container_id):
if container_id and container_id.isalnum():
try:
for container in docker_client.containers.list(all=True, filters={"id": container_id}):
for container in docker_client.containers.list(all=True, filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
return container.attrs
except Exception as e:
return jsonify(type='danger', msg=e)
else:
return jsonify(type='danger', msg='no or invalid id defined')
class container_logs(Resource):
def get(self, container_id, lines):
if container_id and container_id.isalnum() and lines:
try:
for container in docker_client.containers.list(all=True, filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
return container.logs(stdout=True, stderr=True, stream=False, tail=lines)
except Exception as e:
return jsonify(type='danger', msg=e)
else:
return jsonify(type='danger', msg='no or invalid id defined')
class container_post(Resource):
def post(self, container_id, post_action):
if container_id and container_id.isalnum() and post_action:
if post_action == 'stop':
try:
for container in docker_client.containers.list(all=True, filters={"id": container_id}):
for container in docker_client.containers.list(all=True, filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
container.stop()
return jsonify(type='success', msg='command completed successfully')
except Exception as e:
@ -45,7 +59,7 @@ class container_post(Resource):
elif post_action == 'start':
try:
for container in docker_client.containers.list(all=True, filters={"id": container_id}):
for container in docker_client.containers.list(all=True, filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
container.start()
return jsonify(type='success', msg='command completed successfully')
except Exception as e:
@ -53,7 +67,7 @@ class container_post(Resource):
elif post_action == 'restart':
try:
for container in docker_client.containers.list(all=True, filters={"id": container_id}):
for container in docker_client.containers.list(all=True, filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
container.restart()
return jsonify(type='success', msg='command completed successfully')
except Exception as e:
@ -66,16 +80,28 @@ class container_post(Resource):
if request.json['cmd'] == 'sieve_list' and request.json['username']:
try:
for container in docker_client.containers.list(filters={"id": container_id}):
for container in docker_client.containers.list(filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
return container.exec_run(["/bin/bash", "-c", "/usr/local/bin/doveadm sieve list -u '" + request.json['username'].replace("'", "'\\''") + "'"], user='vmail')
except Exception as e:
return jsonify(type='danger', msg=e)
elif request.json['cmd'] == 'sieve_print' and request.json['script_name'] and request.json['username']:
try:
for container in docker_client.containers.list(filters={"id": container_id}):
for container in docker_client.containers.list(filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
return container.exec_run(["/bin/bash", "-c", "/usr/local/bin/doveadm sieve get -u '" + request.json['username'].replace("'", "'\\''") + "' '" + request.json['script_name'].replace("'", "'\\''") + "'"], user='vmail')
except Exception as e:
return jsonify(type='danger', msg=e)
elif request.json['cmd'] == 'worker_password' and request.json['raw']:
try:
for container in docker_client.containers.list(filters={"label": "com.docker.compose.project=" + os.environ['COMPOSE_PROJECT_NAME'], "id": container_id}):
hash = container.exec_run(["/bin/bash", "-c", "/usr/bin/rspamadm pw -e -p '" + request.json['raw'].replace("'", "'\\''") + "'"], user='_rspamd')
f = open("/access.inc", "w")
f.write('enable_password = "' + re.sub('[^0-9a-zA-Z\$]+', '', hash.rstrip()) + '";\n')
f.close()
container.restart()
return jsonify(type='success', msg='command completed successfully')
except Exception as e:
return jsonify(type='danger', msg=e)
else:
return jsonify(type='danger', msg='Unknown command')
@ -99,6 +125,7 @@ def startFlaskAPI():
api.add_resource(containers_get, '/containers/json')
api.add_resource(container_get, '/containers/<string:container_id>/json')
api.add_resource(container_logs, '/containers/<string:container_id>/logs/<int:lines>')
api.add_resource(container_post, '/containers/<string:container_id>/<string:post_action>')
if __name__ == '__main__':

4
data/Dockerfiles/phpfpm/Dockerfile
View File

@ -41,8 +41,8 @@ RUN apk add -U --no-cache libxml2-dev \
Net_Sieve \
NET_SMTP \
Mail_mime \
&& pecl install redis-${REDIS_PECL} memcached-${MEMCACHED_PECL} APCu-${APCU_PECL} imagick-${IMAGICK_PECL} \
&& docker-php-ext-enable redis apcu memcached imagick \
&& pecl install redis-${REDIS_PECL} memcached-${MEMCACHED_PECL} APCu-${APCU_PECL} imagick-${IMAGICK_PECL} mailparse \
&& docker-php-ext-enable redis apcu memcached imagick mailparse \
&& pecl clear-cache \
&& docker-php-ext-configure intl \
&& docker-php-ext-install -j 4 intl gettext ldap sockets soap pdo pdo_mysql xmlrpc gd zip pcntl opcache \

2
data/Dockerfiles/postfix/whitelist_forwardinghosts.sh
View File

@ -6,7 +6,7 @@ while read QUERY; do
echo "500 dunno"
continue
fi
result=$(curl -s http://172.22.1.251:8081/forwardinghosts.php?host=${QUERY[1]})
result=$(curl -s http://nginx:8081/forwardinghosts.php?host=${QUERY[1]})
logger -t whitelist_forwardinghosts -p mail.info "Look up ${QUERY[1]} on whitelist, result $result"
echo ${result}
done

48
data/Dockerfiles/watchdog/watchdog.sh
View File

@ -28,21 +28,19 @@ progress() {
[[ ${CURRENT} -gt ${TOTAL} ]] && return
[[ ${CURRENT} -lt 0 ]] && CURRENT=0
PERCENT=$(( 200 * ${CURRENT} / ${TOTAL} % 2 + 100 * ${CURRENT} / ${TOTAL} ))
log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}"
log_data "$(printf "%d,%d,%d,%d" ${PERCENT} ${CURRENT} ${TOTAL} ${DIFF})" "${SERVICE}"
redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null
log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}" no_redis
}
log_msg() {
redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}")\"}" > /dev/null
if [[ ${2} != "no_redis" ]]; then
redis-cli -h redis LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
tr '%&;$"_[]{}-\r\n' ' ')\"}" > /dev/null
fi
redis-cli -h redis LTRIM WATCHDOG_LOG 0 9999 > /dev/null
echo $(date) $(printf '%s\n' "${1}")
}
log_data() {
[[ -z ${1} ]] && return 1
[[ -z ${2} ]] && return 2
redis-cli -h redis LPUSH WATCHDOG_DATA "{\"time\":\"$(date +%s)\",\"service\":\"data\",\"$(printf '%s' "${2}")\":\"$(printf '%s' "${1}")\"}" > /dev/null
}
function mail_error() {
[[ -z ${1} ]] && return 1
[[ -z ${2} ]] && return 2
@ -234,27 +232,6 @@ Empty
return 1
}
dns_checks() {
err_count=0
diff_c=0
THRESHOLD=28
# Reduce error count by 2 after restarting an unhealthy container
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
while [ ${err_count} -lt ${THRESHOLD} ]; do
host_ip=$(get_container_ip unbound-mailcow)
err_c_cur=${err_count}
/usr/lib/nagios/plugins/check_dns -H google.com 1>&2; err_count=$(( ${err_count} + ($? * 2)))
/usr/lib/nagios/plugins/check_dns -s ${host_ip} -H google.com 1>&2; err_count=$(( ${err_count} + ($? * 2)))
dig +dnssec org. @${host_ip} | grep -E 'flags:.+ad' 1>&2; err_count=$(( ${err_count} + ($? * 2)))
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
progress "Unbound" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
diff_c=0
sleep $(( ( RANDOM % 30 ) + 10 ))
done
return 1
}
# Create watchdog agents
(
while true; do
@ -322,17 +299,6 @@ done
) &
BACKGROUND_TASKS+=($!)
(
while true; do
if ! dns_checks; then
log_msg "Unbound hit error limit"
[[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${WATCHDOG_NOTIFY_EMAIL}" "unbound-mailcow"
#echo unbound-mailcow > /tmp/com_pipe
fi
done
) &
BACKGROUND_TASKS+=($!)
(
while true; do
if ! rspamd_checks; then