From 5ae05b5ec05312a66f1606d31ce8f94962b28f79 Mon Sep 17 00:00:00 2001 From: andryyy Date: Mon, 23 Sep 2019 21:42:47 +0200 Subject: [PATCH] [SQL Upgrade] Fix process of SQL upgrade, wait for command to complete and do not use pipe communication model --- data/Dockerfiles/dockerapi/server.py | 23 ++++--- data/Dockerfiles/phpfpm/docker-entrypoint.sh | 70 +++++++++++--------- docker-compose.yml | 4 +- 3 files changed, 54 insertions(+), 43 deletions(-) diff --git a/data/Dockerfiles/dockerapi/server.py b/data/Dockerfiles/dockerapi/server.py index 8d6a1c66..027c2f00 100644 --- a/data/Dockerfiles/dockerapi/server.py +++ b/data/Dockerfiles/dockerapi/server.py @@ -216,18 +216,19 @@ class container_post(Resource): # api call: container_post - post_action: exec - cmd: system - task: mysql_upgrade def container_post__exec__system__mysql_upgrade(self, container_id): for container in docker_client.containers.list(filters={"id": container_id}): - cmd = "/usr/bin/mysql_upgrade -uroot -p'" + os.environ['DBROOT'].replace("'", "'\\''") + "'\n" - cmd_response = exec_cmd_container(container, cmd, user='mysql') - - matched = False - for line in cmd_response.split("\n"): - if 'is already upgraded to' in line: - matched = True - if matched: - return jsonify(type='success', msg='mysql_upgrade: already upgraded') + sql_return = container.exec_run(["/bin/bash", "-c", "/usr/bin/mysql_upgrade -uroot -p'" + os.environ['DBROOT'].replace("'", "'\\''") + "'\n"], user='mysql') + if sql_return.exit_code == 0: + matched = False + for line in sql_return.output.decode('utf-8').split("\n"): + if 'is already upgraded to' in line: + matched = True + if matched: + return jsonify(type='success', msg='mysql_upgrade: already upgraded', text=sql_return.output.decode('utf-8')) + else: + container.restart() + return jsonify(type='warning', msg='mysql_upgrade: upgrade was applied', text=sql_return.output.decode('utf-8')) else: - container.restart() - return jsonify(type='warning', msg='mysql_upgrade: upgrade was applied') + return jsonify(type='error', msg='mysql_upgrade: error running command') # api call: container_post - post_action: exec - cmd: reload - task: dovecot diff --git a/data/Dockerfiles/phpfpm/docker-entrypoint.sh b/data/Dockerfiles/phpfpm/docker-entrypoint.sh index dd7413fa..11088ad8 100755 --- a/data/Dockerfiles/phpfpm/docker-entrypoint.sh +++ b/data/Dockerfiles/phpfpm/docker-entrypoint.sh @@ -28,50 +28,60 @@ fi # Check of mysql_upgrade CONTAINER_ID= -# Todo: Better check if upgrade failed -# This can happen due to a broken sogo_view -[ -s /mysql_upgrade_loop ] && SQL_LOOP_C=$(cat /mysql_upgrade_loop) -[ -z ${SQL_LOOP_C} ] && SQL_LOOP_C=0 until [[ ! -z "${CONTAINER_ID}" ]] && [[ "${CONTAINER_ID}" =~ ^[[:alnum:]]*$ ]]; do CONTAINER_ID=$(curl --silent --insecure https://dockerapi/containers/json | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], id: .Id}" 2> /dev/null | jq -rc "select( .name | tostring | contains(\"mysql-mailcow\")) | .id" 2> /dev/null) done echo "MySQL @ ${CONTAINER_ID}" -SQL_UPGRADE_RETURN=$(curl --silent --insecure -XPOST https://dockerapi/containers/${CONTAINER_ID}/exec -d '{"cmd":"system", "task":"mysql_upgrade"}' --silent -H 'Content-type: application/json' | jq -r .type) -if [[ ${SQL_UPGRADE_RETURN} == 'warning' ]]; then - sleep 2 - if [ ${SQL_LOOP_C} -lt 2 ]; then - echo $((SQL_LOOP_C+1)) > /mysql_upgrade_loop - echo "MySQL applied an upgrade" - POSTFIX=($(curl --silent --insecure https://dockerapi/containers/json | jq -r '.[] | {name: .Config.Labels["com.docker.compose.service"], id: .Id}' | jq -rc 'select( .name | tostring | contains("postfix-mailcow")) | .id' | tr "\n" " ")) - if [[ -z ${POSTFIX} ]]; then - echo "Could not determine Postfix container ID, skipping Postfix restart." - else - echo "Restarting Postfix" - curl -X POST --silent --insecure https://dockerapi/containers/${POSTFIX}/restart | jq -r '.msg' - echo "Sleeping 10 seconds..." - sleep 10 - fi - echo "Restarting PHP-FPM, bye" - exit 1 - else - rm /mysql_upgrade_loop - echo "MySQL upgrade was not applied previously, skipping. Restart php-fpm-mailcow to retry or run mysql_upgrade manually." +SQL_LOOP_C=0 +SQL_CHANGED=0 +until [[ ${SQL_UPGRADE_STATUS} == 'success' ]]; do + if [ ${SQL_LOOP_C} -gt 4 ]; then + echo "Tried to upgrade MySQL and failed, giving up after ${SQL_LOOP_C} retries and starting container (oops, not good)" + break + fi + SQL_FULL_UPGRADE_RETURN=$(curl --silent --insecure -XPOST https://dockerapi/containers/${CONTAINER_ID}/exec -d '{"cmd":"system", "task":"mysql_upgrade"}' --silent -H 'Content-type: application/json') + SQL_UPGRADE_STATUS=$(echo ${SQL_FULL_UPGRADE_RETURN} | jq -r .type) + SQL_LOOP_C=$((SQL_LOOP_C+1)) + echo "SQL upgrade iteration #${SQL_LOOP_C}" + if [[ ${SQL_UPGRADE_STATUS} == 'warning' ]]; then + SQL_CHANGED=1 + echo "MySQL applied an upgrade, debug output:" + echo ${SQL_FULL_UPGRADE_RETURN} + sleep 3 while ! mysqladmin status --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do - echo "Waiting for SQL to return..." + echo "Waiting for SQL to return, please wait" sleep 2 done + continue + elif [[ ${SQL_UPGRADE_STATUS} == 'success' ]]; then + echo "MySQL is up-to-date - debug output:" + echo ${SQL_FULL_UPGRADE_RETURN} + else + echo "No valid reponse for mysql_upgrade was received" + fi +done + +# doing post-installation stuff, if SQL was upgraded +if [ ${SQL_CHANGED} -eq 1 ]; then + POSTFIX=($(curl --silent --insecure https://dockerapi/containers/json | jq -r '.[] | {name: .Config.Labels["com.docker.compose.service"], id: .Id}' | jq -rc 'select( .name | tostring | contains("postfix-mailcow")) | .id' | tr "\n" " ")) + if [[ -z ${POSTFIX} ]]; then + echo "Could not determine Postfix container ID, skipping Postfix restart." + else + echo "Restarting Postfix" + curl -X POST --silent --insecure https://dockerapi/containers/${POSTFIX}/restart | jq -r '.msg' + echo "Sleeping 5 seconds..." + sleep 5 fi -else - echo "MySQL is up-to-date" fi # Trigger db init echo "Running DB init..." php -c /usr/local/etc/php -f /web/inc/init_db.inc.php -# Migrate domain map +# Recreating domain map +echo "Rebuilding domain map in Redis..." declare -a DOMAIN_ARR -redis-cli -h redis-mailcow DEL DOMAIN_MAP + redis-cli -h redis-mailcow DEL DOMAIN_MAP > /dev/null while read line do DOMAIN_ARR+=("$line") @@ -83,7 +93,7 @@ done < <(mysql --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${D if [[ ! -z ${DOMAIN_ARR} ]]; then for domain in "${DOMAIN_ARR[@]}"; do - redis-cli -h redis-mailcow HSET DOMAIN_MAP ${domain} 1 + redis-cli -h redis-mailcow HSET DOMAIN_MAP ${domain} 1 > /dev/null done fi diff --git a/docker-compose.yml b/docker-compose.yml index f2fb2464..a1f1e4a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,7 +95,7 @@ services: - rspamd php-fpm-mailcow: - image: mailcow/phpfpm:1.45 + image: mailcow/phpfpm:1.46 build: ./data/Dockerfiles/phpfpm command: "php-fpm -d date.timezone=${TZ} -d expose_php=0" depends_on: @@ -394,7 +394,7 @@ services: - watchdog dockerapi-mailcow: - image: mailcow/dockerapi:1.32 + image: mailcow/dockerapi:1.33 restart: always build: ./data/Dockerfiles/dockerapi oom_kill_disable: true