From 78aafe9b33690986492ad3f041ea72ee2a0431db Mon Sep 17 00:00:00 2001 From: andryyy Date: Mon, 18 Oct 2021 12:45:22 +0200 Subject: [PATCH] [Helper] Cold standby script (WIP, docs incoming) --- create_cold_standby.sh | 7 + helper-scripts/_cold-standby.sh | 251 ++++++++++++++++++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 create_cold_standby.sh create mode 100755 helper-scripts/_cold-standby.sh diff --git a/create_cold_standby.sh b/create_cold_standby.sh new file mode 100644 index 00000000..924339af --- /dev/null +++ b/create_cold_standby.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +export REMOTE_SSH_KEY=/root/.ssh/id_rsa +export REMOTE_SSH_PORT=22 +export REMOTE_SSH_HOST=my.remote.host + +/opt/mailcow-dockerized/helper-scripts/_cold-standby.sh diff --git a/helper-scripts/_cold-standby.sh b/helper-scripts/_cold-standby.sh new file mode 100755 index 00000000..3924268e --- /dev/null +++ b/helper-scripts/_cold-standby.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash + +PATH=$PATH:/opt/bin +DATE=$(date +%Y-%m-%d_%H_%M_%S) +export LC_ALL=C + +echo +echo "If this script is run automatically by cron or a timer AND you are using block-level snapshots on your backup destination, make sure both do not run at the same time." +echo "The snapshots of your backup destination should run AFTER the cold standby script finished to ensure consistent snapshots." +echo + +function docker_garbage() { + IMGS_TO_DELETE=() + + for container in $(grep -oP "image: \Kmailcow.+" docker-compose.yml); do + + REPOSITORY=${container/:*} + TAG=${container/*:} + V_MAIN=${container/*.} + V_SUB=${container/*.} + EXISTING_TAGS=$(docker images | grep ${REPOSITORY} | awk '{ print $2 }') + + for existing_tag in ${EXISTING_TAGS[@]}; do + + V_MAIN_EXISTING=${existing_tag/*.} + V_SUB_EXISTING=${existing_tag/*.} + + # Not an integer + [[ ! $V_MAIN_EXISTING =~ ^[0-9]+$ ]] && continue + [[ ! $V_SUB_EXISTING =~ ^[0-9]+$ ]] && continue + + if [[ $V_MAIN_EXISTING == "latest" ]]; then + echo "Found deprecated label \"latest\" for repository $REPOSITORY, it should be deleted." + IMGS_TO_DELETE+=($REPOSITORY:$existing_tag) + elif [[ $V_MAIN_EXISTING -lt $V_MAIN ]]; then + echo "Found tag $existing_tag for $REPOSITORY, which is older than the current tag $TAG and should be deleted." + IMGS_TO_DELETE+=($REPOSITORY:$existing_tag) + elif [[ $V_SUB_EXISTING -lt $V_SUB ]]; then + echo "Found tag $existing_tag for $REPOSITORY, which is older than the current tag $TAG and should be deleted." + IMGS_TO_DELETE+=($REPOSITORY:$existing_tag) + fi + + done + + done + + if [[ ! -z ${IMGS_TO_DELETE[*]} ]]; then + docker rmi ${IMGS_TO_DELETE[*]} + fi +} + +function preflight_local_checks() { + if [[ -z "${REMOTE_SSH_KEY}" ]]; then + echo -e "\e[31mREMOTE_SSH_KEY is not set\e[0m" + exit 1 + fi + + if [[ ! -s "${REMOTE_SSH_KEY}" ]]; then + echo -e "\e[31mKeyfile ${REMOTE_SSH_KEY} is empty\e[0m" + exit 1 + fi + + if [[ $(stat -c "%a" "${REMOTE_SSH_KEY}") -ne 600 ]]; then + echo -e "\e[31mKeyfile ${REMOTE_SSH_KEY} has insecure permissions\e[0m" + exit 1 + fi + + if [[ ! -z "${REMOTE_SSH_PORT}" ]]; then + if [[ ${REMOTE_SSH_PORT} != ?(-)+([0-9]) ]] || [[ ${REMOTE_SSH_PORT} -gt 65535 ]]; then + echo -e "\e[31mREMOTE_SSH_PORT is set but not an integer < 65535\e[0m" + exit 1 + fi + fi + + if [[ -z "${REMOTE_SSH_HOST}" ]]; then + echo -e "\e[31mREMOTE_SSH_HOST cannot be empty\e[0m" + exit 1 + fi + + for bin in rsync docker-compose docker grep cut; do + if [[ -z $(which ${bin}) ]]; then + echo -e "\e[31mCannot find ${bin} in local PATH, exiting...\e[0m" + exit 1 + fi + done + + if grep --help 2>&1 | head -n 1 | grep -q -i "busybox"; then + echo -e "\e[31mBusyBox grep detected on local system, please install GNU grep\e[0m" + exit 1 + fi +} + +function preflight_remote_checks() { + + ssh -o StrictHostKeyChecking=no \ + -i "${REMOTE_SSH_KEY}" \ + ${REMOTE_SSH_HOST} \ + -p ${REMOTE_SSH_PORT} \ + rsync -V > /dev/null + + if [ $? -ne 0 ]; then + echo -e "\e[31mCould not verify connection to ${REMOTE_SSH_HOST}\e[0m" + echo -e "\e[31mPlease check the output above (is rsync >= 3.1.0 installed on the remote system?)\e[0m" + exit 1 + fi + + if ssh -o StrictHostKeyChecking=no \ + -i "${REMOTE_SSH_KEY}" \ + ${REMOTE_SSH_HOST} \ + -p ${REMOTE_SSH_PORT} \ + grep --help 2>&1 | head -n 1 | grep -q -i "busybox" ; then + echo -e "\e[31mBusyBox grep detected on remote system ${REMOTE_SSH_HOST}, please install GNU grep\e[0m" + exit 1 + fi + + for bin in rsync docker-compose docker; do + if ! ssh -o StrictHostKeyChecking=no \ + -i "${REMOTE_SSH_KEY}" \ + ${REMOTE_SSH_HOST} \ + -p ${REMOTE_SSH_PORT} \ + which ${bin} > /dev/null ; then + echo -e "\e[31mCannot find ${bin} in remote PATH, exiting...\e[0m" + exit 1 + fi + done + +} + +preflight_local_checks +preflight_remote_checks + +SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +COMPOSE_FILE="${SCRIPT_DIR}/../docker-compose.yml" +source "${SCRIPT_DIR}/../mailcow.conf" +CMPS_PRJ=$(echo $COMPOSE_PROJECT_NAME | tr -cd "[A-Za-z-_]") +SQLIMAGE=$(grep -iEo '(mysql|mariadb)\:.+' "${COMPOSE_FILE}") + +echo +echo -e "\033[1mFound compose project name ${CMPS_PRJ} for ${MAILCOW_HOSTNAME}\033[0m" +echo -e "\033[1mFound SQL ${SQLIMAGE}\033[0m" +echo + +# Make sure destination exists, rsync can fail under some circumstances +echo -e "\033[1mPreparing remote...\033[0m" +ssh -o StrictHostKeyChecking=no \ + -i "${REMOTE_SSH_KEY}" \ + ${REMOTE_SSH_HOST} \ + -p ${REMOTE_SSH_PORT} \ + mkdir -p "${SCRIPT_DIR}/../" + +# Syncing the mailcow base directory +echo -e "\033[1mSynchronizing mailcow base directory...\033[0m" +rsync --delete -aH -e "ssh -o StrictHostKeyChecking=no \ + -i \"${REMOTE_SSH_KEY}\" \ + -p ${REMOTE_SSH_PORT}" \ + "${SCRIPT_DIR}/../" root@${REMOTE_SSH_HOST}:"${SCRIPT_DIR}/../" + +# Trigger a Redis save for a consistent Redis copy +echo -ne "\033[1mRunning redis-cli save... \033[0m" +docker exec $(docker ps -qf name=redis-mailcow) redis-cli save + +# Syncing volumes related to compose project +# Same here: make sure destination exists +for vol in $(docker volume ls -qf name="${CMPS_PRJ}"); do + + mountpoint="$(docker inspect $vol | grep Mountpoint | cut -d '"' -f4)" + + echo -e "\033[1mCreating remote mountpoint ${mountpoint} for ${vol}...\033[0m" + + ssh -o StrictHostKeyChecking=no \ + -i "${REMOTE_SSH_KEY}" \ + ${REMOTE_SSH_HOST} \ + -p ${REMOTE_SSH_PORT} \ + mkdir -p "${mountpoint}" + + if [[ "${vol}" =~ "mysql-vol-1" ]]; then + + # Make sure a previous backup does not exist + rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/" + + echo -e "\033[1mCreating consistent backup for MariaDB volume...\033[0m" + if ! docker run --rm \ + --network $(docker network ls -qf name=${CMPS_PRJ}_) \ + -v $(docker volume ls -qf name=${CMPS_PRJ}_mysql-vol-1):/var/lib/mysql/:ro \ + --entrypoint= \ + -v "${SCRIPT_DIR}/../_tmp_mariabackup":/backup \ + ${SQLIMAGE} mariabackup --host mysql --user root --password ${DBROOT} --backup --target-dir=/backup 2>/dev/null ; then + echo -e "\e[31mFATAL: mariabackup failed, exiting\e[0m" + rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/" + exit 1 + fi + + if ! docker run --rm \ + --network $(docker network ls -qf name=${CMPS_PRJ}_) \ + --entrypoint= \ + -v "${SCRIPT_DIR}/../_tmp_mariabackup":/backup \ + ${SQLIMAGE} mariabackup --prepare --target-dir=/backup 2> /dev/null ; then + echo -e "\e[31mFATAL: mariabackup failed, exiting\e[0m" + rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/" + exit 1 + fi + + chown -R 999:999 "${SCRIPT_DIR}/../_tmp_mariabackup" + + echo -e "\033[1mSynchronizing MariaDB backup...\033[0m" + rsync --delete --info=progress2 -aH -e "ssh -o StrictHostKeyChecking=no \ + -i \"${REMOTE_SSH_KEY}\" \ + -p ${REMOTE_SSH_PORT}" \ + "${SCRIPT_DIR}/../_tmp_mariabackup/" root@${REMOTE_SSH_HOST}:"${mountpoint}" + + # Cleanup + rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/" + + else + + echo -e "\033[1mSynchronizing ${vol} from local ${mountpoint}...\033[0m" + rsync --delete --info=progress2 -aH -e "ssh -o StrictHostKeyChecking=no \ + -i \"${REMOTE_SSH_KEY}\" \ + -p ${REMOTE_SSH_PORT}" \ + "${mountpoint}/" root@${REMOTE_SSH_HOST}:"${mountpoint}" + + fi + + echo -e "\e[32mCompleted\e[0m" + +done + +# Restart Dockerd on destination +echo -ne "\033[1mRestarting Docker daemon on remote to detect new volumes... \033[0m" +ssh -o StrictHostKeyChecking=no \ + -i "${REMOTE_SSH_KEY}" \ + ${REMOTE_SSH_HOST} \ + -p ${REMOTE_SSH_PORT} \ + systemctl restart docker +echo "OK" + +echo -e "\033[1mPulling images on remote...\033[0m" +ssh -o StrictHostKeyChecking=no \ + -i "${REMOTE_SSH_KEY}" \ + ${REMOTE_SSH_HOST} \ + -p ${REMOTE_SSH_PORT} \ + docker-compose -f "${SCRIPT_DIR}/../docker-compose.yml" pull --no-parallel + +echo -e "\033[1mForcing garbage cleanup on remote...\033[0m" +ssh -o StrictHostKeyChecking=no \ + -i "${REMOTE_SSH_KEY}" \ + ${REMOTE_SSH_HOST} \ + -p ${REMOTE_SSH_PORT} \ + ${SCRIPT_DIR}/../update.sh -f --gc + +echo -e "\e[32mDone\e[0m"