diff --git a/README.md b/README.md index 7666878..46024a6 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,8 @@ Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis servers. - [Manual Backups](#manual-backups) - [Restoring Databases](#restoring-databases) - [Custom Scripts](#custom-scripts) + - [Pre Backup](#pre-backup) + - [Post backup](#post-backup) - [Support](#support) - [Usage](#usage) - [Bugfixes](#bugfixes) @@ -104,10 +106,11 @@ Images are built primarily for `amd64` architecture, and may also include builds ### Persistent Storage The following directories are used for configuration and can be mapped for persistent storage. -| Directory | Description | -| ------------------------ | ---------------------------------------------------------------------------------- | -| `/backup` | Backups | -| `/assets/custom-scripts` | *Optional* Put custom scripts in this directory to execute after backup operations | +| Directory | Description | +| ---------------------------- | ----------------------------------------------------------------------------------- | +| `/backup` | Backups | +| `/assets/custom-scripts/pre` | *Optional* Put custom scripts in this directory to execute before backup operations | +| `/assets/custom-scripts` | *Optional* Put custom scripts in this directory to execute after backup operations | ### Environment Variables @@ -130,21 +133,24 @@ Be sure to view the following repositories to understand all the customizable op | `MANUAL_RUN_FOREVER` | `TRUE` or `FALSE` if you wish to try to make the container exit after the backup | `TRUE` | | `TEMP_LOCATION` | Perform Backups and Compression in this temporary directory | `/tmp/backups/` | | `DEBUG_MODE` | If set to `true`, print copious shell script messages to the container log. Otherwise only basic messages are printed. | `FALSE` | +| `PRE_SCRIPT` | Fill this variable in with a command to execute post the script backing up | | | `POST_SCRIPT` | Fill this variable in with a command to execute post the script backing up | | -| `SPLIT_DB` | For each backup, create a new archive. `TRUE` or `FALSE` (MySQL and Postgresql Only) | `TRUE` +| `SPLIT_DB` | For each backup, create a new archive. `TRUE` or `FALSE` (MySQL and Postgresql Only) | `TRUE` | ### Database Specific Options -| Parameter | Description | Default | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -| `DB_AUTH` | (Mongo Only - Optional) Authentication Database | | -| `DB_TYPE` | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` `sqlite3` | | -| `DB_HOST` | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3` | | -| `DB_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. Backup multiple by seperating with commas eg `db1,db2` | | -| `DB_NAME_EXCLUDE` | If using `ALL` - use this as to exclude databases seperated via commas from being backed up | | -| `DB_USER` | username for the database(s) - Can use `root` for MySQL | | -| `DB_PASS` | (optional if DB doesn't require it) password for the database | | -| `DB_PORT` | (optional) Set port to connect to DB_HOST. Defaults are provided | varies | -| `INFLUX_VERSION` | What Version of Influx are you backing up from `1`.x or `2` series - AMD64 and ARM64 only for `2` | | +| Parameter | Description | Default | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | --------- | +| `DB_AUTH` | (Mongo Only - Optional) Authentication Database | | +| `DB_TYPE` | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` `sqlite3` | | +| `DB_HOST` | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3` | | +| `DB_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. Backup multiple by seperating with commas eg `db1,db2` | | +| `DB_NAME_EXCLUDE` | If using `ALL` - use this as to exclude databases seperated via commas from being backed up | | +| `DB_USER` | username for the database(s) - Can use `root` for MySQL | | +| `DB_PASS` | (optional if DB doesn't require it) password for the database | | +| `DB_PORT` | (optional) Set port to connect to DB_HOST. Defaults are provided | varies | +| `INFLUX_VERSION` | What Version of Influx are you backing up from `1`.x or `2` series - AMD64 and ARM64 only for `2` | | +| `MONGO_HOST_TYPE` | Connect to regular `mongodb` or `atlas` | `mongodb` | +| | You can also skip this and override the uri prefix with `MONGO_URI_PREFIX=mongodb+srv://` or whatever you would like | | #### For Influx DB2: Your Organization will be mapped to `DB_USER` and your root token will need to be mapped to `DB_PASS`. You may use `DB_NAME=ALL` to backup the entire set of databases. For `DB_HOST` use syntax of `http(s)://db-name` @@ -231,7 +237,33 @@ If you only enter some of the arguments you will be prompted to fill them in. ### Custom Scripts -If you want to execute a custom script at the end of backup, you can drop bash scripts with the extension of `.sh` in this directory. See the following example to utilize: +#### Pre Backup +If you want to execute a custom script before a backup starts, you can drop bash scripts with the extension of `.sh` in `/assets/custom/pre`. See the following example to utilize: + +````bash +$ cat pre-script.sh +##!/bin/bash + +# #### Example Pre Script +# #### $2=DB_TYPE (Type of Backup) +# #### $3=DB_HOST (Backup Host) +# #### #4=DB_NAME (Name of Database backed up +# #### $5=BACKUP START TIME (Seconds since Epoch) +# #### $8=BACKUP FILENAME (Filename) + +echo "${2} Backup Starting on ${3} for ${4} on ${5} ending ${6} for a duration of ${7} seconds. Filename: ${8} Size: ${9} bytes MD5: ${10}" +```` + + ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME STARTEPOCH BACKUP_FILENAME + ${f} "${exit_code}" "${dbtype}" "${dbhost}" "${dbname}" "${backup_start_time}" "${target}" + + +Outputs the following on the console: + +`mysql Backup Starting on example-db for example on 1647370800. Filename: mysql_example_example-db_202200315-000000.sql.bz2 + +#### Post backup +If you want to execute a custom script at the end of backup, you can drop bash scripts with the extension of `.sh` in `/assets/custom`. See the following example to utilize: ````bash $ cat post-script.sh diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index e0fee09..2047c28 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -10,7 +10,10 @@ bootstrap_variables() { ;; influx* ) dbtype=influx - DB_PORT=${DB_PORT:-8088} + case "${INFLUX_VERSION}" in + 1) DB_PORT=${DB_PORT:-8088} ;; + 2) DB_PORT=${DB_PORT:-8086} ;; + esac file_env 'DB_USER' file_env 'DB_PASS' sanity_var INFLUX_VERSION "What InfluxDB version you are backing up from '1' or '2'" @@ -78,9 +81,10 @@ bootstrap_variables() { } backup_couch() { - pre_dbbackup + prepare_dbbackup target=couch_${DB_NAME}_${DB_HOST#*//}_${now}.txt compression + pre_dbbackup ${DB_NAME} print_notice "Dumping CouchDB database: '${DB_NAME}' ${compression_string}" curl -sSL -X GET ${DB_HOST}:${DB_PORT}/${DB_NAME}/_all_docs?include_docs=true ${compress_cmd} | $compress_cmd > "${TEMP_LOCATION}"/"${target}" exit_code=$? @@ -101,10 +105,11 @@ backup_influx() { case "${INFLUX_VERSION,,}" in 1 ) for db in ${db_names}; do - pre_dbbackup + prepare_dbbackup if [ "${db}" != "justbackupeverything" ] ; then bucket="-db ${db}" ; else db=all ; fi target=influx_${db}_${DB_HOST#*//}_${now} compression + pre_dbbackup $db print_notice "Dumping Influx database: '${db}'" influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS} "${TEMP_LOCATION}"/"${target_dir}" exit_code=$? @@ -119,10 +124,11 @@ backup_influx() { ;; 2 ) for db in ${db_names}; do - pre_dbbackup + prepare_dbbackup if [ "${db}" != "justbackupeverything" ] ; then bucket="--bucket $db" ; else db=all ; fi target=influx2_${db}_${DB_HOST#*//}_${now} compression + pre_dbbackup $db print_notice "Dumping Influx2 database: '${db}'" influx backup --org ${DB_USER} ${bucket} --host ${DB_HOST}:${DB_PORT} --token ${DB_PASS} ${EXTRA_OPTS} --compression none "${TEMP_LOCATION}"/"${target_dir}" exit_code=$? @@ -138,7 +144,7 @@ backup_influx() { } backup_mongo() { - pre_dbbackup + prepare_dbbackup if [ "${ENABLE_COMPRESSION,,}" = "none" ] || [ "${ENABLE_COMPRESSION,,}" = "false" ] ; then target=${dbtype}_${DB_NAME,,}_${DB_HOST,,}_${now}.archive else @@ -146,8 +152,14 @@ backup_mongo() { mongo_compression="--gzip" compression_string="and compressing with gzip" fi + if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then + MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb+srv://"} + else + MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb://"} + fi + pre_dbbackup "${DB_NAME}" print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" - mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --host ${DB_HOST} --port ${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} + mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri="${MONGO_URI_PREFIX}${DB_HOST}:${DB_PORT}" ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} exit_code=$? check_exit_code $target generate_checksum @@ -156,9 +168,10 @@ backup_mongo() { } backup_mssql() { - pre_dbbackup + prepare_dbbackup target=mssql_${DB_NAME,,}_${DB_HOST,,}_${now}.bak compression + pre_dbbackup "${DB_NAME}" print_notice "Dumping MSSQL database: '${DB_NAME}'" /opt/mssql-tools/bin/sqlcmd -E -C -S ${DB_HOST}\,${DB_PORT} -U ${DB_USER} -P ${DB_PASS} –Q "BACKUP DATABASE \[${DB_NAME}\] TO DISK = N'${TEMP_LOCATION}/${target}' WITH NOFORMAT, NOINIT, NAME = '${DB_NAME}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10" exit_code=$? @@ -194,9 +207,10 @@ backup_mysql() { if var_true "${SPLIT_DB}" ; then for db in ${db_names} ; do - pre_dbbackup + prepare_dbbackup target=mysql_${db}_${DB_HOST,,}_${now}.sql compression + pre_dbbackup $db print_notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}" mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${EXTRA_OPTS} --databases $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}" exit_code=$? @@ -207,9 +221,10 @@ backup_mysql() { done else print_debug "Not splitting database dumps into their own files" - pre_dbbackup + prepare_dbbackup target=mysql_all_${DB_HOST,,}_${now}.sql compression + pre_dbbackup all print_notice "Dumping all MySQL / MariaDB databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}" mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${EXTRA_OPTS} --databases $(echo ${db_names} | xargs) | $compress_cmd > "${TEMP_LOCATION}"/"${target}" exit_code=$? @@ -241,9 +256,10 @@ backup_pgsql() { if var_true "${SPLIT_DB}" ; then for db in ${db_names} ; do - pre_dbbackup + prepare_dbbackup target=pgsql_${db}_${DB_HOST,,}_${now}.sql compression + pre_dbbackup $db print_notice "Dumping PostgresSQL database: '${db}' ${compression_string}" pg_dump -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} $db ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} exit_code=$? @@ -254,9 +270,10 @@ backup_pgsql() { done else print_debug "Not splitting database dumps into their own files" - pre_dbbackup + prepare_dbbackup target=pgsql_all_${DB_HOST,,}_${now}.sql compression + pre_dbbackup all print_notice "Dumping all PostgreSQL databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}" tmp_db_names=$(psql -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' ) for r_db_name in $(echo $db_names | xargs); do @@ -276,7 +293,7 @@ backup_pgsql() { } backup_redis() { - pre_dbbackup + prepare_dbbackup print_notice "Dumping Redis - Flushing Redis Cache First" target=redis_all_${DB_HOST,,}_${now}.rdb echo bgsave | silent redis-cli -h ${DB_HOST} -p ${DB_PORT} ${REDIS_PASS_STR} --rdb ${TEMP_LOCATION}/${target} ${EXTRA_OPTS} @@ -296,6 +313,7 @@ backup_redis() { done target_original=${target} compression + pre_dbbackup all $compress_cmd "${TEMP_LOCATION}/${target_original}" check_exit_code $target generate_checksum @@ -304,11 +322,12 @@ backup_redis() { } backup_sqlite3() { - pre_dbbackup + prepare_dbbackup db=$(basename "${DB_HOST}") db="${db%.*}" target=sqlite3_${db}_${now}.sqlite3 compression + pre_dbbackup $db print_notice "Dumping sqlite3 database: '${DB_HOST}' ${compression_string}" sqlite3 "${DB_HOST}" ".backup '${TEMP_LOCATION}/backup.sqlite3'" exit_code=$? @@ -623,7 +642,7 @@ move_dbbackup() { rm -rf "${TEMP_LOCATION}"/* } -pre_dbbackup() { +prepare_dbbackup() { dbbackup_start_time=$(date +"%s") now=$(date +"%Y%m%d-%H%M%S") now_time=$(date +"%H:%M:%S") @@ -631,6 +650,39 @@ pre_dbbackup() { target=${dbtype}_${DB_NAME,,}_${DB_HOST,,}_${now}.sql } +pre_dbbackup() { + ### Pre Script Support + if [ -n "${PRE_SCRIPT}" ] ; then + if var_true "${PRE_SCRIPT_SKIP_X_VERIFY}" ; then + eval "${PRE_SCRIPT}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${target}" + else + if [ -x "${PRE_SCRIPT}" ] ; then + print_notice "Found PRE_SCRIPT environment variable. Executing '${PRE_SCRIPT}" + eval "${PRE_SCRIPT}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${target}" + else + print_error "Can't execute PRE_SCRIPT environment variable '${PRE_SCRIPT}' as its filesystem bit is not executible!" + fi + fi + fi + + ### Pre Backup Custom Script Support + if [ -d "/assets/custom-scripts/pre/" ] ; then + for f in $(find /assets/custom-scripts/pre/ -name \*.sh -type f); do + if var_true "${PRE_SCRIPT_SKIP_X_VERIFY}" ; then + ${f} "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${target}" + else + if [ -x "${f}" ] ; then + print_notice "Executing pre backup custom script : '${f}'" + ## script DB_TYPE DB_HOST DB_NAME STARTEPOCH BACKUP_FILENAME + ${f} "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${target}" + else + print_error "Can't run pre backup custom script: '${f}' as its filesystem bit is not executible!" + fi + fi + done + fi +} + post_dbbackup() { dbbackup_finish_time=$(date +"%s") dbbackup_total_time=$(echo $((dbbackup_finish_time-dbbackup_start_time)))