Release 3.4.0 - See CHANGELOG.md

Add --rsyncable for gzip compression

CHANGELOG.md

@@ -1,3 +1,14 @@
+## 3.4.0 2022-09-12 <dave at tiredofit dot ca>
+
+   ### Added
+      - Add GZ_RSYNCABLE environment variable for better rsync compatibility (Credit teun95@github)
+      - Add Pre Backup Script Support
+      - Add MongoDB Atlas Support
+
+   ### Changed
+      - Fix Default Port for Influx 2 DB Hosts
+
+
 ## 3.3.12 2022-08-15 <dave at tiredofit dot ca>
 
    ### Changed

README.md

@@ -59,6 +59,8 @@ Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis servers.
   - [Manual Backups](#manual-backups)
   - [Restoring Databases](#restoring-databases)
   - [Custom Scripts](#custom-scripts)
+    - [Pre Backup](#pre-backup)
+    - [Post backup](#post-backup)
 - [Support](#support)
   - [Usage](#usage)
   - [Bugfixes](#bugfixes)
@@ -105,8 +107,9 @@ Images are built primarily for `amd64` architecture, and may also include builds
 
 The following directories are used for configuration and can be mapped for persistent storage.
 | Directory                    | Description                                                                         |
-| ------------------------ | ---------------------------------------------------------------------------------- |
+| ---------------------------- | ----------------------------------------------------------------------------------- |
 | `/backup`                    | Backups                                                                             |
+| `/assets/custom-scripts/pre` | *Optional* Put custom scripts in this directory to execute before backup operations |
 | `/assets/custom-scripts`     | *Optional* Put custom scripts in this directory to execute after backup operations  |
 
 ### Environment Variables
@@ -130,12 +133,13 @@ Be sure to view the following repositories to understand all the customizable op
 | `MANUAL_RUN_FOREVER` | `TRUE` or `FALSE` if you wish to try to make the container exit after the backup                                                 | `TRUE`          |
 | `TEMP_LOCATION`      | Perform Backups and Compression in this temporary directory                                                                      | `/tmp/backups/` |
 | `DEBUG_MODE`         | If set to `true`, print copious shell script messages to the container log. Otherwise only basic messages are printed.           | `FALSE`         |
+| `PRE_SCRIPT`         | Fill this variable in with a command to execute post the script backing up                                                       |                 |
 | `POST_SCRIPT`        | Fill this variable in with a command to execute post the script backing up                                                       |                 |
-| `SPLIT_DB`           | For each backup, create a new archive. `TRUE` or `FALSE` (MySQL and Postgresql Only) | `TRUE`
+| `SPLIT_DB`           | For each backup, create a new archive. `TRUE` or `FALSE` (MySQL and Postgresql Only)                                             | `TRUE`          |
 
 ### Database Specific Options
 | Parameter         | Description                                                                                                                                 | Default   |
-| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ------- |
+| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | --------- |
 | `DB_AUTH`         | (Mongo Only - Optional) Authentication Database                                                                                             |           |
 | `DB_TYPE`         | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` `sqlite3`                                                      |           |
 | `DB_HOST`         | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3`                                               |           |
@@ -144,7 +148,9 @@ Be sure to view the following repositories to understand all the customizable op
 | `DB_USER`         | username for the database(s) - Can use `root` for MySQL                                                                                     |           |
 | `DB_PASS`         | (optional if DB doesn't require it) password for the database                                                                               |           |
 | `DB_PORT`         | (optional) Set port to connect to DB_HOST. Defaults are provided                                                                            | varies    |
-| `INFLUX_VERSION`  | What Version of Influx are you backing up from `1`.x or `2.x` series - AMD64 and ARM64 only for `2`                                         |         |
+| `INFLUX_VERSION`  | What Version of Influx are you backing up from `1`.x or `2` series - AMD64 and ARM64 only for `2`                                           |           |
+| `MONGO_HOST_TYPE` | Connect to regular `mongodb` or `atlas`                                                                                                     | `mongodb` |
+|                   | You can also skip this and override the uri prefix with `MONGO_URI_PREFIX=mongodb+srv://` or whatever you would like                        |           |
 
 #### For Influx DB2:
 Your Organization will be mapped to `DB_USER` and your root token will need to be mapped to `DB_PASS`. You may use `DB_NAME=ALL` to backup the entire set of databases. For `DB_HOST` use syntax of `http(s)://db-name`
@@ -166,6 +172,7 @@ Your Organization will be mapped to `DB_USER` and your root token will need to b
 | `COMPRESSION_LEVEL`            | Numberical value of what level of compression to use, most allow `1` to `9` except for `ZSTD` which allows for `1` to `19` - | `3`            |
 | `ENABLE_PARALLEL_COMPRESSION`  | Use multiple cores when compressing backups `TRUE` or `FALSE`                                                                | `TRUE`         |
 | `PARALLEL_COMPRESSION_THREADS` | Maximum amount of threads to use when compressing - Integer value e.g. `8`                                                   | `autodetected` |
+| `GZ_RSYNCABLE`                 | Use `--rsyncable` (gzip only) for faster rsync transfers and incremental backup deduplication. e.g. `TRUE`                   | `FALSE`        |
 | `ENABLE_CHECKSUM`              | Generate either a MD5 or SHA1 in Directory, `TRUE` or `FALSE`                                                                | `TRUE`         |
 | `CHECKSUM`                     | Either `MD5` or `SHA1`                                                                                                       | `MD5`          |
 | `EXTRA_OPTS`                   | If you need to pass extra arguments to the backup command, add them here e.g. `--extra-command`                              |                |
@@ -230,7 +237,33 @@ If you only enter some of the arguments you will be prompted to fill them in.
 
 ### Custom Scripts
 
-If you want to execute a custom script at the end of backup, you can drop bash scripts with the extension of `.sh` in this directory. See the following example to utilize:
+#### Pre Backup
+If you want to execute a custom script before a backup starts, you can drop bash scripts with the extension of `.sh` in `/assets/custom/pre`. See the following example to utilize:
+
+````bash
+$ cat pre-script.sh
+##!/bin/bash
+
+# #### Example Pre Script
+# #### $2=DB_TYPE (Type of Backup)
+# #### $3=DB_HOST (Backup Host)
+# #### #4=DB_NAME (Name of Database backed up
+# #### $5=BACKUP START TIME (Seconds since Epoch)
+# #### $8=BACKUP FILENAME (Filename)
+
+echo "${2} Backup Starting on ${3} for ${4} on ${5} ending ${6} for a duration of ${7} seconds. Filename: ${8} Size: ${9} bytes MD5: ${10}"
+````
+
+      ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME STARTEPOCH BACKUP_FILENAME
+      ${f} "${exit_code}" "${dbtype}" "${dbhost}" "${dbname}" "${backup_start_time}" "${target}"
+
+
+Outputs the following on the console:
+
+`mysql Backup Starting on example-db for example on 1647370800. Filename: mysql_example_example-db_202200315-000000.sql.bz2
+
+#### Post backup
+If you want to execute a custom script at the end of backup, you can drop bash scripts with the extension of `.sh` in `/assets/custom`. See the following example to utilize:
 
 ````bash
 $ cat post-script.sh

install/assets/functions/10-db-backup

@@ -10,7 +10,10 @@ bootstrap_variables() {
         ;;
         influx* )
             dbtype=influx
-            DB_PORT=${DB_PORT:-8088}
+            case "${INFLUX_VERSION}" in
+                1) DB_PORT=${DB_PORT:-8088} ;;
+                2) DB_PORT=${DB_PORT:-8086} ;;
+            esac
             file_env 'DB_USER'
             file_env 'DB_PASS'
             sanity_var INFLUX_VERSION "What InfluxDB version you are backing up from '1' or '2'"
@@ -78,9 +81,10 @@ bootstrap_variables() {
 }
 
 backup_couch() {
-    pre_dbbackup
+    prepare_dbbackup
     target=couch_${DB_NAME}_${DB_HOST#*//}_${now}.txt
     compression
+    pre_dbbackup ${DB_NAME}
     print_notice "Dumping CouchDB database: '${DB_NAME}' ${compression_string}"
     curl -sSL -X GET ${DB_HOST}:${DB_PORT}/${DB_NAME}/_all_docs?include_docs=true ${compress_cmd} | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
     exit_code=$?
@@ -101,10 +105,11 @@ backup_influx() {
     case "${INFLUX_VERSION,,}" in
         1 )
             for db in ${db_names}; do
-                pre_dbbackup
+                prepare_dbbackup
                 if [ "${db}" != "justbackupeverything" ] ; then bucket="-db ${db}" ; else db=all ; fi
                 target=influx_${db}_${DB_HOST#*//}_${now}
                 compression
+                pre_dbbackup $db
                 print_notice "Dumping Influx database: '${db}'"
                 influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS} "${TEMP_LOCATION}"/"${target_dir}"
                 exit_code=$?
@@ -119,10 +124,11 @@ backup_influx() {
         ;;
         2 )
             for db in ${db_names}; do
-                pre_dbbackup
+                prepare_dbbackup
                 if [ "${db}" != "justbackupeverything" ] ; then bucket="--bucket $db" ; else db=all ; fi
                 target=influx2_${db}_${DB_HOST#*//}_${now}
                 compression
+                pre_dbbackup $db
                 print_notice "Dumping Influx2 database: '${db}'"
                 influx backup --org ${DB_USER} ${bucket} --host ${DB_HOST}:${DB_PORT} --token ${DB_PASS} ${EXTRA_OPTS} --compression none "${TEMP_LOCATION}"/"${target_dir}"
                 exit_code=$?
@@ -138,7 +144,7 @@ backup_influx() {
 }
 
 backup_mongo() {
-    pre_dbbackup
+    prepare_dbbackup
     if [ "${ENABLE_COMPRESSION,,}" = "none" ] || [ "${ENABLE_COMPRESSION,,}" = "false" ] ; then
         target=${dbtype}_${DB_NAME,,}_${DB_HOST,,}_${now}.archive
     else
@@ -146,8 +152,14 @@ backup_mongo() {
         mongo_compression="--gzip"
         compression_string="and compressing with gzip"
     fi
+    if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then
+        MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb+srv://"}
+    else
+        MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb://"}
+    fi
+    pre_dbbackup "${DB_NAME}"
     print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}"
-    mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --host ${DB_HOST} --port ${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS}
+    mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri="${MONGO_URI_PREFIX}${DB_HOST}:${DB_PORT}" ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS}
     exit_code=$?
     check_exit_code $target
     generate_checksum
@@ -156,9 +168,10 @@ backup_mongo() {
 }
 
 backup_mssql() {
-    pre_dbbackup
+    prepare_dbbackup
     target=mssql_${DB_NAME,,}_${DB_HOST,,}_${now}.bak
     compression
+    pre_dbbackup "${DB_NAME}"
     print_notice "Dumping MSSQL database: '${DB_NAME}'"
     /opt/mssql-tools/bin/sqlcmd -E -C -S ${DB_HOST}\,${DB_PORT} -U ${DB_USER} -P ${DB_PASS} –Q "BACKUP DATABASE \[${DB_NAME}\] TO DISK = N'${TEMP_LOCATION}/${target}' WITH NOFORMAT, NOINIT, NAME = '${DB_NAME}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10"
     exit_code=$?
@@ -194,9 +207,10 @@ backup_mysql() {
 
     if var_true "${SPLIT_DB}" ; then
         for db in ${db_names} ; do
-                pre_dbbackup
+                prepare_dbbackup
                 target=mysql_${db}_${DB_HOST,,}_${now}.sql
                 compression
+                pre_dbbackup $db
                 print_notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}"
                 mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${EXTRA_OPTS} --databases $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
                 exit_code=$?
@@ -207,9 +221,10 @@ backup_mysql() {
         done
     else
         print_debug "Not splitting database dumps into their own files"
-        pre_dbbackup
+        prepare_dbbackup
         target=mysql_all_${DB_HOST,,}_${now}.sql
         compression
+        pre_dbbackup all
         print_notice "Dumping all MySQL / MariaDB databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}"
         mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${EXTRA_OPTS} --databases $(echo ${db_names} | xargs) | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
         exit_code=$?
@@ -241,9 +256,10 @@ backup_pgsql() {
 
     if var_true "${SPLIT_DB}" ; then
         for db in ${db_names} ; do
-                pre_dbbackup
+                prepare_dbbackup
                 target=pgsql_${db}_${DB_HOST,,}_${now}.sql
                 compression
+                pre_dbbackup $db
                 print_notice "Dumping PostgresSQL database: '${db}' ${compression_string}"
                 pg_dump -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} $db ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target}
                 exit_code=$?
@@ -254,9 +270,10 @@ backup_pgsql() {
         done
     else
         print_debug "Not splitting database dumps into their own files"
-        pre_dbbackup
+        prepare_dbbackup
         target=pgsql_all_${DB_HOST,,}_${now}.sql
         compression
+        pre_dbbackup all
         print_notice "Dumping all PostgreSQL databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}"
         tmp_db_names=$(psql -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
         for r_db_name in $(echo $db_names  | xargs); do
@@ -276,7 +293,7 @@ backup_pgsql() {
 }
 
 backup_redis() {
-    pre_dbbackup
+    prepare_dbbackup
     print_notice "Dumping Redis - Flushing Redis Cache First"
     target=redis_all_${DB_HOST,,}_${now}.rdb
     echo bgsave | silent redis-cli -h ${DB_HOST} -p ${DB_PORT} ${REDIS_PASS_STR} --rdb ${TEMP_LOCATION}/${target} ${EXTRA_OPTS}
@@ -296,6 +313,7 @@ backup_redis() {
     done
     target_original=${target}
     compression
+    pre_dbbackup all
     $compress_cmd "${TEMP_LOCATION}/${target_original}"
     check_exit_code $target
     generate_checksum
@@ -304,11 +322,12 @@ backup_redis() {
 }
 
 backup_sqlite3() {
-    pre_dbbackup
+    prepare_dbbackup
     db=$(basename "${DB_HOST}")
     db="${db%.*}"
     target=sqlite3_${db}_${now}.sqlite3
     compression
+    pre_dbbackup $db
     print_notice "Dumping sqlite3 database: '${DB_HOST}' ${compression_string}"
     sqlite3 "${DB_HOST}" ".backup '${TEMP_LOCATION}/backup.sqlite3'"
     exit_code=$?
@@ -468,7 +487,10 @@ compression() {
 
    case "${COMPRESSION,,}" in
         gz* )
-            compress_cmd="pigz -q -${COMPRESSION_LEVEL} -p ${PARALLEL_COMPRESSION_THREADS} "
+            if var_true "${GZ_RSYNCABLE}" ; then
+                gz_rsyncable=--rsyncable
+            fi
+            compress_cmd="pigz -q -${COMPRESSION_LEVEL} -p ${PARALLEL_COMPRESSION_THREADS} ${gz_rsyncable}"
             compression_type="gzip"
             extension=".gz"
             dir_compress_cmd=${compress_cmd}
@@ -620,7 +642,7 @@ move_dbbackup() {
     rm -rf "${TEMP_LOCATION}"/*
 }
 
-pre_dbbackup() {
+prepare_dbbackup() {
     dbbackup_start_time=$(date +"%s")
     now=$(date +"%Y%m%d-%H%M%S")
     now_time=$(date +"%H:%M:%S")
@@ -628,6 +650,39 @@ pre_dbbackup() {
     target=${dbtype}_${DB_NAME,,}_${DB_HOST,,}_${now}.sql
 }
 
+pre_dbbackup() {
+    ### Pre Script Support
+    if [ -n "${PRE_SCRIPT}" ] ; then
+        if var_true "${PRE_SCRIPT_SKIP_X_VERIFY}" ; then
+            eval "${PRE_SCRIPT}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${target}"
+        else
+            if [ -x "${PRE_SCRIPT}" ] ; then
+                print_notice "Found PRE_SCRIPT environment variable. Executing '${PRE_SCRIPT}"
+                eval "${PRE_SCRIPT}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${target}"
+            else
+                print_error "Can't execute PRE_SCRIPT environment variable '${PRE_SCRIPT}' as its filesystem bit is not executible!"
+            fi
+        fi
+    fi
+
+    ### Pre Backup Custom Script Support
+    if [ -d "/assets/custom-scripts/pre/" ] ; then
+        for f in $(find /assets/custom-scripts/pre/ -name \*.sh -type f); do
+            if var_true "${PRE_SCRIPT_SKIP_X_VERIFY}" ; then
+                ${f} "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${target}"
+            else
+                if [ -x "${f}" ] ; then
+                    print_notice "Executing pre backup custom script : '${f}'"
+                    ## script DB_TYPE DB_HOST DB_NAME STARTEPOCH BACKUP_FILENAME
+                    ${f} "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${target}"
+                else
+                    print_error "Can't run pre backup custom script: '${f}' as its filesystem bit is not executible!"
+                fi
+            fi
+        done
+    fi
+}
+
 post_dbbackup() {
     dbbackup_finish_time=$(date +"%s")
     dbbackup_total_time=$(echo $((dbbackup_finish_time-dbbackup_start_time)))