From a377f570f16716b553d1493c44d7a78bd70e5226 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Tue, 20 Sep 2022 09:38:50 -0700 Subject: [PATCH 01/16] Introduce MONGO_CUSTOM_URI support --- CHANGELOG.md | 5 +++ README.md | 44 +++++++++++++------------ install/assets/functions/10-db-backup | 47 ++++++++++++++++++++------- 3 files changed, 63 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75cc7ee..fbc91e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## develop 2022-09-20 + + ### Added + - MONGO_CUSTOM_URI support to ignore the seperate environment variables - Parses the URI and populates DB_HOST and DB_NAME for filenaming. Can be overridden + ## 3.4.2 2022-09-19 ### Changed diff --git a/README.md b/README.md index 147c756..8203b03 100644 --- a/README.md +++ b/README.md @@ -139,32 +139,34 @@ Be sure to view the following repositories to understand all the customizable op | `SPLIT_DB` | For each backup, create a new archive. `TRUE` or `FALSE` (MySQL and Postgresql Only) | `TRUE` | ### Database Specific Options -| Parameter | Description | Default | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | --------- | -| `DB_AUTH` | (Mongo Only - Optional) Authentication Database | | -| `DB_TYPE` | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` `sqlite3` | | -| `DB_HOST` | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3` | | -| `DB_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. Backup multiple by seperating with commas eg `db1,db2` | | -| `DB_NAME_EXCLUDE` | If using `ALL` - use this as to exclude databases seperated via commas from being backed up | | -| `DB_USER` | username for the database(s) - Can use `root` for MySQL | | -| `DB_PASS` | (optional if DB doesn't require it) password for the database | | -| `DB_PORT` | (optional) Set port to connect to DB_HOST. Defaults are provided | varies | -| `INFLUX_VERSION` | What Version of Influx are you backing up from `1`.x or `2` series - AMD64 and ARM64 only for `2` | | -| `MONGO_HOST_TYPE` | Connect to regular `mongodb` or `atlas` | `mongodb` | -| | You can also skip this and override the uri prefix with `MONGO_URI_PREFIX=mongodb+srv://` or whatever you would like | | +| Parameter | Description | Default | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------- | +| `DB_AUTH` | (Mongo Only - Optional) Authentication Database | | +| `DB_TYPE` | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` `sqlite3` | | +| `DB_HOST` | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3` | | +| `DB_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. Backup multiple by seperating with commas eg `db1,db2` | | +| `DB_NAME_EXCLUDE` | If using `ALL` - use this as to exclude databases seperated via commas from being backed up | | +| `DB_USER` | username for the database(s) - Can use `root` for MySQL | | +| `DB_PASS` | (optional if DB doesn't require it) password for the database | | +| `DB_PORT` | (optional) Set port to connect to DB_HOST. Defaults are provided | varies | +| `INFLUX_VERSION` | What Version of Influx are you backing up from `1`.x or `2` series - AMD64 and ARM64 only for `2` | | +| `MONGO_HOST_TYPE` | Connect to regular `mongodb` or `atlas` | `mongodb` | +| | You can also skip this and override the uri prefix with `MONGO_URI_PREFIX=mongodb+srv://` or whatever you would like | | +| `MONGO_CUSTOM_URI` | If you wish to override the MongoDB Connection string enter it here e.g. `mongodb+srv://username:password@cluster.id.mongodb.net` | | +| | This environment variable will be parsed and populate the `DB_NAME` and `DB_HOST` variables to properly build your backup filenames. You can overrde them by making your own entries | #### For Influx DB2: Your Organization will be mapped to `DB_USER` and your root token will need to be mapped to `DB_PASS`. You may use `DB_NAME=ALL` to backup the entire set of databases. For `DB_HOST` use syntax of `http(s)://db-name` ### Scheduling Options -| Parameter | Description | Default | -|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------| -| `DB_DUMP_FREQ` | How often to do a dump, in minutes after the first backup. Defaults to 1440 minutes, or once per day. | `1440` | -| `DB_DUMP_BEGIN` | What time to do the first dump. Defaults to immediate. Must be in one of two formats | | -| | Absolute HHMM, e.g. `2330` or `0415` | | -| | Relative +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half | | -| `DB_DUMP_TARGET` | Directory where the database dumps are kept. | `/backup` | -| `DB_CLEANUP_TIME` | Value in minutes to delete old backups (only fired when dump freqency fires). 1440 would delete anything above 1 day old. You don't need to set this variable if you want to hold onto everything. | `FALSE` | +| Parameter | Description | Default | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | +| `DB_DUMP_FREQ` | How often to do a dump, in minutes after the first backup. Defaults to 1440 minutes, or once per day. | `1440` | +| `DB_DUMP_BEGIN` | What time to do the first dump. Defaults to immediate. Must be in one of two formats | | +| | Absolute HHMM, e.g. `2330` or `0415` | | +| | Relative +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half | | +| `DB_DUMP_TARGET` | Directory where the database dumps are kept. | `/backup` | +| `DB_CLEANUP_TIME` | Value in minutes to delete old backups (only fired when dump freqency fires). 1440 would delete anything above 1 day old. You don't need to set this variable if you want to hold onto everything. | `FALSE` | - You may need to wrap your `DB_DUMP_BEGIN` value in quotes for it to properly parse. There have been reports of backups that start with a `0` get converted into a different format which will not allow the timer to start at the correct time. diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index daef113..5370c0c 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -21,8 +21,10 @@ bootstrap_variables() { mongo* ) dbtype=mongo DB_PORT=${DB_PORT:-27017} - [[ ( -n "${DB_USER}" ) || ( -n "${DB_USER_FILE}" ) ]] && file_env 'DB_USER' - [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS' + if [ -z "${MONGO_CUSTOM_URI}" ] ; then + [[ ( -n "${DB_USER}" ) || ( -n "${DB_USER_FILE}" ) ]] && file_env 'DB_USER' + [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS' + fi ;; "mysql" | "mariadb" ) dbtype=mysql @@ -60,13 +62,27 @@ bootstrap_variables() { file_env 'S3_KEY_SECRET' fi - ### Set the Database Authentication Details + ### Set the Database Authentication Details or parse any custom URIs case "$dbtype" in "mongo" ) - [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}" - [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}" - [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}" - [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}" + if [ -z "${MONGO_CUSTOM_URI}" ] ; then + [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}" + [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}" + [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}" + [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}" + else + mongo_uri_proto=$(echo ${MONGO_CUSTOM_URI} | grep :// | sed -e's,^\(.*://\).*,\1,g') + mongo_uri_scratch="${MONGO_CUSTOM_URI/${mongo_uri_proto}/}" + mongo_uri_username_password=$(echo ${mongo_uri_scratch} | grep @ | rev | cut -d@ -f2- | rev) + if [ -n "${mongo_uri_username_password}" ]; then mongo_uri_scratch=$(echo ${mongo_uri_scratch} | rev | cut -d@ -f1 | rev) ; fi + mongo_uri_port=$(echo ${mongo_uri_scratch} | grep : | rev | cut -d: -f2- | rev) + if [ -n "${mongo_uri_port}" ]; then mongo_uri_port=$(echo ${mongo_uri_scratch} | rev | cut -d: -f1 | cut -d/ -f2 | rev) ; fi + mongo_uri_hostname=$(echo ${mongo_uri_scratch} | cut -d/ -f1 | cut -d: -f1 ) + mongo_uri_database=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f1 ) + mongo_uri_options=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f2 ) + DB_NAME=${DB_NAME:-"${mongo_uri_database}"} + DB_HOST=${DB_HOST:-"${mongo_uri_hostname}"} + fi ;; "mysql" ) [[ ( -n "${DB_PASS}" ) ]] && export MYSQL_PWD=${DB_PASS} @@ -152,14 +168,21 @@ backup_mongo() { mongo_compression="--gzip" compression_string="and compressing with gzip" fi - if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then - MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb+srv://"} + + if [ -n "${MONGO_CUSTOM_URI}" ] ; then + mongo_generated_uri=${MONGO_CUSTOM_URI} else - MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb://"} + if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then + MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb+srv://"} + mongo_generated_uri=${MONGO_URI_PREFIX}${DB_HOST} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} + else + MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb://"} + mongo_generated_uri=${MONGO_URI_PREFIX}${DB_HOST}:${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} + fi fi pre_dbbackup "${DB_NAME}" print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" - mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri="${MONGO_URI_PREFIX}${DB_HOST}:${DB_PORT}" ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} + mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri=${mongo_generated_uri} exit_code=$? check_exit_code $target generate_checksum @@ -375,7 +398,7 @@ check_availability() { esac ;; "mongo" ) - if [ "${MONGO_HOST_TYPE,,}" != "atlas" ] ; then + if [ "${MONGO_HOST_TYPE,,}" != "atlas" ] || [ -n "${MONGO_CUSTOM_URI}" ]; then counter=0 while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do sleep 5 From 73c4003dc408bdb938d6e82372e0b68f174238cd Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Tue, 20 Sep 2022 12:06:13 -0700 Subject: [PATCH 02/16] Fix environment variable check for Mongo --- install/assets/functions/10-db-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 5370c0c..669c4e7 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -398,7 +398,7 @@ check_availability() { esac ;; "mongo" ) - if [ "${MONGO_HOST_TYPE,,}" != "atlas" ] || [ -n "${MONGO_CUSTOM_URI}" ]; then + if [ "${MONGO_HOST_TYPE,,}" != "atlas" ] || [ -z "${MONGO_CUSTOM_URI}" ]; then counter=0 while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do sleep 5 From 73ad356ebf69d47c37a8ea0d314a6a8f2a66a4b9 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Tue, 20 Sep 2022 12:08:53 -0700 Subject: [PATCH 03/16] Force lowercase for filenames and hostnames for filename generation --- install/assets/functions/10-db-backup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 669c4e7..10728ec 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -80,8 +80,8 @@ bootstrap_variables() { mongo_uri_hostname=$(echo ${mongo_uri_scratch} | cut -d/ -f1 | cut -d: -f1 ) mongo_uri_database=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f1 ) mongo_uri_options=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f2 ) - DB_NAME=${DB_NAME:-"${mongo_uri_database}"} - DB_HOST=${DB_HOST:-"${mongo_uri_hostname}"} + DB_NAME=${DB_NAME:-"${mongo_uri_database,,}"} + DB_HOST=${DB_HOST:-"${mongo_uri_hostname,,}"} fi ;; "mysql" ) From 8706d3a91c2cd54d15584fc432359a78399b0126 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Tue, 20 Sep 2022 12:49:43 -0700 Subject: [PATCH 04/16] Skip availability checks by default --- install/assets/defaults/10-db-backup | 1 + install/assets/functions/10-db-backup | 176 +++++++++++++------------- 2 files changed, 90 insertions(+), 87 deletions(-) diff --git a/install/assets/defaults/10-db-backup b/install/assets/defaults/10-db-backup index 6403e6a..6dc6717 100755 --- a/install/assets/defaults/10-db-backup +++ b/install/assets/defaults/10-db-backup @@ -20,5 +20,6 @@ S3_PROTOCOL=${S3_PROTOCOL:-"https"} SCRIPT_LOCATION_PRE=${SCRIPT_LOCATION_PRE:-"/assets/scripts/pre/"} SCRIPT_LOCATION_POST=${SCRIPT_LOCATION_POST:-"/assets/scripts/post/"} SIZE_VALUE=${SIZE_VALUE:-"bytes"} +SKIP_AVAILABILITY_CHECK=${SKIP_AVAILABILITY_CHECK:-"TRUE"} SPLIT_DB=${SPLIT_DB:-"TRUE"} TEMP_LOCATION=${TEMP_LOCATION:-"/tmp/backups"} diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 10728ec..4e02c3d 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -363,101 +363,103 @@ backup_sqlite3() { check_availability() { ### Set the Database Type - case "$dbtype" in - "couch" ) - counter=0 - code_received=0 - while [ "${code_received}" != "200" ]; do - code_received=$(curl -XGET -sSL -o /dev/null -L -w ''%{http_code}'' ${DB_HOST}:${DB_PORT}) - if [ "${code_received}" = "200" ] ; then break ; fi - sleep 5 - (( counter+=5 )) - print_warn "CouchDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" - done - ;; - "influx" ) - counter=0 - case "${INFLUX_VERSION,,}" in - 1 ) - while ! (nc -z ${DB_HOST#*//} ${DB_PORT}) ; do + if var_false "${SKIP_AVAILABILITY_CHECK}" ; then + case "$dbtype" in + "couch" ) + counter=0 + code_received=0 + while [ "${code_received}" != "200" ]; do + code_received=$(curl -XGET -sSL -o /dev/null -L -w ''%{http_code}'' ${DB_HOST}:${DB_PORT}) + if [ "${code_received}" = "200" ] ; then break ; fi + sleep 5 + (( counter+=5 )) + print_warn "CouchDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + done + ;; + "influx" ) + counter=0 + case "${INFLUX_VERSION,,}" in + 1 ) + while ! (nc -z ${DB_HOST#*//} ${DB_PORT}) ; do + sleep 5 + (( counter+=5 )) + print_warn "InfluxDB Host '${DB_HOST#*//}' is not accessible, retrying.. ($counter seconds so far)" + done + ;; + 2 ) + code_received=0 + while [ "${code_received}" != "200" ]; do + code_received=$(curl -XGET -sSL -o /dev/null -w ''%{http_code}'' ${DB_HOST}:${DB_PORT}/health) + if [ "${code_received}" = "200" ] ; then break ; fi + sleep 5 + (( counter+=5 )) + print_warn "InfluxDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + done + ;; + esac + ;; + "mongo" ) + if [ "${MONGO_HOST_TYPE,,}" != "atlas" ] || [ -z "${MONGO_CUSTOM_URI}" ]; then + counter=0 + while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do sleep 5 (( counter+=5 )) - print_warn "InfluxDB Host '${DB_HOST#*//}' is not accessible, retrying.. ($counter seconds so far)" + print_warn "Mongo Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" done - ;; - 2 ) - code_received=0 - while [ "${code_received}" != "200" ]; do - code_received=$(curl -XGET -sSL -o /dev/null -w ''%{http_code}'' ${DB_HOST}:${DB_PORT}/health) - if [ "${code_received}" = "200" ] ; then break ; fi - sleep 5 - (( counter+=5 )) - print_warn "InfluxDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" - done - ;; - esac - ;; - "mongo" ) - if [ "${MONGO_HOST_TYPE,,}" != "atlas" ] || [ -z "${MONGO_CUSTOM_URI}" ]; then + fi + ;; + "mysql" ) + counter=0 + export MYSQL_PWD=${DB_PASS} + while ! (mysqladmin -u"${DB_USER}" -P"${DB_PORT}" -h"${DB_HOST}" status > /dev/null 2>&1) ; do + sleep 5 + (( counter+=5 )) + print_warn "MySQL/MariaDB Server '${DB_HOST}' is not accessible, retrying.. (${counter} seconds so far)" + done + ;; + "mssql" ) counter=0 while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do sleep 5 (( counter+=5 )) - print_warn "Mongo Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + print_warn "MSSQL Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" done - fi - ;; - "mysql" ) - counter=0 - export MYSQL_PWD=${DB_PASS} - while ! (mysqladmin -u"${DB_USER}" -P"${DB_PORT}" -h"${DB_HOST}" status > /dev/null 2>&1) ; do - sleep 5 - (( counter+=5 )) - print_warn "MySQL/MariaDB Server '${DB_HOST}' is not accessible, retrying.. (${counter} seconds so far)" - done - ;; - "mssql" ) - counter=0 - while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do - sleep 5 - (( counter+=5 )) - print_warn "MSSQL Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" - done - ;; - "pgsql" ) - counter=0 - export PGPASSWORD=${DB_PASS} - until pg_isready --dbname=${DB_NAME} --host=${DB_HOST} --port=${DB_PORT} --username=${DB_USER} -q - do - sleep 5 - (( counter+=5 )) - print_warn "Postgres Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" - done - ;; - "redis" ) - counter=0 - while ! (nc -z "${DB_HOST}" "${DB_PORT}") ; do - sleep 5 - (( counter+=5 )) - print_warn "Redis Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" - done - ;; - "sqlite3" ) - if [[ ! -e "${DB_HOST}" ]]; then - print_error "File '${DB_HOST}' does not exist." - exit_code=2 - exit $exit_code - elif [[ ! -f "${DB_HOST}" ]]; then - print_error "File '${DB_HOST}' is not a file." - exit_code=2 - exit $exit_code - elif [[ ! -r "${DB_HOST}" ]]; then - print_error "File '${DB_HOST}' is not readable." - exit_code=2 - exit $exit_code - fi - ;; - esac + ;; + "pgsql" ) + counter=0 + export PGPASSWORD=${DB_PASS} + until pg_isready --dbname=${DB_NAME} --host=${DB_HOST} --port=${DB_PORT} --username=${DB_USER} -q + do + sleep 5 + (( counter+=5 )) + print_warn "Postgres Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + done + ;; + "redis" ) + counter=0 + while ! (nc -z "${DB_HOST}" "${DB_PORT}") ; do + sleep 5 + (( counter+=5 )) + print_warn "Redis Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + done + ;; + "sqlite3" ) + if [[ ! -e "${DB_HOST}" ]]; then + print_error "File '${DB_HOST}' does not exist." + exit_code=2 + exit $exit_code + elif [[ ! -f "${DB_HOST}" ]]; then + print_error "File '${DB_HOST}' is not a file." + exit_code=2 + exit $exit_code + elif [[ ! -r "${DB_HOST}" ]]; then + print_error "File '${DB_HOST}' is not readable." + exit_code=2 + exit $exit_code + fi + ;; + esac + fi } check_exit_code() { From c4dbf53ced10e2eaf565024a5dbe3f3dd1a627c7 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Tue, 20 Sep 2022 13:51:54 -0700 Subject: [PATCH 05/16] Final Availability check --- install/assets/defaults/10-db-backup | 2 +- install/assets/functions/10-db-backup | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/install/assets/defaults/10-db-backup b/install/assets/defaults/10-db-backup index 6dc6717..c29be2d 100755 --- a/install/assets/defaults/10-db-backup +++ b/install/assets/defaults/10-db-backup @@ -20,6 +20,6 @@ S3_PROTOCOL=${S3_PROTOCOL:-"https"} SCRIPT_LOCATION_PRE=${SCRIPT_LOCATION_PRE:-"/assets/scripts/pre/"} SCRIPT_LOCATION_POST=${SCRIPT_LOCATION_POST:-"/assets/scripts/post/"} SIZE_VALUE=${SIZE_VALUE:-"bytes"} -SKIP_AVAILABILITY_CHECK=${SKIP_AVAILABILITY_CHECK:-"TRUE"} +SKIP_AVAILABILITY_CHECK=${SKIP_AVAILABILITY_CHECK:-"FALSE"} SPLIT_DB=${SPLIT_DB:-"TRUE"} TEMP_LOCATION=${TEMP_LOCATION:-"/tmp/backups"} diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 4e02c3d..9c62114 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -399,7 +399,9 @@ check_availability() { esac ;; "mongo" ) - if [ "${MONGO_HOST_TYPE,,}" != "atlas" ] || [ -z "${MONGO_CUSTOM_URI}" ]; then + if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] || [ -z "${MONGO_CUSTOM_URI}" ]; then + print_debug "Skipping Connectivity Check" + else counter=0 while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do sleep 5 From 9d8cfd69cb760a8b19735c63cbe93b50c5287491 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Tue, 20 Sep 2022 17:45:05 -0700 Subject: [PATCH 06/16] Fix a couple variable issues and add silencing --- install/assets/functions/10-db-backup | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 9c62114..363b9be 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -182,12 +182,12 @@ backup_mongo() { fi pre_dbbackup "${DB_NAME}" print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" - mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri=${mongo_generated_uri} + silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri=${mongo_generated_uri} exit_code=$? check_exit_code $target generate_checksum move_dbbackup - post_dbbackup + post_dbbackup "${DB_NAME}" } backup_mssql() { @@ -196,7 +196,7 @@ backup_mssql() { compression pre_dbbackup "${DB_NAME}" print_notice "Dumping MSSQL database: '${DB_NAME}'" - /opt/mssql-tools/bin/sqlcmd -E -C -S ${DB_HOST}\,${DB_PORT} -U ${DB_USER} -P ${DB_PASS} –Q "BACKUP DATABASE \[${DB_NAME}\] TO DISK = N'${TEMP_LOCATION}/${target}' WITH NOFORMAT, NOINIT, NAME = '${DB_NAME}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10" + silent /opt/mssql-tools/bin/sqlcmd -E -C -S ${DB_HOST}\,${DB_PORT} -U ${DB_USER} -P ${DB_PASS} –Q "BACKUP DATABASE \[${DB_NAME}\] TO DISK = N'${TEMP_LOCATION}/${target}' WITH NOFORMAT, NOINIT, NAME = '${DB_NAME}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10" exit_code=$? check_exit_code $target generate_checksum @@ -235,7 +235,7 @@ backup_mysql() { compression pre_dbbackup $db print_notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}" - mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${EXTRA_OPTS} --databases $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}" + silent mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${EXTRA_OPTS} --databases $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}" exit_code=$? check_exit_code $target generate_checksum @@ -249,7 +249,7 @@ backup_mysql() { compression pre_dbbackup all print_notice "Dumping all MySQL / MariaDB databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}" - mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${EXTRA_OPTS} --databases $(echo ${db_names} | xargs) | $compress_cmd > "${TEMP_LOCATION}"/"${target}" + silent mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${EXTRA_OPTS} --databases $(echo ${db_names} | xargs) | $compress_cmd > "${TEMP_LOCATION}"/"${target}" exit_code=$? check_exit_code $target generate_checksum @@ -284,7 +284,7 @@ backup_pgsql() { compression pre_dbbackup $db print_notice "Dumping PostgresSQL database: '${db}' ${compression_string}" - pg_dump -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} $db ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} + silent pg_dump -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} $db ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} exit_code=$? check_exit_code $target generate_checksum @@ -306,7 +306,7 @@ backup_pgsql() { for x_db_name in ${tmp_db_names} ; do pgexclude_arg=$(echo ${pgexclude_arg} --exclude-database=${x_db_name}) done - pg_dumpall -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} ${pgexclude_arg} ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} + silent pg_dumpall -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} ${pgexclude_arg} ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} exit_code=$? check_exit_code $target generate_checksum @@ -352,7 +352,7 @@ backup_sqlite3() { compression pre_dbbackup $db print_notice "Dumping sqlite3 database: '${DB_HOST}' ${compression_string}" - sqlite3 "${DB_HOST}" ".backup '${TEMP_LOCATION}/backup.sqlite3'" + silent sqlite3 "${DB_HOST}" ".backup '${TEMP_LOCATION}/backup.sqlite3'" exit_code=$? check_exit_code $target cat "${TEMP_LOCATION}"/backup.sqlite3 | ${dir_compress_cmd} > "${TEMP_LOCATION}/${target}" From 789aa96113060d11a2c46ebb9144b0e7cc25d6cf Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Wed, 21 Sep 2022 09:28:01 -0700 Subject: [PATCH 07/16] Alternate way of solving Host check --- install/assets/defaults/10-db-backup | 2 ++ install/assets/functions/10-db-backup | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/install/assets/defaults/10-db-backup b/install/assets/defaults/10-db-backup index c29be2d..37de24e 100755 --- a/install/assets/defaults/10-db-backup +++ b/install/assets/defaults/10-db-backup @@ -23,3 +23,5 @@ SIZE_VALUE=${SIZE_VALUE:-"bytes"} SKIP_AVAILABILITY_CHECK=${SKIP_AVAILABILITY_CHECK:-"FALSE"} SPLIT_DB=${SPLIT_DB:-"TRUE"} TEMP_LOCATION=${TEMP_LOCATION:-"/tmp/backups"} + +if [ -n "${MONGO_CUSTOM_URI}" ]; then MONGO_DB_TYPE="atlas" ; fi diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 363b9be..add097d 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -399,7 +399,7 @@ check_availability() { esac ;; "mongo" ) - if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] || [ -z "${MONGO_CUSTOM_URI}" ]; then + if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then print_debug "Skipping Connectivity Check" else counter=0 From 1efb2d43a8e0b3cbb9d08a6ae987c8917698c829 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Wed, 21 Sep 2022 10:51:25 -0700 Subject: [PATCH 08/16] Set proper environment variable --- install/assets/defaults/10-db-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/assets/defaults/10-db-backup b/install/assets/defaults/10-db-backup index 37de24e..c158a51 100755 --- a/install/assets/defaults/10-db-backup +++ b/install/assets/defaults/10-db-backup @@ -24,4 +24,4 @@ SKIP_AVAILABILITY_CHECK=${SKIP_AVAILABILITY_CHECK:-"FALSE"} SPLIT_DB=${SPLIT_DB:-"TRUE"} TEMP_LOCATION=${TEMP_LOCATION:-"/tmp/backups"} -if [ -n "${MONGO_CUSTOM_URI}" ]; then MONGO_DB_TYPE="atlas" ; fi +if [ -n "${MONGO_CUSTOM_URI}" ]; then MONGO_HOST_TYPE="atlas" ; fi From 2d017e26c5275ab748b3dc4ff881c4abb9f5a95f Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Sun, 25 Sep 2022 10:00:33 -0700 Subject: [PATCH 09/16] Cleanup README --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 8203b03..a0a6271 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,6 @@ Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis servers. - [License](#license) > **NOTE**: If you are using this with a docker-compose file along with a seperate SQL container, take care not to set the variables to backup immediately, more so have it delay execution for a minute, otherwise you will get a failed first backup. -### Persistent Storage ## Prerequisites and Assumptions * You must have a working connection to one of the supported DB Servers and appropriate credentials From 6de0cc7c037c949c1a20ae2a5e9f7cc2c23196d8 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Thu, 29 Sep 2022 16:06:06 -0700 Subject: [PATCH 10/16] Wrap mongo_generated_uri in quotes --- install/assets/functions/10-db-backup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index add097d..88d2f0f 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -170,14 +170,14 @@ backup_mongo() { fi if [ -n "${MONGO_CUSTOM_URI}" ] ; then - mongo_generated_uri=${MONGO_CUSTOM_URI} + mongo_generated_uri="${MONGO_CUSTOM_URI}" else if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb+srv://"} - mongo_generated_uri=${MONGO_URI_PREFIX}${DB_HOST} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} + mongo_generated_uri="${MONGO_URI_PREFIX}${DB_HOST} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS}" else MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb://"} - mongo_generated_uri=${MONGO_URI_PREFIX}${DB_HOST}:${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} + mongo_generated_uri="${MONGO_URI_PREFIX}${DB_HOST}:${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS}" fi fi pre_dbbackup "${DB_NAME}" From 4a8f85ddf5f0b8794da6c98df589fa6cea0563b3 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Thu, 29 Sep 2022 19:35:01 -0700 Subject: [PATCH 11/16] Add EXTRA_OPTS to Mongo string --- install/assets/functions/10-db-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 88d2f0f..9c899c5 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -182,7 +182,7 @@ backup_mongo() { fi pre_dbbackup "${DB_NAME}" print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" - silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri=${mongo_generated_uri} + silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri=${mongo_generated_uri} ${EXTRA_OPTS} exit_code=$? check_exit_code $target generate_checksum From 8b41f5efcf105428df625788cc4ca2751fabde35 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Fri, 7 Oct 2022 08:03:24 -0700 Subject: [PATCH 12/16] Allow EXTRA_OPTS with MONGO_CUSTOM_URI --- install/assets/functions/10-db-backup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 9c899c5..f47a3bc 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -170,7 +170,7 @@ backup_mongo() { fi if [ -n "${MONGO_CUSTOM_URI}" ] ; then - mongo_generated_uri="${MONGO_CUSTOM_URI}" + mongo_generated_uri="${MONGO_CUSTOM_URI} ${EXTRA_OPTS}" else if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb+srv://"} @@ -182,7 +182,7 @@ backup_mongo() { fi pre_dbbackup "${DB_NAME}" print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" - silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri=${mongo_generated_uri} ${EXTRA_OPTS} + silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri="${mongo_generated_uri}" exit_code=$? check_exit_code $target generate_checksum From 2b441f11e1fc5c2c664aad52370aca877f981ccf Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Fri, 7 Oct 2022 08:26:54 -0700 Subject: [PATCH 13/16] Restore old functionality for Mongo backup when not using MONGO_CUSTOM_URI --- README.md | 5 +--- install/assets/defaults/10-db-backup | 1 - install/assets/functions/10-db-backup | 34 +++++++++++++++++++-------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a0a6271..4cfa835 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,6 @@ Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis servers. - [About](#about) - [Maintainer](#maintainer) - [Table of Contents](#table-of-contents) - - [Persistent Storage](#persistent-storage) - [Prerequisites and Assumptions](#prerequisites-and-assumptions) - [Installation](#installation) - [Build from Source](#build-from-source) @@ -45,7 +44,7 @@ Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis servers. - [Multi Architecture](#multi-architecture) - [Configuration](#configuration) - [Quick Start](#quick-start) - - [Persistent Storage](#persistent-storage-1) + - [Persistent Storage](#persistent-storage) - [Environment Variables](#environment-variables) - [Base Images used](#base-images-used) - [Container Options](#container-options) @@ -149,8 +148,6 @@ Be sure to view the following repositories to understand all the customizable op | `DB_PASS` | (optional if DB doesn't require it) password for the database | | | `DB_PORT` | (optional) Set port to connect to DB_HOST. Defaults are provided | varies | | `INFLUX_VERSION` | What Version of Influx are you backing up from `1`.x or `2` series - AMD64 and ARM64 only for `2` | | -| `MONGO_HOST_TYPE` | Connect to regular `mongodb` or `atlas` | `mongodb` | -| | You can also skip this and override the uri prefix with `MONGO_URI_PREFIX=mongodb+srv://` or whatever you would like | | | `MONGO_CUSTOM_URI` | If you wish to override the MongoDB Connection string enter it here e.g. `mongodb+srv://username:password@cluster.id.mongodb.net` | | | | This environment variable will be parsed and populate the `DB_NAME` and `DB_HOST` variables to properly build your backup filenames. You can overrde them by making your own entries | diff --git a/install/assets/defaults/10-db-backup b/install/assets/defaults/10-db-backup index c158a51..fd35b46 100755 --- a/install/assets/defaults/10-db-backup +++ b/install/assets/defaults/10-db-backup @@ -24,4 +24,3 @@ SKIP_AVAILABILITY_CHECK=${SKIP_AVAILABILITY_CHECK:-"FALSE"} SPLIT_DB=${SPLIT_DB:-"TRUE"} TEMP_LOCATION=${TEMP_LOCATION:-"/tmp/backups"} -if [ -n "${MONGO_CUSTOM_URI}" ]; then MONGO_HOST_TYPE="atlas" ; fi diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index f47a3bc..3c0f662 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -170,19 +170,13 @@ backup_mongo() { fi if [ -n "${MONGO_CUSTOM_URI}" ] ; then - mongo_generated_uri="${MONGO_CUSTOM_URI} ${EXTRA_OPTS}" + mongo_backup_parameter="--uri=${MONGO_CUSTOM_URI} ${EXTRA_OPTS}" else - if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then - MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb+srv://"} - mongo_generated_uri="${MONGO_URI_PREFIX}${DB_HOST} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS}" - else - MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb://"} - mongo_generated_uri="${MONGO_URI_PREFIX}${DB_HOST}:${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS}" - fi + mongo_backup_parameter="--host ${DB_HOST} --port ${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS}" fi pre_dbbackup "${DB_NAME}" print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" - silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri="${mongo_generated_uri}" + silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} "${mongo_backup_parameter}" exit_code=$? check_exit_code $target generate_checksum @@ -399,7 +393,7 @@ check_availability() { esac ;; "mongo" ) - if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then + if [ -n "${MONGO_CUSTOM_URI}" ] ; then print_debug "Skipping Connectivity Check" else counter=0 @@ -828,3 +822,23 @@ EOF fi } +urlencode() { + old_lc_collate=$LC_COLLATE + LC_COLLATE=C + + local length="${#1}" + for (( i = 0; i < length; i++ )); do + local c="${1:$i:1}" + case $c in + [a-zA-Z0-9.~_-]) printf '%s' "$c" ;; + *) printf '%%%02X' "'$c" ;; + esac + done + + LC_COLLATE=$old_lc_collate +} + +urldecode() { + local url_encoded="${1//+/ }" + printf '%b' "${url_encoded//%/\\x}" +} \ No newline at end of file From 2022158a4e5c124e2862525ced9fee3914b49c68 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Fri, 7 Oct 2022 08:33:19 -0700 Subject: [PATCH 14/16] Swap around some environment checks --- install/assets/functions/10-db-backup | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 3c0f662..4fbaf5a 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -65,12 +65,7 @@ bootstrap_variables() { ### Set the Database Authentication Details or parse any custom URIs case "$dbtype" in "mongo" ) - if [ -z "${MONGO_CUSTOM_URI}" ] ; then - [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}" - [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}" - [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}" - [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}" - else + if [ -n "${MONGO_CUSTOM_URI}" ] ; then mongo_uri_proto=$(echo ${MONGO_CUSTOM_URI} | grep :// | sed -e's,^\(.*://\).*,\1,g') mongo_uri_scratch="${MONGO_CUSTOM_URI/${mongo_uri_proto}/}" mongo_uri_username_password=$(echo ${mongo_uri_scratch} | grep @ | rev | cut -d@ -f2- | rev) @@ -82,6 +77,11 @@ bootstrap_variables() { mongo_uri_options=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f2 ) DB_NAME=${DB_NAME:-"${mongo_uri_database,,}"} DB_HOST=${DB_HOST:-"${mongo_uri_hostname,,}"} + else + [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}" + [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}" + [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}" + [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}" fi ;; "mysql" ) From 231dd63a38916982936f1dcbdf0eeeef95199e21 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Sat, 8 Oct 2022 11:57:40 -0700 Subject: [PATCH 15/16] Remove quote --- install/assets/functions/10-db-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 4fbaf5a..2f2b965 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -176,7 +176,7 @@ backup_mongo() { fi pre_dbbackup "${DB_NAME}" print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" - silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} "${mongo_backup_parameter}" + silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} ${mongo_backup_parameter} exit_code=$? check_exit_code $target generate_checksum From be490b3f4b8c9bd3e29e2ed72fa365494c824a54 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Mon, 10 Oct 2022 11:53:09 -0700 Subject: [PATCH 16/16] Remove url [en|de]code functions --- install/assets/functions/10-db-backup | 78 ++++++++------------------- install/etc/cont-init.d/10-db-backup | 1 + 2 files changed, 22 insertions(+), 57 deletions(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 2f2b965..3a30f2d 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -20,16 +20,33 @@ bootstrap_variables() { ;; mongo* ) dbtype=mongo - DB_PORT=${DB_PORT:-27017} - if [ -z "${MONGO_CUSTOM_URI}" ] ; then + if [ -n "${MONGO_CUSTOM_URI}" ] ; then + mongo_uri_proto=$(echo ${MONGO_CUSTOM_URI} | grep :// | sed -e's,^\(.*://\).*,\1,g') + mongo_uri_scratch="${MONGO_CUSTOM_URI/${mongo_uri_proto}/}" + mongo_uri_username_password=$(echo ${mongo_uri_scratch} | grep @ | rev | cut -d@ -f2- | rev) + if [ -n "${mongo_uri_username_password}" ]; then mongo_uri_scratch=$(echo ${mongo_uri_scratch} | rev | cut -d@ -f1 | rev) ; fi + mongo_uri_port=$(echo ${mongo_uri_scratch} | grep : | rev | cut -d: -f2- | rev) + if [ -n "${mongo_uri_port}" ]; then mongo_uri_port=$(echo ${mongo_uri_scratch} | rev | cut -d: -f1 | cut -d/ -f2 | rev) ; fi + mongo_uri_hostname=$(echo ${mongo_uri_scratch} | cut -d/ -f1 | cut -d: -f1 ) + mongo_uri_database=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f1 ) + mongo_uri_options=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f2 ) + DB_NAME=${DB_NAME:-"${mongo_uri_database,,}"} + DB_HOST=${DB_HOST:-"${mongo_uri_hostname,,}"} + else + DB_PORT=${DB_PORT:-27017} [[ ( -n "${DB_USER}" ) || ( -n "${DB_USER_FILE}" ) ]] && file_env 'DB_USER' [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS' + [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}" + [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}" + [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}" + [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}" fi ;; "mysql" | "mariadb" ) dbtype=mysql DB_PORT=${DB_PORT:-3306} [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS' + [[ ( -n "${DB_PASS}" ) ]] && export MYSQL_PWD=${DB_PASS} sanity_var DB_NAME "Database Name to backup. Multiple seperated by commas" ;; "mssql" | "microsoftsql" ) @@ -45,12 +62,14 @@ bootstrap_variables() { dbtype=pgsql DB_PORT=${DB_PORT:-5432} [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS' + [[ ( -n "${DB_PASS}" ) ]] && POSTGRES_PASS_STR="PGPASSWORD=${DB_PASS}" sanity_var DB_NAME "Database Name to backup. Multiple seperated by commas" ;; "redis" ) dbtype=redis DB_PORT=${DB_PORT:-6379} [[ ( -n "${DB_PASS}" || ( -n "${DB_PASS_FILE}" ) ) ]] && file_env 'DB_PASS' + [[ ( -n "${DB_PASS}" ) ]] && REDIS_PASS_STR=" -a ${DB_PASS}" ;; sqlite* ) dbtype=sqlite3 @@ -61,39 +80,6 @@ bootstrap_variables() { file_env 'S3_KEY_ID' file_env 'S3_KEY_SECRET' fi - - ### Set the Database Authentication Details or parse any custom URIs - case "$dbtype" in - "mongo" ) - if [ -n "${MONGO_CUSTOM_URI}" ] ; then - mongo_uri_proto=$(echo ${MONGO_CUSTOM_URI} | grep :// | sed -e's,^\(.*://\).*,\1,g') - mongo_uri_scratch="${MONGO_CUSTOM_URI/${mongo_uri_proto}/}" - mongo_uri_username_password=$(echo ${mongo_uri_scratch} | grep @ | rev | cut -d@ -f2- | rev) - if [ -n "${mongo_uri_username_password}" ]; then mongo_uri_scratch=$(echo ${mongo_uri_scratch} | rev | cut -d@ -f1 | rev) ; fi - mongo_uri_port=$(echo ${mongo_uri_scratch} | grep : | rev | cut -d: -f2- | rev) - if [ -n "${mongo_uri_port}" ]; then mongo_uri_port=$(echo ${mongo_uri_scratch} | rev | cut -d: -f1 | cut -d/ -f2 | rev) ; fi - mongo_uri_hostname=$(echo ${mongo_uri_scratch} | cut -d/ -f1 | cut -d: -f1 ) - mongo_uri_database=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f1 ) - mongo_uri_options=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f2 ) - DB_NAME=${DB_NAME:-"${mongo_uri_database,,}"} - DB_HOST=${DB_HOST:-"${mongo_uri_hostname,,}"} - else - [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}" - [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}" - [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}" - [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}" - fi - ;; - "mysql" ) - [[ ( -n "${DB_PASS}" ) ]] && export MYSQL_PWD=${DB_PASS} - ;; - "postgres" ) - [[ ( -n "${DB_PASS}" ) ]] && POSTGRES_PASS_STR="PGPASSWORD=${DB_PASS}" - ;; - "redis" ) - [[ ( -n "${DB_PASS}" ) ]] && REDIS_PASS_STR=" -a ${DB_PASS}" - ;; - esac } backup_couch() { @@ -168,7 +154,6 @@ backup_mongo() { mongo_compression="--gzip" compression_string="and compressing with gzip" fi - if [ -n "${MONGO_CUSTOM_URI}" ] ; then mongo_backup_parameter="--uri=${MONGO_CUSTOM_URI} ${EXTRA_OPTS}" else @@ -821,24 +806,3 @@ EOF fi fi } - -urlencode() { - old_lc_collate=$LC_COLLATE - LC_COLLATE=C - - local length="${#1}" - for (( i = 0; i < length; i++ )); do - local c="${1:$i:1}" - case $c in - [a-zA-Z0-9.~_-]) printf '%s' "$c" ;; - *) printf '%%%02X' "'$c" ;; - esac - done - - LC_COLLATE=$old_lc_collate -} - -urldecode() { - local url_encoded="${1//+/ }" - printf '%b' "${url_encoded//%/\\x}" -} \ No newline at end of file diff --git a/install/etc/cont-init.d/10-db-backup b/install/etc/cont-init.d/10-db-backup index 4c30eed..46a75d4 100755 --- a/install/etc/cont-init.d/10-db-backup +++ b/install/etc/cont-init.d/10-db-backup @@ -6,6 +6,7 @@ prepare_service 03-monitoring PROCESS_NAME="db-backup" output_off +bootstrap_variables sanity_test setup_mode create_zabbix dbbackup