From a377f570f16716b553d1493c44d7a78bd70e5226 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Tue, 20 Sep 2022 09:38:50 -0700 Subject: [PATCH] Introduce MONGO_CUSTOM_URI support --- CHANGELOG.md | 5 +++ README.md | 44 +++++++++++++------------ install/assets/functions/10-db-backup | 47 ++++++++++++++++++++------- 3 files changed, 63 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75cc7ee..fbc91e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## develop 2022-09-20 + + ### Added + - MONGO_CUSTOM_URI support to ignore the seperate environment variables - Parses the URI and populates DB_HOST and DB_NAME for filenaming. Can be overridden + ## 3.4.2 2022-09-19 ### Changed diff --git a/README.md b/README.md index 147c756..8203b03 100644 --- a/README.md +++ b/README.md @@ -139,32 +139,34 @@ Be sure to view the following repositories to understand all the customizable op | `SPLIT_DB` | For each backup, create a new archive. `TRUE` or `FALSE` (MySQL and Postgresql Only) | `TRUE` | ### Database Specific Options -| Parameter | Description | Default | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | --------- | -| `DB_AUTH` | (Mongo Only - Optional) Authentication Database | | -| `DB_TYPE` | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` `sqlite3` | | -| `DB_HOST` | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3` | | -| `DB_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. Backup multiple by seperating with commas eg `db1,db2` | | -| `DB_NAME_EXCLUDE` | If using `ALL` - use this as to exclude databases seperated via commas from being backed up | | -| `DB_USER` | username for the database(s) - Can use `root` for MySQL | | -| `DB_PASS` | (optional if DB doesn't require it) password for the database | | -| `DB_PORT` | (optional) Set port to connect to DB_HOST. Defaults are provided | varies | -| `INFLUX_VERSION` | What Version of Influx are you backing up from `1`.x or `2` series - AMD64 and ARM64 only for `2` | | -| `MONGO_HOST_TYPE` | Connect to regular `mongodb` or `atlas` | `mongodb` | -| | You can also skip this and override the uri prefix with `MONGO_URI_PREFIX=mongodb+srv://` or whatever you would like | | +| Parameter | Description | Default | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------- | +| `DB_AUTH` | (Mongo Only - Optional) Authentication Database | | +| `DB_TYPE` | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` `sqlite3` | | +| `DB_HOST` | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3` | | +| `DB_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. Backup multiple by seperating with commas eg `db1,db2` | | +| `DB_NAME_EXCLUDE` | If using `ALL` - use this as to exclude databases seperated via commas from being backed up | | +| `DB_USER` | username for the database(s) - Can use `root` for MySQL | | +| `DB_PASS` | (optional if DB doesn't require it) password for the database | | +| `DB_PORT` | (optional) Set port to connect to DB_HOST. Defaults are provided | varies | +| `INFLUX_VERSION` | What Version of Influx are you backing up from `1`.x or `2` series - AMD64 and ARM64 only for `2` | | +| `MONGO_HOST_TYPE` | Connect to regular `mongodb` or `atlas` | `mongodb` | +| | You can also skip this and override the uri prefix with `MONGO_URI_PREFIX=mongodb+srv://` or whatever you would like | | +| `MONGO_CUSTOM_URI` | If you wish to override the MongoDB Connection string enter it here e.g. `mongodb+srv://username:password@cluster.id.mongodb.net` | | +| | This environment variable will be parsed and populate the `DB_NAME` and `DB_HOST` variables to properly build your backup filenames. You can overrde them by making your own entries | #### For Influx DB2: Your Organization will be mapped to `DB_USER` and your root token will need to be mapped to `DB_PASS`. You may use `DB_NAME=ALL` to backup the entire set of databases. For `DB_HOST` use syntax of `http(s)://db-name` ### Scheduling Options -| Parameter | Description | Default | -|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------| -| `DB_DUMP_FREQ` | How often to do a dump, in minutes after the first backup. Defaults to 1440 minutes, or once per day. | `1440` | -| `DB_DUMP_BEGIN` | What time to do the first dump. Defaults to immediate. Must be in one of two formats | | -| | Absolute HHMM, e.g. `2330` or `0415` | | -| | Relative +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half | | -| `DB_DUMP_TARGET` | Directory where the database dumps are kept. | `/backup` | -| `DB_CLEANUP_TIME` | Value in minutes to delete old backups (only fired when dump freqency fires). 1440 would delete anything above 1 day old. You don't need to set this variable if you want to hold onto everything. | `FALSE` | +| Parameter | Description | Default | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | +| `DB_DUMP_FREQ` | How often to do a dump, in minutes after the first backup. Defaults to 1440 minutes, or once per day. | `1440` | +| `DB_DUMP_BEGIN` | What time to do the first dump. Defaults to immediate. Must be in one of two formats | | +| | Absolute HHMM, e.g. `2330` or `0415` | | +| | Relative +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half | | +| `DB_DUMP_TARGET` | Directory where the database dumps are kept. | `/backup` | +| `DB_CLEANUP_TIME` | Value in minutes to delete old backups (only fired when dump freqency fires). 1440 would delete anything above 1 day old. You don't need to set this variable if you want to hold onto everything. | `FALSE` | - You may need to wrap your `DB_DUMP_BEGIN` value in quotes for it to properly parse. There have been reports of backups that start with a `0` get converted into a different format which will not allow the timer to start at the correct time. diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index daef113..5370c0c 100755 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -21,8 +21,10 @@ bootstrap_variables() { mongo* ) dbtype=mongo DB_PORT=${DB_PORT:-27017} - [[ ( -n "${DB_USER}" ) || ( -n "${DB_USER_FILE}" ) ]] && file_env 'DB_USER' - [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS' + if [ -z "${MONGO_CUSTOM_URI}" ] ; then + [[ ( -n "${DB_USER}" ) || ( -n "${DB_USER_FILE}" ) ]] && file_env 'DB_USER' + [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS' + fi ;; "mysql" | "mariadb" ) dbtype=mysql @@ -60,13 +62,27 @@ bootstrap_variables() { file_env 'S3_KEY_SECRET' fi - ### Set the Database Authentication Details + ### Set the Database Authentication Details or parse any custom URIs case "$dbtype" in "mongo" ) - [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}" - [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}" - [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}" - [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}" + if [ -z "${MONGO_CUSTOM_URI}" ] ; then + [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}" + [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}" + [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}" + [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}" + else + mongo_uri_proto=$(echo ${MONGO_CUSTOM_URI} | grep :// | sed -e's,^\(.*://\).*,\1,g') + mongo_uri_scratch="${MONGO_CUSTOM_URI/${mongo_uri_proto}/}" + mongo_uri_username_password=$(echo ${mongo_uri_scratch} | grep @ | rev | cut -d@ -f2- | rev) + if [ -n "${mongo_uri_username_password}" ]; then mongo_uri_scratch=$(echo ${mongo_uri_scratch} | rev | cut -d@ -f1 | rev) ; fi + mongo_uri_port=$(echo ${mongo_uri_scratch} | grep : | rev | cut -d: -f2- | rev) + if [ -n "${mongo_uri_port}" ]; then mongo_uri_port=$(echo ${mongo_uri_scratch} | rev | cut -d: -f1 | cut -d/ -f2 | rev) ; fi + mongo_uri_hostname=$(echo ${mongo_uri_scratch} | cut -d/ -f1 | cut -d: -f1 ) + mongo_uri_database=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f1 ) + mongo_uri_options=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f2 ) + DB_NAME=${DB_NAME:-"${mongo_uri_database}"} + DB_HOST=${DB_HOST:-"${mongo_uri_hostname}"} + fi ;; "mysql" ) [[ ( -n "${DB_PASS}" ) ]] && export MYSQL_PWD=${DB_PASS} @@ -152,14 +168,21 @@ backup_mongo() { mongo_compression="--gzip" compression_string="and compressing with gzip" fi - if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then - MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb+srv://"} + + if [ -n "${MONGO_CUSTOM_URI}" ] ; then + mongo_generated_uri=${MONGO_CUSTOM_URI} else - MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb://"} + if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then + MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb+srv://"} + mongo_generated_uri=${MONGO_URI_PREFIX}${DB_HOST} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} + else + MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb://"} + mongo_generated_uri=${MONGO_URI_PREFIX}${DB_HOST}:${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} + fi fi pre_dbbackup "${DB_NAME}" print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" - mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri="${MONGO_URI_PREFIX}${DB_HOST}:${DB_PORT}" ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} + mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri=${mongo_generated_uri} exit_code=$? check_exit_code $target generate_checksum @@ -375,7 +398,7 @@ check_availability() { esac ;; "mongo" ) - if [ "${MONGO_HOST_TYPE,,}" != "atlas" ] ; then + if [ "${MONGO_HOST_TYPE,,}" != "atlas" ] || [ -n "${MONGO_CUSTOM_URI}" ]; then counter=0 while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do sleep 5