diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..59c288a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Declare files that will always have LF line endings on checkout. +*.* text eol=lf \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 75cc7ee..468b6f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,42 @@ +## 3.5.4 2022-10-13 + + ### Changed + - Fix for Influx DB 1 backups when compression enabled + + +## 3.5.3 2022-10-12 + + ### Changed + - Remove build dependencies for blobxfer making image size smaller + - Remove silencing commands limiting Postgres backups from working without DEBUG_MODE=TRUE + + +## 3.5.2 2022-10-11 + + ### Added + - Update Influxdb client to 2.4.0 (jauderho@github) + + +## 3.5.1 2022-10-11 + + ### Changed + - Tighten up cleanup routines to not call blobxfer unless absolutely necessary + + +## 3.5.0 2022-10-10 + + ### Added + - Blobxfer / Microsoft Azure Support (credit: eoehen@github) + - Introduce MONGO_CUSTOM_URI environment variable for those not wanting to use DB_* variables + + ### Changed + - Force filenames to be in lowercase + - Fix S3 Database cleanups (credit greenatwork@github) + - Remove MONGO_DB_TYPE variable as MONGO_CUSTOM_URI overrides + - Fix MSSQL Backups (credit eoehen@github) + - Seperate examples for MySQL and MSSQL + + ## 3.4.2 2022-09-19 ### Changed diff --git a/Dockerfile b/Dockerfile index 78fbb33..610b908 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ LABEL maintainer="Dave Conroy (github.com/tiredofit)" ### Set Environment Variables -ENV INFLUX2_VERSION=2.2.1 \ +ENV INFLUX2_VERSION=2.4.0 \ MSSQL_VERSION=18.0.1.1-1 \ CONTAINER_ENABLE_MESSAGING=FALSE \ CONTAINER_ENABLE_MONITORING=TRUE \ @@ -20,6 +20,10 @@ RUN set -ex && \ bzip2-dev \ git \ libarchive-dev \ + libressl-dev \ + libffi-dev \ + python3-dev \ + py3-pip \ xz-dev \ && \ \ @@ -36,6 +40,7 @@ RUN set -ex && \ postgresql \ postgresql-client \ pv \ + py3-cryptography \ redis \ sqlite \ xz \ @@ -68,6 +73,8 @@ RUN set -ex && \ make && \ make install && \ \ + pip3 install blobxfer && \ + \ ### Cleanup apk del .db-backup-build-deps && \ rm -rf /usr/src/* && \ diff --git a/README.md b/README.md index 8d6b32d..30395bc 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ This will build a container for backing up multiple types of DB Servers Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis servers. -* dump to local filesystem or backup to S3 Compatible services +* dump to local filesystem or backup to S3 Compatible services, and Azure. * select database user and password * backup all databases, single, or multiple databases * backup all to seperate files or one singular file @@ -37,7 +37,6 @@ Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis servers. - [About](#about) - [Maintainer](#maintainer) - [Table of Contents](#table-of-contents) - - [Persistent Storage](#persistent-storage) - [Prerequisites and Assumptions](#prerequisites-and-assumptions) - [Installation](#installation) - [Build from Source](#build-from-source) @@ -45,7 +44,7 @@ Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis servers. - [Multi Architecture](#multi-architecture) - [Configuration](#configuration) - [Quick Start](#quick-start) - - [Persistent Storage](#persistent-storage-1) + - [Persistent Storage](#persistent-storage) - [Environment Variables](#environment-variables) - [Base Images used](#base-images-used) - [Container Options](#container-options) @@ -70,7 +69,6 @@ Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis servers. - [License](#license) > **NOTE**: If you are using this with a docker-compose file along with a seperate SQL container, take care not to set the variables to backup immediately, more so have it delay execution for a minute, otherwise you will get a failed first backup. -### Persistent Storage ## Prerequisites and Assumptions * You must have a working connection to one of the supported DB Servers and appropriate credentials @@ -104,6 +102,7 @@ Images are built primarily for `amd64` architecture, and may also include builds * Set various [environment variables](#environment-variables) to understand the capabilities of this image. * Map [persistent storage](#data-volumes) for access to configuration and data files for backup. * Make [networking ports](#networking) available for public access if necessary + ### Persistent Storage The following directories are used for configuration and can be mapped for persistent storage. @@ -139,32 +138,32 @@ Be sure to view the following repositories to understand all the customizable op | `SPLIT_DB` | For each backup, create a new archive. `TRUE` or `FALSE` (MySQL and Postgresql Only) | `TRUE` | ### Database Specific Options -| Parameter | Description | Default | -| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | --------- | -| `DB_AUTH` | (Mongo Only - Optional) Authentication Database | | -| `DB_TYPE` | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` `sqlite3` | | -| `DB_HOST` | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3` | | -| `DB_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. Backup multiple by seperating with commas eg `db1,db2` | | -| `DB_NAME_EXCLUDE` | If using `ALL` - use this as to exclude databases seperated via commas from being backed up | | -| `DB_USER` | username for the database(s) - Can use `root` for MySQL | | -| `DB_PASS` | (optional if DB doesn't require it) password for the database | | -| `DB_PORT` | (optional) Set port to connect to DB_HOST. Defaults are provided | varies | -| `INFLUX_VERSION` | What Version of Influx are you backing up from `1`.x or `2` series - AMD64 and ARM64 only for `2` | | -| `MONGO_HOST_TYPE` | Connect to regular `mongodb` or `atlas` | `mongodb` | -| | You can also skip this and override the uri prefix with `MONGO_URI_PREFIX=mongodb+srv://` or whatever you would like | | +| Parameter | Description | Default | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------- | +| `DB_AUTH` | (Mongo Only - Optional) Authentication Database | | +| `DB_TYPE` | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` `sqlite3` | | +| `DB_HOST` | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3` | | +| `DB_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. Backup multiple by seperating with commas eg `db1,db2` | | +| `DB_NAME_EXCLUDE` | If using `ALL` - use this as to exclude databases seperated via commas from being backed up | | +| `DB_USER` | username for the database(s) - Can use `root` for MySQL | | +| `DB_PASS` | (optional if DB doesn't require it) password for the database | | +| `DB_PORT` | (optional) Set port to connect to DB_HOST. Defaults are provided | varies | +| `INFLUX_VERSION` | What Version of Influx are you backing up from `1`.x or `2` series - AMD64 and ARM64 only for `2` | | +| `MONGO_CUSTOM_URI` | If you wish to override the MongoDB Connection string enter it here e.g. `mongodb+srv://username:password@cluster.id.mongodb.net` | | +| | This environment variable will be parsed and populate the `DB_NAME` and `DB_HOST` variables to properly build your backup filenames. You can overrde them by making your own entries | #### For Influx DB2: Your Organization will be mapped to `DB_USER` and your root token will need to be mapped to `DB_PASS`. You may use `DB_NAME=ALL` to backup the entire set of databases. For `DB_HOST` use syntax of `http(s)://db-name` ### Scheduling Options -| Parameter | Description | Default | -|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------| -| `DB_DUMP_FREQ` | How often to do a dump, in minutes after the first backup. Defaults to 1440 minutes, or once per day. | `1440` | -| `DB_DUMP_BEGIN` | What time to do the first dump. Defaults to immediate. Must be in one of two formats | | -| | Absolute HHMM, e.g. `2330` or `0415` | | -| | Relative +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half | | -| `DB_DUMP_TARGET` | Directory where the database dumps are kept. | `/backup` | -| `DB_CLEANUP_TIME` | Value in minutes to delete old backups (only fired when dump freqency fires). 1440 would delete anything above 1 day old. You don't need to set this variable if you want to hold onto everything. | `FALSE` | +| Parameter | Description | Default | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | +| `DB_DUMP_FREQ` | How often to do a dump, in minutes after the first backup. Defaults to 1440 minutes, or once per day. | `1440` | +| `DB_DUMP_BEGIN` | What time to do the first dump. Defaults to immediate. Must be in one of two formats | | +| | Absolute HHMM, e.g. `2330` or `0415` | | +| | Relative +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half | | +| `DB_DUMP_TARGET` | Directory where the database dumps are kept. | `/backup` | +| `DB_CLEANUP_TIME` | Value in minutes to delete old backups (only fired when dump freqency fires). 1440 would delete anything above 1 day old. You don't need to set this variable if you want to hold onto everything. | `FALSE` | - You may need to wrap your `DB_DUMP_BEGIN` value in quotes for it to properly parse. There have been reports of backups that start with a `0` get converted into a different format which will not allow the timer to start at the correct time. @@ -204,6 +203,22 @@ If `BACKUP_LOCATION` = `S3` then the following options are used. | _*OR*_ | | | | `S3_CERT_SKIP_VERIFY` | Skip verifying self signed certificates when connecting | `TRUE` | +#### Upload to a Azure storage account by `blobxfer` + +Support to upload backup files with [blobxfer](https://github.com/Azure/blobxfer) to the Azure fileshare storage. + + +If `BACKUP_LOCATION` = `blobxfer` then the following options are used. + +| Parameter | Description | Default | +| ------------------------------- | ------------------------------------------------------------------------ | -------------------- | +| `BLOBXFER_STORAGE_ACCOUNT` | Microsoft Azure Cloud storage account name. | | +| `BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key. | | +| `BLOBXFER_REMOTE_PATH` | Remote Azure path | `/docker-db-backup` | + +> This service uploads files from backup targed directory `DB_DUMP_TARGET`. +> If the a cleanup configuration in `DB_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically. + ## Maintenance ### Shell Access diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 0000000..6180330 --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1,5 @@ +# See http://help.github.com/ignore-files/ for more about ignoring files. + +# Example container mounted folders +**/backups/ +**/db/ \ No newline at end of file diff --git a/examples/mssql-blobxfer/docker-compose.yml b/examples/mssql-blobxfer/docker-compose.yml new file mode 100644 index 0000000..efe648f --- /dev/null +++ b/examples/mssql-blobxfer/docker-compose.yml @@ -0,0 +1,69 @@ +# +# Example for Microsoft SQL Server +# upload with blobxfer to azure storage +# + +version: '2' + +networks: + example-mssql-blobxfer-net: + name: example-mssql-blobxfer-net + +services: + example-mssql-s3-db: + hostname: example-db-host + image: mcr.microsoft.com/mssql/server:2019-latest + container_name: example-mssql-s3-db + restart: unless-stopped + ports: + - "127.0.0.1:11433:1433" + networks: + example-mssql-blobxfer-net: + volumes: + - ./tmp/backups:/tmp/backups # shared tmp backup directory + environment: + ACCEPT_EULA: Y + MSSQL_SA_PASSWORD: 5hQa0utRFBpIY3yhoIyE + MSSQL_PID: Express + + example-mssql-blobxfer-db-backup: + container_name: example-mssql-blobxfer-db-backup + # if you want to build and use image from current source + # execute in terminal --> docker build -t tiredofit/db-backup-mssql-blobxfer . + # replace --> image: tiredofit/db-backup-mssql + # image: tiredofit/db-backup + image: tiredofit/db-backup-mssql-blobxfer + links: + - example-mssql-s3-db + volumes: + - ./backups:/backup + - ./tmp/backups:/tmp/backups # shared tmp backup directory + #- ./post-script.sh:/assets/custom-scripts/post-script.sh + environment: + # - DEBUG_MODE=TRUE + - DB_TYPE=mssql + - DB_HOST=example-db-host + # - DB_PORT=1488 + # - DB_NAME=ALL # [ALL] not working on sql server. + # create database with name `test1` manually first + - DB_NAME=test1 # Create this database + - DB_USER=sa + - DB_PASS=5hQa0utRFBpIY3yhoIyE + - DB_DUMP_FREQ=1 # backup every 5 minute + # - DB_DUMP_BEGIN=0000 # backup starts immediately + - DB_CLEANUP_TIME=3 # clean backups they are older than 60 minutes + - ENABLE_CHECKSUM=TRUE + - CHECKSUM=SHA1 + - COMPRESSION=GZ + - SPLIT_DB=FALSE + - CONTAINER_ENABLE_MONITORING=FALSE + # === S3 Blobxfer === + - BACKUP_LOCATION=blobxfer + # Add here azure storage account + - BLOBXFER_STORAGE_ACCOUNT={TODO Add Storage Name} + # Add here azure storage account key + - BLOBXFER_STORAGE_ACCOUNT_KEY={TODO Add Key} + - BLOBXFER_REMOTE_PATH=docker-db-backup + restart: always + networks: + example-mssql-blobxfer-net: \ No newline at end of file diff --git a/examples/mssql/docker-compose.yml b/examples/mssql/docker-compose.yml new file mode 100644 index 0000000..4e9554c --- /dev/null +++ b/examples/mssql/docker-compose.yml @@ -0,0 +1,61 @@ +# +# Example for Microsoft SQL Server +# + +version: '2' + +networks: + example-mssql-net: + name: example-mssql-net + +services: + example-mssql-db: + hostname: example-db-host + image: mcr.microsoft.com/mssql/server:2019-latest + container_name: example-mssql-db + restart: unless-stopped + ports: + - "127.0.0.1:11433:1433" + networks: + example-mssql-net: + volumes: + - ./tmp/backups:/tmp/backups # shared tmp backup directory + environment: + ACCEPT_EULA: Y + MSSQL_SA_PASSWORD: 5hQa0utRFBpIY3yhoIyE + MSSQL_PID: Express + + example-mssql-db-backup: + container_name: example-mssql-db-backup + # if you want to build and use image from current source + # execute in terminal --> docker build -t tiredofit/db-backup-mssql . + # replace --> image: tiredofit/db-backup-mssql + # image: tiredofit/db-backup + image: tiredofit/db-backup-mssql + links: + - example-mssql-db + volumes: + - ./backups:/backup + - ./tmp/backups:/tmp/backups # shared tmp backup directory + #- ./post-script.sh:/assets/custom-scripts/post-script.sh + environment: + # - DEBUG_MODE=TRUE + - DB_TYPE=mssql + - DB_HOST=example-db-host + # - DB_PORT=1488 + # - DB_NAME=ALL # [ALL] not working on sql server. + # create database with name `test1` manually first + - DB_NAME=test1 + - DB_USER=sa + - DB_PASS=5hQa0utRFBpIY3yhoIyE + - DB_DUMP_FREQ=1 # backup every minute + # - DB_DUMP_BEGIN=0000 # backup starts immediately + - DB_CLEANUP_TIME=5 # clean backups they are older than 5 minute + - ENABLE_CHECKSUM=FALSE + - CHECKSUM=SHA1 + - COMPRESSION=GZ + - SPLIT_DB=FALSE + - CONTAINER_ENABLE_MONITORING=FALSE + restart: always + networks: + example-mssql-net: \ No newline at end of file diff --git a/examples/docker-compose.yml b/examples/mysql/docker-compose.yml old mode 100755 new mode 100644 similarity index 55% rename from examples/docker-compose.yml rename to examples/mysql/docker-compose.yml index 66a93ed..f65b18b --- a/examples/docker-compose.yml +++ b/examples/mysql/docker-compose.yml @@ -1,9 +1,16 @@ version: '2' +networks: + example-db-network: + name: example-db-network + services: example-db: + hostname: example-db-host container_name: example-db image: mariadb:latest + ports: + - 13306:3306 volumes: - ./db:/var/lib/mysql environment: @@ -12,6 +19,8 @@ services: - MYSQL_USER=example - MYSQL_PASSWORD=examplepassword restart: always + networks: + - example-db-network example-db-backup: container_name: example-db-backup @@ -22,17 +31,21 @@ services: - ./backups:/backup #- ./post-script.sh:/assets/custom-scripts/post-script.sh environment: + # - DEBUG_MODE=TRUE - DB_TYPE=mariadb - - DB_HOST=example-db + - DB_HOST=example-db-host - DB_NAME=example - DB_USER=example - - DB_PASS="examplepassword" - - DB_DUMP_FREQ=1440 - - DB_DUMP_BEGIN=0000 - - DB_CLEANUP_TIME=8640 + - DB_PASS=examplepassword + - DB_DUMP_FREQ=1 # backup every minute + # - DB_DUMP_BEGIN=0000 # backup starts immediately + - DB_CLEANUP_TIME=5 # clean backups they are older than 5 minute - CHECKSUM=SHA1 - - COMPRESSION=ZSTD + - COMPRESSION=GZ - SPLIT_DB=FALSE + - CONTAINER_ENABLE_MONITORING=FALSE restart: always + networks: + - example-db-network diff --git a/examples/post-script.sh b/examples/mysql/post-script.sh old mode 100755 new mode 100644 similarity index 100% rename from examples/post-script.sh rename to examples/mysql/post-script.sh diff --git a/install/assets/defaults/10-db-backup b/install/assets/defaults/10-db-backup old mode 100755 new mode 100644 index 6403e6a..324940d --- a/install/assets/defaults/10-db-backup +++ b/install/assets/defaults/10-db-backup @@ -1,6 +1,7 @@ #!/command/with-contenv bash BACKUP_LOCATION=${BACKUP_LOCATION:-"FILESYSTEM"} +BLOBXFER_REMOTE_PATH=${BLOBXFER_REMOTE_PATH:-"/docker-db-backup"} CHECKSUM=${CHECKSUM:-"MD5"} COMPRESSION=${COMPRESSION:-"ZSTD"} COMPRESSION_LEVEL=${COMPRESSION_LEVEL:-"3"} @@ -20,5 +21,6 @@ S3_PROTOCOL=${S3_PROTOCOL:-"https"} SCRIPT_LOCATION_PRE=${SCRIPT_LOCATION_PRE:-"/assets/scripts/pre/"} SCRIPT_LOCATION_POST=${SCRIPT_LOCATION_POST:-"/assets/scripts/post/"} SIZE_VALUE=${SIZE_VALUE:-"bytes"} +SKIP_AVAILABILITY_CHECK=${SKIP_AVAILABILITY_CHECK:-"FALSE"} SPLIT_DB=${SPLIT_DB:-"TRUE"} -TEMP_LOCATION=${TEMP_LOCATION:-"/tmp/backups"} +TEMP_LOCATION=${TEMP_LOCATION:-"/tmp/backups"} \ No newline at end of file diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup old mode 100755 new mode 100644 index 2013bbe..ee95db2 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -20,14 +20,33 @@ bootstrap_variables() { ;; mongo* ) dbtype=mongo - DB_PORT=${DB_PORT:-27017} - [[ ( -n "${DB_USER}" ) || ( -n "${DB_USER_FILE}" ) ]] && file_env 'DB_USER' - [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS' + if [ -n "${MONGO_CUSTOM_URI}" ] ; then + mongo_uri_proto=$(echo ${MONGO_CUSTOM_URI} | grep :// | sed -e's,^\(.*://\).*,\1,g') + mongo_uri_scratch="${MONGO_CUSTOM_URI/${mongo_uri_proto}/}" + mongo_uri_username_password=$(echo ${mongo_uri_scratch} | grep @ | rev | cut -d@ -f2- | rev) + if [ -n "${mongo_uri_username_password}" ]; then mongo_uri_scratch=$(echo ${mongo_uri_scratch} | rev | cut -d@ -f1 | rev) ; fi + mongo_uri_port=$(echo ${mongo_uri_scratch} | grep : | rev | cut -d: -f2- | rev) + if [ -n "${mongo_uri_port}" ]; then mongo_uri_port=$(echo ${mongo_uri_scratch} | rev | cut -d: -f1 | cut -d/ -f2 | rev) ; fi + mongo_uri_hostname=$(echo ${mongo_uri_scratch} | cut -d/ -f1 | cut -d: -f1 ) + mongo_uri_database=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f1 ) + mongo_uri_options=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f2 ) + DB_NAME=${DB_NAME:-"${mongo_uri_database,,}"} + DB_HOST=${DB_HOST:-"${mongo_uri_hostname,,}"} + else + DB_PORT=${DB_PORT:-27017} + [[ ( -n "${DB_USER}" ) || ( -n "${DB_USER_FILE}" ) ]] && file_env 'DB_USER' + [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS' + [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}" + [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}" + [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}" + [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}" + fi ;; "mysql" | "mariadb" ) dbtype=mysql DB_PORT=${DB_PORT:-3306} [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS' + [[ ( -n "${DB_PASS}" ) ]] && export MYSQL_PWD=${DB_PASS} sanity_var DB_NAME "Database Name to backup. Multiple seperated by commas" ;; "mssql" | "microsoftsql" ) @@ -43,12 +62,14 @@ bootstrap_variables() { dbtype=pgsql DB_PORT=${DB_PORT:-5432} [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS' + [[ ( -n "${DB_PASS}" ) ]] && POSTGRES_PASS_STR="PGPASSWORD=${DB_PASS}" sanity_var DB_NAME "Database Name to backup. Multiple seperated by commas" ;; "redis" ) dbtype=redis DB_PORT=${DB_PORT:-6379} [[ ( -n "${DB_PASS}" || ( -n "${DB_PASS_FILE}" ) ) ]] && file_env 'DB_PASS' + [[ ( -n "${DB_PASS}" ) ]] && REDIS_PASS_STR=" -a ${DB_PASS}" ;; sqlite* ) dbtype=sqlite3 @@ -59,25 +80,6 @@ bootstrap_variables() { file_env 'S3_KEY_ID' file_env 'S3_KEY_SECRET' fi - - ### Set the Database Authentication Details - case "$dbtype" in - "mongo" ) - [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}" - [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}" - [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}" - [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}" - ;; - "mysql" ) - [[ ( -n "${DB_PASS}" ) ]] && export MYSQL_PWD=${DB_PASS} - ;; - "postgres" ) - [[ ( -n "${DB_PASS}" ) ]] && POSTGRES_PASS_STR="PGPASSWORD=${DB_PASS}" - ;; - "redis" ) - [[ ( -n "${DB_PASS}" ) ]] && REDIS_PASS_STR=" -a ${DB_PASS}" - ;; - esac } backup_couch() { @@ -114,7 +116,7 @@ backup_influx() { influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS} "${TEMP_LOCATION}"/"${target_dir}" exit_code=$? check_exit_code $target_dir - print_notice "Creating archive file of '${target_dir}' with tar ${compresion_string}" + print_notice "Creating archive file of '${target_dir}' with tar ${compression_string}" tar cf - "${TEMP_LOCATION}"/"${target_dir}" | $dir_compress_cmd > "${TEMP_LOCATION}"/"${target_dir}".tar"${extension}" target=influx_${db}_${DB_HOST#*//}_${now}.tar${extension} generate_checksum @@ -152,19 +154,19 @@ backup_mongo() { mongo_compression="--gzip" compression_string="and compressing with gzip" fi - if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then - MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb+srv://"} + if [ -n "${MONGO_CUSTOM_URI}" ] ; then + mongo_backup_parameter="--uri=${MONGO_CUSTOM_URI} ${EXTRA_OPTS}" else - MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb://"} + mongo_backup_parameter="--host ${DB_HOST} --port ${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS}" fi pre_dbbackup "${DB_NAME}" print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" - mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri="${MONGO_URI_PREFIX}${DB_HOST}:${DB_PORT}" ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} + silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} ${mongo_backup_parameter} exit_code=$? check_exit_code $target generate_checksum move_dbbackup - post_dbbackup + post_dbbackup "${DB_NAME}" } backup_mssql() { @@ -173,7 +175,7 @@ backup_mssql() { compression pre_dbbackup "${DB_NAME}" print_notice "Dumping MSSQL database: '${DB_NAME}'" - /opt/mssql-tools/bin/sqlcmd -E -C -S ${DB_HOST}\,${DB_PORT} -U ${DB_USER} -P ${DB_PASS} –Q "BACKUP DATABASE \[${DB_NAME}\] TO DISK = N'${TEMP_LOCATION}/${target}' WITH NOFORMAT, NOINIT, NAME = '${DB_NAME}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10" + silent /opt/mssql-tools18/bin/sqlcmd -C -S ${DB_HOST}\,${DB_PORT} -U ${DB_USER} -P ${DB_PASS} -Q "BACKUP DATABASE [${DB_NAME}] TO DISK = N'${TEMP_LOCATION}/${target}' WITH NOFORMAT, NOINIT, NAME = '${DB_NAME}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10" exit_code=$? check_exit_code $target generate_checksum @@ -329,7 +331,7 @@ backup_sqlite3() { compression pre_dbbackup $db print_notice "Dumping sqlite3 database: '${DB_HOST}' ${compression_string}" - sqlite3 "${DB_HOST}" ".backup '${TEMP_LOCATION}/backup.sqlite3'" + silent sqlite3 "${DB_HOST}" ".backup '${TEMP_LOCATION}/backup.sqlite3'" exit_code=$? check_exit_code $target cat "${TEMP_LOCATION}"/backup.sqlite3 | ${dir_compress_cmd} > "${TEMP_LOCATION}/${target}" @@ -340,101 +342,105 @@ backup_sqlite3() { check_availability() { ### Set the Database Type - case "$dbtype" in - "couch" ) - counter=0 - code_received=0 - while [ "${code_received}" != "200" ]; do - code_received=$(curl -XGET -sSL -o /dev/null -L -w ''%{http_code}'' ${DB_HOST}:${DB_PORT}) - if [ "${code_received}" = "200" ] ; then break ; fi - sleep 5 - (( counter+=5 )) - print_warn "CouchDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" - done - ;; - "influx" ) - counter=0 - case "${INFLUX_VERSION,,}" in - 1 ) - while ! (nc -z ${DB_HOST#*//} ${DB_PORT}) ; do + if var_false "${SKIP_AVAILABILITY_CHECK}" ; then + case "$dbtype" in + "couch" ) + counter=0 + code_received=0 + while [ "${code_received}" != "200" ]; do + code_received=$(curl -XGET -sSL -o /dev/null -L -w ''%{http_code}'' ${DB_HOST}:${DB_PORT}) + if [ "${code_received}" = "200" ] ; then break ; fi + sleep 5 + (( counter+=5 )) + print_warn "CouchDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + done + ;; + "influx" ) + counter=0 + case "${INFLUX_VERSION,,}" in + 1 ) + while ! (nc -z ${DB_HOST#*//} ${DB_PORT}) ; do + sleep 5 + (( counter+=5 )) + print_warn "InfluxDB Host '${DB_HOST#*//}' is not accessible, retrying.. ($counter seconds so far)" + done + ;; + 2 ) + code_received=0 + while [ "${code_received}" != "200" ]; do + code_received=$(curl -XGET -sSL -o /dev/null -w ''%{http_code}'' ${DB_HOST}:${DB_PORT}/health) + if [ "${code_received}" = "200" ] ; then break ; fi + sleep 5 + (( counter+=5 )) + print_warn "InfluxDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + done + ;; + esac + ;; + "mongo" ) + if [ -n "${MONGO_CUSTOM_URI}" ] ; then + print_debug "Skipping Connectivity Check" + else + counter=0 + while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do sleep 5 (( counter+=5 )) - print_warn "InfluxDB Host '${DB_HOST#*//}' is not accessible, retrying.. ($counter seconds so far)" + print_warn "Mongo Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" done - ;; - 2 ) - code_received=0 - while [ "${code_received}" != "200" ]; do - code_received=$(curl -XGET -sSL -o /dev/null -w ''%{http_code}'' ${DB_HOST}:${DB_PORT}/health) - if [ "${code_received}" = "200" ] ; then break ; fi - sleep 5 - (( counter+=5 )) - print_warn "InfluxDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" - done - ;; - esac - ;; - "mongo" ) - if [ "${MONGO_HOST_TYPE,,}" != "atlas" ] ; then + fi + ;; + "mysql" ) + counter=0 + export MYSQL_PWD=${DB_PASS} + while ! (mysqladmin -u"${DB_USER}" -P"${DB_PORT}" -h"${DB_HOST}" status > /dev/null 2>&1) ; do + sleep 5 + (( counter+=5 )) + print_warn "MySQL/MariaDB Server '${DB_HOST}' is not accessible, retrying.. (${counter} seconds so far)" + done + ;; + "mssql" ) counter=0 while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do sleep 5 (( counter+=5 )) - print_warn "Mongo Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + print_warn "MSSQL Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" done - fi - ;; - "mysql" ) - counter=0 - export MYSQL_PWD=${DB_PASS} - while ! (mysqladmin -u"${DB_USER}" -P"${DB_PORT}" -h"${DB_HOST}" status > /dev/null 2>&1) ; do - sleep 5 - (( counter+=5 )) - print_warn "MySQL/MariaDB Server '${DB_HOST}' is not accessible, retrying.. (${counter} seconds so far)" - done - ;; - "mssql" ) - counter=0 - while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do - sleep 5 - (( counter+=5 )) - print_warn "MSSQL Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" - done - ;; - "pgsql" ) - counter=0 - export PGPASSWORD=${DB_PASS} - until pg_isready --dbname=${DB_NAME} --host=${DB_HOST} --port=${DB_PORT} --username=${DB_USER} -q - do - sleep 5 - (( counter+=5 )) - print_warn "Postgres Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" - done - ;; - "redis" ) - counter=0 - while ! (nc -z "${DB_HOST}" "${DB_PORT}") ; do - sleep 5 - (( counter+=5 )) - print_warn "Redis Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" - done - ;; - "sqlite3" ) - if [[ ! -e "${DB_HOST}" ]]; then - print_error "File '${DB_HOST}' does not exist." - exit_code=2 - exit $exit_code - elif [[ ! -f "${DB_HOST}" ]]; then - print_error "File '${DB_HOST}' is not a file." - exit_code=2 - exit $exit_code - elif [[ ! -r "${DB_HOST}" ]]; then - print_error "File '${DB_HOST}' is not readable." - exit_code=2 - exit $exit_code - fi - ;; - esac + ;; + "pgsql" ) + counter=0 + export PGPASSWORD=${DB_PASS} + until pg_isready --dbname=${DB_NAME} --host=${DB_HOST} --port=${DB_PORT} --username=${DB_USER} -q + do + sleep 5 + (( counter+=5 )) + print_warn "Postgres Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + done + ;; + "redis" ) + counter=0 + while ! (nc -z "${DB_HOST}" "${DB_PORT}") ; do + sleep 5 + (( counter+=5 )) + print_warn "Redis Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + done + ;; + "sqlite3" ) + if [[ ! -e "${DB_HOST}" ]]; then + print_error "File '${DB_HOST}' does not exist." + exit_code=2 + exit $exit_code + elif [[ ! -f "${DB_HOST}" ]]; then + print_error "File '${DB_HOST}' is not a file." + exit_code=2 + exit $exit_code + elif [[ ! -r "${DB_HOST}" ]]; then + print_error "File '${DB_HOST}' is not readable." + exit_code=2 + exit $exit_code + fi + ;; + esac + fi } check_exit_code() { @@ -455,12 +461,16 @@ cleanup_old_data() { if [ "${master_exit_code}" != 1 ]; then case "${BACKUP_LOCATION,,}" in "file" | "filesystem" ) - print_info "Cleaning up old backups" + print_info "Cleaning up old backups on filesystem" mkdir -p "${DB_DUMP_TARGET}" find "${DB_DUMP_TARGET}"/ -mmin +"${DB_CLEANUP_TIME}" -iname "*" -exec rm {} \; + if [ "${BACKUP_LOCATION,,}" = "blobxfer" ] ; then + print_info "Syncing changes via blobxfer" + silent blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET} --delete --delete-only + fi ;; "s3" | "minio" ) - print_info "Cleaning up old backups" + print_info "Cleaning up old backups on S3 storage" aws ${PARAM_AWS_ENDPOINT_URL} s3 ls s3://${S3_BUCKET}/${S3_PATH}/ ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} | grep " DIR " -v | grep " PRE " -v | while read -r s3_file; do s3_createdate=$(echo $s3_file | awk {'print $1" "$2'}) s3_createdate=$(date -d "$s3_createdate" "+%s") @@ -633,6 +643,18 @@ move_dbbackup() { silent aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_LOCATION}/*.${checksum_extension} s3://${S3_BUCKET}/${S3_PATH}/ ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} fi + rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}" + rm -rf "${TEMP_LOCATION}"/"${target}" + ;; + "blobxfer" ) + print_info "Moving backup to S3 Bucket with blobxfer" + + mkdir -p "${DB_DUMP_TARGET}" + mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/ + mv "${TEMP_LOCATION}"/"${target}" "${DB_DUMP_TARGET}"/"${target}" + + silent blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET} + rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}" rm -rf "${TEMP_LOCATION}"/"${target}" ;; diff --git a/install/etc/cont-init.d/10-db-backup b/install/etc/cont-init.d/10-db-backup index 4c30eed..46a75d4 100755 --- a/install/etc/cont-init.d/10-db-backup +++ b/install/etc/cont-init.d/10-db-backup @@ -6,6 +6,7 @@ prepare_service 03-monitoring PROCESS_NAME="db-backup" output_off +bootstrap_variables sanity_test setup_mode create_zabbix dbbackup