diff --git a/Dockerfile b/Dockerfile index 9b7278d..eb62348 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,64 +5,69 @@ FROM docker.io/tiredofit/${DISTRO}:${DISTRO_VARIANT} LABEL maintainer="Dave Conroy (github.com/tiredofit)" ### Set Environment Variables -ENV INFLUX_VERSION=1.8.0 \ - INFLUX2_VERSION=2.4.0 \ +ENV INFLUX1_CLIENT_VERSION=1.8.0 \ + INFLUX2_CLIENT_VERSION=2.7.3 \ MSODBC_VERSION=18.3.2.1-1 \ MSSQL_VERSION=18.3.1.1-1 \ - AWS_CLI_VERSION=1.25.97 \ + AWS_CLI_VERSION=1.29.78 \ CONTAINER_ENABLE_MESSAGING=FALSE \ CONTAINER_ENABLE_MONITORING=TRUE \ - CONTAINER_PROCESS_RUNAWAY_PROTECTOR=FALSE \ IMAGE_NAME="tiredofit/db-backup" \ IMAGE_REPO_URL="https://github.com/tiredofit/docker-db-backup/" ### Dependencies RUN source /assets/functions/00-container && \ set -ex && \ + addgroup -S -g 10000 dbbackup && \ + adduser -S -D -H -u 10000 -G dbbackup -g "Tired of I.T! DB Backup" dbbackup && \ + \ package update && \ package upgrade && \ package install .db-backup-build-deps \ - build-base \ - bzip2-dev \ - cargo \ - git \ - go \ - libarchive-dev \ - openssl-dev \ - libffi-dev \ - python3-dev \ - py3-pip \ - xz-dev \ - && \ + build-base \ + bzip2-dev \ + cargo \ + git \ + go \ + libarchive-dev \ + openssl-dev \ + libffi-dev \ + python3-dev \ + py3-pip \ + xz-dev \ + && \ \ package install .db-backup-run-deps \ - bzip2 \ - groff \ - libarchive \ - mariadb-client \ - mariadb-connector-c \ - mongodb-tools \ - openssl \ - pigz \ - postgresql16 \ - postgresql16-client \ - pv \ - py3-botocore \ - py3-colorama \ - py3-cryptography \ - py3-docutils \ - py3-jmespath \ - py3-rsa \ - py3-setuptools \ - py3-s3transfer \ - py3-yaml \ - python3 \ - redis \ - sqlite \ - xz \ - zip \ - zstd \ - && \ + bzip2 \ + coreutils \ + gpg \ + gpg-agent \ + groff \ + libarchive \ + mariadb-client \ + mariadb-connector-c \ + mongodb-tools \ + openssl \ + pigz \ + postgresql16 \ + postgresql16-client \ + pv \ + py3-botocore \ + py3-colorama \ + py3-cryptography \ + py3-docutils \ + py3-jmespath \ + py3-rsa \ + py3-setuptools \ + py3-s3transfer \ + py3-yaml \ + python3 \ + redis \ + sqlite \ + xz \ + zip \ + zstd \ + && \ \ apkArch="$(uname -m)"; \ case "$apkArch" in \ @@ -72,10 +77,10 @@ RUN source /assets/functions/00-container && \ esac; \ \ if [ $mssql = "true" ] ; then curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk ; curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; ls -l ; echo y | apk add --allow-untrusted msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; else echo >&2 "Detected non x86_64 or ARM64 build variant, skipping MSSQL installation" ; fi; \ - if [ $influx2 = "true" ] ; then curl -sSL https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX2_VERSION}-linux-${influx_arch}.tar.gz | tar xvfz - --strip=1 -C /usr/src/ ; chmod +x /usr/src/influx ; mv /usr/src/influx /usr/sbin/ ; else echo >&2 "Unable to build Influx 2 on this system" ; fi ; \ + if [ $influx2 = "true" ] ; then curl -sSL https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX2_CLIENT_VERSION}-linux-${influx_arch}.tar.gz | tar xvfz - --strip=1 -C /usr/src/ ; chmod +x /usr/src/influx ; mv /usr/src/influx /usr/sbin/ ; else echo >&2 "Unable to build Influx 2 on this system" ; fi ; \ clone_git_repo https://github.com/aws/aws-cli "${AWS_CLI_VERSION}" && \ python3 setup.py install --prefix=/usr && \ - clone_git_repo https://github.com/influxdata/influxdb "${INFLUX_VERSION}" && \ + clone_git_repo https://github.com/influxdata/influxdb "${INFLUX1_CLIENT_VERSION}" && \ go build -o /usr/sbin/influxd ./cmd/influxd && \ strip /usr/sbin/influxd && \ mkdir -p /usr/src/pbzip2 && \ @@ -106,5 +111,4 @@ RUN source /assets/functions/00-container && \ /tmp/* \ /usr/src/* - COPY install / diff --git a/LICENSE b/LICENSE index 5dd7a07..35e8726 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2022 Dave Conroy +Copyright (c) 2023 Dave Conroy Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 5e408f1..bec7b82 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Build Status](https://img.shields.io/github/actions/workflow/status/tiredofit/docker-db-backup/main.yml?branch=main&style=flat-square)](https://github.com/tiredofit/docker-db-backup/actions) [![Docker Stars](https://img.shields.io/docker/stars/tiredofit/db-backup.svg?style=flat-square&logo=docker)](https://hub.docker.com/r/tiredofit/db-backup/) [![Docker Pulls](https://img.shields.io/docker/pulls/tiredofit/db-backup.svg?style=flat-square&logo=docker)](https://hub.docker.com/r/tiredofit/db-backup/) -[![Become a sponsor](https://img.shields.io/badge/sponsor-tiredofit-181717.svg?logo=github&style=flat-square)](https://github.com/sponsors/tiredofit) +[![Become a sponsor](https://img.shields.io/badge/sponsor-tiredofit-181717.svg?logo=github&style=flat-square)](https://www.tiredofit.ca/sponsor) [![Paypal Donate](https://img.shields.io/badge/donate-paypal-00457c.svg?logo=paypal&style=flat-square)](https://www.paypal.me/tiredofit) --- @@ -13,21 +13,24 @@ This will build a container for backing up multiple types of DB Servers -Currently backs up CouchDB, InfluxDB, MySQL, Microsoft SQL, MongoDB, Postgres, Redis servers. +Backs up CouchDB, InfluxDB, MySQL/MariaDB, Microsoft SQL, MongoDB, Postgres, Redis servers. - dump to local filesystem or backup to S3 Compatible services, and Azure. -- select database user and password -- backup all databases, single, or multiple databases -- backup all to separate files or one singular file -- choose to have an MD5 or SHA1 sum after backup for verification -- delete old backups after specific amount of time -- choose compression type (none, gz, bz, xz, zstd) -- connect to any container running on the same system -- Script to perform restores -- Zabbix Monitoring capabilities -- select how often to run a dump -- select when to start the first dump, whether time of day or relative to container start time -- Execute script after backup for monitoring/alerting purposes +- multiple backup job support + - selectable when to start the first dump, whether time of day or relative to container start time + - selectable interval + - selectable omit scheduling during periods of time + - selectable database user and password + - selectable cleanup and archive capabilities + - selectable database name support - all databases, single, or multiple databases + - backup all to separate files or one singular file +- checksum support choose to have an MD5 or SHA1 hash generated after backup for verification +- compression support (none, gz, bz, xz, zstd) +- encryption support (passphrase and public key) +- notify upon job failure to email, matrix, mattermost, rocketchat, custom script +- zabbix metrics support +- hooks to execute pre and post backup job for customization purposes +- companion script to aid in restores ## Maintainer @@ -35,43 +38,77 @@ Currently backs up CouchDB, InfluxDB, MySQL, Microsoft SQL, MongoDB, Postgres, R ## Table of Contents -- [github.com/tiredofit/docker-db-backup](#githubcomtiredofitdocker-db-backup) - - [About](#about) - - [Maintainer](#maintainer) - - [Table of Contents](#table-of-contents) - - [Prerequisites and Assumptions](#prerequisites-and-assumptions) - - [Installation](#installation) - - [Build from Source](#build-from-source) - - [Prebuilt Images](#prebuilt-images) - - [Multi Architecture](#multi-architecture) - - [Configuration](#configuration) - - [Quick Start](#quick-start) - - [Persistent Storage](#persistent-storage) - - [Environment Variables](#environment-variables) - - [Base Images used](#base-images-used) - - [Container Options](#container-options) - - [Database Specific Options](#database-specific-options) - - [For Influx DB2](#for-influx-db2) - - [Scheduling Options](#scheduling-options) - - [Backup Options](#backup-options) - - [Backing Up to S3 Compatible Services](#backing-up-to-s3-compatible-services) - - [Upload to a Azure storage account by `blobxfer`](#upload-to-a-azure-storage-account-by-blobxfer) - - [Maintenance](#maintenance) - - [Shell Access](#shell-access) - - [Manual Backups](#manual-backups) - - [Restoring Databases](#restoring-databases) - - [Custom Scripts](#custom-scripts) - - [Path Options](#path-options) - - [Pre Backup](#pre-backup) - - [Post backup](#post-backup) - - [Support](#support) - - [Usage](#usage) - - [Bugfixes](#bugfixes) - - [Feature Requests](#feature-requests) - - [Updates](#updates) - - [License](#license) - -> **NOTE**: If you are using this with a docker-compose file along with a seperate SQL container, take care not to set the variables to backup immediately, more so have it delay execution for a minute, otherwise you will get a failed first backup. +- [About](#about) +- [Maintainer](#maintainer) +- [Table of Contents](#table-of-contents) +- [Prerequisites and Assumptions](#prerequisites-and-assumptions) +- [Installation](#installation) + - [Build from Source](#build-from-source) + - [Prebuilt Images](#prebuilt-images) + - [Multi Architecture](#multi-architecture) +- [Configuration](#configuration) + - [Quick Start](#quick-start) + - [Persistent Storage](#persistent-storage) + - [Environment Variables](#environment-variables) + - [Base Images used](#base-images-used) + - [Container Options](#container-options) + - [Job Defaults](#job-defaults) + - [Compression Options](#compression-options) + - [Encryption Options](#encryption-options) + - [Scheduling Options](#scheduling-options) + - [Default Database Options](#default-database-options) + - [CouchDB](#couchdb) + - [InfluxDB](#influxdb) + - [MariaDB/MySQL](#mariadbmysql) + - [Microsoft SQL](#microsoft-sql) + - [MongoDB](#mongodb) + - [Postgresql](#postgresql) + - [Redis](#redis) + - [Default Storage Options](#default-storage-options) + - [Filesystem](#filesystem) + - [S3](#s3) + - [Azure](#azure) + - [Hooks](#hooks) + - [Path Options](#path-options) + - [Pre Backup](#pre-backup) + - [Post backup](#post-backup) + - [Job Backup Options](#job-backup-options) + - [Compression Options](#compression-options-1) + - [Encryption Options](#encryption-options-1) + - [Scheduling Options](#scheduling-options-1) + - [Specific Database Options](#specific-database-options) + - [CouchDB](#couchdb-1) + - [InfluxDB](#influxdb-1) + - [MariaDB/MySQL](#mariadbmysql-1) + - [Microsoft SQL](#microsoft-sql-1) + - [MongoDB](#mongodb-1) + - [Postgresql](#postgresql-1) + - [Redis](#redis-1) + - [SQLite](#sqlite) + - [Specific Storage Options](#specific-storage-options) + - [Filesystem](#filesystem-1) + - [S3](#s3-1) + - [Azure](#azure-1) + - [Hooks](#hooks-1) + - [Path Options](#path-options-1) + - [Pre Backup](#pre-backup-1) + - [Post backup](#post-backup-1) + - [Notifications](#notifications) + - [Custom Notifications](#custom-notifications) + - [Email Notifications](#email-notifications) + - [Matrix Notifications](#matrix-notifications) + - [Mattermost Notifications](#mattermost-notifications) + - [Rocketchat Notifications](#rocketchat-notifications) +- [Maintenance](#maintenance) + - [Shell Access](#shell-access) + - [Manual Backups](#manual-backups) + - [Restoring Databases](#restoring-databases) +- [Support](#support) + - [Usage](#usage) + - [Bugfixes](#bugfixes) + - [Feature Requests](#feature-requests) + - [Updates](#updates) +- [License](#license) ## Prerequisites and Assumptions @@ -105,17 +142,16 @@ docker pull docker.io/tiredofit/db-backup:(imagetag) #### Multi Architecture -Images are built primarily for `amd64` architecture, and may also include builds for `arm/v7`, `arm64` and others. These variants are all unsupported. Consider [sponsoring](https://github.com/sponsors/tiredofit) my work so that I can work with various hardware. To see if this image supports multiple architectures, type `docker manifest (image):(tag)` +Images are built primarily for `amd64` architecture, and may also include builds for `arm/v7`, `arm64` and others. These variants are all unsupported. Consider [sponsoring](https://www.tiredofit.ca/sponsor) my work so that I can work with various hardware. To see if this image supports multiple architectures, type `docker manifest (image):(tag)` ## Configuration ### Quick Start -- The quickest way to get started is using [docker-compose](https://docs.docker.com/compose/). See the examples folder for a working [docker-compose.yml](examples/docker-compose.yml) that can be modified for development or production use. +- The quickest way to get started is using [docker-compose](https://docs.docker.com/compose/). See the examples folder for a series of example compose.yml that can be modified for development or production use. - Set various [environment variables](#environment-variables) to understand the capabilities of this image. - Map [persistent storage](#data-volumes) for access to configuration and data files for backup. -- Make [networking ports](#networking) available for public access if necessary ### Persistent Storage @@ -125,6 +161,7 @@ The following directories are used for configuration and can be mapped for persi | `/backup` | Backups | | `/assets/scripts/pre` | _Optional_ Put custom scripts in this directory to execute before backup operations | | `/assets/scripts/post` | _Optional_ Put custom scripts in this directory to execute after backup operations | +| `/logs` | _Optional_ Logfiles for backup jobs | ### Environment Variables @@ -140,113 +177,602 @@ Be sure to view the following repositories to understand all the customizable op #### Container Options -| Parameter | Description | Default | -| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------- | --------------- | -| `BACKUP_LOCATION` | Backup to `FILESYSTEM`, `blobxfer` or `S3` compatible services like S3, Minio, Wasabi | `FILESYSTEM` | -| `MODE` | `AUTO` mode to use internal scheduling routines or `MANUAL` to simply use this as manual backups only executed by your own means | `AUTO` | -| `MANUAL_RUN_FOREVER` | `TRUE` or `FALSE` if you wish to try to make the container exit after the backup | `TRUE` | -| `TEMP_LOCATION` | Perform Backups and Compression in this temporary directory | `/tmp/backups/` | -| `DEBUG_MODE` | If set to `true`, print copious shell script messages to the container log. Otherwise only basic messages are printed. | `FALSE` | -| `CREATE_LATEST_SYMLINK` | Create a symbolic link pointing to last backup in this format: `latest-(DB_TYPE)-(DB_NAME)-(DB_HOST)` | `TRUE` | -| `PRE_SCRIPT` | Fill this variable in with a command to execute pre backing up | | -| `POST_SCRIPT` | Fill this variable in with a command to execute post backing up | | -| `SPLIT_DB` | For each backup, create a new archive. `TRUE` or `FALSE` (MySQL and Postgresql Only) | `TRUE` | +| Parameter | Description | Default | +| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------- | --------------- | +| `MODE` | `AUTO` mode to use internal scheduling routines or `MANUAL` to simply use this as manual backups only executed by your own means | `AUTO` | +| `USER_DBBACKUP` | The uid that the image should read and write files as (username is `dbbackup`) | `10000` | +| `GROUP_DBBACKUP` | The gid that the image should read and write files as (groupname is `dbbackup`) | `10000` | +| `LOG_PATH` | Path to log files | `/logs` | +| `TEMP_PATH` | Perform Backups and Compression in this temporary directory | `/tmp/backups/` | +| `MANUAL_RUN_FOREVER` | `TRUE` or `FALSE` if you wish to try to make the container exit after the backup | `TRUE` | +| `DEBUG_MODE` | If set to `true`, print copious shell script messages to the container log. Otherwise only basic messages are printed. | `FALSE` | +| `BACKUP_JOB_CONCURRENCY` | How many backup jobs to run concurrently | `1` | -### Database Specific Options +#### Job Defaults +If these are set and no other defaults or variables are set explicitly, they will be added to any of the backup jobs. -| Parameter | Description | Default | `_FILE` | -| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | ------- | -| `DB_AUTH` | (Mongo/PGSQL Only - Optional) Authentication Database | | | -| `DB_TYPE` | Type of DB Server to backup `couch` `influx` `mysql` `mssql` `pgsql` `mongo` `redis` `sqlite3` | | | -| `DB_HOST` | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3` | | x | -| `DB_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. Backup multiple by separating with commas eg `db1,db2` | | x | -| `DB_NAME_EXCLUDE` | If using `ALL` - use this as to exclude databases separated via commas from being backed up | | x | -| `DB_USER` | username for the database(s) - Can use `root` for MySQL | | x | -| `DB_PASS` | (optional if DB doesn't require it) password for the database | | x | -| `DB_PORT` | (optional) Set port to connect to DB_HOST. Defaults are provided | varies | x | -| `INFLUX_VERSION` | What Version of Influx are you backing up from `1`.x or `2` series - AMD64 and ARM64 only for `2` | | | -| `MONGO_CUSTOM_URI` | If you wish to override the MongoDB Connection string enter it here e.g. `mongodb+srv://username:password@cluster.id.mongodb.net` | | x | -| | This environment variable will be parsed and populate the `DB_NAME` and `DB_HOST` variables to properly build your backup filenames. You can override them by making your own entries | | | +| Variable | Description | Default | +| --------------------------------- | ------------------------------------------------------------------------------------- | ------------ | +| `DEFAULT_BACKUP_LOCATION` | Backup to `FILESYSTEM`, `blobxfer` or `S3` compatible services like S3, Minio, Wasabi | `FILESYSTEM` | +| `DEFAULT_CHECKSUM` | Either `MD5` or `SHA1` or `NONE` | `MD5` | +| `DEFAULT_LOG_LEVEL` | Log output on screen and in files `INFO` `NOTICE` `ERROR` `WARN` `DEBUG` | `notice` | +| `DEFAULT_RESOURCE_OPTIMIZED` | Perform operations at a lower priority to the CPU and IO scheduler | `FALSE` | +| `DEFAULT_SKIP_AVAILABILITY_CHECK` | Before backing up - skip connectivity check | `FALSE` | -#### For Influx DB2 +##### Compression Options -Your Organization will be mapped to `DB_USER` and your root token will need to be mapped to `DB_PASS`. You may use `DB_NAME=ALL` to backup the entire set of databases. For `DB_HOST` use syntax of `http(s)://db-name` +| Variable | Description | Default | +| -------------------------------------- | ---------------------------------------------------------------------------------------------- | -------------- | +| `DEFAULT_COMPRESSION` | Use either Gzip `GZ`, Bzip2 `BZ`, XZip `XZ`, ZSTD `ZSTD` or none `NONE` | `ZSTD` | +| `DEFAULT_COMPRESSION_LEVEL` | Numerical value of what level of compression to use, most allow `1` to `9` | `3` | +| | except for `ZSTD` which allows for `1` to `19` | | +| `DEFAULT_GZ_RSYNCABLE` | Use `--rsyncable` (gzip only) for faster rsync transfers and incremental backup deduplication. | `FALSE` | +| `DEFAULT_ENABLE_PARALLEL_COMPRESSION` | Use multiple cores when compressing backups `TRUE` or `FALSE` | `TRUE` | +| `DEFAULT_PARALLEL_COMPRESSION_THREADS` | Maximum amount of threads to use when compressing - Integer value e.g. `8` | `autodetected` | -### Scheduling Options +##### Encryption Options -| Parameter | Description | Default | -| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | -| `DB_DUMP_FREQ` | How often to do a dump, in minutes after the first backup. Defaults to 1440 minutes, or once per day. | `1440` | -| `DB_DUMP_BEGIN` | What time to do the first dump. Defaults to immediate. Must be in one of two formats | | -| | Absolute HHMM, e.g. `2330` or `0415` | | -| | Relative +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half | | -| `DB_DUMP_TARGET` | Directory where the database dumps are kept. | `${DB_DUMP_TARGET}/archive/` | -| `DB_DUMP_TARGET_ARCHIVE` | Optional Directory where the database dumps archives are kept. | | -| `DB_CLEANUP_TIME` | Value in minutes to delete old backups (only fired when dump frequency fires). 1440 would delete anything above 1 day old. You don't need to set this variable if you want to hold onto everything. | `FALSE` | -| `DB_ARCHIVE_TIME` | Value in minutes to move all files files older than (x) from `DB_DUMP_TARGET` to `DB_DUMP_TARGET_ARCHIVE` - which is useful when pairing against an external backup system. | | +| Variable | Description | Default | +| ---------------------------- | ------------------------------------------- | ------- | +| `DEFAULT_ENCRYPT` | Encrypt file after backing up with GPG | `FALSE` | +| `DEFAULT_ENCRYPT_PASSPHRASE` | Passphrase to encrypt file with GPG | | +| *or* | | | +| `DEFAULT_ENCRYPT_PUBKEY` | Path of public key to encrypt file with GPG | | -- You may need to wrap your `DB_DUMP_BEGIN` value in quotes for it to properly parse. There have been reports of backups that start with a `0` get converted into a different format which will not allow the timer to start at the correct time. +##### Scheduling Options -### Backup Options +| Variable | Description | Default | +| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| `DEFAULT_BACKUP_INTERVAL` | How often to do a backup, in minutes after the first backup. Defaults to 1440 minutes, or once per day. | `1440` | +| `DEFAULT_BACKUP_BEGIN` | What time to do the initial backup. Defaults to immediate. (`+1`) | `+0` | +| | Must be in one of four formats: | | +| | Absolute HHMM, e.g. `2330` or `0415` | | +| | Relative +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half | | +| | Full datestamp e.g. `2023-12-21 23:30:00` | | +| | Cron expression e.g. `30 23 * * *` - [Understand the format](https://en.wikipedia.org/wiki/ Cron) - *BACKUP_INTERVAL is ignored* | | +| `DEFAULT_CLEANUP_TIME` | Value in minutes to delete old backups (only fired when backup interval executes) | `FALSE` | +| | 1440 would delete anything above 1 day old. You don't need to set this variable if you want to hold onto everything. | | +| `DEFAULT_ARCHIVE_TIME` | Value in minutes to move all files files older than (x) from | | +| `DEFAULT_BACKUP_BLACKOUT_BEGIN` | Use `HHMM` notation to start a blackout period where no backups occur eg `0420` | | +| `DEFAULT_BACKUP_BLACKOUT_END` | Use `HHMM` notation to set the end period where no backups occur eg `0430` | | -| Parameter | Description | Default | `_FILE` | -| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------- | ------------------------- | ------- | -| `COMPRESSION` | Use either Gzip `GZ`, Bzip2 `BZ`, XZip `XZ`, ZSTD `ZSTD` or none `NONE` | `ZSTD` | | -| `COMPRESSION_LEVEL` | Numerical value of what level of compression to use, most allow `1` to `9` except for `ZSTD` which allows for `1` to `19` - | `3` | | -| `ENABLE_PARALLEL_COMPRESSION` | Use multiple cores when compressing backups `TRUE` or `FALSE` | `TRUE` | | -| `PARALLEL_COMPRESSION_THREADS` | Maximum amount of threads to use when compressing - Integer value e.g. `8` | `autodetected` | | -| `GZ_RSYNCABLE` | Use `--rsyncable` (gzip only) for faster rsync transfers and incremental backup deduplication. e.g. `TRUE` | `FALSE` | | -| `ENABLE_CHECKSUM` | Generate either a MD5 or SHA1 in Directory, `TRUE` or `FALSE` | `TRUE` | | -| `CHECKSUM` | Either `MD5` or `SHA1` | `MD5` | | -| `EXTRA_OPTS` | If you need to pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | | -| `EXTRA_DUMP_OPTS` | If you need to pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | | -| `EXTRA_ENUMERATION_OPTS` | If you need to pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | | -| `MYSQL_MAX_ALLOWED_PACKET` | Max allowed packet if backing up MySQL / MariaDB | `512M` | | -| `MYSQL_SINGLE_TRANSACTION` | Backup in a single transaction with MySQL / MariaDB | `TRUE` | | -| `MYSQL_STORED_PROCEDURES` | Backup stored procedures with MySQL / MariaDB | `TRUE` | | -| `MYSQL_ENABLE_TLS` | Enable TLS functionality for MySQL client | `FALSE` | | -| `MYSQL_TLS_VERIFY` | (optional) If using TLS (by means of MYSQL_TLS_* variables) verify remote host | `FALSE` | | -| `MYSQL_TLS_VERSION` | What TLS `v1.1` `v1.2` `v1.3` version to utilize | `TLSv1.1,TLSv1.2,TLSv1.3` | | -| `MYSQL_TLS_CA_FILE` | Filename to load custom CA certificate for connecting via TLS | `/etc/ssl/cert.pem` | x | -| `MYSQL_TLS_CERT_FILE` | Filename to load client certificate for connecting via TLS | | x | -| `MYSQL_TLS_KEY_FILE` | Filename to load client key for connecting via TLS | | x | +> You may need to wrap your `DEFAULT_BACKUP_BEGIN` value in quotes for it to properly parse. There have been reports of backups that start with a `0` get converted into a different format which will not allow the timer to start at the correct time. -- When using compression with MongoDB, only `GZ` compression is possible. -#### Backing Up to S3 Compatible Services +##### Default Database Options -If `BACKUP_LOCATION` = `S3` then the following options are used. +###### CouchDB -| Parameter | Description | Default | `_FILE` | -| --------------------- | ----------------------------------------------------------------------------------------- | ------- | ------- | -| `S3_BUCKET` | S3 Bucket name e.g. `mybucket` | | x | -| `S3_KEY_ID` | S3 Key ID (Optional) | | x | -| `S3_KEY_SECRET` | S3 Key Secret (Optional) | | x | -| `S3_PATH` | S3 Pathname to save to (must NOT end in a trailing slash e.g. '`backup`') | | x | -| `S3_REGION` | Define region in which bucket is defined. Example: `ap-northeast-2` | | x | -| `S3_HOST` | Hostname (and port) of S3-compatible service, e.g. `minio:8080`. Defaults to AWS. | | x | -| `S3_PROTOCOL` | Protocol to connect to `S3_HOST`. Either `http` or `https`. Defaults to `https`. | `https` | x | -| `S3_EXTRA_OPTS` | Add any extra options to the end of the `aws-cli` process execution | | x | -| `S3_CERT_CA_FILE` | Map a volume and point to your custom CA Bundle for verification e.g. `/certs/bundle.pem` | | x | -| _*OR*_ | | | | -| `S3_CERT_SKIP_VERIFY` | Skip verifying self signed certificates when connecting | `TRUE` | | +| Variable | Description | Default | `_FILE` | +| -------------- | ------------ | ------- | ------- | +| `DEFAULT_PORT` | CouchDB Port | `5984` | x | -- When `S3_KEY_ID` and/or `S3_KEY_SECRET` is not set, will try to use IAM role assigned (if any) for uploading the backup files to S3 bucket. +###### InfluxDB -#### Upload to a Azure storage account by `blobxfer` +| Variable | Description | Default | `_FILE` | +| ------------------------ | ------------------------------------------------------------------------------------------------------- | ------- | ------- | +| `DEFAULT_PORT` | InfluxDB Port | | x | +| | Version 1.x | `8088` | | +| | Version 2.x | `8086` | | +| `DEFAULT_INFLUX_VERSION` | What Version of Influx are you backing up from `1`.x or `2` series - amd64 and aarch/armv8 only for `2` | `2` | | -Support to upload backup files with [blobxfer](https://github.com/Azure/blobxfer) to the Azure file share storage. +###### MariaDB/MySQL -If `BACKUP_LOCATION` = `blobxfer` then the following options are used. +| Variable | Description | Default | `_FILE` | +| ---------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------- | ------- | +| `DEFAULT_PORT` | MySQL / MariaDB Port | `3306` | x | +| `DEFAULT_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | | +| `DEFAULT_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | | +| `DEFAULT_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | | +| `DEFAULT_MYSQL_EVENTS` | Backup Events | `TRUE` | | +| `DEFAULT_MYSQL_MAX_ALLOWED_PACKET` | Max allowed packet | `512M` | | +| `DEFAULT_MYSQL_SINGLE_TRANSACTION` | Backup in a single transaction | `TRUE` | | +| `DEFAULT_MYSQL_STORED_PROCEDURES` | Backup stored procedures | `TRUE` | | +| `DEFAULT_MYSQL_ENABLE_TLS` | Enable TLS functionality | `FALSE` | | +| `DEFAULT_MYSQL_TLS_VERIFY` | (optional) If using TLS (by means of MYSQL_TLS_* variables) verify remote host | `FALSE` | | +| `DEFAULT_MYSQL_TLS_VERSION` | What TLS `v1.1` `v1.2` `v1.3` version to utilize | `TLSv1.1,TLSv1.2,TLSv1.3` | | +| `DEFAULT_MYSQL_TLS_CA_FILE` | Filename to load custom CA certificate for connecting via TLS | `/etc/ssl/cert.pem` | x | +| `DEFAULT_MYSQL_TLS_CERT_FILE` | Filename to load client certificate for connecting via TLS | | x | +| `DEFAULT_MYSQL_TLS_KEY_FILE` | Filename to load client key for connecting via TLS | | x | -| Parameter | Description | Default | `_FILE` | -| ------------------------------ | ------------------------------------------- | ------------------- | ------- | -| `BLOBXFER_STORAGE_ACCOUNT` | Microsoft Azure Cloud storage account name. | | x | -| `BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key. | | x | -| `BLOBXFER_REMOTE_PATH` | Remote Azure path | `/docker-db-backup` | x | -> This service uploads files from backup targed directory `DB_DUMP_TARGET`. -> If the a cleanup configuration in `DB_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically. +###### Microsoft SQL + +| Variable | Description | Default | `_FILE` | +| -------------------- | --------------------------------------- | ---------- | ------- | +| `DEFAULT_PORT` | Microsoft SQL Port | `1433` | x | +| `DEFAULT_MSSQL_MODE` | Backup `DATABASE` or `TRANSACTION` logs | `DATABASE` | + +###### MongoDB + +| Variable | Description | Default | `_FILE` | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------ | ------- | ------- | +| `DEFAULT_AUTH` | (Optional) Authentication Database | | x | +| `DEFAULT_PORT` | MongoDB Port | `27017` | x | +| `MONGO_CUSTOM_URI` | If you wish to override the MongoDB Connection string enter it here e.g. `mongodb+srv://username:password@cluster.id.mongodb.net` | | x | +| | This environment variable will be parsed and populate the `DB_NAME` and `DB_HOST` variables to properly build your backup filenames. | | | +| | You can override them by making your own entries | | | + +###### Postgresql + +| Variable | Description | Default | `_FILE` | +| -------------------------------- | --------------------------------------------------------------------------------------------------------- | ------- | ------- | +| `DEFAULT_AUTH` | (Optional) Authentication Database | | x | +| `DEFAULT_BACKUP_GLOBALS` | Backup Globals as part of backup procedure | | | +| `DEFAULT_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | | +| `DEFAULT_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | | +| `DEFAULT_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | | +| `DEFAULT_PORT` | PostgreSQL Port | `5432` | x | + +###### Redis + +| Variable | Description | Default | `_FILE` | +| -------------------------------- | --------------------------------------------------------------------------------------------------- | ------- | ------- | +| `DEFAULT_PORT` | Default Redis Port | `6379` | x | +| `DEFAULT_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | | + + +##### Default Storage Options + +Options that are related to the value of `DEFAULT_BACKUP_LOCATION` + +###### Filesystem + +If `DEFAULT_BACKUP_LOCTION` = `FILESYSTEM` then the following options are used: + +| Variable | Description | Default | +| --------------------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------- | +| `DEFAULT_CREATE_LATEST_SYMLINK` | Create a symbolic link pointing to last backup in this format: `latest-(DB_TYPE)-(DB_NAME)-(DB_HOST)` | `TRUE` | +| `DEFAULT_FILESYSTEM_PATH` | Directory where the database dumps are kept. | `/backup` | +| `DEFAULT_FILESYSTEM_ARCHIVE_PATH` | Optional Directory where the database dumps archives are kept | `${DEFAULT_FILESYSTEM_PATH}/archive/` | +| `DEFAULT_FILESYSTEM_PERMISSION` | Directory and File permissions to apply to files. | `700` | + +###### S3 + +If `DEFAULT_BACKUP_LOCATION` = `S3` then the following options are used: + +| Parameter | Description | Default | `_FILE` | +| ----------------------------- | ----------------------------------------------------------------------------------------- | ------- | ------- | +| `DEFAULT_S3_BUCKET` | S3 Bucket name e.g. `mybucket` | | x | +| `DEFAULT_S3_KEY_ID` | S3 Key ID (Optional) | | x | +| `DEFAULT_S3_KEY_SECRET` | S3 Key Secret (Optional) | | x | +| `DEFAULT_S3_PATH` | S3 Pathname to save to (must NOT end in a trailing slash e.g. '`backup`') | | x | +| `DEFAULT_S3_REGION` | Define region in which bucket is defined. Example: `ap-northeast-2` | | x | +| `DEFAULT_S3_HOST` | Hostname (and port) of S3-compatible service, e.g. `minio:8080`. Defaults to AWS. | | x | +| `DEFAULT_S3_PROTOCOL` | Protocol to connect to `DEFAULT_S3_HOST`. Either `http` or `https`. Defaults to `https`. | `https` | x | +| `DEFAULT_S3_EXTRA_OPTS` | Add any extra options to the end of the `aws-cli` process execution | | x | +| `DEFAULT_S3_CERT_CA_FILE` | Map a volume and point to your custom CA Bundle for verification e.g. `/certs/bundle.pem` | | x | +| _*OR*_ | | | | +| `DEFAULT_S3_CERT_SKIP_VERIFY` | Skip verifying self signed certificates when connecting | `TRUE` | | + +- When `DEFAULT_S3_KEY_ID` and/or `DEFAULT_S3_KEY_SECRET` is not set, will try to use IAM role assigned (if any) for uploading the backup files to S3 bucket. + +###### Azure + +If `DEFAULT_BACKUP_LOCATION` = `blobxfer` then the following options are used:. + +| Parameter | Description | Default | `_FILE` | +| -------------------------------------- | ------------------------------------------- | ------------------- | ------- | +| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT` | Microsoft Azure Cloud storage account name. | | x | +| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key. | | x | +| `DEFAULT_BLOBXFER_REMOTE_PATH` | Remote Azure path | `/docker-db-backup` | x | + +> This service uploads files from backup targed directory `DEFAULT_FILESYSTEM_PATH`. +> If the a cleanup configuration in `DEFAULT_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically. + +##### Hooks + +###### Path Options + +| Parameter | Description | Default | +| ------------------------------ | --------------------------------------------------------------------------- | ----------------------- | +| `DEFAULT_SCRIPT_LOCATION_PRE` | Location on filesystem inside container to execute bash scripts pre backup | `/assets/scripts/pre/` | +| `DEFAULT_SCRIPT_LOCATION_POST` | Location on filesystem inside container to execute bash scripts post backup | `/assets/scripts/post/` | +| `DEFAULT_PRE_SCRIPT` | Fill this variable in with a command to execute pre backing up | | +| `DEFAULT_POST_SCRIPT` | Fill this variable in with a command to execute post backing up | | + +###### Pre Backup + +If you want to execute a custom script before a backup starts, you can drop bash scripts with the extension of `.sh` in the location defined in `DB01_SCRIPT_LOCATION_PRE`. See the following example to utilize: + +```bash +$ cat pre-script.sh +##!/bin/bash + +# #### Example Pre Script +# #### $1=DBXX_TYPE (Type of Backup) +# #### $2=DBXX_HOST (Backup Host) +# #### $3=DBXX_NAME (Name of Database backed up +# #### $4=BACKUP START TIME (Seconds since Epoch) +# #### $5=BACKUP FILENAME (Filename) + +echo "${1} Backup Starting on ${2} for ${3} at ${4}. Filename: ${5}" +``` + + ## script DBXX_TYPE DBXX_HOST DBXX_NAME STARTEPOCH BACKUP_FILENAME + ${f} "${backup_job_db_type}" "${backup_job_db_host}" "${backup_job_db_name}" "${backup_routines_start_time}" "${target}" + +Outputs the following on the console: + +`mysql Backup Starting on example-db for example at 1647370800. Filename: mysql_example_example-db_202200315-000000.sql.bz2 + +###### Post backup + +If you want to execute a custom script at the end of a backup, you can drop bash scripts with the extension of `.sh` in the location defined in `DB01_SCRIPT_LOCATION_POST`. Also to support legacy users `/assets/custom-scripts` is also scanned and executed.See the following example to utilize: + +```bash +$ cat post-script.sh +##!/bin/bash + +# #### Example Post Script +# #### $1=EXIT_CODE (After running backup routine) +# #### $2=DBXX_TYPE (Type of Backup) +# #### $3=DBXX_HOST (Backup Host) +# #### #4=DBXX_NAME (Name of Database backed up +# #### $5=BACKUP START TIME (Seconds since Epoch) +# #### $6=BACKUP FINISH TIME (Seconds since Epoch) +# #### $7=BACKUP TOTAL TIME (Seconds between Start and Finish) +# #### $8=BACKUP FILENAME (Filename) +# #### $9=BACKUP FILESIZE +# #### $10=HASH (If CHECKSUM enabled) +# #### $11=MOVE_EXIT_CODE + +echo "${1} ${2} Backup Completed on ${3} for ${4} on ${5} ending ${6} for a duration of ${7} seconds. Filename: ${8} Size: ${9} bytes MD5: ${10}" +``` + + ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME STARTEPOCH FINISHEPOCH DURATIONEPOCH BACKUP_FILENAME FILESIZE CHECKSUMVALUE + ${f} "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${backup_job_db_name}" "${backup_routines_start_time}" "${backup_routines_finish_time}" "${backup_routines_total_time}" "${target}" "${FILESIZE}" "${checksum_value}" "${move_exit_code} + +Outputs the following on the console: + +`0 mysql Backup Completed on example-db for example on 1647370800 ending 1647370920 for a duration of 120 seconds. Filename: mysql_example_example-db_202200315-000000.sql.bz2 Size: 7795 bytes Hash: 952fbaafa30437494fdf3989a662cd40 0` + +If you wish to change the size value from bytes to megabytes set environment variable `DB01_SIZE_VALUE=megabytes` + +You must make your scripts executable otherwise there is an internal check that will skip trying to run it otherwise. +If for some reason your filesystem or host is not detecting it right, use the environment variable `DB01_POST_SCRIPT_SKIP_X_VERIFY=TRUE` to bypass. + + +#### Job Backup Options + +If `DEFAULT_` variables are set and you do not wish for the settings to carry over into your jobs, you can set the appropriate environment variable with the value of `unset`. +Otherwise, override them per backup job. Additional backup jobs can be scheduled by using `DB02_`,`DB03_`,`DB04_` ... prefixes. See [Specific Database Options](#specific-database-options) which may overrule this list. + +| Parameter | Description | Default | `_FILE` | +| ----------- | ---------------------------------------------------------------------------------------------- | ------- | ------- | +| `DB01_TYPE` | Type of DB Server to backup `couch` `influx` `mysql` `mssql` `pgsql` `mongo` `redis` `sqlite3` | | | +| `DB01_HOST` | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3` | | x | +| `DB01_NAME` | Schema Name e.g. `database` | | x | +| `DB01_USER` | username for the database(s) - Can use `root` for MySQL | | x | +| `DB01_PASS` | (optional if DB doesn't require it) password for the database | | x | + + +| Variable | Description | Default | +| ------------------------------ | --------------------------------------------------------------------------------------------------------- | ------------ | +| `DB01_BACKUP_LOCATION` | Backup to `FILESYSTEM`, `blobxfer` or `S3` compatible services like S3, Minio, Wasabi | `FILESYSTEM` | +| `DB01_CHECKSUM` | Either `MD5` or `SHA1` or `NONE` | `MD5` | +| `DB01_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | +| `DB01_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | +| `DB01_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | +| `DB01_LOG_LEVEL` | Log output on screen and in files `INFO` `NOTICE` `ERROR` `WARN` `DEBUG` | `debug` | +| `DB01_RESOURCE_OPTIMIZED` | Perform operations at a lower priority to the CPU and IO scheduler | `FALSE` | +| `DB01_SKIP_AVAILABILITY_CHECK` | Before backing up - skip connectivity check | `FALSE` | + +##### Compression Options + +| Variable | Description | Default | +| ----------------------------------- | ---------------------------------------------------------------------------------------------- | -------------- | +| `DB01_COMPRESSION` | Use either Gzip `GZ`, Bzip2 `BZ`, XZip `XZ`, ZSTD `ZSTD` or none `NONE` | `ZSTD` | +| `DB01_COMPRESSION_LEVEL` | Numerical value of what level of compression to use, most allow `1` to `9` | `3` | +| | except for `ZSTD` which allows for `1` to `19` | | +| `DB01_GZ_RSYNCABLE` | Use `--rsyncable` (gzip only) for faster rsync transfers and incremental backup deduplication. | `FALSE` | +| `DB01_ENABLE_PARALLEL_COMPRESSION` | Use multiple cores when compressing backups `TRUE` or `FALSE` | `TRUE` | +| `DB01_PARALLEL_COMPRESSION_THREADS` | Maximum amount of threads to use when compressing - Integer value e.g. `8` | `autodetected` | + +##### Encryption Options + +| Variable | Description | Default | +| ------------------------- | ------------------------------------------- | ------- | +| `DB01_ENCRYPT` | Encrypt file after backing up with GPG | `FALSE` | +| `DB01_ENCRYPT_PASSPHRASE` | Passphrase to encrypt file with GPG | | +| *or* | | | +| `DB01_ENCRYPT_PUBKEY` | Path of public key to encrypt file with GPG | | + +##### Scheduling Options + +| Variable | Description | Default | +| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| `DB01_BACKUP_INTERVAL` | How often to do a backup, in minutes after the first backup. Defaults to 1440 minutes, or once per day. | `1440` | +| `DB01_BACKUP_BEGIN` | What time to do the initial backup. Defaults to immediate. (`+1`) | `+0` | +| | Must be in one of four formats: | | +| | Absolute HHMM, e.g. `2330` or `0415` | | +| | Relative +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half | | +| | Full datestamp e.g. `2023-12-21 23:30:00` | | +| | Cron expression e.g. `30 23 * * *` - [Understand the format](https://en.wikipedia.org/wiki/ Cron) - *BACKUP_INTERVAL is ignored* | | +| `DB01_CLEANUP_TIME` | Value in minutes to delete old backups (only fired when backup interval executes) | `FALSE` | +| | 1440 would delete anything above 1 day old. You don't need to set this variable if you want to hold onto everything. | | +| `DB01_ARCHIVE_TIME` | Value in minutes to move all files files older than (x) from `DB01_BACKUP_FILESYSTEM_PATH` | | +| | to `DB01_BACKUP_FILESYSTEM_ARCHIVE_PATH` - which is useful when pairing against an external backup system. | | +| `DB01_BACKUP_BLACKOUT_BEGIN` | Use `HHMM` notation to start a blackout period where no backups occur eg `0420` | | +| `DB01_BACKUP_BLACKOUT_END` | Use `HHMM` notation to set the end period where no backups occur eg `0430` | | + +##### Specific Database Options + +###### CouchDB + +| Variable | Description | Default | `_FILE` | +| ----------- | ------------ | ------- | ------- | +| `DB01_PORT` | CouchDB Port | `5984` | x | + +###### InfluxDB + +| Variable | Description | Default | `_FILE` | +| --------------------- | ------------------------------------------------------------------------------------------------------- | ------- | ------- | +| `DB01_PORT` | InfluxDB Port | | x | +| | Version 1.x | `8088` | | +| | Version 2.x | `8086` | | +| `DB01_INFLUX_VERSION` | What Version of Influx are you backing up from `1`.x or `2` series - amd64 and aarch/armv8 only for `2` | `2` | | + +> Your Organization will be mapped to `DB_USER` and your root token will need to be mapped to `DB_PASS`. +> You may use `DB_NAME=ALL` to backup the entire set of databases. +> For `DB_HOST` use syntax of `http(s)://db-name` + + +###### MariaDB/MySQL + +| Variable | Description | Default | `_FILE` | +| ------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------- | ------- | +| `DB01_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | | +| `DB01_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | | +| `DB01_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | | +| `DB01_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. | | | +| | Backup multiple by separating with commas eg `db1,db2` | | x | +| `DB01_NAME_EXCLUDE` | If using `ALL` - use this as to exclude databases separated via commas from being backed up | | x | +| `DB01_PORT` | MySQL / MariaDB Port | `3306` | x | +| `DB01_MYSQL_EVENTS` | Backup Events for | `TRUE` | | +| `DB01_MYSQL_MAX_ALLOWED_PACKET` | Max allowed packet | `512M` | | +| `DB01_MYSQL_SINGLE_TRANSACTION` | Backup in a single transaction | `TRUE` | | +| `DB01_MYSQL_STORED_PROCEDURES` | Backup stored procedures | `TRUE` | | +| `DB01_MYSQL_ENABLE_TLS` | Enable TLS functionality | `FALSE` | | +| `DB01_MYSQL_TLS_VERIFY` | (optional) If using TLS (by means of MYSQL_TLS_* variables) verify remote host | `FALSE` | | +| `DB01_MYSQL_TLS_VERSION` | What TLS `v1.1` `v1.2` `v1.3` version to utilize | `TLSv1.1,TLSv1.2,TLSv1.3` | | +| `DB01_MYSQL_TLS_CA_FILE` | Filename to load custom CA certificate for connecting via TLS | `/etc/ssl/cert.pem` | x | +| `DB01_MYSQL_TLS_CERT_FILE` | Filename to load client certificate for connecting via TLS | | x | +| `DB01_MYSQL_TLS_KEY_FILE` | Filename to load client key for connecting via TLS | | x | + + +###### Microsoft SQL + +| Variable | Description | Default | `_FILE` | +| ----------------- | --------------------------------------- | ---------- | ------- | +| `DB01_PORT` | Microsoft SQL Port | `1433` | x | +| `DB01_MSSQL_MODE` | Backup `DATABASE` or `TRANSACTION` logs | `DATABASE` | + +###### MongoDB + +| Variable | Description | Default | `_FILE` | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | ------- | ------- | +| `DB01_AUTH` | (Optional) Authentication Database | | | +| `DB01_PORT` | MongoDB Port | `27017` | x | +| `DB01_MONGO_CUSTOM_URI` | If you wish to override the MongoDB Connection string enter it here e.g. `mongodb+srv://username:password@cluster.id.mongodb.net` | | x | +| | This environment variable will be parsed and populate the `DB_NAME` and `DB_HOST` variables to properly build your backup filenames. | | | +| | You can override them by making your own entries | | | + +###### Postgresql + +| Variable | Description | Default | `_FILE` | +| ----------------------------- | --------------------------------------------------------------------------------------------------------- | ------- | ------- | +| `DB01_AUTH` | (Optional) Authentication Database | | | +| `DB01_BACKUP_GLOBALS` | Backup Globals after backing up database (forces `TRUE` if `_NAME=ALL``) | `FALSE` | | +| `DB01_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | | +| `DB01_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | | +| `DB01_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | | +| `DB01_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. | | | +| | Backup multiple by separating with commas eg `db1,db2` | | x | +| `DB01_PORT` | PostgreSQL Port | `5432` | x | + +###### Redis + +| Variable | Description | Default | `_FILE` | +| ------------------------ | --------------------------------------------------------------------------------------------------------- | ------- | ------- | +| `DB01_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | | +| `DB01_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | | +| `DB01_PORT` | Redis Port | `6379` | x | + +###### SQLite + +| Variable | Description | Default | `_FILE` | +| ----------- | -------------------------------------------------------- | ------- | ------- | +| `DB01_HOST` | Enter the full path to DB file e.g. `/backup/db.sqlite3` | | x | + +##### Specific Storage Options + +Options that are related to the value of `DB01_BACKUP_LOCATION` + +###### Filesystem + +If `DB01_BACKUP_LOCTION` = `FILESYSTEM` then the following options are used: + +| Variable | Description | Default | +| ------------------------------ | ----------------------------------------------------------------------------------------------------- | --------------------------------- | +| `DB01_CREATE_LATEST_SYMLINK` | Create a symbolic link pointing to last backup in this format: `latest-(DB_TYPE)-(DB_NAME)-(DB_HOST)` | `TRUE` | +| `DB01_FILESYSTEM_PATH` | Directory where the database dumps are kept. | `/backup` | +| `DB01_FILESYSTEM_ARCHIVE_PATH` | Optional Directory where the database dumps archives are kept | `${DB01_FILESYSTEM_PATH/archive/` | +| `DB01_FILESYSTEM_PERMISSION` | Directory and File permissions to apply to files. | `700` | + +###### S3 + +If `DB01_BACKUP_LOCATION` = `S3` then the following options are used: + +| Parameter | Description | Default | `_FILE` | +| -------------------------- | ----------------------------------------------------------------------------------------- | ------- | ------- | +| `DB01_S3_BUCKET` | S3 Bucket name e.g. `mybucket` | | x | +| `DB01_S3_KEY_ID` | S3 Key ID (Optional) | | x | +| `DB01_S3_KEY_SECRET` | S3 Key Secret (Optional) | | x | +| `DB01_S3_PATH` | S3 Pathname to save to (must NOT end in a trailing slash e.g. '`backup`') | | x | +| `DB01_S3_REGION` | Define region in which bucket is defined. Example: `ap-northeast-2` | | x | +| `DB01_S3_HOST` | Hostname (and port) of S3-compatible service, e.g. `minio:8080`. Defaults to AWS. | | x | +| `DB01_S3_PROTOCOL` | Protocol to connect to `DB01_S3_HOST`. Either `http` or `https`. Defaults to `https`. | `https` | x | +| `DB01_S3_EXTRA_OPTS` | Add any extra options to the end of the `aws-cli` process execution | | x | +| `DB01_S3_CERT_CA_FILE` | Map a volume and point to your custom CA Bundle for verification e.g. `/certs/bundle.pem` | | x | +| _*OR*_ | | | | +| `DB01_S3_CERT_SKIP_VERIFY` | Skip verifying self signed certificates when connecting | `TRUE` | | + +> When `DB01_S3_KEY_ID` and/or `DB01_S3_KEY_SECRET` is not set, will try to use IAM role assigned (if any) for uploading the backup files to S3 bucket. + +###### Azure + +If `DB01_BACKUP_LOCATION` = `blobxfer` then the following options are used:. + +| Parameter | Description | Default | `_FILE` | +| ----------------------------------- | ------------------------------------------- | ------------------- | ------- | +| `DB01_BLOBXFER_STORAGE_ACCOUNT` | Microsoft Azure Cloud storage account name. | | x | +| `DB01_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key. | | x | +| `DB01_BLOBXFER_REMOTE_PATH` | Remote Azure path | `/docker-db-backup` | x | + +> This service uploads files from backup target directory `DB01_BACKUP_FILESYSTEM_PATH`. +> If the a cleanup configuration in `DB01_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically. + +##### Hooks + +###### Path Options + +| Parameter | Description | Default | +| --------------------------- | --------------------------------------------------------------------------- | ----------------------- | +| `DB01_SCRIPT_LOCATION_PRE` | Location on filesystem inside container to execute bash scripts pre backup | `/assets/scripts/pre/` | +| `DB01_SCRIPT_LOCATION_POST` | Location on filesystem inside container to execute bash scripts post backup | `/assets/scripts/post/` | +| `DB01_PRE_SCRIPT` | Fill this variable in with a command to execute pre backing up | | +| `DB01_POST_SCRIPT` | Fill this variable in with a command to execute post backing up | | + +###### Pre Backup + +If you want to execute a custom script before a backup starts, you can drop bash scripts with the extension of `.sh` in the location defined in `DB01_SCRIPT_LOCATION_PRE`. See the following example to utilize: + +```bash +$ cat pre-script.sh +##!/bin/bash + +# #### Example Pre Script +# #### $1=DB01_TYPE (Type of Backup) +# #### $2=DB01_HOST (Backup Host) +# #### $3=DB01_NAME (Name of Database backed up +# #### $4=BACKUP START TIME (Seconds since Epoch) +# #### $5=BACKUP FILENAME (Filename) + +echo "${1} Backup Starting on ${2} for ${3} at ${4}. Filename: ${5}" +``` + + ## script DB01_TYPE DB01_HOST DB01_NAME STARTEPOCH BACKUP_FILENAME + ${f} "${backup_job_db_type}" "${backup_job_db_host}" "${backup_job_db_name}" "${backup_routines_start_time}" "${target}" + +Outputs the following on the console: + +`mysql Backup Starting on example-db for example at 1647370800. Filename: mysql_example_example-db_202200315-000000.sql.bz2 + +###### Post backup + +If you want to execute a custom script at the end of a backup, you can drop bash scripts with the extension of `.sh` in the location defined in `DB01_SCRIPT_LOCATION_POST`. Also to support legacy users `/assets/custom-scripts` is also scanned and executed.See the following example to utilize: + +```bash +$ cat post-script.sh +##!/bin/bash + +# #### Example Post Script +# #### $1=EXIT_CODE (After running backup routine) +# #### $2=DB_TYPE (Type of Backup) +# #### $3=DB_HOST (Backup Host) +# #### #4=DB_NAME (Name of Database backed up +# #### $5=BACKUP START TIME (Seconds since Epoch) +# #### $6=BACKUP FINISH TIME (Seconds since Epoch) +# #### $7=BACKUP TOTAL TIME (Seconds between Start and Finish) +# #### $8=BACKUP FILENAME (Filename) +# #### $9=BACKUP FILESIZE +# #### $10=HASH (If CHECKSUM enabled) +# #### $11=MOVE_EXIT_CODE + +echo "${1} ${2} Backup Completed on ${3} for ${4} on ${5} ending ${6} for a duration of ${7} seconds. Filename: ${8} Size: ${9} bytes MD5: ${10}" +``` + + ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME STARTEPOCH FINISHEPOCH DURATIONEPOCH BACKUP_FILENAME FILESIZE CHECKSUMVALUE + ${f} "${exit_code}" "${dbtype}" "${dbhost}" "${dbname}" "${backup_routines_start_time}" "${backup_routines_finish_time}" "${backup_routines_total_time}" "${target}" "${FILESIZE}" "${checksum_value}" "${move_exit_code} + +Outputs the following on the console: + +`0 mysql Backup Completed on example-db for example on 1647370800 ending 1647370920 for a duration of 120 seconds. Filename: mysql_example_example-db_202200315-000000.sql.bz2 Size: 7795 bytes Hash: 952fbaafa30437494fdf3989a662cd40 0` + +If you wish to change the size value from bytes to megabytes set environment variable `DB01_SIZE_VALUE=megabytes` + +You must make your scripts executable otherwise there is an internal check that will skip trying to run it otherwise. +If for some reason your filesystem or host is not detecting it right, use the environment variable `DB01_POST_SCRIPT_SKIP_X_VERIFY=TRUE` to bypass. + + +#### Notifications + +This image has capabilities on sending notifications via a handful of services when a backup job fails. This is a global option that cannot be individually set per backup job. + +| Parameter | Description | Default | +| ---------------------- | --------------------------------------------------------------------------------- | ------- | +| `ENABLE_NOTIFICATIONS` | Enable Notifications | `FALSE` | +| `NOTIFICATION_TYPE` | `CUSTOM` `EMAIL` `MATRIX` `MATTERMOST` `ROCKETCHAT` - Seperate Multiple by commas | | + +##### Custom Notifications + +The following is sent to the custom script. Use how you wish: + +```` +$1 unix timestamp +$2 logfile +$3 errorcode +$4 subject +$5 body/error message +```` + +| Parameter | Description | Default | +| ---------------------------- | ------------------------------------------------------- | ------- | +| `NOTIFICATION_CUSTOM_SCRIPT` | Path and name of custom script to execute notification. | | + + +##### Email Notifications +| Parameter | Description | Default | `_FILE` | +| ----------- | ----------------------------------------------------------------------------------------- | ------- | ------- | +| `MAIL_FROM` | What email address to send mail from for errors | | | +| `MAIL_TO` | What email address to send mail to for errors. Send to multiple by seperating with comma. | | | +| `SMTP_HOST` | What SMTP server to use for sending mail | | x | +| `SMTP_PORT` | What SMTP port to use for sending mail | | x | + +##### Matrix Notifications + +Fetch a `MATRIX_ACCESS_TOKEN`: + +```` +curl -XPOST -d '{"type":"m.login.password", "user":"myuserid", "password":"mypass"}' "https://matrix.org/_matrix/client/r0/login" +```` + +Copy the JSON response `access_token` that will look something like this: + +```` +{"access_token":"MDAxO...blahblah","refresh_token":"MDAxO...blahblah","home_server":"matrix.org","user_id":"@myuserid:matrix.org"} +```` + +| Parameter | Description | Default | `_FILE` | +| --------------------- | ---------------------------------------------------------------------------------------- | ------- | ------- | +| `MATRIX_HOST` | URL (https://matrix.example.com) of Matrix Homeserver | | x | +| `MATRIX_ROOM` | Room ID eg `\!abcdef:example.com` to send to. Send to multiple by seperating with comma. | | x | +| `MATRIX_ACCESS_TOKEN` | Access token of user authorized to send to room | | x | + +##### Mattermost Notifications +| Parameter | Description | Default | `_FILE` | +| ------------------------ | -------------------------------------------------------------------------------------------- | ------- | ------- | +| `MATTERMOST_WEBHOOK_URL` | Full URL to send webhook notifications to | | x | +| `MATTERMOST_RECIPIENT` | Channel or User to send Webhook notifications to. Send to multiple by seperating with comma. | | x | +| `MATTERMOST_USERNAME` | Username to send as eg `tiredofit` | | x | + +##### Rocketchat Notifications +| Parameter | Description | Default | `_FILE` | +| ------------------------ | -------------------------------------------------------------------------------------------- | ------- | ------- | +| `ROCKETCHAT_WEBHOOK_URL` | Full URL to send webhook notifications to | | x | +| `ROCKETCHAT_RECIPIENT` | Channel or User to send Webhook notifications to. Send to multiple by seperating with comma. | | x | +| `ROCKETCHAT_USERNAME` | Username to send as eg `tiredofit` | | x | ## Maintenance @@ -260,7 +786,7 @@ docker exec -it (whatever your container name is) bash ### Manual Backups -Manual Backups can be performed by entering the container and typing `backup-now` +Manual Backups can be performed by entering the container and typing `backup-now`. This will execute all the backup tasks that are scheduled by means of the `BACKUPXX_` variables. Alternatively if you wanted to execute a job on its own you could simply type `backup01-now` (or whatever your number would be). There is no concurrency, and jobs will be executed sequentially. - Recently there was a request to have the container work with Kubernetes cron scheduling. This can theoretically be accomplished by setting the container `MODE=MANUAL` and then setting `MANUAL_RUN_FOREVER=FALSE` - You would also want to disable a few features from the upstream base images specifically `CONTAINER_ENABLE_SCHEDULING` and `CONTAINER_ENABLE_MONITORING`. This should allow the container to start, execute a backup by executing and then exit cleanly. An alternative way to running the script is to execute `/etc/services.available/10-db-backup/run`. @@ -287,75 +813,7 @@ The script can also be executed skipping the interactive mode by using the follo If you only enter some of the arguments you will be prompted to fill them in. -### Custom Scripts -#### Path Options - -| Parameter | Description | Default | -| ---------------------- | --------------------------------------------------------------------------- | ----------------------- | -| `SCRIPT_LOCATION_PRE` | Location on filesystem inside container to execute bash scripts pre backup | `/assets/scripts/pre/` | -| `SCRIPT_LOCATION_POST` | Location on filesystem inside container to execute bash scripts post backup | `/assets/scripts/post/` | - -#### Pre Backup - -If you want to execute a custom script before a backup starts, you can drop bash scripts with the extension of `.sh` in the location defined in `SCRIPT_LOCATION_PRE`. See the following example to utilize: - -```bash -$ cat pre-script.sh -##!/bin/bash - -# #### Example Pre Script -# #### $1=DB_TYPE (Type of Backup) -# #### $2=DB_HOST (Backup Host) -# #### $3=DB_NAME (Name of Database backed up -# #### $4=BACKUP START TIME (Seconds since Epoch)ff -# #### $5=BACKUP FILENAME (Filename) - -echo "${1} Backup Starting on ${2} for ${3} at ${4}. Filename: ${5}" -``` - - ## script DB_TYPE DB_HOST DB_NAME STARTEPOCH BACKUP_FILENAME - ${f} "${dbtype}" "${dbhost}" "${dbname}" "${backup_start_time}" "${target}" - -Outputs the following on the console: - -`mysql Backup Starting on example-db for example at 1647370800. Filename: mysql_example_example-db_202200315-000000.sql.bz2 - -#### Post backup - -If you want to execute a custom script at the end of a backup, you can drop bash scripts with the extension of `.sh` in the location defined in `SCRIPT_LOCATION_POST`. Also to support legacy users `/assets/custom-scripts` is also scanned and executed.See the following example to utilize: - -```bash -$ cat post-script.sh -##!/bin/bash - -# #### Example Post Script -# #### $1=EXIT_CODE (After running backup routine) -# #### $2=DB_TYPE (Type of Backup) -# #### $3=DB_HOST (Backup Host) -# #### #4=DB_NAME (Name of Database backed up -# #### $5=BACKUP START TIME (Seconds since Epoch) -# #### $6=BACKUP FINISH TIME (Seconds since Epoch) -# #### $7=BACKUP TOTAL TIME (Seconds between Start and Finish) -# #### $8=BACKUP FILENAME (Filename) -# #### $9=BACKUP FILESIZE -# #### $10=HASH (If CHECKSUM enabled) -# #### $11=MOVE_EXIT_CODE - -echo "${1} ${2} Backup Completed on ${3} for ${4} on ${5} ending ${6} for a duration of ${7} seconds. Filename: ${8} Size: ${9} bytes MD5: ${10}" -``` - - ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME STARTEPOCH FINISHEPOCH DURATIONEPOCH BACKUP_FILENAME FILESIZE CHECKSUMVALUE - ${f} "${exit_code}" "${dbtype}" "${dbhost}" "${dbname}" "${backup_start_timme}" "${backup_finish_time}" "${backup_total_time}" "${target}" "${FILESIZE}" "${checksum_value}" "${move_exit_code} - -Outputs the following on the console: - -`0 mysql Backup Completed on example-db for example on 1647370800 ending 1647370920 for a duration of 120 seconds. Filename: mysql_example_example-db_202200315-000000.sql.bz2 Size: 7795 bytes Hash: 952fbaafa30437494fdf3989a662cd40 0` - -If you wish to change the size value from bytes to megabytes set environment variable `SIZE_VALUE=megabytes` - -You must make your scripts executable otherwise there is an internal check that will skip trying to run it otherwise. -If for some reason your filesystem or host is not detecting it right, use the environment variable `POST_SCRIPT_SKIP_X_VERIFY=TRUE` to bypass. ## Support @@ -364,7 +822,7 @@ These images were built to serve a specific need in a production environment and ### Usage - The [Discussions board](../../discussions) is a great place for working with the community on tips and tricks of using this image. -- Consider [sponsoring me](https://github.com/sponsors/tiredofit) for personalized support +- [Sponsor me](https://www.tiredofit.ca/sponsor) for personalized support ### Bugfixes @@ -373,12 +831,12 @@ These images were built to serve a specific need in a production environment and ### Feature Requests - Feel free to submit a feature request, however there is no guarantee that it will be added, or at what timeline. -- Consider [sponsoring me](https://github.com/sponsors/tiredofit) regarding development of features. +- [Sponsor me](https://www.tiredofit.ca/sponsor) regarding development of features. ### Updates - Best effort to track upstream changes, More priority if I am actively using the image in a production environment. -- Consider [sponsoring me](https://github.com/sponsors/tiredofit) for up to date releases. +- [Sponsor me](https://www.tiredofit.ca/sponsor) for up to date releases. ## License diff --git a/examples/compose.yml b/examples/compose.yml new file mode 100644 index 0000000..8c3f108 --- /dev/null +++ b/examples/compose.yml @@ -0,0 +1,67 @@ +services: + example-db: + hostname: example-db-host + container_name: example-db + image: tiredofit/mariadb:10.11 + ports: + - 3306:3306 + volumes: + - ./db:/var/lib/mysql + environment: + - ROOT_PASS=examplerootpassword + - DB_NAME=example + - DB_USER=example + - DB_PASS=examplepassword + restart: always + networks: + - example-db-network + + example-db-backup: + container_name: example-db-backup + image: tiredofit/db-backup + volumes: + - ./backups:/backup + #- ./post-script.sh:/assets/custom-scripts/post-script.sh + environment: + - TIMEZONE=America/Vancouver + - CONTAINER_NAME=example-db-backup + - CONTAINER_ENABLE_MONITORING=FALSE + # - DEBUG_MODE=TRUE + + - BACKUP_JOB_CONCURRENCY=1 # Only run one job at a time + - DEFAULT_CHECKSUM=NONE # Don't create checksums + - DEFAULT_COMPRESSION=ZSTD # Compress all with ZSTD + - DEFAULT_DUMP_INTERVAL=1440 # Backup every 1440 minutes + - DEFAULT_DUMP_BEGIN=0000 # Start backing up at midnight + - DEFAULT_CLEANUP_TIME=8640 # Cleanup backups after a week + + - DB01_TYPE=mariadb + - DB01_HOST=example-db-host + - DB01_NAME=example + - DB01_USER=example + - DB01_PASS=examplepassword + - DB01_DUMP_INTERVAL=30 # (override) Backup every 30 minutes + - DB01_DUMP_BEGIN=+1 # (override) Backup starts immediately + - DB01_CLEANUP_TIME=180 # (override) Cleanup backups they are older than 180 minutes + - DB01_CHECKSUM=SHA1 # (override) Create a SHA1 checksum + - DB01_COMPRESSION=GZ # (override) Compress with GZIP + + #- DB02_TYPE=postgres + #- DB02_HOST=example-postgres-host + #- DB02_NAME=example + #- DB02_USER=example + #- DB02_PASS=examplepassword + #- DB02_DUMP_INTERVAL=60 # (override) Backup every 60 minutes + #- DB02_DUMP_BEGIN=+10 # (override) Backup starts in ten minutes + #- DB02_CLEANUP_TIME=240 # (override) Cleanup backups they are older than 240 minutes + #- DB02_CHECKSUM=MD5 # (override) Create a SHA1 checksum + #- DB02_COMPRESSION=BZ # (override) Compress with BZIP + + restart: always + networks: + - example-db-network + +networks: + example-db-network: + name: example-db-network + diff --git a/examples/mssql-blobxfer/docker-compose.yml b/examples/mssql-blobxfer/compose.yml similarity index 60% rename from examples/mssql-blobxfer/docker-compose.yml rename to examples/mssql-blobxfer/compose.yml index efe648f..462834a 100644 --- a/examples/mssql-blobxfer/docker-compose.yml +++ b/examples/mssql-blobxfer/compose.yml @@ -1,13 +1,7 @@ -# +# # Example for Microsoft SQL Server # upload with blobxfer to azure storage -# - -version: '2' - -networks: - example-mssql-blobxfer-net: - name: example-mssql-blobxfer-net +# services: example-mssql-s3-db: @@ -32,7 +26,7 @@ services: # execute in terminal --> docker build -t tiredofit/db-backup-mssql-blobxfer . # replace --> image: tiredofit/db-backup-mssql # image: tiredofit/db-backup - image: tiredofit/db-backup-mssql-blobxfer + image: tiredofit/db-backup links: - example-mssql-s3-db volumes: @@ -40,30 +34,35 @@ services: - ./tmp/backups:/tmp/backups # shared tmp backup directory #- ./post-script.sh:/assets/custom-scripts/post-script.sh environment: - # - DEBUG_MODE=TRUE - - DB_TYPE=mssql - - DB_HOST=example-db-host - # - DB_PORT=1488 - # - DB_NAME=ALL # [ALL] not working on sql server. - # create database with name `test1` manually first - - DB_NAME=test1 # Create this database - - DB_USER=sa - - DB_PASS=5hQa0utRFBpIY3yhoIyE - - DB_DUMP_FREQ=1 # backup every 5 minute - # - DB_DUMP_BEGIN=0000 # backup starts immediately - - DB_CLEANUP_TIME=3 # clean backups they are older than 60 minutes - - ENABLE_CHECKSUM=TRUE - - CHECKSUM=SHA1 - - COMPRESSION=GZ - - SPLIT_DB=FALSE + - TIMEZONE=America/Vancouver - CONTAINER_ENABLE_MONITORING=FALSE + - CONTAINER_NAME=example-mssql-blobxfer-db-backup + # - DEBUG_MODE=TRUE + - DB01_TYPE=mssql + - DB01_HOST=example-db-host + # - DB01_PORT=1488 + # create database with name `test1` manually first + - DB01_NAME=test1 # Create this database + - DB01_USER=sa + - DB01_PASS=5hQa0utRFBpIY3yhoIyE + - DB01_DUMP_INTERVAL=5 # backup every 5 minute + # - DB01_DUMP_BEGIN=0000 # backup starts at midnight vs not set immediately + - DB01_CLEANUP_TIME=60 # clean backups they are older than 60 minutes + - DB01_CHECKSUM=SHA1 # Set Checksum to be SHA1 + - DB01_COMPRESSION=GZ # Set compression to use GZIP + + # === S3 Blobxfer === - - BACKUP_LOCATION=blobxfer + - DB01_BACKUP_LOCATION=blobxfer # Add here azure storage account - - BLOBXFER_STORAGE_ACCOUNT={TODO Add Storage Name} + - DB01_BLOBXFER_STORAGE_ACCOUNT={TODO Add Storage Name} # Add here azure storage account key - - BLOBXFER_STORAGE_ACCOUNT_KEY={TODO Add Key} - - BLOBXFER_REMOTE_PATH=docker-db-backup + - SB01_BLOBXFER_STORAGE_ACCOUNT_KEY={TODO Add Key} + - DB01_BLOBXFER_REMOTE_PATH=docker-db-backup restart: always networks: - example-mssql-blobxfer-net: \ No newline at end of file + example-mssql-blobxfer-net: + +networks: + example-mssql-blobxfer-net: + name: example-mssql-blobxfer-net diff --git a/examples/mssql/docker-compose.yml b/examples/mssql/compose.yml similarity index 69% rename from examples/mssql/docker-compose.yml rename to examples/mssql/compose.yml index 4e9554c..b2ae491 100644 --- a/examples/mssql/docker-compose.yml +++ b/examples/mssql/compose.yml @@ -1,12 +1,6 @@ -# +# # Example for Microsoft SQL Server -# - -version: '2' - -networks: - example-mssql-net: - name: example-mssql-net +# services: example-mssql-db: @@ -31,7 +25,7 @@ services: # execute in terminal --> docker build -t tiredofit/db-backup-mssql . # replace --> image: tiredofit/db-backup-mssql # image: tiredofit/db-backup - image: tiredofit/db-backup-mssql + image: tiredofit/db-backup links: - example-mssql-db volumes: @@ -39,23 +33,28 @@ services: - ./tmp/backups:/tmp/backups # shared tmp backup directory #- ./post-script.sh:/assets/custom-scripts/post-script.sh environment: + - TIMEZONE=America/Vancouver + - CONTAINER_ENABLE_MONITORING=FALSE + - CONTAINER_NAME=example-mssql-blobxfer-db-backup + # - DEBUG_MODE=TRUE - - DB_TYPE=mssql - - DB_HOST=example-db-host + - DB01_TYPE=mssql + - DB01_HOST=example-db-host # - DB_PORT=1488 # - DB_NAME=ALL # [ALL] not working on sql server. # create database with name `test1` manually first - - DB_NAME=test1 - - DB_USER=sa - - DB_PASS=5hQa0utRFBpIY3yhoIyE - - DB_DUMP_FREQ=1 # backup every minute - # - DB_DUMP_BEGIN=0000 # backup starts immediately - - DB_CLEANUP_TIME=5 # clean backups they are older than 5 minute - - ENABLE_CHECKSUM=FALSE - - CHECKSUM=SHA1 - - COMPRESSION=GZ - - SPLIT_DB=FALSE - - CONTAINER_ENABLE_MONITORING=FALSE + - DB01_NAME=test1 + - DB01_USER=sa + - DB01_PASS=5hQa0utRFBpIY3yhoIyE + - DB01_DUMP_INTERVAL=1 # backup every minute + # - DB01_DUMP_BEGIN=0000 # backup starts at midnight vs unset immediately + - DB01_CLEANUP_TIME=5 # clean backups they are older than 5 minute + - DB01_CHECKSUM=NONE + - DB01_COMPRESSION=GZ restart: always networks: - example-mssql-net: \ No newline at end of file + example-mssql-net: + +networks: + example-mssql-net: + name: example-mssql-net diff --git a/examples/mysql/docker-compose.yml b/examples/mysql/docker-compose.yml deleted file mode 100644 index f306aae..0000000 --- a/examples/mysql/docker-compose.yml +++ /dev/null @@ -1,53 +0,0 @@ -version: '2' - -networks: - example-db-network: - name: example-db-network - -services: - example-db: - hostname: example-db-host - container_name: example-db - image: mariadb:latest - ports: - - 13306:3306 - volumes: - - ./db:/var/lib/mysql - environment: - - MYSQL_ROOT_PASSWORD=examplerootpassword - - MYSQL_DATABASE=example - - MYSQL_USER=example - - MYSQL_PASSWORD=examplepassword - restart: always - networks: - - example-db-network - - example-db-backup: - container_name: example-db-backup - image: tiredofit/db-backup - links: - - example-db - volumes: - - ./backups:/backup - #- ./post-script.sh:/assets/custom-scripts/post-script.sh - environment: - - TIMEZONE=America/Vancouver - - CONTAINER_ENABLE_MONITORING=FALSE - # - DEBUG_MODE=TRUE - - DB_TYPE=mariadb - - DB_HOST=example-db-host - - DB_NAME=example - - DB_USER=example - - DB_PASS=examplepassword - - DB_DUMP_FREQ=1 # backup every minute - # - DB_DUMP_BEGIN=0000 # backup starts immediately - - DB_CLEANUP_TIME=5 # clean backups they are older than 5 minute - - CHECKSUM=SHA1 - - COMPRESSION=GZ - - SPLIT_DB=FALSE - - restart: always - networks: - - example-db-network - - diff --git a/examples/mysql/post-script.sh b/examples/post-script.sh similarity index 92% rename from examples/mysql/post-script.sh rename to examples/post-script.sh index 4307134..98e8e09 100644 --- a/examples/mysql/post-script.sh +++ b/examples/post-script.sh @@ -4,7 +4,7 @@ # #### $1=EXIT_CODE (After running backup routine) # #### $2=DB_TYPE (Type of Backup) # #### $3=DB_HOST (Backup Host) -# #### #4=DB_NAME (Name of Database backed up +# #### #4=DB_NAME (Name of Database backed up) # #### $5=BACKUP START TIME (Seconds since Epoch) # #### $6=BACKUP FINISH TIME (Seconds since Epoch) # #### $7=BACKUP TOTAL TIME (Seconds between Start and Finish) diff --git a/install/assets/dbbackup/template-dbbackup/run b/install/assets/dbbackup/template-dbbackup/run new file mode 100755 index 0000000..e468c84 --- /dev/null +++ b/install/assets/dbbackup/template-dbbackup/run @@ -0,0 +1,107 @@ +#!/command/with-contenv bash + +source /assets/functions/00-container +PROCESS_NAME="db-backup{{BACKUP_NUMBER}}-scheduler" +check_container_initialized +check_service_initialized init 10-db-backup +source /assets/functions/10-db-backup +source /assets/defaults/10-db-backup +bootstrap_variables backup_init {{BACKUP_NUMBER}} +bootstrap_variables parse_variables {{BACKUP_NUMBER}} +PROCESS_NAME="{{BACKUP_NUMBER}}-${backup_job_db_host}__${backup_job_db_name}" + + +trap ctrl_c INT + +if [[ "${MODE,,}" =~ "standalone" ]] || [ "${1,,}" = "manual" ] || [ "${1,,}" = "now" ]; then + print_debug "Detected Manual Mode" + persist=false + backup_job_backup_begin=+0 +else + silent sleep {{BACKUP_NUMBER}} + time_last_run=0 + time_current=$(date +'%s') + + if [[ "${backup_job_backup_begin}" =~ ^\+(.*)$ ]]; then + print_debug "BACKUP_BEGIN is a jump of minute starting with +" + timer plusvalue + elif [[ "${backup_job_backup_begin}" =~ ^[0-9]{4}$ ]]; then + print_debug "BACKUP_BEGIN is a HHMM value" + timer time + elif [[ "${backup_job_backup_begin}" =~ ([0-9]{4})-([0-9]{2})-([0-9]{2})[[:space:]]([0-9]{2}):([0-9]{2}):([0-9]{2}) ]]; then + print_debug "BACKUP_BEGIN is a full date timestamp" + timer datetime + elif echo "${backup_job_backup_begin//\*/#}" | grep -qP "^(((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|#) ?){5}$" ; then + print_debug "BACKUP_BEGIN is a cron expression" + time_last_run=$(date +"%s") + timer cron "${backup_job_backup_begin}" "${time_current}" "${time_last_run}" + else + print_error "_BACKUP_BEGIN is invalid - Unable to perform scheduling" + cat < "${backup_job_blackout_start}" ]] && [[ "${time_current_hour_minute}" < "${backup_job_blackout_finish}" ]] ; then + blackout=true + else + blackout=false + fi + fi + + if var_true "${blackout}" ; then + print_notice "Detected Blackout Period - Not performing backup operations" + else + timer job start + process_limiter + echo "{{BACKUP_NUMBER}}" >> /tmp/.container/db-backup-backups + print_debug "Backup {{BACKUP_NUMBER}} routines started time: $(date +'%Y-%m-%d %T %Z')" + bootstrap_filesystem + check_availability + backup_"${dbtype,,}" + timer job stop + if [ -z "${exitcode_backup}" ] ; then exitcode_backup="0" ; fi + print_info "Backup {{BACKUP_NUMBER}} routines finish time: $(date -d @"${backup_job_finish_time}" +'%Y-%m-%d %T %Z') with exit code ${exitcode_backup}" + print_notice "Backup {{BACKUP_NUMBER}} routines time taken: $(echo "${backup_job_total_time}" | awk '{printf "Hours: %d Minutes: %02d Seconds: %02d", $1/3600, ($1/60)%60, $1%60}')" + sed -i "/^{{BACKUP_NUMBER}}/d" /tmp/.container/db-backup-backups + fi + + symlink_log + + if var_false "${persist}" ; then + print_debug "Exiting due to manual mode" + exit "${exitcode_backup}"; + else + if var_true "${stop_scheduler_backup}" ; then + print_error "Stopping backup_scheduler {{BACKUP_NUMBER}} due to detected errors. Fix and restart container." + s6-svc -d /var/run/s6/legacy-services/dbbackup-{{BACKUP_NUMBER}} + else + if [ ! "${time_cron}" = "true" ]; then + print_notice "Sleeping for another $(($backup_job_backup_interval*60-backup_job_total_time)) seconds. Waking up at $(date -d@"$(( $(date +%s)+$(($backup_job_backup_interval*60-backup_job_total_time))))" +'%Y-%m-%d %T %Z') " + silent sleep $(($backup_job_backup_interval*60-backup_job_total_time)) + else + time_last_run=$(date +"%s") + timer cron "${backup_job_backup_begin}" "${time_current}" "${time_last_run}" + print_notice "Sleeping for another ${time_wait} seconds. Waking up at $(date -d@"${time_future}" +'%Y-%m-%d %T %Z') " + silent sleep "${time_wait}" + fi + fi + fi +done diff --git a/install/assets/defaults/10-db-backup b/install/assets/defaults/10-db-backup index 60280d8..4f2b465 100644 --- a/install/assets/defaults/10-db-backup +++ b/install/assets/defaults/10-db-backup @@ -1,32 +1,43 @@ #!/command/with-contenv bash -BACKUP_LOCATION=${BACKUP_LOCATION:-"FILESYSTEM"} -BLOBXFER_REMOTE_PATH=${BLOBXFER_REMOTE_PATH:-"/docker-db-backup"} -CHECKSUM=${CHECKSUM:-"MD5"} -COMPRESSION=${COMPRESSION:-"ZSTD"} -COMPRESSION_LEVEL=${COMPRESSION_LEVEL:-"3"} -CREATE_LATEST_SYMLINK=${CREATE_LATEST_SYMLINK:-"TRUE"} -DB_DUMP_BEGIN=${DB_DUMP_BEGIN:-+0} -DB_DUMP_FREQ=${DB_DUMP_FREQ:-1440} -DB_DUMP_TARGET=${DB_DUMP_TARGET:-"/backup"} -DB_DUMP_TARGET_ARCHIVE=${DB_DUMP_TARGET_ARCHIVE:-"${DB_DUMP_TARGET}/archive/"} -ENABLE_CHECKSUM=${ENABLE_CHECKSUM:-"TRUE"} -ENABLE_PARALLEL_COMPRESSION=${ENABLE_PARALLEL_COMPRESSION:-"TRUE"} +BACKUP_JOB_CONCURRENCY=${BACKUP_JOB_CONCURRENCY:-"1"} +DEFAULT_BACKUP_BEGIN=${DEFAULT_BACKUP_BEGIN:-+0} +DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440} +DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440} +DEFAULT_BACKUP_LOCATION=${DEFAULT_BACKUP_LOCATION:-"FILESYSTEM"} +DEFAULT_BLOBXFER_REMOTE_PATH=${DEFAULT_BLOBXFER_REMOTE_PATH:-"/docker-db-backup"} +DEFAULT_CHECKSUM=${DEFAULT_CHECKSUM:-"MD5"} +DEFAULT_COMPRESSION=${DEFAULT_COMPRESSION:-"ZSTD"} +DEFAULT_COMPRESSION_LEVEL=${DEFAULT_COMPRESSION_LEVEL:-"3"} +DEFAULT_CREATE_LATEST_SYMLINK=${DEFAULT_CREATE_LATEST_SYMLINK:-"TRUE"} +DEFAULT_ENABLE_PARALLEL_COMPRESSION=${DEFAULT_ENABLE_PARALLEL_COMPRESSION:-"TRUE"} +DEFAULT_ENCRYPT=${DEFAULT_ENCRYPT:-"FALSE"} +DEFAULT_FILESYSTEM_PATH=${DEFAULT_FILESYSTEM_PATH:-"/backup"} +DEFAULT_FILESYSTEM_PERMISSION=${DEFAULT_FILESYSTEM_PERMISSION:-"700"} +DEFAULT_FILESYSTEM_ARCHIVE_PATH=${DEFAULT_FILESYSTEM_ARCHIVE_PATH:-"${DEFAULT_FILESYSTEM_PATH}/archive/"} +DEFAULT_LOG_LEVEL=${DEFAULT_LOG_LEVEL:-"notice"} +DEFAULT_MYSQL_ENABLE_TLS=${DEFAULT_MYSQL_ENABLE_TLS:-"FALSE"} +DEFAULT_MYSQL_EVENTS=${DEFAULT_MYSQL_EVENTS:-"TRUE"} +DEFAULT_MYSQL_MAX_ALLOWED_PACKET=${DEFAULT_MYSQL_MAX_ALLOWED_PACKET:-"512M"} +DEFAULT_MYSQL_SINGLE_TRANSACTION=${DEFAULT_MYSQL_SINGLE_TRANSACTION:-"TRUE"} +DEFAULT_MYSQL_STORED_PROCEDURES=${DEFAULT_MYSQL_STORED_PROCEDURES:-"TRUE"} +DEFAULT_MYSQL_TLS_CA_FILE=${DEFAULT_MYSQL_TLS_CA_FILE:-"/etc/ssl/cert.pem"} +DEFAULT_MYSQL_TLS_VERIFY=${DEFAULT_MYSQL_TLS_VERIFY:-"FALSE"} +DEFAULT_MYSQL_TLS_VERSION=${DEFAULT_MYSQL_TLS_VERSION:-"TLSv1.1,TLSv1.2,TLSv1.3"} +DEFAULT_MSSQL_MODE=${DEFAULT_MSSQL_MODE:-"database"} +DEFAULT_PARALLEL_COMPRESSION_THREADS=${DEFAULT_PARALLEL_COMPRESSION_THREADS:-"$(nproc)"} +DEFAULT_RESOURCE_OPTIMIZED=${DEFAULT_RESOURCE_OPTIMIZED:-"FALSE"} +DEFAULT_S3_CERT_SKIP_VERIFY=${DEFAULT_S3_CERT_SKIP_VERIFY:-"TRUE"} +DEFAULT_S3_PROTOCOL=${DEFAULT_S3_PROTOCOL:-"https"} +DEFAULT_SCRIPT_LOCATION_PRE=${DEFAULT_SCRIPT_LOCATION_PRE:-"/assets/scripts/pre/"} +DEFAULT_SCRIPT_LOCATION_POST=${DEFAULT_SCRIPT_LOCATION_POST:-"/assets/scripts/post/"} +DEFAULT_SIZE_VALUE=${DEFAULT_SIZE_VALUE:-"bytes"} +DEFAULT_SKIP_AVAILABILITY_CHECK=${DEFAULT_SKIP_AVAILABILITY_CHECK:-"FALSE"} +DEFAULT_SPLIT_DB=${DEFAULT_SPLIT_DB:-"TRUE"} +LOG_PATH=${LOG_PATH:-"/logs"} MANUAL_RUN_FOREVER=${MANUAL_RUN_FOREVER:-"TRUE"} MODE=${MODE:-"AUTO"} -MYSQL_ENABLE_TLS=${MYSQL_ENABLE_TLS:-"FALSE"} -MYSQL_MAX_ALLOWED_PACKET=${MYSQL_MAX_ALLOWED_PACKET:-"512M"} -MYSQL_SINGLE_TRANSACTION=${MYSQL_SINGLE_TRANSACTION:-"TRUE"} -MYSQL_STORED_PROCEDURES=${MYSQL_STORED_PROCEDURES:-"TRUE"} -MYSQL_TLS_CA_FILE=${MYSQL_TLS_CA_FILE:-"/etc/ssl/cert.pem"} -MYSQL_TLS_VERIFY=${MYSQL_TLS_VERIFY:-"FALSE"} -MYSQL_TLS_VERSION=${MYSQL_TLS_VERSION:-"TLSv1.1,TLSv1.2,TLSv1.3"} -PARALLEL_COMPRESSION_THREADS=${PARALLEL_COMPRESSION_THREADS:-"$(nproc)"} -S3_CERT_SKIP_VERIFY=${S3_CERT_SKIP_VERIFY:-"TRUE"} -S3_PROTOCOL=${S3_PROTOCOL:-"https"} -SCRIPT_LOCATION_PRE=${SCRIPT_LOCATION_PRE:-"/assets/scripts/pre/"} -SCRIPT_LOCATION_POST=${SCRIPT_LOCATION_POST:-"/assets/scripts/post/"} -SIZE_VALUE=${SIZE_VALUE:-"bytes"} -SKIP_AVAILABILITY_CHECK=${SKIP_AVAILABILITY_CHECK:-"FALSE"} -SPLIT_DB=${SPLIT_DB:-"TRUE"} -TEMP_LOCATION=${TEMP_LOCATION:-"/tmp/backups"} + +TEMP_PATH=${TEMP_PATH:-"/tmp/backups"} +if [ -n "${TEMP_LOCATION}" ] ; then TEMP_PATH=${TEMP_LOCATION:-"/tmp/backups"} ; fi # To be removed 4.3.0 + diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 9116abb..f617ebb 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -1,196 +1,460 @@ #!/command/with-contenv bash +bootstrap_filesystem() { + if var_true "${DEBUG_BOOTSTRAP_FILESYSTEM}" ; then debug on; fi + if [ ! -d "${backup_job_filesystem_path}" ]; then + mkdir -p "${backup_job_filesystem_path}" + fi + if [ "$(stat -c %U "${backup_job_filesystem_path}")" != "dbbackup" ] ; then chown -R dbbackup:dbbackup "${backup_job_filesystem_path}" ; fi + if [ "$(stat -c %a "${backup_job_filesystem_path}")" != "${backup_job_filesystem_permission}" ] ; then chmod -R "${backup_job_filesystem_permission}" "${backup_job_filesystem_path}" ; fi + + if [ -d "${backup_job_filesystem_archive_path}" ]; then + if [ "$(stat -c %U "${backup_job_filesystem_archive_path}")" != "dbbackup" ] ; then chown -R dbbackup:dbbackup "${backup_job_filesystem_archive_path}" ; fi + if [ "$(stat -c %a "${backup_job_filesystem_archive_path}")" != "${backup_job_filesystem_permission}" ] ; then chmod -R "${backup_job_filesystem_permission}" "${backup_job_filesystem_archive_path}" ; fi + fi + + if [ ! -d "${LOG_PATH}" ]; then + mkdir -p "${LOG_PATH}" + fi + + if [ "$(stat -c %U "${LOG_PATH}")" != "dbbackup" ] ; then chown dbbackup:dbbackup "${LOG_PATH}" ; fi + if [ ! -d "${LOG_PATH}"/"$(date +'%Y%m%d')" ]; then run_as_user mkdir -p "${LOG_PATH}"/"$(date +'%Y%m%d')"; fi + if [ "$(stat -c %a "${LOG_PATH}")" != "755" ] ; then chmod -R 755 "${LOG_PATH}" ; fi + + if [ ! -d "${TEMP_PATH}" ]; then + mkdir -p "${TEMP_PATH}" + fi + if [ "$(stat -c %U "${TEMP_PATH}")" != "dbbackup" ] ; then chown -R dbbackup:dbbackup "${TEMP_PATH}" ; fi + if var_true "${DEBUG_BOOTSTRAP_FILESYSTEM}" ; then debug off; fi +} + bootstrap_variables() { - sanity_var DB_TYPE "Set appropriate DB_TYPE" - transform_file_var \ - DB_HOST \ - DB_NAME \ - DB_PORT \ - DB_USER \ - DB_PASS + if var_true "${DEBUG_BOOTSTRAP_VARIABLES}" ; then debug on; fi + backup_init() { + backup_instance_number=${1} + backup_instance_vars=$(mktemp) + set -o posix ; set | grep -oE "^backup_job_.*=" | grep -oE ".*=" | sed "/--/d" > "${backup_instance_vars}" + while read -r backup_instance_var ; do + unset "$(echo "${backup_instance_var}" | cut -d = -f 1)" + done < "${backup_instance_vars}" - case "${DB_TYPE,,}" in - couch* ) - dbtype=couch - DB_PORT=${DB_PORT:-5984} - sanity_var DB_USER - sanity_var DB_PASS - ;; - influx* ) - dbtype=influx - case "${INFLUX_VERSION}" in - 1) DB_PORT=${DB_PORT:-8088} ;; - 2) DB_PORT=${DB_PORT:-8086} ;; - esac - sanity_var DB_USER - sanity_var DB_PASS - sanity_var INFLUX_VERSION "What InfluxDB version you are backing up from '1' or '2'" - ;; - mongo* ) - dbtype=mongo - transform_file_var MONGO_CUSTOM_URI - if [ -n "${MONGO_CUSTOM_URI}" ] ; then - mongo_uri_proto=$(echo "${MONGO_CUSTOM_URI}" | grep :// | sed -e's,^\(.*://\).*,\1,g') - mongo_uri_scratch="${MONGO_CUSTOM_URI/${mongo_uri_proto}/}" - mongo_uri_username_password="$(echo "${mongo_uri_scratch}" | grep @ | rev | cut -d@ -f2- | rev)" - if [ -n "${mongo_uri_username_password}" ]; then mongo_uri_scratch="$(echo "${mongo_uri_scratch}" | rev | cut -d@ -f1 | rev)" ; fi - mongo_uri_port="$(echo "${mongo_uri_scratch}" | grep : | rev | cut -d: -f2- | rev)" - if [ -n "${mongo_uri_port}" ]; then mongo_uri_port="$(echo "${mongo_uri_scratch}" | rev | cut -d: -f1 | cut -d/ -f2 | rev)" ; fi - mongo_uri_hostname="$(echo "${mongo_uri_scratch}" | cut -d/ -f1 | cut -d: -f1 )" - mongo_uri_database="$(echo "${mongo_uri_scratch}" | cut -d/ -f2 | cut -d? -f1 )" - mongo_uri_options="$(echo "${mongo_uri_scratch}" | cut -d/ -f2 | cut -d? -f2 )" - DB_NAME=${DB_NAME:-"${mongo_uri_database,,}"} - DB_HOST=${DB_HOST:-"${mongo_uri_hostname,,}"} + transform_file_var \ + DEFAULT_AUTH \ + DEFAULT_TYPE \ + DEFAULT_HOST \ + DEFAULT_PORT \ + DEFAULT_NAME \ + DEFAULT_NAME_EXCLUDE \ + DEFAULT_USER \ + DEFAULT_PASS \ + DEFAULT_ENCRYPT_PASSPHRASE \ + DEFAULT_ENCRYPT_PUBKEY \ + DEFAULT_MONGO_CUSTOM_URI \ + DEFAULT_MYSQL_TLS_CA_FILE \ + DEFAULT_MYSQL_TLS_backup_job_filenameCERT_FILE \ + DEFAULT_MYSQL_TLS_KEY_FILE \ + DEFAULT_S3_BUCKET \ + DEFAULT_S3_KEY_ID \ + DEFAULT_S3_KEY_SECRET \ + DEFAULT_S3_PATH \ + DEFAULT_S3_REGION \ + DEFAULT_S3_HOST \ + DEFAULT_S3_PROTOCOL \ + DEFAULT_S3_EXTRA_OPTS \ + DEFAULT_S3_CERT_CA_FILE \ + DEFAULT_BLOBXFER_STORAGE_ACCOUNT \ + DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY \ + DEFAULT_BLOBXFER_REMOTE_PATH \ + DB"${backup_instance_number}"_AUTH \ + DB"${backup_instance_number}"_TYPE \ + DB"${backup_instance_number}"_HOST \ + DB"${backup_instance_number}"_PORT \ + DB"${backup_instance_number}"_NAME \ + DB"${backup_instance_number}"_NAME_EXCLUDE \ + DB"${backup_instance_number}"_USER \ + DB"${backup_instance_number}"_PASS \ + DB"${backup_instance_number}"_ENCRYPT_PASSPHRASE \ + DB"${backup_instance_number}"_ENCRYPT_PUBKEY \ + DB"${backup_instance_number}"_MONGO_CUSTOM_URI \ + DB"${backup_instance_number}"_MYSQL_TLS_CA_FILE \ + DB"${backup_instance_number}"_MYSQL_TLS_CERT_FILE \ + DB"${backup_instance_number}"_MYSQL_TLS_KEY_FILE \ + DB"${backup_instance_number}"_S3_BUCKET \ + DB"${backup_instance_number}"_S3_KEY_ID \ + DB"${backup_instance_number}"_S3_KEY_SECRET \ + DB"${backup_instance_number}"_S3_PATH \ + DB"${backup_instance_number}"_S3_REGION \ + DB"${backup_instance_number}"_S3_HOST \ + DB"${backup_instance_number}"_S3_PROTOCOL \ + DB"${backup_instance_number}"_S3_EXTRA_OPTS \ + DB"${backup_instance_number}"_S3_CERT_CA_FILE \ + DB"${backup_instance_number}"_BLOBXFER_STORAGE_ACCOUNT \ + DB"${backup_instance_number}"_BLOBXFER_STORAGE_ACCOUNT_KEY \ + DB"${backup_instance_number}"_BLOBXFER_REMOTE_PATH \ + BLOBXFER_STORAGE_ACCOUNT \ + BLOBXFER_STORAGE_KEY \ + DB_HOST \ + DB_NAME \ + DB_PORT \ + DB_USER \ + DB_PASS \ + MONGO_CUSTOM_URI \ + DB_AUTH \ + S3_BUCKET \ + S3_KEY_ID \ + S3_KEY_SECRET \ + S3_PATH \ + S3_REGION \ + S3_HOST \ + S3_PROTOCOL \ + S3_EXTRA_OPTS + ## Legacy after DEFAULT + set -o posix ; set | grep -E "^DB${backup_instance_number}_|^DEFAULT_|^DB_|^ARCHIVE|^BACKUP_|^BLOBXFER_|^CHECKSUM|^COMPRESSION|^CREATE_|^ENABLE_|^EXTRA_|^GZ_|^INFLUX_|^MYSQL_|^MONGO_|^PARALLEL|^PRE_|^POST_|^S3|^SKIP|^SPLIT" > "${backup_instance_vars}" + + ## Legacy checks from removed variables + if [ -n "${ENABLE_CHECKSUM}" ]; then + print_warn "Deprecated and unsupported variable 'ENABLE_CHECKSUM' detected - Please upgrade your variables as they will be removed in version 4.3.0" + if var_false "${ENABLE_CHECKSUM}" ; then + sed -i "/DEFAULT_CHECKSUM=/d" "${backup_instance_vars}" + echo "DEFAULT_CHECKSUM=NONE" >> "${backup_instance_vars}" + fi + fi + + if [ -n "${DB_DUMP_BEGIN}" ]; then + print_warn "Deprecated and unsupported variable 'DB_DUMP_BEGIN' detected - Please upgrade your variables as they will be removed in version 4.3.0" + sed -i "/DEFAULT_BACKUP_BEGIN=/d" "${backup_instance_vars}" + echo "DEFAULT_BACKUP_BEGIN=${DB_DUMP_BEGIN}" >> "${backup_instance_vars}" + fi + + if [ -n "${DB_DUMP_FREQ}" ]; then + print_warn "Deprecated and unsupported variable 'DB_DUMP_FREQ' detected - Please upgrade your variables as they will be removed in version 4.3.0" + sed -i "/DEFAULT_BACKUP_INTERVAL=/d" "${backup_instance_vars}" + echo "DEFAULT_BACKUP_INTERVAL=${DB_DUMP_FREQ}" >> "${backup_instance_vars}" + fi + + if [ -n "${DB_DUMP_TARGET}" ]; then + print_warn "Deprecated and unsupported variable 'DB_DUMP_TARGET' detected - Please upgrade your variables as they will be removed in version 4.3.0" + sed -i "/DEFAULT_FILESYSTEM_PATH=/d" "${backup_instance_vars}" + echo "DEFAULT_FILESYSTEM_PATH=${DB_DUMP_TARGET}" >> "${backup_instance_vars}" + fi + + if [ -n "${DB_DUMP_TARGET_ARCHIVE}" ]; then + print_warn "Deprecated and unsupported variable 'DB_DUMP_TARGET_ACRHIVE' detected - Please upgrade your variables as they will be removed in version 4.3.0" + sed -i "/DEFAULT_FILESYSTEM_ARCHIVE_PATH=/d" "${backup_instance_vars}" + echo "DEFAULT_FILESYSTEM_ARCHIVE_PATH=${DB_DUMP_TARGET_ARCHIVE}" >> "${backup_instance_vars}" + fi + + if [ -n "${EXTRA_DUMP_OPTS}" ]; then + print_warn "Deprecated and unsupported variable 'EXTRA_DUMP_OPTS' detected - Please upgrade your variables as they will be removed in version 4.3.0" + sed -i "/DEFAULT_EXTRA_BACKUP_OPTS=/d" "${backup_instance_vars}" + echo "DEFAULT_EXTRA_BACKUP_OPTS=${EXTRA_DUMP_OPTS}" >> "${backup_instance_vars}" + fi + ## + + transform_backup_instance_variable() { + if grep -q "^DB${1}_${2}=" "${backup_instance_vars}" && [ "$(grep "^DB${1}_${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then + export "$3"="$(grep "^DB${1}_${2}=" "${backup_instance_vars}" | cut -d = -f2)" + elif grep -q "^DB_${2}=" "${backup_instance_vars}" && [ "$(grep "^DB_${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then + # Allow old legacy work, perhaps remove old DB_ functionality in future? This should allow for seamless upgrades + #print_warn "Legacy Variable 'DB_${2}'' detected - Please upgrade your variables as they will be removed in version 4.3.0" + export "$3"="$(grep "^DB_${2}=" "${backup_instance_vars}" | cut -d = -f2)" + elif grep -q "^${2}=" "${backup_instance_vars}" && [ "$(grep "^${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then + print_warn "Legacy unsupported variable '${2}' detected - Please upgrade your variables as they will be removed in version 4.3.0" + export "$3"="$(grep "^${2}=" "${backup_instance_vars}" | cut -d = -f2)" + elif grep -q "^DEFAULT_${2}=" "${backup_instance_vars}" && [ "$(grep "^DEFAULT_${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then + export "$3"="$(grep "^DEFAULT_${2}=" "${backup_instance_vars}" | cut -d = -f2)" + fi + } + + transform_backup_instance_variable "${backup_instance_number}" ARCHIVE_TIME backup_job_archive_time + transform_backup_instance_variable "${backup_instance_number}" AUTH backup_job_db_auth + transform_backup_instance_variable "${backup_instance_number}" BACKUP_BEGIN backup_job_backup_begin + transform_backup_instance_variable "${backup_instance_number}" BACKUP_INTERVAL backup_job_backup_interval + transform_backup_instance_variable "${backup_instance_number}" BACKUP_GLOBALS backup_job_backup_pgsql_globals + transform_backup_instance_variable "${backup_instance_number}" BACKUP_LOCATION backup_job_backup_location + transform_backup_instance_variable "${backup_instance_number}" BLACKOUT_BEGIN backup_job_snapshot_blackout_start + transform_backup_instance_variable "${backup_instance_number}" BLACKOUT_END backup_job_snapshot_blackout_finish + transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_REMOTE_PATH backup_job_blobxfer_remote_path + transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_ACCOUNT backup_job_blobxfer_storage_account + transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_KEY backup_job_blobxfer_storage_key + transform_backup_instance_variable "${backup_instance_number}" CHECKSUM backup_job_checksum + transform_backup_instance_variable "${backup_instance_number}" CLEANUP_TIME backup_job_cleanup_time + transform_backup_instance_variable "${backup_instance_number}" COMPRESSION backup_job_compression + transform_backup_instance_variable "${backup_instance_number}" COMPRESSION_LEVEL backup_job_compression_level + transform_backup_instance_variable "${backup_instance_number}" CREATE_LATEST_SYMLINK backup_job_create_latest_symlink + transform_backup_instance_variable "${backup_instance_number}" ENABLE_PARALLEL_COMPRESSION backup_job_parallel_compression + transform_backup_instance_variable "${backup_instance_number}" ENCRYPT backup_job_encrypt + transform_backup_instance_variable "${backup_instance_number}" ENCRYPT_PASSPHRASE backup_job_encrypt_passphrase + transform_backup_instance_variable "${backup_instance_number}" ENCRYPT_PUBKEY backup_job_encrypt_pubkey + transform_backup_instance_variable "${backup_instance_number}" EXTRA_DUMP_OPTS backup_job_extra_dump_opts + transform_backup_instance_variable "${backup_instance_number}" EXTRA_ENUMERATION_OPTS backup_job_extra_enumeration_opts + transform_backup_instance_variable "${backup_instance_number}" EXTRA_OPTS backup_job_extra_opts + transform_backup_instance_variable "${backup_instance_number}" FILESYSTEM_ARCHIVE_PATH backup_job_filesystem_archive_path + transform_backup_instance_variable "${backup_instance_number}" FILESYSTEM_PATH backup_job_filesystem_path + transform_backup_instance_variable "${backup_instance_number}" FILESYSTEM_PERMISSION backup_job_filesystem_permission + transform_backup_instance_variable "${backup_instance_number}" GZ_RSYNCABLE backup_job_gz_rsyncable + transform_backup_instance_variable "${backup_instance_number}" HOST backup_job_db_host + transform_backup_instance_variable "${backup_instance_number}" INFLUX_VERSION backup_job_influx_version + transform_backup_instance_variable "${backup_instance_number}" LOG_LEVEL backup_job_log_level + transform_backup_instance_variable "${backup_instance_number}" MONGO_CUSTOM_URI backup_job_mongo_custom_uri + transform_backup_instance_variable "${backup_instance_number}" MYSQL_ENABLE_TLS backup_job_mysql_enable_tls + transform_backup_instance_variable "${backup_instance_number}" MYSQL_EVENTS backup_job_mysql_events + transform_backup_instance_variable "${backup_instance_number}" MYSQL_MAX_ALLOWED_PACKET backup_job_mysql_max_allowed_packet + transform_backup_instance_variable "${backup_instance_number}" MYSQL_SINGLE_TRANSACTION backup_job_mysql_single_transaction + transform_backup_instance_variable "${backup_instance_number}" MYSQL_STORED_PROCEDURES backup_job_mysql_stored_procedures + transform_backup_instance_variable "${backup_instance_number}" MYSQL_TLS_CA_FILE backup_job_mysql_tls_ca_file + transform_backup_instance_variable "${backup_instance_number}" MYSQL_TLS_CERT_FILE backup_job_mysql_tls_cert_file + transform_backup_instance_variable "${backup_instance_number}" MYSQL_TLS_KEY_FILE backup_job_mysql_tls_key_file + transform_backup_instance_variable "${backup_instance_number}" MYSQL_TLS_VERIFY backup_job_mysql_tls_verify + transform_backup_instance_variable "${backup_instance_number}" MYSQL_TLS_VERSION backup_job_mysql_tls_version + transform_backup_instance_variable "${backup_instance_number}" MSSQL_MODE backup_job_mssql_mode + transform_backup_instance_variable "${backup_instance_number}" NAME backup_job_db_name + transform_backup_instance_variable "${backup_instance_number}" NAME_EXCLUDE backup_job_db_name_exclude + transform_backup_instance_variable "${backup_instance_number}" PARALLEL_COMPRESSION_THREADS backup_job_parallel_compression_threads + transform_backup_instance_variable "${backup_instance_number}" PASS backup_job_db_pass + transform_backup_instance_variable "${backup_instance_number}" PORT backup_job_db_port + transform_backup_instance_variable "${backup_instance_number}" POST_SCRIPT_X_VERIFY backup_job_post_script_x_verify + transform_backup_instance_variable "${backup_instance_number}" POST_SCRIPT backup_job_post_script + transform_backup_instance_variable "${backup_instance_number}" PRE_SCRIPT backup_job_pre_script + transform_backup_instance_variable "${backup_instance_number}" PRE_SCRIPT_X_VERIFY backup_job_pre_script_x_verify + transform_backup_instance_variable "${backup_instance_number}" RESOURCE_OPTIMIZED backup_job_resource_optimized + transform_backup_instance_variable "${backup_instance_number}" S3_BUCKET backup_job_s3_bucket + transform_backup_instance_variable "${backup_instance_number}" S3_CERT_CA_FILE backup_job_s3_cert_ca_file + transform_backup_instance_variable "${backup_instance_number}" S3_CERT_SKIP_VERIFY backup_job_s3_cert_skip_verify + transform_backup_instance_variable "${backup_instance_number}" S3_EXTRA_OPTS backup_job_s3_extra_opts + transform_backup_instance_variable "${backup_instance_number}" S3_HOST backup_job_s3_host + transform_backup_instance_variable "${backup_instance_number}" S3_KEY_ID backup_job_s3_key_id + transform_backup_instance_variable "${backup_instance_number}" S3_KEY_SECRET backup_job_s3_key_secret + transform_backup_instance_variable "${backup_instance_number}" S3_PATH backup_job_s3_path + transform_backup_instance_variable "${backup_instance_number}" S3_PROTOCOL backup_job_s3_protocol + transform_backup_instance_variable "${backup_instance_number}" S3_REGION backup_job_s3_region + transform_backup_instance_variable "${backup_instance_number}" SCRIPT_LOCATION_POST backup_job_script_location_post + transform_backup_instance_variable "${backup_instance_number}" SCRIPT_LOCATION_PRE backup_job_script_location_pre + transform_backup_instance_variable "${backup_instance_number}" SIZE_VALUE backup_job_size_value + transform_backup_instance_variable "${backup_instance_number}" SKIP_AVAILABILITY_CHECK backup_job_skip_availability_check + transform_backup_instance_variable "${backup_instance_number}" SPLIT_DB backup_job_split_db + transform_backup_instance_variable "${backup_instance_number}" TYPE backup_job_db_type + transform_backup_instance_variable "${backup_instance_number}" USER backup_job_db_user + + backup_job_backup_begin=$(echo "${backup_job_backup_begin}" | sed -e "s|'||g" -e 's|"||g') + rm -rf "${backup_instance_vars}" + } + + upgrade_lonely_variables() { + upgrade_lonely_variables_tmp=$(mktemp) + set -o posix; set | grep "^$1" | grep -v "^$1[0-9]._" > "${upgrade_lonely_variables_tmp}" + while read -r exist_var ; do + if [ ! -v "${1}"01_"$(echo "${exist_var}" | cut -d = -f1 | cut -d _ -f2)" ] ; then + export "${1}"01_"$(echo "${exist_var}" | cut -d = -f1 | cut -d _ -f2-9)"="$(echo "${exist_var}" | cut -d = -f2)" else - DB_PORT=${DB_PORT:-27017} - [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}" - [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}" - [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}" - transform_file_var DB_AUTH - [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}" + print_error "Variable ${1}01_$(echo "${exist_var}" | cut -d = -f1 | cut -d _ -f2) already exists, fix your variables and start again.." + exit 1 fi - ;; - "mysql" | "mariadb" ) - dbtype=mysql - DB_PORT=${DB_PORT:-3306} - sanity_var DB_NAME "Database Name to backup. Multiple seperated by commas" - transform_file_var DB_PASS - if [ -n "${DB_PASS}" ] ; then export MYSQL_PWD=${DB_PASS} ; fi - if var_true "${MYSQL_ENABLE_TLS}" ; then - if [ -n "${MYSQL_TLS_CA_FILE}" ] ; then - mysql_tls_args="--ssl_ca=${MYSQL_TLS_CA_FILE}" - fi - if [ -n "${MYSQL_TLS_CERT_FILE}" ] ; then - mysql_tls_args="${mysql_tls_args} --ssl_cert=${MYSQL_TLS_CERT_FILE}" - fi - if [ -n "${MYSQL_TLS_KEY_FILE}" ] ; then - mysql_tls_args="${mysql_tls_args} --ssl_key=${MYSQL_TLS_KEY_FILE}" - fi + done < "${upgrade_lonely_variables_tmp}" + rm -rf "${upgrade_lonely_variables_tmp}" + } - if var_true "${TLS_VERIFY}" ; then - mysql_tls_args="${mysql_tls_args} --sslverify-server-cert" - fi - - if [ -n "${MYSQL_TLS_VERSION}" ] ; then - mysql_tls_args="${mysql_tls_args} --tls_version=${MYSQL_TLS_VERSION}" - fi + parse_variables() { + local v_instance=${1} + check_var() { + ## Check is Variable is Defined + ## Usage: check_var transformed_varname real_varname "Description" + output_off + print_debug "Looking for existence of $2 environment variable" + if [ ! -v "$1" ]; then + print_error "No '$3' Entered! - Set '\$$2' environment variable - Halting Backup Number ${v_instance}" + s6-svc -d /var/run/s6/legacy-services/dbbackup-"${v_instance}" + exit 1 fi - ;; - "mssql" | "microsoftsql" ) - apkArch="$(apk --print-arch)"; \ - case "$apkArch" in - x86_64) mssql=true ;; - *) print_error "MSSQL cannot operate on $apkArch processor!" ; exit 1 ;; - esac - dbtype=mssql - DB_PORT=${DB_PORT:-1433} - ;; - postgres* | "pgsql" ) - dbtype=pgsql - DB_PORT=${DB_PORT:-5432} - [[ ( -n "${DB_PASS}" ) ]] && POSTGRES_PASS_STR="PGPASSWORD=${DB_PASS}" - sanity_var DB_NAME "Database Name to backup. Multiple seperated by commas" - ;; - "redis" ) - dbtype=redis - DB_PORT=${DB_PORT:-6379} - [[ ( -n "${DB_PASS}" ) ]] && REDIS_PASS_STR=" -a ${DB_PASS}" - ;; - sqlite* ) - dbtype=sqlite3 - ;; - * ) - print_error "I don't recognize 'DB_TYPE=${DB_TYPE}' - Exitting.." - exit 99 - ;; + output_on + } + + check_var backup_job_db_type DB"${v_instance}"_TYPE "appropriate database type" + + case "${backup_job_db_type,,}" in + couch* ) + dbtype=couch + backup_job_backup_job_db_port=${backup_job_db_port:-5984} + check_var backup_job_db_user DB"${v_instance}"_USER "database username" + check_var backup_job_db_pass DB"${v_instance}"_PASS "database password" + ;; + influx* ) + dbtype=influx + case "${backup_job_influx_version}" in + 1) backup_job_db_port=${backup_job_db_port:-8088} ;; + 2) backup_job_db_port=${backup_job_db_port:-8086} ;; + esac + check_var backup_job_db_user DB"${v_instance}"_USER "database username" + check_var backup_job_db_pass DB"${v_instance}"_PASS "database password" + check_var backup_job_influx_version DB"${v_instance}"_INFLUX_VERSION "InfluxDB version you are backing up from" + ;; + mongo* ) + dbtype=mongo + if [ -n "${backup_job_mongo_custom_uri}" ] ; then + mongo_uri_proto=$(echo "${backup_job_mongo_custom_uri}" | grep :// | sed -e's,^\(.*://\).*,\1,g') + mongo_uri_scratch="${backup_job_mongo_custom_uri/${mongo_uri_proto}/}" + mongo_uri_username_password="$(echo "${mongo_uri_scratch}" | grep @ | rev | cut -d@ -f2- | rev)" + if [ -n "${mongo_uri_username_password}" ]; then mongo_uri_scratch="$(echo "${mongo_uri_scratch}" | rev | cut -d@ -f1 | rev)" ; fi + mongo_uri_port="$(echo "${mongo_uri_scratch}" | grep : | rev | cut -d: -f2- | rev)" + if [ -n "${mongo_uri_port}" ]; then mongo_uri_port="$(echo "${mongo_uri_scratch}" | rev | cut -d: -f1 | cut -d/ -f2 | rev)" ; fi + mongo_uri_hostname="$(echo "${mongo_uri_scratch}" | cut -d/ -f1 | cut -d: -f1 )" + mongo_uri_database="$(echo "${mongo_uri_scratch}" | cut -d/ -f2 | cut -d? -f1 )" + mongo_uri_options="$(echo "${mongo_uri_scratch}" | cut -d/ -f2 | cut -d? -f2 )" + backup_job_db_name=${backup_job_db_name:-"${mongo_uri_database,,}"} + backup_job_db_host=${backup_job_db_host:-"${mongo_uri_hostname,,}"} + else + backup_job_db_port=${backup_job_db_port:-27017} + [[ ( -n "${backup_job_db_user}" ) ]] && MONGO_USER_STR=" --username ${backup_job_db_user}" + [[ ( -n "${backup_job_db_pass}" ) ]] && MONGO_PASS_STR=" --password ${backup_job_db_pass}" + [[ ( -n "${backup_job_db_name}" ) ]] && MONGO_DB_STR=" --db ${backup_job_db_name}" + [[ ( -n "${backup_job_db_auth}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${backup_job_db_auth}" + fi + ;; + "mysql" | "mariadb" ) + dbtype=mysql + backup_job_db_port=${backup_job_db_port:-3306} + check_var backup_job_db_name DB"${v_instance}"_NAME "database name. Seperate multiple with commas" + + if [ -n "${backup_job_db_pass}" ] ; then export MYSQL_PWD=${backup_job_db_pass} ; fi + if var_true "${backup_job_mysql_enable_tls}" ; then + if [ -n "${backup_job_mysql_tls_ca_file}" ] ; then + mysql_tls_args="--ssl_ca=${backup_job_mysql_tls_ca_file}" + fi + if [ -n "${backup_job_mysql_tls_cert_file}" ] ; then + mysql_tls_args="${mysql_tls_args} --ssl_cert=${backup_job_mysql_tls_cert_file}" + fi + if [ -n "${backup_job_mysql_tls_key_file}" ] ; then + mysql_tls_args="${mysql_tls_args} --ssl_key=${backup_job_mysql_tls_key_file}" + fi + + if var_true "${backup_job_mysql_tls_verify}" ; then + mysql_tls_args="${mysql_tls_args} --sslverify-server-cert" + fi + + if [ -n "${backup_job_mysql_tls_version}" ] ; then + mysql_tls_args="${mysql_tls_args} --tls_version=${backup_job_mysql_tls_version}" + fi + fi + ;; + "mssql" | "microsoftsql" ) + apkArch="$(apk --print-arch)"; \ + case "$apkArch" in + x86_64) mssql=true ;; + *) write_log error "MSSQL cannot operate on $apkArch processor!" ; exit 1 ;; + esac + dbtype=mssql + backup_job_db_port=${backup_job_db_port:-1433} + ;; + postgres* | "pgsql" ) + dbtype=pgsql + backup_job_db_port=${backup_job_db_port:-5432} + [[ ( -n "${backup_job_db_pass}" ) ]] && POSTGRES_PASS_STR="PGPASSWORD=${backup_job_db_pass}" + check_var backup_job_db_name DB"${v_instance}"_NAME "database name. Seperate multiple with commas" + ;; + "redis" ) + dbtype=redis + backup_job_db_port=${backup_job_db_port:-6379} + [[ ( -n "${backup_job_db_pass}" ) ]] && REDIS_PASS_STR=" -a ${backup_job_db_pass}" + ;; + sqlite* ) + dbtype=sqlite3 + ;; + * ) + write_log error "I don't recognize 'DB${v_instance}_TYPE=${backup_job_db_type}' - Exitting.." + exit 99 + ;; + esac + + if var_true "${backup_job_resource_optimized}" ; then play_fair="nice -19 ionice -c2" ; fi + } + + case "${1}" in + backup_init ) backup_init "$2" ;; + parse_variables) parse_variables "$2" ;; + upgrade ) upgrade_lonely_variables "$2" ;; esac - if [ "${BACKUP_LOCATION,,}" = "s3" ] || [ "${BACKUP_LOCATION,,}" = "minio" ] ; then - transform_file_var \ - S3_BUCKET \ - S3_KEY_ID \ - S3_KEY_SECRET \ - S3_PATH \ - S3_REGION \ - S3_HOST \ - S3_PROTOCOL \ - S3_EXTRA_OPTS \ - S3_CERT_CA_FILE - fi - - if [ "${BACKUP_LOCATION,,}" = "blobxfer" ] ; then - transform_file_var \ - BLOBXFER_STORAGE_ACCOUNT \ - BLOBXFER_STORAGE_KEY - fi + if var_true "${DEBUG_BOOTSTRAP_VARIABLES}" ; then debug off; fi } backup_couch() { prepare_dbbackup - target=couch_${DB_NAME}_${DB_HOST#*//}_${now}.txt - ltarget=couch_${DB_NAME}_${DB_HOST#*//} + backup_job_filename=couch_${backup_job_db_name}_${backup_job_db_host#*//}_${now}.txt + backup_job_filename_base=couch_${backup_job_db_name}_${backup_job_db_host#*//} compression - pre_dbbackup ${DB_NAME} - print_notice "Dumping CouchDB database: '${DB_NAME}' ${compression_string}" - curl -sSL -X GET ${DB_HOST}:${DB_PORT}/${DB_NAME}/_all_docs?include_docs=true ${compress_cmd} | $compress_cmd > "${TEMP_LOCATION}"/"${target}" + pre_dbbackup ${backup_job_db_name} + write_log notice "Dumping CouchDB database: '${backup_job_db_name}' ${compression_string}" + if var_true "${DEBUG_BACKUP_COUCH}" ; then debug on; fi + run_as_user curl -sSL -X GET ${backup_job_db_host}:${backup_job_db_port}/${backup_job_db_name}/_all_docs?include_docs=true | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${backup_job_filename}" > /dev/null exit_code=$? - check_exit_code backup $target + if var_true "${DEBUG_BACKUP_COUCH}" ; then debug off; fi + check_exit_code backup "${backup_job_filename}" + timer backup finish + file_encryption generate_checksum move_dbbackup - check_exit_code move $target - post_dbbackup ${DB_NAME} + check_exit_code move "${backup_job_filename}" + post_dbbackup ${backup_job_db_name} } backup_influx() { - if [ "${DB_NAME,,}" = "all" ] ; then - print_debug "Preparing to back up everything" + if [ "${backup_job_db_name,,}" = "all" ] ; then + write_log debug "Preparing to back up everything" db_names=justbackupeverything else - db_names=$(echo "${DB_NAME}" | tr ',' '\n') + db_names=$(echo "${backup_job_db_name}" | tr ',' '\n') fi - case "${INFLUX_VERSION,,}" in + case "${backup_job_db_influx_version,,}" in 1 ) for db in ${db_names}; do prepare_dbbackup if [ "${db}" != "justbackupeverything" ] ; then bucket="-db ${db}" ; else db=all ; fi - target=influx_${db}_${DB_HOST#*//}_${now} - ltarget=influx_${db}_${DB_HOST#*//} + backup_job_filename=influx_${db}_${backup_job_db_host#*//}_${now} + backup_job_filename_base=influx_${db}_${backup_job_db_host#*//} compression - pre_dbbackup $db - print_notice "Dumping Influx database: '${db}'" - influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} "${TEMP_LOCATION}"/"${target_dir}" + pre_dbbackup "${db}" + write_log notice "Dumping Influx database: '${db}'" + if var_true "${DEBUG_BACKUP_INFLUX}" ; then debug on; fi + run_as_user influxd backup ${influx_compression} ${bucket} -portable -host ${backup_job_db_host}:${backup_job_db_port} ${backup_job_extra_opts} ${backup_job_extra_dump_opts} "${TEMP_PATH}"/"${backup_job_filename_dir}" exit_code=$? - check_exit_code backup $target_dir - print_notice "Creating archive file of '${target_dir}' with tar ${compression_string}" - tar cf - "${TEMP_LOCATION}"/"${target_dir}" | $dir_compress_cmd > "${TEMP_LOCATION}"/"${target_dir}".tar"${extension}" - target=influx_${db}_${DB_HOST#*//}_${now}.tar${extension} - ltarget=influx_${db}_${DB_HOST#*//} + check_exit_code backup "${backup_job_filename_dir}" + write_log notice "Creating archive file of '${backup_job_filename_dir}' with tar ${compression_string}" + run_as_user tar cf - "${TEMP_PATH}"/"${backup_job_filename_dir}" | ${dir_compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${backup_job_filename_dir}".tar"${extension}" > /dev/null + backup_job_filename=influx_${db}_${backup_job_db_host#*//}_${now}.tar${extension} + backup_job_filename_base=influx_${db}_${backup_job_db_host#*//} + if var_true "${DEBUG_BACKUP_INFLUX}" ; then debug off; fi + timer backup finish + file_encryption generate_checksum move_dbbackup - check_exit_code move $target_dir - post_dbbackup $db + check_exit_code move "${backup_job_filename_dir}" + post_dbbackup "${db}" done ;; 2 ) for db in ${db_names}; do prepare_dbbackup if [ "${db}" != "justbackupeverything" ] ; then bucket="--bucket $db" ; else db=all ; fi - target=influx2_${db}_${DB_HOST#*//}_${now} - ltarget=influx2_${db}_${DB_HOST#*//} + backup_job_filename=influx2_${db}_${backup_job_db_host#*//}_${now} + backup_job_filename_base=influx2_${db}_${backup_job_db_host#*//} compression - pre_dbbackup $db - print_notice "Dumping Influx2 database: '${db}'" - influx backup --org ${DB_USER} ${bucket} --host ${DB_HOST}:${DB_PORT} --token ${DB_PASS} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} --compression none "${TEMP_LOCATION}"/"${target_dir}" + pre_dbbackup "${db}" + write_log notice "Dumping Influx2 database: '${db}'" + if var_true "${DEBUG_BACKUP_INFLUX}" ; then debug on; fi + run_as_user influx backup --org ${backup_job_db_user} ${bucket} --host ${backup_job_db_host}:${backup_job_db_port} --token ${backup_job_db_pass} ${backup_job_extra_opts} ${backup_job_extra_dump_opts} --compression none "${TEMP_PATH}"/"${backup_job_filename_dir}" exit_code=$? - check_exit_code backup $target_dir + check_exit_code backup "${backup_job_filename_dir}" create_archive - target=influx2_${db}_${DB_HOST#*//}_${now}.tar${extension} - ltarget=influx2_${db}_${DB_HOST#*//} + backup_job_filename=influx2_${db}_${backup_job_db_host#*//}_${now}.tar${extension} + backup_job_filename_base=influx2_${db}_${backup_job_db_host#*//} + if var_true "${DEBUG_BACKUP_INFLUX}" ; then debug off; fi + timer backup finish + file_encryption generate_checksum move_dbbackup - check_exit_code move $target_dir - post_dbbackup $db + check_exit_code move "${backup_job_filename_dir}" + post_dbbackup "${db}" done ;; esac @@ -198,166 +462,231 @@ backup_influx() { backup_mongo() { prepare_dbbackup - if [ "${ENABLE_COMPRESSION,,}" = "none" ] || [ "${ENABLE_COMPRESSION,,}" = "false" ] ; then - target=${dbtype}_${DB_NAME,,}_${DB_HOST,,}_${now}.archive - ltarget=${dbtype}_${DB_NAME,,}_${DB_HOST,,} + if [ "${backup_job_compression,,}" = "none" ] ; then + backup_job_filename=${dbtype}_${backup_job_db_name,,}_${backup_job_db_host,,}_${now}.archive + backup_job_filename_base=${dbtype}_${backup_job_db_name,,}_${backup_job_db_host,,} else - target=${dbtype}_${DB_NAME,,}_${DB_HOST,,}_${now}.archive.gz - ltarget=${dbtype}_${DB_NAME,,}_${DB_HOST,,} + backup_job_filename=${dbtype}_${backup_job_db_name,,}_${backup_job_db_host,,}_${now}.archive.gz + backup_job_filename_base=${dbtype}_${backup_job_db_name,,}_${backup_job_db_host,,} mongo_compression="--gzip" compression_string="and compressing with gzip" fi - if [ -n "${MONGO_CUSTOM_URI}" ] ; then - mongo_backup_parameter="--uri=${MONGO_CUSTOM_URI} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS}" + if [ -n "${backup_job_mongo_custom_uri}" ] ; then + mongo_backup_parameter="--uri=${backup_job_mongo_custom_uri} ${backup_job_extra_opts} ${backup_job_extra_dump_opts}" else - mongo_backup_parameter="--host ${DB_HOST} --port ${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS}" + mongo_backup_parameter="--host ${backup_job_db_host} --port ${backup_job_db_port} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${backup_job_extra_opts} ${backup_job_extra_dump_opts}" fi - pre_dbbackup "${DB_NAME}" - print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" - silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} ${mongo_backup_parameter} + pre_dbbackup "${backup_job_db_name}" + write_log notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" + if var_true "${DEBUG_BACKUP_MONGO}" ; then debug on; fi + silent run_as_user ${play_fair} mongodump --archive=${TEMP_PATH}/${backup_job_filename} ${mongo_compression} ${mongo_backup_parameter} exit_code=$? - check_exit_code backup $target + if var_true "${DEBUG_BACKUP_MONGO}" ; then debug off; fi + check_exit_code backup "${backup_job_filename}" + timer backup finish + file_encryption generate_checksum move_dbbackup - check_exit_code move $target - post_dbbackup "${DB_NAME}" + check_exit_code move "${backup_job_filename}" + post_dbbackup "${backup_job_db_name}" + if var_true "${DEBUG_BACKUP_MONGO}" ; then debug off; fi } backup_mssql() { - prepare_dbbackup - target=mssql_${DB_NAME,,}_${DB_HOST,,}_${now}.bak - ltarget=mssql_${DB_NAME,,}_${DB_HOST,,} - compression - pre_dbbackup "${DB_NAME}" - print_notice "Dumping MSSQL database: '${DB_NAME}'" - silent /opt/mssql-tools18/bin/sqlcmd -C -S ${DB_HOST}\,${DB_PORT} -U ${DB_USER} -P ${DB_PASS} -Q "BACKUP DATABASE [${DB_NAME}] TO DISK = N'${TEMP_LOCATION}/${target}' WITH NOFORMAT, NOINIT, NAME = '${DB_NAME}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10" - exit_code=$? - check_exit_code backup $target - generate_checksum - move_dbbackup - check_exit_code move $target - post_dbbackup $DB_NAME + case "${backup_job_mssql_mode,,}" in + db|database ) + prepare_dbbackup + backup_job_filename=mssql_${backup_job_db_name,,}_${backup_job_db_host,,}_${now}.bak + backup_job_filename_base=mssql_${backup_job_db_name,,}_${backup_job_db_host,,} + pre_dbbackup "${backup_job_db_name}" + write_log notice "Dumping MSSQL database: '${DB_NAME}'" + if var_true "${DEBUG_BACKUP_MSSQL}" ; then debug on; fi + silent run_as_user ${play_fair} /opt/mssql-tools18/bin/sqlcmd -C -S ${backup_job_db_host}\,${backup_job_db_port} -U ${backup_job_db_user} -P ${backup_job_db_pass} -Q "BACKUP DATABASE [${backup_job_db_name}] TO DISK = N'${TEMP_PATH}/${backup_job_filename}' WITH NOFORMAT, NOINIT, NAME = '${backup_job_db_name}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10" + exit_code=$? + if var_true "${DEBUG_BACKUP_MSSQL}" ; then debug off; fi + backup_job_filename_original=${backup_job_filename} + compression + pre_dbbackup all + run_as_user ${compress_cmd} "${TEMP_PATH}/${backup_job_filename_original}" + check_exit_code backup "${backup_job_filename}" + timer backup finish + file_encryption + generate_checksum + move_dbbackup + check_exit_code move "${backup_job_filename}" + post_dbbackup "${backup_job_db_name}" + ;; + trn|transaction ) + prepare_dbbackup + backup_job_filename=mssql_${backup_job_db_name,,}_${backup_job_db_host,,}_${now}.trn + backup_job_filename_base=mssql_${backup_job_db_name,,}_trn_${backup_job_db_host,,} + pre_dbbackup "${backup_job_db_name}" + write_log notice "Dumping MSSQL database: '${DB_NAME}'" + if var_true "${DEBUG_BACKUP_MSSQL}" ; then debug on; fi + silent run_as_user ${play_fair} /opt/mssql-tools18/bin/sqlcmd -C -S ${backup_job_db_host}\,${backup_job_db_port} -U ${backup_job_db_user} -P ${backup_job_db_pass} -Q "BACKUP LOG [${backup_job_db_name}] TO DISK = N'${TEMP_PATH}/${backup_job_filename}' WITH NOFORMAT, NOINIT, NAME = '${backup_job_db_name}-log', SKIP, NOREWIND, NOUNLOAD, STATS = 10" + exit_code=$? + if var_true "${DEBUG_BACKUP_MSSQL}" ; then debug off; fi + backup_job_filename_original=${backup_job_filename} + compression + pre_dbbackup all + run_as_user ${compress_cmd} "${TEMP_PATH}/${backup_job_filename_original}" + check_exit_code backup "${backup_job_filename}" + file_encryption + timer backup finish + generate_checksum + move_dbbackup + check_exit_code move "${backup_job_filename}" + post_dbbackup "${backup_job_db_name}" + ;; + esac } backup_mysql() { - if var_true "${MYSQL_SINGLE_TRANSACTION}" ; then + + if var_true "${backup_job_mysql_events}" ; then + events="--events" + fi + if var_true "${backup_job_mysql_single_transaction}" ; then single_transaction="--single-transaction" fi - if var_true "${MYSQL_STORED_PROCEDURES}" ; then + if var_true "${backup_job_mysql_stored_procedures}" ; then stored_procedures="--routines" fi - if [ "${DB_NAME,,}" = "all" ] ; then - print_debug "Preparing to back up everything except for information_schema and _* prefixes" - db_names=$(mysql -h ${DB_HOST} -P $DB_PORT -u$DB_USER ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_ENUMERATION_OPTS} --batch -e "SHOW DATABASES;" | grep -v Database | grep -v schema ) - if [ -n "${DB_NAME_EXCLUDE}" ] ; then - db_names_exclusions=$(echo "${DB_NAME_EXCLUDE}" | tr ',' '\n') + if [ "${backup_job_db_name,,}" = "all" ] ; then + write_log debug "Preparing to back up everything except for information_schema and _* prefixes" + db_names=$(run_as_user mysql -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_enumeration_opts} --batch -e "SHOW DATABASES;" | grep -v Database | grep -v schema ) + if [ -n "${backup_job_db_name_exclude}" ] ; then + db_names_exclusions=$(echo "${backup_job_db_name_exclude}" | tr ',' '\n') for db_exclude in ${db_names_exclusions} ; do - print_debug "Excluding '${db_exclude}' from ALL DB_NAME backups" + write_log debug "Excluding '${db_exclude}' from ALL DB_NAME backups" db_names=$(echo "$db_names" | sed "/${db_exclude}/d" ) done fi else - db_names=$(echo "${DB_NAME}" | tr ',' '\n') + db_names=$(echo "${backup_job_db_name}" | tr ',' '\n') fi - print_debug "Databases Found: $(echo ${db_names} | xargs | tr ' ' ',')" + write_log debug "Databases Found: $(echo ${db_names} | xargs | tr ' ' ',')" - if var_true "${SPLIT_DB}" ; then + if var_true "${backup_job_split_db}" ; then for db in ${db_names} ; do prepare_dbbackup - target=mysql_${db}_${DB_HOST,,}_${now}.sql - ltarget=mysql_${db}_${DB_HOST,,} + backup_job_filename=mysql_${db}_${backup_job_db_host,,}_${now}.sql + backup_job_filename_base=mysql_${db}_${backup_job_db_host,,} compression - pre_dbbackup $db - print_notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}" - mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}" + pre_dbbackup "${db}" + write_log notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}" + if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug on; fi + run_as_user ${play_fair} mysqldump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${events} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_dump_opts} $db | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${backup_job_filename}" > /dev/null exit_code=$? - check_exit_code backup $target + if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug off; fi + check_exit_code backup "${backup_job_filename}" + timer backup finish + file_encryption generate_checksum move_dbbackup - check_exit_code move $target - post_dbbackup $db + check_exit_code move "${backup_job_filename}" + post_dbbackup "${db}" done else - print_debug "Not splitting database dumps into their own files" + write_log debug "Not splitting database dumps into their own files" prepare_dbbackup - target=mysql_all_${DB_HOST,,}_${now}.sql - ltarget=mysql_all_${DB_HOST,,} + backup_job_filename=mysql_all_${backup_job_db_host,,}_${now}.sql + backup_job_filename_base=mysql_all_${backup_job_db_host,,} compression pre_dbbackup all - print_notice "Dumping all MySQL / MariaDB databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}" - mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} --databases $(echo ${db_names} | xargs) | $compress_cmd > "${TEMP_LOCATION}"/"${target}" + write_log notice "Dumping all MySQL / MariaDB databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}" + if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug on; fi + run_as_user ${play_fair} mysqldump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${events} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_dump_opts} --databases $(echo ${db_names} | xargs) | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${backup_job_filename}" > /dev/null exit_code=$? - check_exit_code backup $target + if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug off; fi + check_exit_code backup "${backup_job_filename}" + timer backup finish + file_encryption generate_checksum move_dbbackup - check_exit_code move $target + check_exit_code move "${backup_job_filename}" post_dbbackup all fi } backup_pgsql() { - export PGPASSWORD=${DB_PASS} - if [ -n "${DB_AUTH}" ] ; then - authdb=${DB_AUTH} - else - authdb=${DB_USER} - fi - if [ "${DB_NAME,,}" = "all" ] ; then - print_debug "Preparing to back up all databases" - db_names=$(psql -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' ) - if [ -n "${DB_NAME_EXCLUDE}" ] ; then - db_names_exclusions=$(echo "${DB_NAME_EXCLUDE}" | tr ',' '\n') - for db_exclude in ${db_names_exclusions} ; do - print_debug "Excluding '${db_exclude}' from ALL DB_NAME backups" - db_names=$(echo "$db_names" | sed "/${db_exclude}/d" ) - done - fi - else - db_names=$(echo "${DB_NAME}" | tr ',' '\n') - fi - - print_debug "Databases Found: $(echo ${db_names} | xargs | tr ' ' ',')" - - if var_true "${SPLIT_DB}" ; then - for db in ${db_names} ; do - prepare_dbbackup - target=pgsql_${db}_${DB_HOST,,}_${now}.sql - ltarget=pgsql_${db}_${DB_HOST,,} - compression - pre_dbbackup $db - print_notice "Dumping PostgresSQL database: '${db}' ${compression_string}" - pg_dump -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} $db ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} - exit_code=$? - check_exit_code backup $target - generate_checksum - move_dbbackup - check_exit_code move $target - post_dbbackup $db - done + backup_pgsql_globals() { prepare_dbbackup - target=pgsql_globals_${DB_HOST,,}_${now}.sql - ltarget=pgsql_globals_${DB_HOST,,} + backup_job_filename=pgsql_globals_${backup_job_db_host,,}_${now}.sql compression pre_dbbackup "globals" - print_notice "Dumping PostgresSQL globals: 'pg_dumpall -g' ${compression_string}" - pg_dumpall -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} -g | $compress_cmd > ${TEMP_LOCATION}/${target} + print_notice "Dumping PostgresSQL globals: with 'pg_dumpall -g' ${compression_string}" + if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug on; fi + run_as_user ${play_fair} pg_dumpall -h "${backup_job_db_host}" -U "${backup_job_db_user}" -p "${backup_job_db_port}" -g ${backup_job_extra_opts} ${backup_job_extra_dump_opts} | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${backup_job_filename}" > /dev/null exit_code=$? - check_exit_code $target + if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug off; fi + check_exit_code "${backup_job_filename}" + timer backup finish + file_encryption generate_checksum move_dbbackup - check_exit_code "move backup file" + check_exit_code move "${backup_job_filename}" post_dbbackup "globals" + } + + export PGPASSWORD=${backup_job_db_pass} + if [ -n "${backup_job_db_auth}" ] ; then + authdb=${backup_job_db_auth} else - print_debug "Not splitting database dumps into their own files" + authdb=${backup_job_db_user} + fi + if [ "${backup_job_db_name,,}" = "all" ] ; then + write_log debug "Preparing to back up all databases" + db_names=$(run_as_user psql -h ${backup_job_db_host} -U ${backup_job_db_user} -p ${backup_job_db_port} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' ) + if [ -n "${backup_job_db_name_exclude}" ] ; then + db_names_exclusions=$(echo "${backup_job_db_name_exclude}" | tr ',' '\n') + for db_exclude in ${db_names_exclusions} ; do + write_log debug "Excluding '${db_exclude}' from ALL DB_NAME backups" + db_names=$(echo "$db_names" | sed "/${db_exclude}/d" ) + done + _postgres_backup_globals=true + fi + else + db_names=$(echo "${backup_job_db_name}" | tr ',' '\n') + _postgres_backup_globals=false + fi + + if var_false "${_postgres_backup_globals}" && var_true "${backup_job_backup_pgsql_globals}" ; then _postgres_backup_globals=true; fi + + write_log debug "Databases Found: $(echo ${db_names} | xargs | tr ' ' ',')" + + if var_true "${backup_job_split_db}" ; then + for db in ${db_names} ; do + prepare_dbbackup + backup_job_filename=pgsql_${db}_${backup_job_db_host,,}_${now}.sql + backup_job_filename_base=pgsql_${db}_${backup_job_db_host,,} + compression + pre_dbbackup "${db}" + write_log notice "Dumping PostgresSQL database: '${db}' ${compression_string}" + if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug on; fi + run_as_user ${play_fair} pg_dump -h "${backup_job_db_host}" -p "${backup_job_db_port}" -U "${backup_job_db_user}" $db ${backup_job_extra_opts} ${backup_job_extra_dump_opts} | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${backup_job_filename}" > /dev/null + exit_code=$? + if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug off; fi + check_exit_code backup "${backup_job_filename}" + timer backup finish + file_encryption + generate_checksum + move_dbbackup + check_exit_code move "${backup_job_filename}" + post_dbbackup "${db}" + done + if var_true "${_postgres_backup_globals}" ; then backup_pgsql_globals; fi + else + write_log debug "Not splitting database dumps into their own files" prepare_dbbackup - target=pgsql_all_${DB_HOST,,}_${now}.sql - ltarget=pgsql_${db}_${DB_HOST,,} + backup_job_filename=pgsql_all_${backup_job_db_host,,}_${now}.sql + backup_job_filename_base=pgsql_${db}_${backup_job_db_host,,} compression pre_dbbackup all - print_notice "Dumping all PostgreSQL databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}" - tmp_db_names=$(psql -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' ) + write_log notice "Dumping all PostgreSQL databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}" + tmp_db_names=$(run_as_user psql -h ${backup_job_db_host} -p ${backup_job_db_port} -U ${backup_job_db_user} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' ) for r_db_name in $(echo $db_names | xargs); do tmp_db_names=$(echo "$tmp_db_names" | xargs | sed "s|${r_db_name}||g" ) done @@ -365,293 +694,323 @@ backup_pgsql() { for x_db_name in ${tmp_db_names} ; do pgexclude_arg=$(echo ${pgexclude_arg} --exclude-database=${x_db_name}) done - pg_dumpall -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} ${pgexclude_arg} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} + if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug on; fi + run_as_user ${play_fair} pg_dumpall -h ${backup_job_db_host} -p ${backup_job_db_port} -U ${backup_job_db_user} ${pgexclude_arg} ${backup_job_extra_opts} ${backup_job_extra_dump_opts} | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${backup_job_filename}" > /dev/null exit_code=$? - check_exit_code backup $target + if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug off; fi + check_exit_code backup "${backup_job_filename}" + timer backup finish + file_encryption generate_checksum move_dbbackup - check_exit_code move $target + check_exit_code move "${backup_job_filename}" post_dbbackup all + if var_true "${_postgres_backup_globals}" ; then backup_pgsql_globals; fi fi } backup_redis() { prepare_dbbackup - print_notice "Dumping Redis - Flushing Redis Cache First" - target=redis_all_${DB_HOST,,}_${now}.rdb - ltarget=redis_${DB_HOST,,} - echo bgsave | silent redis-cli -h ${DB_HOST} -p ${DB_PORT} ${REDIS_PASS_STR} --rdb ${TEMP_LOCATION}/${target} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} + write_log notice "Dumping Redis - Flushing Redis Cache First" + backup_job_filename=redis_all_${backup_job_db_host,,}_${now}.rdb + backup_job_filename_base=redis_${backup_job_db_host,,} + if var_true "${DEBUG_BACKUP_REDIS}" ; then debug on; fi + echo bgsave | silent run_as_user ${play_fair} redis-cli -h ${backup_job_db_host} -p ${backup_job_db_port} ${REDIS_PASS_STR} --rdb ${TEMP_PATH}/${backup_job_filename} ${backup_job_extra_opts} ${backup_job_extra_dump_opts} sleep 10 try=5 while [ $try -gt 0 ] ; do - saved=$(echo 'info Persistence' | redis-cli -h ${DB_HOST} -p ${DB_PORT} ${REDIS_PASS_STR} | awk '/rdb_bgsave_in_progress:0/{print "saved"}') - ok=$(echo 'info Persistence' | redis-cli -h ${DB_HOST} -p ${DB_PORT} ${REDIS_PASS_STR} | awk '/rdb_last_bgsave_status:ok/{print "ok"}') + saved=$(echo 'info Persistence' | redis-cli -h ${backup_job_db_host} -p ${backup_job_db_port} ${REDIS_PASS_STR} | awk '/rdb_bgsave_in_progress:0/{print "saved"}') + ok=$(echo 'info Persistence' | redis-cli -h ${backup_job_db_host} -p ${backup_job_db_port} ${REDIS_PASS_STR} | awk '/rdb_last_bgsave_status:ok/{print "ok"}') if [[ "$saved" = "saved" ]] && [[ "$ok" = "ok" ]]; then - print_notice "Redis Backup Complete" + write_log notice "Redis Backup Complete" exit_code=0 break fi try=$((try - 1)) - print_warn "Redis Busy - Waiting and retrying in 5 seconds" + write_log warn "Redis Busy - Waiting and retrying in 5 seconds" sleep 5 done - target_original=${target} + backup_job_filename_original=${backup_job_filename} compression pre_dbbackup all - $compress_cmd "${TEMP_LOCATION}/${target_original}" - check_exit_code backup $target + run_as_user ${compress_cmd} "${TEMP_PATH}/${backup_job_filename_original}" + timer backup finish + if var_true "${DEBUG_BACKUP_REDIS}" ; then debug on; fi + check_exit_code backup "${backup_job_filename}" + file_encryption generate_checksum move_dbbackup - check_exit_code move $target + check_exit_code move "${backup_job_filename}" post_dbbackup all } backup_sqlite3() { prepare_dbbackup - db=$(basename "${DB_HOST}") + db=$(basename "${backup_job_db_host}") db="${db%.*}" - target=sqlite3_${db}_${now}.sqlite3 - ltarget=sqlite3_${db}.sqlite3 + backup_job_filename=sqlite3_${db}_${now}.sqlite3 + backup_job_filename_base=sqlite3_${db}.sqlite3 compression - pre_dbbackup $db - print_notice "Dumping sqlite3 database: '${DB_HOST}' ${compression_string}" - silent sqlite3 "${DB_HOST}" ".backup '${TEMP_LOCATION}/backup.sqlite3'" + pre_dbbackup "${db}" + write_log notice "Dumping sqlite3 database: '${backup_job_db_host}' ${compression_string}" + if var_true "${DEBUG_BACKUP_SQLITE3}" ; then debug on; fi + silent run_as_user ${play_fair} sqlite3 "${backup_job_db_host}" ".backup '${TEMP_PATH}/backup.sqlite3'" exit_code=$? - check_exit_code backup $target - cat "${TEMP_LOCATION}"/backup.sqlite3 | ${dir_compress_cmd} > "${TEMP_LOCATION}/${target}" + check_exit_code backup "${backup_job_filename}" + run_as_user ${play_fair} cat "${TEMP_PATH}"/backup.sqlite3 | ${dir_compress_cmd} | run_as_user tee "${TEMP_PATH}/${backup_job_filename}" > /dev/null + timer backup finish + if var_true "${DEBUG_BACKUP_SQLITE3}" ; then debug off; fi + file_encryption generate_checksum move_dbbackup - check_exit_code move $target - post_dbbackup $db + check_exit_code move "${backup_job_filename}" + post_dbbackup "${db}" } check_availability() { -### Set the Database Type - if var_false "${SKIP_AVAILABILITY_CHECK}" ; then - case "$dbtype" in + if var_true "${DEBUG_CHECK_AVAILABILITY}" ; then debug on; fi + ### Set the Database Type + if var_false "${backup_job_skip_availability_check}" ; then + case "${dbtype}" in "couch" ) counter=0 code_received=0 while [ "${code_received}" != "200" ]; do - code_received=$(curl -XGET -sSL -o /dev/null -L -w ''%{http_code}'' ${DB_HOST}:${DB_PORT}) + code_received=$(run_as_user curl -XGET -sSL -o /dev/null -L -w ''%{http_code}'' ${backup_job_db_host}:${backup_job_db_port}) if [ "${code_received}" = "200" ] ; then break ; fi sleep 5 (( counter+=5 )) - print_warn "CouchDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + write_log warn "CouchDB Host '${backup_job_db_host}' is not accessible, retrying.. ($counter seconds so far)" done ;; "influx" ) counter=0 - case "${INFLUX_VERSION,,}" in + case "${backup_job_influx_version,,}" in 1 ) - while ! (nc -z ${DB_HOST#*//} ${DB_PORT}) ; do + while ! (run_as_user nc -z ${backup_job_db_host#*//} ${backup_job_db_port}) ; do sleep 5 (( counter+=5 )) - print_warn "InfluxDB Host '${DB_HOST#*//}' is not accessible, retrying.. ($counter seconds so far)" + write_log warn "InfluxDB Host '${backup_job_db_host#*//}' is not accessible, retrying.. ($counter seconds so far)" done ;; 2 ) code_received=0 while [ "${code_received}" != "200" ]; do - code_received=$(curl -XGET -sSL -o /dev/null -w ''%{http_code}'' ${DB_HOST}:${DB_PORT}/health) + code_received=$(run_as_user curl -XGET -sSL -o /dev/null -w ''%{http_code}'' ${backup_job_db_host}:${backup_job_db_port}/health) if [ "${code_received}" = "200" ] ; then break ; fi sleep 5 (( counter+=5 )) - print_warn "InfluxDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + write_log warn "InfluxDB Host '${backup_job_db_host}' is not accessible, retrying.. ($counter seconds so far)" done ;; esac ;; "mongo" ) - if [ -n "${MONGO_CUSTOM_URI}" ] ; then - print_debug "Skipping Connectivity Check" + if [ -n "${backup_job_mongo_custom_uri}" ] ; then + write_log debug "Skipping Connectivity Check" else counter=0 - while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do + while ! (run_as_user nc -z ${backup_job_db_host} ${backup_job_db_port}) ; do sleep 5 (( counter+=5 )) - print_warn "Mongo Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + write_log warn "Mongo Host '${backup_job_db_host}' is not accessible, retrying.. ($counter seconds so far)" done fi ;; "mysql" ) counter=0 - transform_file_var DB_PASS - export MYSQL_PWD=${DB_PASS} - while ! (mysqladmin -u"${DB_USER}" -P"${DB_PORT}" -h"${DB_HOST}" ${mysql_tls_args} status > /dev/null 2>&1) ; do + export MYSQL_PWD=${backup_job_db_pass} + while ! (run_as_user mysqladmin -u"${backup_job_db_user}" -P"${backup_job_db_port}" -h"${backup_job_db_host}" ${mysql_tls_args} status > /dev/null 2>&1) ; do sleep 5 (( counter+=5 )) - print_warn "MySQL/MariaDB Server '${DB_HOST}' is not accessible, retrying.. (${counter} seconds so far)" + write_log warn "MySQL/MariaDB Server '${backup_job_db_host}' is not accessible, retrying.. (${counter} seconds so far)" done ;; "mssql" ) counter=0 - while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do + while ! (run_as_user nc -z ${backup_job_db_host} ${backup_job_db_port}) ; do sleep 5 (( counter+=5 )) - print_warn "MSSQL Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + write_log warn "MSSQL Host '${backup_job_db_host}' is not accessible, retrying.. ($counter seconds so far)" done ;; "pgsql" ) counter=0 - until pg_isready --host=${DB_HOST} --port=${DB_PORT} -q + until run_as_user pg_isready --host=${backup_job_db_host} --port=${backup_job_db_port} -q do sleep 5 (( counter+=5 )) - print_warn "Postgres Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + write_log warn "Postgres Host '${backup_job_db_host}' is not accessible, retrying.. ($counter seconds so far)" done ;; "redis" ) counter=0 - while ! (nc -z "${DB_HOST}" "${DB_PORT}") ; do + while ! (run_as_user nc -z "${backup_job_db_host}" "${backup_job_db_port}") ; do sleep 5 (( counter+=5 )) - print_warn "Redis Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)" + write_log warn "Redis Host '${backup_job_db_host}' is not accessible, retrying.. ($counter seconds so far)" done ;; "sqlite3" ) - if [[ ! -e "${DB_HOST}" ]]; then - print_error "File '${DB_HOST}' does not exist." + if [[ ! -e "${backup_job_db_host}" ]]; then + write_log error "File '${backup_job_db_host}' does not exist." exit_code=2 exit $exit_code - elif [[ ! -f "${DB_HOST}" ]]; then - print_error "File '${DB_HOST}' is not a file." + elif [[ ! -f "${backup_job_db_host}" ]]; then + write_log error "File '${backup_job_db_host}' is not a file." exit_code=2 exit $exit_code - elif [[ ! -r "${DB_HOST}" ]]; then - print_error "File '${DB_HOST}' is not readable." + elif [[ ! -r "${backup_job_db_host}" ]]; then + write_log error "File '${backup_job_db_host}' is not readable." exit_code=2 exit $exit_code fi ;; esac fi + if var_true "${DEBUG_CHECK_AVAILABILITY}" ; then debug off; fi } check_exit_code() { - print_debug "DB Backup Exit Code is ${exit_code}" + if var_true "${DEBUG_CHECK_EXIT_CODE}" ; then debug on; fi case "${1}" in backup ) + write_log debug "DB Backup exit Code is ${exit_code}" case "${exit_code}" in 0 ) - print_info "DB Backup of '${2}' completed successfully" + write_log info "DB Backup of '${2}' completed successfully" ;; * ) - print_error "DB Backup of '${2}' reported errors" + write_log error "DB Backup of '${2}' reported errors" + notify \ + "$(date -d @"${backup_job_start_time}" +'%Y%m%d_%H%M%S')" \ + "${LOG_PATH}/$(date +'%Y%m%d')/$(date -d @${backup_job_start_time} +'%Y%m%d_%H%M%S')-${backup_job_filename_base}.log" \ + "${exit_code}" \ + "[FATAL] Host: ${backup_job_db_host} Name: ${backup_job_db_name} - Backup failed completely" \ + "DB Backup is failing to backup the '${backup_job_db_host}-${backup_job_db_name}' job." master_exit_code=1 ;; esac ;; move ) + write_log debug "Move exit Code is ${exit_code}" case "${move_exit_code}" in 0 ) - print_debug "Moving of backup '${2}' completed successfully" + write_log debug "Moving of backup '${2}' completed successfully" ;; * ) - print_error "Moving of backup '${2}' reported errors" + write_log error "Moving of backup '${2}' reported errors" + notify \ + "$(date -d @"${backup_job_start_time}" +'%Y%m%d_%H%M%S')" \ + "${LOG_PATH}/$(date +'%Y%m%d')/$(date -d @${backup_job_start_time} +'%Y%m%d_%H%M%S')-${backup_job_filename_base}.log" \ + "${move_exit_code}" \ + "[FATAL] Host: ${backup_job_db_host} Name: ${backup_job_db_name} - Backup failed to move to destination" \ + "DB Backup is failing to backup the '${backup_job_db_host}-${backup_job_db_name}' job." master_exit_code=1 ;; esac ;; esac + if var_true "${DEBUG_CHECK_EXIT_CODE}" ; then debug off; fi } cleanup_old_data() { - if [ -n "${DB_CLEANUP_TIME}" ]; then + if var_true "${DEBUG_CLEANUP_OLD_DATA}" ; then debug on; fi + if [ -n "${backup_job_cleanup_time}" ]; then if [ "${master_exit_code}" != 1 ]; then - case "${BACKUP_LOCATION,,}" in + case "${backup_job_backup_location,,}" in "blobxfer" ) - print_info "Cleaning up old backups on filesystem" - mkdir -p "${DB_DUMP_TARGET}" - find "${DB_DUMP_TARGET}"/ -mmin +"${DB_CLEANUP_TIME}" -iname "*" -exec rm {} \; - print_info "Syncing changes via blobxfer" - silent blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET} --delete --delete-only + write_log info "Cleaning up old backups on filesystem" + run_as_user mkdir -p "${backup_job_filesystem_path}" + find "${backup_job_filesystem_path}"/ -type f -mmin +"${backup_job_cleanup_time}" -iname "${backup_job_filename_base}*" -exec rm -f {} \; + write_log info "Syncing changes via blobxfer" + silent run_as_user blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --local-path ${backup_job_filesystem_path} --delete --delete-only ;; "file" | "filesystem" ) - print_info "Cleaning up old backups on filesystem" - mkdir -p "${DB_DUMP_TARGET}" - find "${DB_DUMP_TARGET}"/ -mmin +"${DB_CLEANUP_TIME}" -iname "*" -exec rm {} \; + write_log info "Cleaning up old backups on filesystem" + run_as_user mkdir -p "${backup_job_filesystem_path}" + run_as_user find "${backup_job_filesystem_path}"/ -type f -mmin +"${backup_job_cleanup_time}" -iname "${backup_job_filename_base}*" -exec rm -f {} \; ;; "s3" | "minio" ) - print_info "Cleaning up old backups on S3 storage" - aws ${PARAM_AWS_ENDPOINT_URL} s3 ls s3://${S3_BUCKET}/${S3_PATH}/ ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} | grep " DIR " -v | grep " PRE " -v | while read -r s3_file; do + write_log info "Cleaning up old backups on S3 storage" + run_as_user aws ${PARAM_AWS_ENDPOINT_URL} s3 ls s3://${backup_job_s3_bucket}/${backup_job_s3_path}/ ${s3_ssl} ${s3_ca_cert} ${backup_job_s3_extra_opts} | grep " DIR " -v | grep " PRE " -v | while read -r s3_file; do s3_createdate=$(echo $s3_file | awk {'print $1" "$2'}) s3_createdate=$(date -d "$s3_createdate" "+%s") - s3_olderthan=$(echo $(( $(date +%s)-${DB_CLEANUP_TIME}*60 ))) + s3_olderthan=$(echo $(( $(date +%s)-${backup_job_cleanup_time}*60 ))) if [[ $s3_createdate -le $s3_olderthan ]] ; then s3_filename=$(echo $s3_file | awk {'print $4'}) if [ "$s3_filename" != "" ] ; then - print_debug "Deleting $s3_filename" - aws ${PARAM_AWS_ENDPOINT_URL} s3 rm s3://${S3_BUCKET}/${S3_PATH}/${s3_filename} ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} + write_log debug "Deleting $s3_filename" + run_as_user aws ${PARAM_AWS_ENDPOINT_URL} s3 rm s3://${backup_job_s3_bucket}/${backup_job_s3_path}/${s3_filename} ${s3_ssl} ${s3_ca_cert} ${backup_job_s3_extra_opts} fi fi - done ;; esac else - print_error "Skipping Cleaning up old backups because there were errors in backing up" + write_log error "Skipping Cleaning up old backups because there were errors in backing up" fi fi + if var_true "${DEBUG_CLEANUP_OLD_DATA}" ; then debug off; fi } - compression() { - if var_false "${ENABLE_PARALLEL_COMPRESSION}" ; then - PARALLEL_COMPRESSION_THREADS=1 + if var_true "${DEBUG_COMPRESSION}" ; then debug on; fi + if var_false "${backup_job_parallel_compression}" ; then + backup_job_parallel_compression_threads=1 fi - if var_true "${GZ_RSYNCABLE}" ; then + if var_true "${backup_job_gz_rsyncable}" ; then gz_rsyncable=--rsyncable fi - case "${COMPRESSION,,}" in - gz* ) - compress_cmd="pigz -q -${COMPRESSION_LEVEL} -p ${PARALLEL_COMPRESSION_THREADS} ${gz_rsyncable}" - compression_type="gzip" - extension=".gz" - dir_compress_cmd=${compress_cmd} - target_dir=${target} - target=${target}.gz - ;; + case "${backup_job_compression,,}" in bz* ) - compress_cmd="pbzip2 -q -${COMPRESSION_LEVEL} -p${PARALLEL_COMPRESSION_THREADS} " + compress_cmd="${play_fair} pbzip2 -q -${backup_job_compression_level} -p${backup_job_parallel_compression_threads} " compression_type="bzip2" dir_compress_cmd=${compress_cmd} extension=".bz2" - target_dir=${target} - target=${target}.bz2 + backup_job_filename_dir=${backup_job_filename} + backup_job_filename=${backup_job_filename}.bz2 + ;; + gz* ) + compress_cmd="${play_fair} pigz -q -${backup_job_compression_level} -p ${backup_job_parallel_compression_threads} ${gz_rsyncable}" + compression_type="gzip" + extension=".gz" + dir_compress_cmd=${compress_cmd} + backup_job_filename_dir=${backup_job_filename} + backup_job_filename=${backup_job_filename}.gz ;; xz* ) - compress_cmd="pixz -${COMPRESSION_LEVEL} -p ${PARALLEL_COMPRESSION_THREADS} " + compress_cmd="${play_fair} pixz -${backup_job_compression_level} -p ${backup_job_parallel_compression_threads} " compression_type="xzip" dir_compress_cmd=${compress_cmd} extension=".xz" - target_dir=${target} - target=${target}.xz + backup_job_filename_dir=${backup_job_filename} + backup_job_filename=${backup_job_filename}.xz ;; zst* ) - compress_cmd="zstd -q -q --rm -${COMPRESSION_LEVEL} -T${PARALLEL_COMPRESSION_THREADS} ${gz_rsyncable}" + compress_cmd="${play_fair} zstd -q -q --rm -${backup_job_compression_level} -T${backup_job_parallel_compression_threads} ${gz_rsyncable}" compression_type="zstd" dir_compress_cmd=${compress_cmd} extension=".zst" - target_dir=${target} - target=${target}.zst + backup_job_filename_dir=${backup_job_filename} + backup_job_filename=${backup_job_filename}.zst ;; "none" | "false") compress_cmd="cat " compression_type="none" dir_compress_cmd="cat " - target_dir=${target} + backup_job_filename_dir=${backup_job_filename} ;; esac case "${CONTAINER_LOG_LEVEL,,}" in "debug" ) if [ "${compression_type}" = "none" ] ; then - compression_string="with '${PARALLEL_COMPRESSION_THREADS}' threads" + compression_string="with '${backup_job_parallel_compression_threads}' threads" else - compression_string="and compressing with '${compression_type}:${COMPRESSION_LEVEL}' with '${PARALLEL_COMPRESSION_THREADS}' threads" + compression_string="and compressing with '${compression_type}:${backup_job_compression_level}' with '${backup_job_parallel_compression_threads}' threads" fi ;; * ) @@ -660,258 +1019,573 @@ compression() { fi ;; esac + if var_true "${DEBUG_COMPRESSION}" ; then debug off; fi } create_archive() { + if var_true "${DEBUG_CREATE_ARCHIVE}" ; then debug on; fi if [ "${exit_code}" = "0" ] ; then - print_notice "Creating archive file of '${target_dir}' with tar ${compression_string}" - tar cf - "${TEMP_LOCATION}"/"${target_dir}" | $dir_compress_cmd > "${TEMP_LOCATION}"/"${target_dir}".tar"${extension}" + write_log notice "Creating archive file of '${backup_job_filename_dir}' with tar ${compression_string}" + run_as_user tar cf - "${TEMP_PATH}"/"${backup_job_filename_dir}" | ${dir_compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${backup_job_filename_dir}".tar"${extension}" > /dev/null else - print_error "Skipping creating archive file because backup did not complete successfully" + write_log error "Skipping creating archive file because backup did not complete successfully" fi + if var_true "${DEBUG_CREATE_ARCHIVE}" ; then debug off; fi +} + +create_schedulers() { + if var_true "${DEBUG_CREATE_SCHEDULERS}" ; then debug on; fi + backup() { + bootstrap_variables upgrade BACKUP + local backup_instances=$(printenv | sort | grep -c "^DB[0-9]._HOST") + if [ -n "${DB_HOST}" ] && [ "${backup_instances}" ]; then + backup_instances=1; + print_debug "Detected using old DB_ variables" + fi + + for (( instance = 01; instance <= backup_instances; )) ; do + instance=$(printf "%02d" $instance) + cp -R /assets/dbbackup/template-dbbackup /etc/services.available/dbbackup-"${instance}" + sed -i "s|{{BACKUP_NUMBER}}|${instance}|g" /etc/services.available/dbbackup-"${instance}"/run + + cat <> /usr/bin/backup"${instance}"-now +#!/bin/bash +source /assets/functions/00-container +PROCESS_NAME=db-backup${instance} +print_info "Starting Manual Backup for db-backup${instance}" +/var/run/s6/legacy-services/dbbackup-${instance}/run now + +EOF + chmod +x /usr/bin/backup"${instance}"-now + if [ "${instance}" = "01" ] ; then + touch /usr/bin/backup-now + chmod +x /usr/bin/backup-now + cat < /usr/bin/backup-now +#!/bin/bash + +/usr/bin/backup${instance}-now + +EOF + else + echo "/usr/bin/backup${instance}-now" >> /usr/bin/backup-now + fi + + instance=$(echo "${instance} +1" | bc) + done + } + + case "${1}" in + backup ) backup ;; + esac + if var_true "${DEBUG_CREATE_SCHEDULERS}" ; then debug off; fi +} + +ctrl_c() { + sed -i "/^{{BACKUP_NUMBER}}/d" /tmp/.container/db-backup-backups + symlink_log + print_warn "User aborted" + exit +} + +db_backup_container_init() { + rm -rf /tmp/.container/db-backup-backups + echo "0 0 * * * /usr/sbin/logrotate_dbbackup >/dev/null 2>&1" > /assets/cron/dbbackup_logrotate + touch /tmp/.container/db-backup-backups +} + +debug() { + case "${1}" in + off) + DEBUG_MODE=${_original_debug_mode} + if var_true "${DEBUG_MODE}" ; then + set -x + else + set +x + fi + ;; + on) + if [ -z "${_original_debug_mode}" ]; then + _original_debug_mode="${DEBUG_MODE}" + fi + set -x + ;; + esac +} + +file_encryption() { + if var_true "${DEBUG_FILE_ENCRYPTION}" ; then debug on; fi + if var_true "${backup_job_encrypt}" ; then + if [ "${exit_code}" = "0" ] ; then + print_debug "Encrypting" + output_off + if [ -n "${backup_job_encrypt_passphrase}" ] && [ -n "${backup_job_encrypt_pubkey}" ]; then + print_error "Can't encrypt as both ENCRYPT_PASSPHRASE and ENCRYPT_PUBKEY exist!" + return + elif [ -n "${backup_job_encrypt_passphrase}" ] && [ -z "${backup_job_encrypt_pubkey}" ]; then + print_notice "Encrypting with GPG Passphrase" + encrypt_routines_start_time=$(date +'%s') + encrypt_tmp_dir=$(run_as_user mktemp -d) + echo "${backup_job_encrypt_passphrase}" | silent run_as_user ${play_fair} gpg --batch --home ${encrypt_tmp_dir} --yes --passphrase-fd 0 -c "${TEMP_PATH}"/"${backup_job_filename}" + rm -rf "${encrypt_tmp_dir}" + elif [ -z "${backup_job_encrypt_passphrase}" ] && [ -n "${backup_job_encrypt_pubkey}" ]; then + if [ -f "${backup_job_encrypt_pubkey}" ]; then + encrypt_routines_start_time=$(date +'%s') + print_notice "Encrypting with GPG Public Key" + encrypt_tmp_dir=$(run_as_user mktemp -d) + silent run_as_user ${play_fair} gpg --batch --yes --home "${encrypt_tmp_dir}" --recipient-file "${backup_job_encrypt_pubkey}" -c "${TEMP_PATH}"/"${backup_job_filename}" + rm -rf "${encrypt_tmp_dir}" + fi + fi + if [ -f "${TEMP_PATH}"/"${backup_job_filename}".gpg ]; then + rm -rf "${TEMP_PATH:?}"/"${backup_job_filename:?}" + backup_job_filename="${backup_job_filename}.gpg" + + encrypt_routines_finish_time=$(date +'%s') + encrypt_routines_total_time=$(echo $((encrypt_routines_finish_time-encrypt_routines_start_time))) + zabbix_encrypt_time=$(cat < /dev/null + chmod ${backup_job_filesystem_permission} "${backup_job_filename}"."${checksum_extension}" + checksum_value=$(run_as_user cat "${backup_job_filename}"."${checksum_extension}" | awk '{print $1}') + checksum_routines_finish_time=$(date +'%s') + checksum_routines_total_time=$(echo $((checksum_routines_finish_time-checksum_routines_start_time))) + zabbix_checksum_time=$(cat < "${target}"."${checksum_extension}" - checksum_value=$(${checksum_command} "${target}" | awk ' { print $1}') - print_debug "${checksum_extension^^}: ${checksum_value} - ${target}" - else - print_error "Skipping Checksum creation because backup did not complete successfully" - fi + done fi + if var_true "${DEBUG_NOTIFY}" ; then debug off; fi } move_dbbackup() { + if var_true "${DEBUG_MOVE_DBBACKUP}" ; then debug on; fi if [ "${exit_code}" = "0" ] ; then - dbbackup_size="$(stat -c%s "${TEMP_LOCATION}"/"${target}")" - dbbackup_date="$(date -r "${TEMP_LOCATION}"/"${target}" +'%s')" + dbbackup_size="$(run_as_user stat -c%s "${TEMP_PATH}"/"${backup_job_filename}")" + dbbackup_date="$(run_as_user date -r "${TEMP_PATH}"/"${backup_job_filename}" +'%s')" - case "${SIZE_VALUE,,}" in + case "${backup_job_size_value,,}" in "b" | "bytes" ) - SIZE_VALUE=1 + backup_job_size_value=1 ;; "[kK]" | "[kK][bB]" | "kilobytes" | "[mM]" | "[mM][bB]" | "megabytes" ) - SIZE_VALUE="-h" + backup_job_size_value="-h" ;; *) - SIZE_VALUE=1 + backup_job_size_value=1 ;; esac - if [ "$SIZE_VALUE" = "1" ] ; then - filesize=$(stat -c%s "${TEMP_LOCATION}"/"${target}") - print_notice "Backup of ${target} created with the size of ${filesize} bytes" + if [ "${backup_job_size_value}" = "1" ] ; then + filesize=$(run_as_user stat -c%s "${TEMP_PATH}"/"${backup_job_filename}") + write_log notice "Backup of '${backup_job_filename}' created with the size of ${filesize} bytes" else - filesize=$(du -h "${TEMP_LOCATION}"/"${target}" | awk '{ print $1}') - print_notice "Backup of ${target} created with the size of ${filesize}" + filesize=$(run_as_user du -h "${TEMP_PATH}"/"${backup_job_filename}" | awk '{ print $1}') + write_log notice "Backup of '${backup_job_filename}' created with the size of ${filesize}" fi - case "${BACKUP_LOCATION,,}" in + chmod "${backup_job_filesystem_permission}" "${TEMP_PATH}"/"${backup_job_filename}" + case "${backup_job_backup_location,,}" in "file" | "filesystem" ) - print_debug "Moving backup to filesystem" - mkdir -p "${DB_DUMP_TARGET}" - if var_true "${ENABLE_CHECKSUM}" ; then mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/ ; fi - mv "${TEMP_LOCATION}"/"${target}" "${DB_DUMP_TARGET}"/"${target}" + write_log debug "Moving backup to filesystem" + run_as_user mkdir -p "${backup_job_filesystem_path}" + if [ "${backup_job_checksum,,}" != "none" ] ; then run_as_user mv "${TEMP_PATH}"/*."${checksum_extension}" "${backup_job_filesystem_path}"/ ; fi + run_as_user mv "${TEMP_PATH}"/"${backup_job_filename}" "${backup_job_filesystem_path}"/"${backup_job_filename}" move_exit_code=$? - if var_true "${CREATE_LATEST_SYMLINK}" ; then - ln -sf "${DB_DUMP_TARGET}"/"${target}" "${DB_DUMP_TARGET}"/latest-"${ltarget}" + if var_true "${backup_job_create_latest_symlink}" ; then + run_as_user ln -sfr "${backup_job_filesystem_path}"/"${backup_job_filename}" "${backup_job_filesystem_path}"/latest-"${backup_job_filename_base}" fi - if [ -n "${DB_ARCHIVE_TIME}" ] ; then - mkdir -p "${DB_DUMP_TARGET_ARCHIVE}" - find "${DB_DUMP_TARGET}"/ -maxdepth 1 -mmin +"${DB_ARCHIVE_TIME}" -iname "*" -exec mv {} "${DB_DUMP_TARGET_ARCHIVE}" \; + if [ -n "${backup_job_archive_time}" ] ; then + run_as_user mkdir -p "${backup_job_filesystem_archive_path}" + run_as_user find "${backup_job_filesystem_path}"/ -type f -maxdepth 1 -mmin +"${backup_job_archive_time}" -iname "${backup_job_filename_base}*" -exec mv {} "${backup_job_filesystem_archive_path}" \; fi ;; "s3" | "minio" ) - print_debug "Moving backup to S3 Bucket" - if [ -n "${S3_KEY_ID}" ] && [ -n "${S3_KEY_SECRET}" ]; then - export AWS_ACCESS_KEY_ID=${S3_KEY_ID} - export AWS_SECRET_ACCESS_KEY=${S3_KEY_SECRET} + write_log debug "Moving backup to S3 Bucket" + if [ -n "${backup_job_s3_key_id}" ] && [ -n "${backup_job_s3_key_secret}" ]; then + export AWS_ACCESS_KEY_ID=${backup_job_s3_key_id} + export AWS_SECRET_ACCESS_KEY=${backup_job_s3_key_secret} else - print_debug "Variable S3_KEY_ID or S3_KEY_SECRET is not set. Please ensure sufficiant IAM role is assigned." + write_log debug "Variable S3_KEY_ID or S3_KEY_SECRET is not set. Please ensure sufficiant IAM role is assigned." fi - export AWS_DEFAULT_REGION=${S3_REGION} - if [ -f "${S3_CERT_CA_FILE}" ] ; then - print_debug "Using Custom CA for S3 Backups" - s3_ca_cert="--ca-bundle ${S3_CERT_CA_FILE}" + export AWS_DEFAULT_REGION=${backup_job_s3_region} + if [ -f "${backup_job_s3_cert_ca_file}" ] ; then + write_log debug "Using Custom CA for S3 Backups" + s3_ca_cert="--ca-bundle ${backup_job_s3_cert_ca_file}" fi - if var_true "${S3_CERT_SKIP_VERIFY}" ; then - print_debug "Skipping SSL verification for HTTPS S3 Hosts" + if var_true "${backup_job_s3_cert_skip_verify}" ; then + write_log debug "Skipping SSL verification for HTTPS S3 Hosts" s3_ssl="--no-verify-ssl" fi - [[ ( -n "${S3_HOST}" ) ]] && PARAM_AWS_ENDPOINT_URL=" --endpoint-url ${S3_PROTOCOL}://${S3_HOST}" + [[ ( -n "${S3_HOST}" ) ]] && PARAM_AWS_ENDPOINT_URL=" --endpoint-url ${backup_job_s3_protocol}://${backup_job_s3_host}" - silent aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_LOCATION}/${target} s3://${S3_BUCKET}/${S3_PATH}/${target} ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} + silent aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_PATH}/${backup_job_filename} s3://${backup_job_s3_bucket}/${backup_job_s3_path}/${backup_job_filename} ${s3_ssl} ${s3_ca_cert} ${backup_job_s3_extra_opts} move_exit_code=$? - if var_true "${ENABLE_CHECKSUM}" ; then - silent aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_LOCATION}/*.${checksum_extension} s3://${S3_BUCKET}/${S3_PATH}/ ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} + if [ "${backup_job_checksum}" != "none" ] ; then + silent run_as_user ${play_fair} aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_PATH}/*.${checksum_extension} s3://${backup_job_s3_bucket}/${backup_job_s3_path}/ ${s3_ssl} ${s3_ca_cert} ${backup_job_s3_extra_opts} + run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}"."${checksum_extension}" fi - if var_true "${ENABLE_CHECKSUM}" ; then rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}"; fi - rm -rf "${TEMP_LOCATION}"/"${target}" + run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}" ;; "blobxfer" ) + write_log info "Synchronize local storage from S3 Bucket with blobxfer" + ${play_fair} blobxfer download --mode file --remote-path ${backup_job_blobxfer_remote_path} --local-path ${backup_job_filesystem_path} --delete - print_info "Synchronize local storage from S3 Bucket with blobxfer" - blobxfer download --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET} --delete + write_log info "Moving backup to external storage with blobxfer" + mkdir -p "${backup_job_filesystem_path}" + if [ "${backup_job_checksum}" != "none" ] ; then run_as_user mv "${TEMP_PATH}"/*."${checksum_extension}" "${backup_job_filesystem_path}"/; fi - print_info "Moving backup to external storage with blobxfer" + run_as_user mv "${TEMP_PATH}"/"${backup_job_filename}" "${backup_job_filesystem_path}"/"${backup_job_filename}" - mkdir -p "${DB_DUMP_TARGET}" - if var_true "${ENABLE_CHECKSUM}" ; then mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/; fi - - mv "${TEMP_LOCATION}"/"${target}" "${DB_DUMP_TARGET}"/"${target}" - - silent blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET} + silent run_as_user ${play_fair} blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --local-path ${backup_job_filesystem_path} move_exit_code=$? - if var_true "${ENABLE_CHECKSUM}" ; then rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}" ; fi - rm -rf "${TEMP_LOCATION}"/"${target}" + if [ "${backup_job_checksum}" != "none" ] ; then run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}"."${checksum_extension}" ; fi + run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}" ;; esac else - print_error "Skipping moving DB Backup to final location because backup did not complete successfully" + write_log error "Skipping moving DB Backup to final location because backup did not complete successfully" fi - rm -rf "${TEMP_LOCATION}"/* + run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}" + if var_true "${DEBUG_MOVE_DBBACKUP}" ; then debug off; fi } prepare_dbbackup() { - dbbackup_start_time=$(date +"%s") - now=$(date +"%Y%m%d-%H%M%S") - now_time=$(date +"%H:%M:%S") - now_date=$(date +"%Y-%m-%d") - ltarget=${dbtype}_${DB_NAME,,}_${DB_HOST,,} - target=${dbtype}_${DB_NAME,,}_${DB_HOST,,}_${now}.sql + timer backup start + now=$(run_as_user date +"%Y%m%d-%H%M%S") + backup_job_filename_base=${dbtype}_${backup_job_db_name,,}_${backup_job_db_host,,} + backup_job_filename=${dbtype}_${backup_job_db_name,,}_${backup_job_db_host,,}_${now}.sql } pre_dbbackup() { + if var_true "${DEBUG_PRE_DBBACKUP}" ; then debug on; fi ### Pre Script Support - if [ -n "${PRE_SCRIPT}" ] ; then - if var_true "${PRE_SCRIPT_SKIP_X_VERIFY}" ; then - eval "${PRE_SCRIPT}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${target}" + if [ -n "${backup_job_pre_script}" ] ; then + if var_true "${backup_job_pre_script_x_verify}" ; then + run_as_user eval "${backup_job_pre_script}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${backup_job_filename}" else - if [ -x "${PRE_SCRIPT}" ] ; then - print_notice "Found PRE_SCRIPT environment variable. Executing '${PRE_SCRIPT}" - eval "${PRE_SCRIPT}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${target}" + if [ -x "${backup_job_pre_script}" ] ; then + write_log notice "Found PRE_SCRIPT environment variable. Executing '${backup_job_pre_script}" + run_as_user eval "${backup_job_pre_script}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${backup_job_filename}" else - print_error "Can't execute PRE_SCRIPT environment variable '${PRE_SCRIPT}' as its filesystem bit is not executible!" + write_log error "Can't execute PRE_SCRIPT environment variable '${backup_job_pre_script}' as its filesystem bit is not executible!" fi fi fi ### Pre Backup Custom Script Support if [ -d "/assets/custom-scripts/pre" ] && dir_notempty "/assets/custom-scripts/pre" ; then - print_warning "Found Custom Post Scripts in /assets/custom-scripts/pre - Automatically moving them to '${SCRIPT_LOCATION_PRE}'" - mkdir -p "${SCRIPT_LOCATION_PRE}" - silent cp /assets/custom-scripts/pre/* "${SCRIPT_LOCATION_PRE}" + write_log warning "Found Custom Post Scripts in /assets/custom-scripts/pre - Automatically moving them to '${backup_job_script_location_pre}'" + run_as_user mkdir -p "${backup_job_script_location_pre}" + silent run_as_user cp /assets/custom-scripts/pre/* "${backup_job_script_location_pre}" fi - if [ -d "${SCRIPT_LOCATION_PRE}" ] && dir_notempty "${SCRIPT_LOCATION_PRE}" ; then - for f in $(find ${SCRIPT_LOCATION_PRE} -name \*.sh -type f); do - if var_true "${PRE_SCRIPT_SKIP_X_VERIFY}" ; then - ${f} "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${target}" + if [ -d "${backup_job_script_location_pre}" ] && dir_notempty "${backup_job_script_location_pre}" ; then + for f in $(find ${backup_job_script_location_pre} -name \*.sh -type f); do + if var_true "${backup_job_pre_script_x_verify}" ; then + run_as_user ${f} "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${backup_job_filename}" else if [ -x "${f}" ] ; then - print_notice "Executing pre backup custom script : '${f}'" + write_log notice "Executing pre backup custom script : '${f}'" ## script DB_TYPE DB_HOST DB_NAME STARTEPOCH BACKUP_FILENAME - ${f} "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${target}" + run_as_user ${f} "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${backup_job_filename}" else - print_error "Can't run pre backup custom script: '${f}' as its filesystem bit is not executible!" + write_log error "Can't run pre backup custom script: '${f}' as its filesystem bit is not executible!" fi fi done fi + if var_true "${DEBUG_PRE_DBBACKUP}" ; then debug off; fi } post_dbbackup() { - dbbackup_finish_time=$(date +"%s") - dbbackup_total_time=$(echo $((dbbackup_finish_time-dbbackup_start_time))) + if var_true "${DEBUG_POST_DBBACKUP}" ; then debug on; fi + dbbackup_finish_time=$(run_as_user date +"%s") + dbbackup_total_time=$(run_as_user echo $((dbbackup_finish_time-dbbackup_start_time))) - if var_true "${CONTAINER_ENABLE_MONITORING}" ; then - print_notice "Sending Backup Statistics to Zabbix" - silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.size -o "${dbbackup_size}" - silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.datetime -o "${dbbackup_date}" - silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.status -o "${exit_code}" - silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.backup_duration -o "$(echo $((dbbackup_finish_time-dbbackup_start_time)))" - if [ "$?" != "0" ] ; then print_error "Error sending statistics, consider disabling with 'CONTAINER_ENABLE_MONITORING=FALSE'" ; fi + if var_true "${CONTAINER_ENABLE_MONITORING}" && [ "${CONTAINER_MONITORING_BACKEND,,}" = "zabbix" ]; then + source /assets/defaults/03-monitoring + write_log notice "Sending Backup Statistics to Zabbix" + silent run_as_user zabbix_sender -c "${ZABBIX_CONFIG_PATH}"/"${ZABBIX_CONFIG_FILE}" -k dbbackup.backup -o '[{"{#NAME}":"'${backup_job_db_host}.${backup_job_db_name}'"}]' + local zabbix_payload=$(run_as_user mktemp) + + cat < /dev/null + ;; + esac + ;; + error ) + case "${_arg_log_level,,}" in + "debug" | "notice" | "warn" | "error") + echo "$(date +'%Y-%m-%d %H:%M:%S %Z') [error] ${_arg_log_message}" | run_as_user tee -a "${LOG_PATH}/$(date +'%Y%m%d')/$(date -d @${backup_job_start_time} +'%Y%m%d_%H%M%S')-${backup_job_filename_base}.log" > /dev/null + ;; + esac + ;; + info ) + echo "$(date +'%Y-%m-%d %H:%M:%S %Z') [info] ${_arg_log_message}" | run_as_user tee -a "${LOG_PATH}/$(date +'%Y%m%d')/$(date -d @${backup_job_start_time} +'%Y%m%d_%H%M%S')-${backup_job_filename_base}.log" > /dev/null + ;; + notice ) + case "${_arg_log_level,,}" in + "debug" | "notice" ) + echo "$(date +'%Y-%m-%d %H:%M:%S %Z') [notice] ${_arg_log_message}" | run_as_user tee -a "${LOG_PATH}/$(date +'%Y%m%d')/$(date -d @${backup_job_start_time} +'%Y%m%d_%H%M%S')-${backup_job_filename_base}.log" > /dev/null + ;; + esac + ;; + warn ) + case "${_arg_log_level,,}" in + "debug" | "notice" | "warn" ) + echo "$(date +'%Y-%m-%d %H:%M:%S %Z') [warn] ${_arg_log_message}" | run_as_user tee -a "${LOG_PATH}/$(date +'%Y%m%d')/$(date -d @${backup_job_start_time} +'%Y%m%d_%H%M%S')-${backup_job_filename_base}.log" > /dev/null + ;; + esac + ;; + esac + + fi + + print_${_arg_log_level} "${_arg_log_message}" + output_on + if var_true "${DEBUG_WRITE_LOG}" ; then debug off; fi +} \ No newline at end of file diff --git a/install/etc/cont-init.d/10-db-backup b/install/etc/cont-init.d/10-db-backup index 46a75d4..23c855b 100755 --- a/install/etc/cont-init.d/10-db-backup +++ b/install/etc/cont-init.d/10-db-backup @@ -6,9 +6,9 @@ prepare_service 03-monitoring PROCESS_NAME="db-backup" output_off -bootstrap_variables -sanity_test setup_mode -create_zabbix dbbackup +db_backup_container_init +create_schedulers backup +create_zabbix dbbackup4 liftoff diff --git a/install/etc/services.available/10-db-backup/run b/install/etc/services.available/10-db-backup/run deleted file mode 100755 index 12f2364..0000000 --- a/install/etc/services.available/10-db-backup/run +++ /dev/null @@ -1,115 +0,0 @@ -#!/command/with-contenv bash - -source /assets/functions/00-container -source /assets/functions/10-db-backup -source /assets/defaults/10-db-backup -PROCESS_NAME="db-backup" - -bootstrap_variables - -if [ "${MODE,,}" = "manual" ] || [ "${1,,}" = "manual" ] || [ "${1,,}" = "now" ]; then - DB_DUMP_BEGIN=+0 - manual=TRUE - print_debug "Detected Manual Mode" -else - sleep 5 - current_time=$(date +"%s") - today=$(date +"%Y%m%d") - - print_debug "******** Begin DB_DUMP_BEGIN ********" - print_debug "Current time $current_time" - - if [[ $DB_DUMP_BEGIN =~ ^\+(.*)$ ]]; then - print_debug "DB_DUMP_BEGIN is a jump of minute starting with +." - waittime=$(( ${BASH_REMATCH[1]} * 60 )) - target_time=$(($current_time + $waittime)) - elif [[ $DB_DUMP_BEGIN =~ ^[0-9]+$ ]]; then - print_debug "DB_DUMP_BEGIN is an integer." - - target_time=$(date --date="${today}${DB_DUMP_BEGIN}" +"%s") - if [[ "$target_time" < "$current_time" ]]; then - target_time=$(($target_time + 24*60*60)) - fi - waittime=$(($target_time - $current_time)) - elif [[ $DB_DUMP_BEGIN =~ ^([0-9]{4})-([0-9]{2})-([0-9]{2})[[:space:]]([0-9]{2}):([0-9]{2}):([0-9]{2})$ ]]; - then - # Calculate DB_DUMP_BEGIN time in seconds - dump_time=$(date -d "${DB_DUMP_BEGIN}" +%s) - print_debug "Dump time = $dump_time" - - # Calculate the difference in seconds - waittime=$((dump_time - current_time)) - print_debug "Difference in seconds: $waittime" - - if (( $waittime < 0 )); then - waittime=$(( ($waittime + ($DB_DUMP_FREQ - 1)) / ($DB_DUMP_FREQ * 60) )) - waittime=$(( $waittime * -1 )) - print_debug "Difference in seconds (rounded) waittime is in the past : $waittime" - fi - - target_time=$(($current_time + $waittime)) - print_debug "Target time = $target_time" - else - print_info "DB_DUMP_BEGIN is not starting with + or is not an integer or is not in the correct format (YYYY-mm-dd hh:mm:ss)." - fi - - print_debug "******** End DB_DUMP_BEGIN ********" - print_info "Wait Time: ${waittime} Target time: $(date -d @${target_time} +"%Y-%m-%d %T %Z") Current Time: $(date -d @${current_time} +"%Y-%m-%d %T %Z")" - sleep $waittime -fi - -while true; do - mkdir -p "${TEMP_LOCATION}" - backup_start_time=$(date +"%s") - print_debug "Backup routines started time: $(date +'%Y-%m-%d %T %Z')" - case "${dbtype,,}" in - "couch" ) - check_availability - backup_couch - ;; - "influx" ) - check_availability - backup_influx - ;; - "mssql" ) - check_availability - backup_mssql - ;; - "mysql" ) - check_availability - backup_mysql - ;; - "mongo" ) - check_availability - backup_mongo - ;; - "pgsql" ) - check_availability - backup_pgsql - ;; - "redis" ) - check_availability - backup_redis - ;; - "sqlite3" ) - check_availability - backup_sqlite3 - ;; - esac - - backup_finish_time=$(date +"%s") - backup_total_time=$(echo $((backup_finish_time-backup_start_time))) - if [ -z "$master_exit_code" ] ; then master_exit_code="0" ; fi - print_info "Backup routines finish time: $(date -d @${backup_finish_time} +"%Y-%m-%d %T %Z") with overall exit code ${master_exit_code}" - print_notice "Backup routines time taken: $(echo ${backup_total_time} | awk '{printf "Hours: %d Minutes: %02d Seconds: %02d", $1/3600, ($1/60)%60, $1%60}')" - - cleanup_old_data - - if var_true "${manual}" ; then - print_debug "Exiting due to manual mode" - exit ${master_exit_code}; - else - print_notice "Sleeping for another $(($DB_DUMP_FREQ*60-backup_total_time)) seconds. Waking up at $(date -d@"$(( $(date +%s)+$(($DB_DUMP_FREQ*60-backup_total_time))))" +"%Y-%m-%d %T %Z") " - sleep $(($DB_DUMP_FREQ*60-backup_total_time)) - fi -done diff --git a/install/usr/local/bin/backup-now b/install/usr/local/bin/backup-now deleted file mode 100755 index cd6e333..0000000 --- a/install/usr/local/bin/backup-now +++ /dev/null @@ -1,4 +0,0 @@ -#!/command/with-contenv bash - -echo '** Performing Manual Backup' -/etc/services.available/10-db-backup/run manual diff --git a/install/usr/local/bin/logrotate_dbbackup b/install/usr/local/bin/logrotate_dbbackup new file mode 100755 index 0000000..e852afa --- /dev/null +++ b/install/usr/local/bin/logrotate_dbbackup @@ -0,0 +1,24 @@ +#!/command/with-contenv bash + +source /assets/functions/00-container +source /assets/defaults/05-logging +source /assets/defaults/10-db-backup + +## Compress each log 2 days old +timestamp_2dayold_unixtime="$(stat -c %Y "${LOG_PATH}"/"$(date --date='2 days ago' +'%Y%m%d')")" +for logfile in "${LOG_PATH}"/"$(date --date='2 days ago' +'%Y%m%d')"/"$(date --date='2 days ago' +'%Y%m%d')"_*.log ; do + sudo -u restic zstd --rm --rsyncable "${logfile}" +done +touch -t $(date -d"@${timestamp_2dayold_unixtime}" +'%Y%m%d%H%m.%S') "${LOG_PATH}"/"$(date --date='2 days ago' +'%Y%m%d')" + +# Look fook files older than certain day and delete +if [ -n "${LOG_PATH}" ] && [ -d "${LOG_PATH}" ] ; then + find "${LOG_PATH}" -mtime +"${LOGROTATE_RETAIN_DAYS}" -type d -exec rm -rf {} + +fi + +# Look for stale symbolic links and delete accordingly +for symbolic_link in "${LOG_PATH}"/latest*.log ; do + if [ ! -e "${symbolic_link}" ] ; then + rm -rf "${symbolic_link}" + fi +done \ No newline at end of file diff --git a/install/usr/local/bin/restore b/install/usr/local/bin/restore index c17edf3..d75faee 100755 --- a/install/usr/local/bin/restore +++ b/install/usr/local/bin/restore @@ -38,7 +38,7 @@ else "-h" ) cat < 4.0.0\n\nSupports auto discovery of backup jobs and creates graphs and triggers", "groups": [ { "name": "DB/Backup" @@ -26,134 +25,260 @@ "name": "Templates/Databases" } ], - "items": [ + "discovery_rules": [ { - "uuid": "72fd00fa2dd24e479f5affe03e8711d8", - "name": "DB Backup: Backup Duration", + "uuid": "94bb6f862e1841f8b2834b04c41c1d86", + "name": "Backup", "type": "TRAP", - "key": "dbbackup.backup_duration", + "key": "dbbackup.backup", "delay": "0", - "history": "7d", - "units": "uptime", - "description": "How long the backup took", - "tags": [ + "item_prototypes": [ { - "tag": "Application", - "value": "DB Backup" - } - ] - }, - { - "uuid": "3549a2c9d56849babc6dc3c855484c1e", - "name": "DB Backup: Backup Time", - "type": "TRAP", - "key": "dbbackup.datetime", - "delay": "0", - "history": "7d", - "units": "unixtime", - "request_method": "POST", - "tags": [ + "uuid": "5a2c4d1cacf844829bc1fbf912e071c5", + "name": "[{#NAME}] Checksum - Duration", + "type": "TRAP", + "key": "dbbackup.backup.checksum.duration.[{#NAME}]", + "delay": "0", + "history": "7d", + "units": "uptime", + "tags": [ + { + "tag": "Application", + "value": "DB Backup" + } + ] + }, { - "tag": "Application", - "value": "DB Backup" + "uuid": "6e49769ec07344a4974b13dab00c3539", + "name": "[{#NAME}] Checksum - Hash", + "type": "TRAP", + "key": "dbbackup.backup.checksum.hash.[{#NAME}]", + "delay": "0", + "history": "30d", + "trends": "0", + "value_type": "TEXT", + "tags": [ + { + "tag": "Application", + "value": "DB Backup" + } + ] + }, + { + "uuid": "bb6472e30bff4d9c908b1d34b893e622", + "name": "[{#NAME}] Backup - Last Backup", + "type": "TRAP", + "key": "dbbackup.backup.datetime.[{#NAME}]", + "delay": "0", + "history": "7d", + "units": "unixtime", + "description": "Datestamp of last database backup", + "tags": [ + { + "tag": "Application", + "value": "DB Backup" + } + ], + "trigger_prototypes": [ + { + "uuid": "3681b56bb882466fb304a48b4beb15f0", + "expression": "fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],172800s)=0 and fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],259200s)<>0 and fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],345600s)<>0 and fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],432800s)<>0", + "name": "[{#NAME}] No backups detected in 2 days", + "priority": "HIGH", + "manual_close": "YES" + }, + { + "uuid": "6c70136c84994197b6396a143b4e956f", + "expression": "fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],172800s)<>0 and fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],259200s)=0 and fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],345600s)<>0 and fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],432800s)<>0", + "name": "[{#NAME}] No backups detected in 3 days", + "priority": "DISASTER", + "manual_close": "YES" + }, + { + "uuid": "d2038025cab643019cb9610c301f0cb9", + "expression": "fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],172800s)<>0 and fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],259200s)<>0 and fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],345600s)=0 and fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],432800s)<>0", + "name": "[{#NAME}] No backups detected in 4 days", + "priority": "DISASTER", + "manual_close": "YES" + }, + { + "uuid": "ea85f02d032c4a1dbc1b6e91a3b2b37b", + "expression": "fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],172800s)<>0 and fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],259200s)<>0 and fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],345600s)<>0 and fuzzytime(/DB Backup4/dbbackup.backup.datetime.[{#NAME}],432800s)=0", + "name": "[{#NAME}] No backups detected in 5 days", + "priority": "DISASTER", + "manual_close": "YES" + } + ] + }, + { + "uuid": "8ec2b2f44ddf4f36b3dbb2aa15e3a32f", + "name": "[{#NAME}] Backup - Duration", + "type": "TRAP", + "key": "dbbackup.backup.duration.[{#NAME}]", + "delay": "0", + "history": "7d", + "units": "uptime", + "description": "How long the DB Backup job took", + "tags": [ + { + "tag": "Application", + "value": "DB Backup" + } + ] + }, + { + "uuid": "3f0dc3c75261447c93482815c3d69524", + "name": "[{#NAME}] Encrypt - Duration", + "type": "TRAP", + "key": "dbbackup.backup.encrypt.duration.[{#NAME}]", + "delay": "0", + "history": "7d", + "units": "uptime", + "tags": [ + { + "tag": "Application", + "value": "DB Backup" + } + ] + }, + { + "uuid": "c3d5ad0789c443859d6a673e03db9cec", + "name": "[{#NAME}] Backup - Filename", + "type": "TRAP", + "key": "dbbackup.backup.filename.[{#NAME}]", + "delay": "0", + "history": "30d", + "trends": "0", + "value_type": "TEXT", + "tags": [ + { + "tag": "Application", + "value": "DB Backup" + } + ] + }, + { + "uuid": "43b700c03897465eb7e49bbfe8fc9fc5", + "name": "[{#NAME}] Backup - Size", + "type": "TRAP", + "key": "dbbackup.backup.size.[{#NAME}]", + "delay": "0", + "history": "7d", + "description": "Backup Size", + "tags": [ + { + "tag": "Application", + "value": "DB Backup" + } + ], + "trigger_prototypes": [ + { + "uuid": "849f8660bee04427aff55af47b6f509c", + "expression": "last(/DB Backup4/dbbackup.backup.size.[{#NAME}])/last(/DB Backup4/dbbackup.backup.size.[{#NAME}],#2)>1.2", + "name": "[{#NAME}] Backup 20% Greater in size", + "priority": "WARNING", + "manual_close": "YES" + }, + { + "uuid": "74d16a7680544c65af22cc568ce3d59d", + "expression": "last(/DB Backup4/dbbackup.backup.size.[{#NAME}])/last(/DB Backup4/dbbackup.backup.size.[{#NAME}],#2)<0.2", + "name": "[{#NAME}] Backup 20% Smaller in Size", + "priority": "WARNING", + "manual_close": "YES" + }, + { + "uuid": "5595d769c73f4eaeadda95c84c2c0f17", + "expression": "last(/DB Backup4/dbbackup.backup.size.[{#NAME}])<1K", + "name": "[{#NAME}] Backup Empty", + "priority": "HIGH", + "manual_close": "YES" + } + ] + }, + { + "uuid": "a6fc542a565c4baba8429ed9ab31b5ae", + "name": "[{#NAME}] Backup - Status", + "type": "TRAP", + "key": "dbbackup.backup.status.[{#NAME}]", + "delay": "0", + "history": "7d", + "description": "Maps exit code by DB Backup procedure", + "valuemap": { + "name": "Backup Status" + }, + "tags": [ + { + "tag": "Application", + "value": "DB Backup" + } + ], + "trigger_prototypes": [ + { + "uuid": "74b91e28453b4c2a84743f5e371495c1", + "expression": "last(/DB Backup4/dbbackup.backup.status.[{#NAME}])=1", + "name": "[{#NAME}] Backup - Failed with errors", + "priority": "WARNING", + "manual_close": "YES" + } + ] } ], - "triggers": [ + "graph_prototypes": [ { - "uuid": "3ac1e074ffea46eb8002c9c08a85e7b4", - "expression": "nodata(/DB Backup/dbbackup.datetime,2d)=1", - "name": "DB-Backup: No backups detected in 2 days", - "priority": "DISASTER", - "manual_close": "YES" + "uuid": "b5e8e9fe0c474fedba2b06366234afdf", + "name": "[{#NAME}] Backup Duration", + "graph_items": [ + { + "color": "199C0D", + "calc_fnc": "ALL", + "item": { + "host": "DB Backup4", + "key": "dbbackup.backup.duration.[{#NAME}]" + } + } + ] }, { - "uuid": "b8b5933dfa1a488c9c37dd7f4784c1ff", - "expression": "fuzzytime(/DB Backup/dbbackup.datetime,172800s)=0 and fuzzytime(/DB Backup/dbbackup.datetime,259200s)<>0 and fuzzytime(/DB Backup/dbbackup.datetime,345600s)<>0 and fuzzytime(/DB Backup/dbbackup.datetime,432800s)<>0", - "name": "DB Backup: No Backups occurred in 2 days", - "priority": "AVERAGE" + "uuid": "99b5deb4e28f40059c50846c7be2ef26", + "name": "[{#NAME}] Backup Size", + "graph_items": [ + { + "color": "199C0D", + "calc_fnc": "ALL", + "item": { + "host": "DB Backup4", + "key": "dbbackup.backup.size.[{#NAME}]" + } + } + ] }, { - "uuid": "35c5f420d0e142cc9601bae38decdc40", - "expression": "fuzzytime(/DB Backup/dbbackup.datetime,172800s)<>0 and fuzzytime(/DB Backup/dbbackup.datetime,259200s)=0 and fuzzytime(/DB Backup/dbbackup.datetime,345600s)<>0 and fuzzytime(/DB Backup/dbbackup.datetime,432800s)<>0", - "name": "DB Backup: No Backups occurred in 3 days", - "priority": "AVERAGE" + "uuid": "8c641e33659e4c8b866da64e252cfc2a", + "name": "[{#NAME}] Checksum Duration", + "graph_items": [ + { + "color": "199C0D", + "calc_fnc": "ALL", + "item": { + "host": "DB Backup4", + "key": "dbbackup.backup.checksum.duration.[{#NAME}]" + } + } + ] }, { - "uuid": "03c3719d82c241e886a0383c7d908a77", - "expression": "fuzzytime(/DB Backup/dbbackup.datetime,172800s)<>0 and fuzzytime(/DB Backup/dbbackup.datetime,259200s)<>0 and fuzzytime(/DB Backup/dbbackup.datetime,345600s)=0 and fuzzytime(/DB Backup/dbbackup.datetime,432800s)<>0", - "name": "DB Backup: No Backups occurred in 4 days", - "priority": "AVERAGE" - }, - { - "uuid": "1634a03e44964e42b7e0101f5f68499c", - "expression": "fuzzytime(/DB Backup/dbbackup.datetime,172800s)<>0 and fuzzytime(/DB Backup/dbbackup.datetime,259200s)<>0 and fuzzytime(/DB Backup/dbbackup.datetime,345600s)<>0 and fuzzytime(/DB Backup/dbbackup.datetime,432800s)=0", - "name": "DB Backup: No Backups occurred in 5 days or more", - "priority": "HIGH" - } - ] - }, - { - "uuid": "467dfec952b34f5aa4cc890b4351b62d", - "name": "DB Backup: Backup Size", - "type": "TRAP", - "key": "dbbackup.size", - "delay": "0", - "history": "7d", - "units": "B", - "request_method": "POST", - "tags": [ - { - "tag": "Application", - "value": "DB Backup" - } - ], - "triggers": [ - { - "uuid": "a41eb49b8a3541afb6de247dca750e38", - "expression": "last(/DB Backup/dbbackup.size)/last(/DB Backup/dbbackup.size,#2)>1.2", - "name": "DB Backup: 20% Greater in Size", - "priority": "WARNING", - "manual_close": "YES" - }, - { - "uuid": "422f66be5049403293f3d96fc53f20cd", - "expression": "last(/DB Backup/dbbackup.size)/last(/DB Backup/dbbackup.size,#2)<0.2", - "name": "DB Backup: 20% Smaller in Size", - "priority": "WARNING", - "manual_close": "YES" - }, - { - "uuid": "d6d9d875b92f4d799d4bc89aabd4e90e", - "expression": "last(/DB Backup/dbbackup.size)<1K", - "name": "DB Backup: empty", - "priority": "HIGH" - } - ] - }, - { - "uuid": "a6b13e8b46a64abab64a4d44d620d272", - "name": "DB Backup: Last Backup Status", - "type": "TRAP", - "key": "dbbackup.status", - "delay": "0", - "history": "7d", - "description": "Maps Exit Codes received by backup applications", - "valuemap": { - "name": "DB Backup Status" - }, - "tags": [ - { - "tag": "Application", - "value": "DB Backup" - } - ], - "triggers": [ - { - "uuid": "23d71e356f96493180f02d4b84a79fd6", - "expression": "last(/DB Backup/dbbackup.status)=1", - "name": "DB Backup: Failed Backup Detected", - "priority": "HIGH", - "manual_close": "YES" + "uuid": "65b8770f71ed4cff9111b82c42b17571", + "name": "[{#NAME}] Encrypt Duration", + "graph_items": [ + { + "color": "199C0D", + "calc_fnc": "ALL", + "item": { + "host": "DB Backup4", + "key": "dbbackup.backup.encrypt.duration.[{#NAME}]" + } + } + ] } ] } @@ -168,38 +293,10 @@ "value": "Database" } ], - "dashboards": [ - { - "uuid": "90c81bb47184401ca9663626784a6f30", - "name": "DB Backup", - "pages": [ - { - "widgets": [ - { - "type": "GRAPH_CLASSIC", - "name": "Backup Size", - "width": "23", - "height": "5", - "fields": [ - { - "type": "GRAPH", - "name": "graphid", - "value": { - "name": "DB Backup: Backup Size", - "host": "DB Backup" - } - } - ] - } - ] - } - ] - } - ], "valuemaps": [ { - "uuid": "82f3a3d01b3c42b8942b59d2363724e0", - "name": "DB Backup Status", + "uuid": "92a87279388b4fd1ac51c1e417e1776e", + "name": "Backup Status", "mappings": [ { "value": "0", @@ -214,36 +311,6 @@ } ] } - ], - "graphs": [ - { - "uuid": "6e02c200b76046bab76062cd1ab086b2", - "name": "DB Backup: Backup Duration", - "graph_items": [ - { - "color": "199C0D", - "item": { - "host": "DB Backup", - "key": "dbbackup.backup_duration" - } - } - ] - }, - { - "uuid": "b881ee18f05c4f4c835982c9dfbb55d6", - "name": "DB Backup: Backup Size", - "type": "STACKED", - "graph_items": [ - { - "sortorder": "1", - "color": "1A7C11", - "item": { - "host": "DB Backup", - "key": "dbbackup.size" - } - } - ] - } ] } } \ No newline at end of file