Compare commits

...

26 Commits

Author SHA1 Message Date
Dave Conroy
f207f375cc Release 2.4.0 - See CHANGELOG.md 2020-12-07 15:27:20 -08:00
Dave Conroy
88b58bffc5 Release 2.3.2 - See CHANGELOG.md 2020-11-14 12:37:58 -08:00
Dave Conroy
738f7fad25 Release 2.3.1 - See CHANGELOG.md 2020-11-11 13:45:05 -08:00
Dave Conroy
8c4733bf7f Merge pull request #52 from bambi73/master
#51 Fix backup of multiple InfluxDB databases failure
2020-11-11 13:43:56 -08:00
Bambi125
be4d8c0747 #51 Fix backup of multiple InfluxDB databases failure 2020-11-11 22:38:04 +01:00
Dave Conroy
a13849df0a Release 2.3.0 - See CHANGELOG.md 2020-10-15 08:15:10 -07:00
Dave Conroy
cb5347afe5 Release 2.2.2 - See CHANGELOG.md 2020-09-22 21:14:37 -07:00
Dave Conroy
ca03c5369d Merge pull request #47 from tpansino/bug/46-fix-docker-secrets
Fix Docker Secrets injection from DB_USER_FILE/DB_PASS_FILE
2020-09-22 21:02:05 -07:00
Tom Pansino
3008d9125f Fix Docker Secrets injection from DB_USER_FILE/DB_PASS_FILE 2020-09-22 20:32:09 -07:00
Dave Conroy
19cf3d007f Release 2.2.1 - See CHANGELOG.md 2020-09-17 21:39:27 -07:00
Dave Conroy
0bbf142349 Merge pull request #45 from alwynpan/fix-backup-now-date-error-message
Fix backup now date error message
2020-09-17 21:38:10 -07:00
Yao (Alwyn) Pan
1bc357866f #42 Update README 2020-09-18 14:34:06 +10:00
Yao (Alwyn) Pan
b38ad7a5cc #44 Remove 'invalid date' error message when performing backup-now 2020-09-18 14:32:08 +10:00
Dave Conroy
8bc02ee6c8 Release 2.2.0 - See CHANGELOG.md 2020-09-14 07:07:44 -07:00
Dave Conroy
3e71c377c6 Merge pull request #43 from alwynpan/fix-optional-vars
#42 Make DB_USER and DB_PASS optional for some dbtypes; update alpine repo URI
2020-09-14 07:05:29 -07:00
Yao (Alwyn) Pan
76a857239f #42 Make DB_USER and DB_PASS optional for some dbtypes; update alpine repo URI 2020-09-14 19:22:57 +10:00
Dave Conroy
02880d6541 Release 5.1.1 - See CHANGELOG.md 2020-09-01 09:57:58 -07:00
Dave Conroy
564613f329 Merge pull request #41 from zicklag/patch-1
Fix POST_SCRIPT Environment Vairable Run
2020-09-01 09:55:37 -07:00
Zicklag
2606d3c4d5 Fix POST_SCRIPT Environment Vairable Run 2020-09-01 09:46:43 -05:00
Dave Conroy
51f0206e17 Release 2.1.0 - See CHANGELOG.md 2020-08-29 07:43:24 -07:00
Dave Conroy
8d7bea3315 Merge branch 'master' of https://github.com/tiredofit/docker-db-backup into master 2020-08-29 07:37:10 -07:00
Dave Conroy
30c56229cf Release 2.1.0 - See CHANGELOG.md 2020-08-29 07:37:03 -07:00
Dave Conroy
04594087ed Create FUNDING.yml 2020-06-24 17:08:09 -07:00
Dave Conroy
b57683e992 Update README.md 2020-06-17 08:53:22 -07:00
Dave Conroy
1323966e22 Update README.md 2020-06-17 08:21:12 -07:00
Dave Conroy
310edda88c Reduce size of temporarily files
Changed way backups are performed to reduce temporary files
Removed Rethink Support
Rework MongoDB compression
Remove function prefix from functions
Rename case on variables for easier reading
2020-06-17 08:15:34 -07:00
6 changed files with 454 additions and 322 deletions

1
.github/FUNDING.yml vendored Normal file
View File

@@ -0,0 +1 @@
github: [tiredofit]

View File

@@ -1,3 +1,71 @@
## 2.4.0 2020-12-07 <dave at tiredofit dot ca>
### Added
- Switch back to packges for Postgresql (now 13.1)
## 2.3.2 2020-11-14 <dave at tiredofit dot ca>
### Changed
- Reapply S6-Overlay into filesystem as Postgresql build is removing S6 files due to edge containing S6 overlay
## 2.3.1 2020-11-11 <bambi73@github>
### Fixed
- Multiple Influx DB's not being backed up correctly
## 2.3.0 2020-10-15 <dave at tiredofit dot ca>
### Added
- Microsoft SQL Server support (experimental)
### Changed
- Compiled Postgresql 13 from source to backup psql/13 hosts
## 2.2.2 2020-09-22 <tpansino@github>
### Fixed
- Patch for 2.2.0 release fixing Docker Secrets Support. Was skipping password check.
## 2.2.1 2020-09-17 <alwynpan@github>
### Fixed
- Ondemand/Manual backup with `backup-now` was throwing errors not being able to find a proper date
## 2.2.0 2020-09-14 <alwynpan@github>
### Fixed
- Allow to use MariaDB and MongoDBs with no username and password while still allowing Docker Secrets
- Changed source of Alpine package repositories
## 2.1.1 2020-09-01 <zicklag@github>
### Fixed
- Add eval to POST_SCRIPT execution
## 2.1.0 2020-08-29 <dave at tiredofit dot ca>
### Added
- Add Exit Code variable to be used for custom scripts - See README.md for placement
- Add POST_SCRIPT environment variable to execute command instead of relying on custom script
## 2.0.0 2020-06-17 <dave at tiredofit dot ca>
### Added
- Reworked compression routines to remove dependency on temporary files
- Changed the way that MongoDB compression works - only supports GZ going forward
### Changed
- Code cleanup (removed function prefixes, added verbosity)
### Reverted
- Removed Rethink Support
## 1.23.0 2020-06-15 <dave at tiredofit dot ca> ## 1.23.0 2020-06-15 <dave at tiredofit dot ca>
### Added ### Added

View File

@@ -1,15 +1,15 @@
FROM tiredofit/alpine:edge FROM tiredofit/alpine:edge
LABEL maintainer="Dave Conroy (dave at tiredofit dot ca)"
### Set Environment Variables ### Set Environment Variables
ENV ENABLE_CRON=FALSE \ ENV MSSQL_VERSION=17.5.2.1-1 \
ENABLE_CRON=FALSE \
ENABLE_SMTP=FALSE \ ENABLE_SMTP=FALSE \
ENABLE_ZABBIX=TRUE \ ENABLE_ZABBIX=TRUE \
ZABBIX_HOSTNAME=db-backup ZABBIX_HOSTNAME=db-backup
### Dependencies ### Dependencies
RUN set -ex && \ RUN set -ex && \
echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \ echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
apk update && \ apk update && \
apk upgrade && \ apk upgrade && \
apk add -t .db-backup-build-deps \ apk add -t .db-backup-build-deps \
@@ -19,7 +19,7 @@ RUN set -ex && \
xz-dev \ xz-dev \
&& \ && \
\ \
apk add -t .db-backup-run-deps \ apk add --no-cache -t .db-backup-run-deps \
bzip2 \ bzip2 \
influxdb \ influxdb \
mariadb-client \ mariadb-client \
@@ -33,10 +33,15 @@ RUN set -ex && \
zstd \ zstd \
&& \ && \
\ \
apk add \ apk add --no-cache \
pixz@testing \ pixz@testing \
&& \ && \
\ \
cd /usr/src && \
curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_${MSSQL_VERSION}_amd64.apk && \
curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_${MSSQL_VERSION}_amd64.apk && \
echo y | apk add --allow-untrusted msodbcsql17_${MSSQL_VERSION}_amd64.apk mssql-tools_${MSSQL_VERSION}_amd64.apk && \
\
mkdir -p /usr/src/pbzip2 && \ mkdir -p /usr/src/pbzip2 && \
curl -ssL https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz | tar xvfz - --strip=1 -C /usr/src/pbzip2 && \ curl -ssL https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz | tar xvfz - --strip=1 -C /usr/src/pbzip2 && \
cd /usr/src/pbzip2 && \ cd /usr/src/pbzip2 && \

128
README.md
View File

@@ -1,16 +1,15 @@
# hub.docker.com/r/tiredofit/db-backup # hub.docker.com/r/tiredofit/db-backup
[![Build Status](https://img.shields.io/docker/build/tiredofit/db-backup.svg)](https://hub.docker.com/r/tiredofit/db-backup) [![Build Status](https://img.shields.io/docker/build/tiredofit/db-backup.svg)](https://hub.docker.com/r/tiredofit/db-backup)
[![Docker Pulls](https://img.shields.io/docker/pulls/tiredofit/db-backup.svg)](https://hub.docker.com/r/tiredofit/db-backup) [![Docker Pulls](https://img.shields.io/docker/pulls/tiredofit/db-backup.svg)](https://hub.docker.com/r/tiredofit/db-backup)
[![Docker Stars](https://img.shields.io/docker/stars/tiredofit/db-backup.svg)](https://hub.docker.com/r/tiredofit/db-backup) [![Docker Stars](https://img.shields.io/docker/stars/tiredofit/db-backup.svg)](https://hub.docker.com/r/tiredofit/db-backup)
[![Docker Layers](https://images.microbadger.com/badges/image/tiredofit/db-backup.svg)](https://microbadger.com/images/tiredofit/db-backup) [![Docker Layers](https://images.microbadger.com/badges/image/tiredofit/db-backup.svg)](https://microbadger.com/images/tiredofit/db-backup)
# Introduction ## Introduction
This will build a container for backing up multiple type of DB Servers This will build a container for backing up multiple type of DB Servers
Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis, Rethink servers. Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis servers.
* dump to local filesystem or backup to S3 Compatible services * dump to local filesystem or backup to S3 Compatible services
* select database user and password * select database user and password
@@ -27,40 +26,50 @@ Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis, Rethink s
[Changelog](CHANGELOG.md) [Changelog](CHANGELOG.md)
# Authors ## Authors
- [Dave Conroy](https://github.com/tiredofit) - [Dave Conroy](https://github.com/tiredofit)
# Table of Contents ## Table of Contents
- [Introduction](#introduction) - [hub.docker.com/r/tiredofit/db-backup](#hubdockercomrtiredofitdb-backup)
- [Changelog](CHANGELOG.md) - [Introduction](#introduction)
- [Prerequisites](#prerequisites) - [Authors](#authors)
- [Installation](#installation) - [Table of Contents](#table-of-contents)
- [Quick Start](#quick-start) - [Prerequisites](#prerequisites)
- [Configuration](#configuration) - [Installation](#installation)
- [Data Volumes](#data-volumes) - [Quick Start](#quick-start)
- [Environment Variables](#environmentvariables) - [Configuration](#configuration)
- [Maintenance](#maintenance) - [Data-Volumes](#data-volumes)
- [Environment Variables](#environment-variables)
- [Maintenance](#maintenance)
- [Shell Access](#shell-access) - [Shell Access](#shell-access)
- [Custom Scripts](#custom-scripts) - [Custom Scripts](#custom-scripts)
- [Example Post Script](#example-post-script)
- [$1=EXIT_CODE (After running backup routine)](#1exit_code-after-running-backup-routine)
- [$2=DB_TYPE (Type of Backup)](#2db_type-type-of-backup)
- [$3=DB_HOST (Backup Host)](#3db_host-backup-host)
- [#4=DB_NAME (Name of Database backed up](#4db_name-name-of-database-backed-up)
- [$5=DATE (Date of Backup)](#5date-date-of-backup)
- [$6=TIME (Time of Backup)](#6time-time-of-backup)
- [$7=BACKUP_FILENAME (Filename of Backup)](#7backup_filename-filename-of-backup)
- [$8=FILESIZE (Filesize of backup)](#8filesize-filesize-of-backup)
- [$9=MD5_RESULT (MD5Sum if enabled)](#9md5_result-md5sum-if-enabled)
# Prerequisites ## Prerequisites
You must have a working DB server or container available for this to work properly, it does not provide server functionality! You must have a working DB server or container available for this to work properly, it does not provide server functionality!
## Installation
# Installation
Automated builds of the image are available on [Docker Hub](https://hub.docker.com/r/tiredofit/db-backup) and is the recommended Automated builds of the image are available on [Docker Hub](https://hub.docker.com/r/tiredofit/db-backup) and is the recommended
method of installation. method of installation.
```bash ```bash
docker pull tiredofit/db-backup:latest docker pull tiredofit/db-backup:latest
``` ```
# Quick Start ### Quick Start
* The quickest way to get started is using [docker-compose](https://docs.docker.com/compose/). See the examples folder for a working [docker-compose.yml](examples/docker-compose.yml) that can be modified for development or production use. * The quickest way to get started is using [docker-compose](https://docs.docker.com/compose/). See the examples folder for a working [docker-compose.yml](examples/docker-compose.yml) that can be modified for development or production use.
@@ -69,68 +78,65 @@ docker pull tiredofit/db-backup:latest
> **NOTE**: If you are using this with a docker-compose file along with a seperate SQL container, take care not to set the variables to backup immediately, more so have it delay execution for a minute, otherwise you will get a failed first backup. > **NOTE**: If you are using this with a docker-compose file along with a seperate SQL container, take care not to set the variables to backup immediately, more so have it delay execution for a minute, otherwise you will get a failed first backup.
# Configuration ## Configuration
## Data-Volumes ### Data-Volumes
The following directories are used for configuration and can be mapped for persistent storage. The following directories are used for configuration and can be mapped for persistent storage.
| Directory | Description | | Directory | Description |
|-----------|-------------| | ------------------------ | ---------------------------------------------------------------------------------- |
| `/backup` | Backups | | `/backup` | Backups |
| `/assets/custom-scripts` | *Optional* Put custom scripts in this directory to execute after backup operations | `/assets/custom-scripts` | *Optional* Put custom scripts in this directory to execute after backup operations |
### Environment Variables
## Environment Variables
*If you are trying to backup a database that doesn't have a user or a password (you should!) make sure you set `CONTAINER_ENABLE_DOCKER_SECRETS=FALSE`*
Along with the Environment Variables from the [Base image](https://hub.docker.com/r/tiredofit/alpine), below is the complete list of available options that can be used to customize your installation. Along with the Environment Variables from the [Base image](https://hub.docker.com/r/tiredofit/alpine), below is the complete list of available options that can be used to customize your installation.
| Parameter | Description | | Parameter | Description |
|-----------|-------------| | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `BACKUP_LOCATION` | Backup to `FILESYSTEM` or `S3` compatible services like S3, Minio, Wasabi - Default `FILESYSTEM` | `BACKUP_LOCATION` | Backup to `FILESYSTEM` or `S3` compatible services like S3, Minio, Wasabi - Default `FILESYSTEM` |
| `COMPRESSION` | Use either Gzip `GZ`, Bzip2 `BZ`, XZip `XZ`, ZSTD `ZSTD` or none `NONE` - Default `GZ` | `COMPRESSION` | Use either Gzip `GZ`, Bzip2 `BZ`, XZip `XZ`, ZSTD `ZSTD` or none `NONE` - Default `GZ` |
| `COMPRESSION_LEVEL` | Numberical value of what level of compression to use, most allow `1` to `9` except for `ZSTD` which allows for `1` to `19` - Default `3` | | `COMPRESSION_LEVEL` | Numberical value of what level of compression to use, most allow `1` to `9` except for `ZSTD` which allows for `1` to `19` - Default `3` |
| `DB_TYPE` | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` `rethink` | `DB_TYPE` | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` |
| `DB_HOST` | Server Hostname e.g. `mariadb` | `DB_HOST` | Server Hostname e.g. `mariadb` |
| `DB_NAME` | Schema Name e.g. `database` | `DB_NAME` | Schema Name e.g. `database` |
| `DB_USER` | username for the database - use `root` to backup all MySQL of them. | `DB_USER` | username for the database - use `root` to backup all MySQL of them. |
| `DB_PASS` | (optional if DB doesn't require it) password for the database | `DB_PASS` | (optional if DB doesn't require it) password for the database |
| `DB_PORT` | (optional) Set port to connect to DB_HOST. Defaults are provided | `DB_PORT` | (optional) Set port to connect to DB_HOST. Defaults are provided |
| `DB_DUMP_FREQ` | How often to do a dump, in minutes. Defaults to 1440 minutes, or once per day. | `DB_DUMP_FREQ` | How often to do a dump, in minutes. Defaults to 1440 minutes, or once per day. |
| `DB_DUMP_BEGIN` | What time to do the first dump. Defaults to immediate. Must be in one of two formats | `DB_DUMP_BEGIN` | What time to do the first dump. Defaults to immediate. Must be in one of two formats |
| | Absolute HHMM, e.g. `2330` or `0415` | | Absolute HHMM, e.g. `2330` or `0415` |
| | Relative +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half | | Relative +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half |
| `DB_CLEANUP_TIME` | Value in minutes to delete old backups (only fired when dump freqency fires). 1440 would delete anything above 1 day old. You don't need to set this variable if you want to hold onto everything. | `DB_CLEANUP_TIME` | Value in minutes to delete old backups (only fired when dump freqency fires). 1440 would delete anything above 1 day old. You don't need to set this variable if you want to hold onto everything. |
| `DEBUG_MODE` | If set to `true`, print copious shell script messages to the container log. Otherwise only basic messages are printed. | `DEBUG_MODE` | If set to `true`, print copious shell script messages to the container log. Otherwise only basic messages are printed. |
| `EXTRA_OPTS` | If you need to pass extra arguments to the backup command, add them here e.g. "--extra-command" | `EXTRA_OPTS` | If you need to pass extra arguments to the backup command, add them here e.g. "--extra-command" |
| `MD5` | Generate MD5 Sum in Directory, `TRUE` or `FALSE` - Default `TRUE` | `MD5` | Generate MD5 Sum in Directory, `TRUE` or `FALSE` - Default `TRUE` |
| `PARALLEL_COMPRESSION` | Use multiple cores when compressing backups `TRUE` or `FALSE` - Default `TRUE` | | `PARALLEL_COMPRESSION` | Use multiple cores when compressing backups `TRUE` or `FALSE` - Default `TRUE` |
| `POST_SCRIPT` | Fill this variable in with a command to execute post the script backing up | |
| `SPLIT_DB` | If using root as username and multiple DBs on system, set to TRUE to create Seperate DB Backups instead of all in one. - Default `FALSE` | | `SPLIT_DB` | If using root as username and multiple DBs on system, set to TRUE to create Seperate DB Backups instead of all in one. - Default `FALSE` |
When using compression with MongoDB, only `GZ` compression is possible.
**Backing Up to S3 Compatible Services** **Backing Up to S3 Compatible Services**
If `BACKUP_LOCATION` = `S3` then the following options are used. If `BACKUP_LOCATION` = `S3` then the following options are used.
| Parameter | Description | | Parameter | Description |
|-----------|-------------| | --------------- | --------------------------------------------------------------------------------------- |
| `S3_BUCKET` | S3 Bucket name e.g. 'mybucket' | | `S3_BUCKET` | S3 Bucket name e.g. 'mybucket' |
| `S3_HOSTNAME` | Hostname of S3 Server e.g "s3.amazonaws.com" - You can also include a port if necessary | `S3_HOSTNAME` | Hostname of S3 Server e.g "s3.amazonaws.com" - You can also include a port if necessary |
| `S3_KEY_ID` | S3 Key ID | | `S3_KEY_ID` | S3 Key ID |
| `S3_KEY_SECRET` | S3 Key Secret | | `S3_KEY_SECRET` | S3 Key Secret |
| `S3_PATH` | S3 Pathname to save to e.g. '`backup`' | | `S3_PATH` | S3 Pathname to save to e.g. '`backup`' |
| `S3_PROTOCOL` | Use either `http` or `https` to access service - Default `https` | | `S3_PROTOCOL` | Use either `http` or `https` to access service - Default `https` |
| `S3_URI_STYLE` | Choose either `VIRTUALHOST` or `PATH` style - Default `VIRTUALHOST` | `S3_URI_STYLE` | Choose either `VIRTUALHOST` or `PATH` style - Default `VIRTUALHOST` |
## Maintenance ## Maintenance
Manual Backups can be performed by entering the container and typing `backup-now` Manual Backups can be performed by entering the container and typing `backup-now`
### Shell Access
#### Shell Access
For debugging and maintenance purposes you may want access the containers shell. For debugging and maintenance purposes you may want access the containers shell.
@@ -147,19 +153,21 @@ $ cat post-script.sh
##!/bin/bash ##!/bin/bash
## Example Post Script ## Example Post Script
## $1=DB_TYPE (Type of Backup) ## $1=EXIT_CODE (After running backup routine)
## $2=DB_HOST (Backup Host) ## $2=DB_TYPE (Type of Backup)
## #3=DB_NAME (Name of Database backed up ## $3=DB_HOST (Backup Host)
## $4=DATE (Date of Backup) ## #4=DB_NAME (Name of Database backed up
## $5=TIME (Time of Backup) ## $5=DATE (Date of Backup)
## $6=BACKUP_FILENAME (Filename of Backup) ## $6=TIME (Time of Backup)
## $7=FILESIZE (Filesize of backup) ## $7=BACKUP_FILENAME (Filename of Backup)
## $8=MD5_RESULT (MD5Sum if enabled) ## $8=FILESIZE (Filesize of backup)
## $9=MD5_RESULT (MD5Sum if enabled)
echo "${1} Backup Completed on ${2} for ${3} on ${4} ${5}. Filename: ${6} Size: ${7} bytes MD5: ${8}" echo "${1} ${2} Backup Completed on ${3} for ${4} on ${5} ${6}. Filename: ${7} Size: ${8} bytes MD5: ${9}"
```` ````
Outputs the following on the console: Outputs the following on the console:
`mysql Backup Completed on example-db for example on 2020-04-22 05:19:10. Filename: mysql_example_example-db_20200422-051910.sql.bz2 Size: 7795 bytes MD5: 952fbaafa30437494fdf3989a662cd40` `0 mysql Backup Completed on example-db for example on 2020-04-22 05:19:10. Filename: mysql_example_example-db_20200422-051910.sql.bz2 Size: 7795 bytes MD5: 952fbaafa30437494fdf3989a662cd40`
If you wish to change the size value from bytes to megabytes set environment variable `SIZE_VALUE=megabytes` If you wish to change the size value from bytes to megabytes set environment variable `SIZE_VALUE=megabytes`

View File

@@ -1,13 +1,14 @@
##!/bin/bash ##!/bin/bash
## Example Post Script ## Example Post Script
## $1=DB_TYPE (Type of Backup) ## $1=EXIT_CODE (After running backup routine)
## $2=DB_HOST (Backup Host) ## $2=DB_TYPE (Type of Backup)
## #3=DB_NAME (Name of Database backed up ## $3=DB_HOST (Backup Host)
## $4=DATE (Date of Backup) ## #4=DB_NAME (Name of Database backed up
## $5=TIME (Time of Backup) ## $5=DATE (Date of Backup)
## $6=BACKUP_FILENAME (Filename of Backup) ## $6=TIME (Time of Backup)
## $7=FILESIZE (Filesize of backup) ## $7=BACKUP_FILENAME (Filename of Backup)
## $8=MD5_RESULT (MD5Sum if enabled) ## $8=FILESIZE (Filesize of backup)
## $9=MD5_RESULT (MD5Sum if enabled)
echo "${1} Backup Completed on ${2} for ${3} on ${4} ${5}. Filename: ${6} Size: ${7} bytes MD5: ${8}" echo "${1} ${2} Backup Completed on ${3} for ${4} on ${5} ${6}. Filename: ${7} Size: ${8} bytes MD5: ${9}"

View File

@@ -14,8 +14,48 @@ fi
sanity_var DB_TYPE "Database Type" sanity_var DB_TYPE "Database Type"
sanity_var DB_HOST "Database Host" sanity_var DB_HOST "Database Host"
file_env 'DB_USER' ### Set the Database Type
file_env 'DB_PASS' dbtype=${DB_TYPE}
case "$dbtype" in
"couch" | "couchdb" | "COUCH" | "COUCHDB" )
dbtype=couch
dbport=${DB_PORT:-5984}
file_env 'DB_USER'
file_env 'DB_PASS'
;;
"influx" | "influxdb" | "INFLUX" | "INFLUXDB" )
dbtype=influx
dbport=${DB_PORT:-8088}
file_env 'DB_USER'
file_env 'DB_PASS'
;;
"mongo" | "mongodb" | "MONGO" | "MONGODB" )
dbtype=mongo
dbport=${DB_PORT:-27017}
[[ ( -n "${DB_USER}" ) || ( -n "${DB_USER_FILE}" ) ]] && file_env 'DB_USER'
[[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS'
;;
"mysql" | "MYSQL" | "mariadb" | "MARIADB")
dbtype=mysql
dbport=${DB_PORT:-3306}
[[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS'
;;
"mssql" | "MSSQL" | "microsoftsql" | "MICROSOFTSQL")
dbtype=mssql
dbport=${DB_PORT:-1433}
;;
"postgres" | "postgresql" | "pgsql" | "POSTGRES" | "POSTGRESQL" | "PGSQL" )
dbtype=pgsql
dbport=${DB_PORT:-5432}
[[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS'
;;
"redis" | "REDIS" )
dbtype=redis
dbport=${DB_PORT:-6379}
[[ ( -n "${DB_PASS}" || ( -n "${DB_PASS_FILE}" ) ) ]] && file_env 'DB_PASS'
;;
esac
### Set Defaults ### Set Defaults
BACKUP_LOCATION=${BACKUP_LOCATION:-"FILESYSTEM"} BACKUP_LOCATION=${BACKUP_LOCATION:-"FILESYSTEM"}
@@ -24,18 +64,17 @@ COMPRESSION_LEVEL=${COMPRESSION_LEVEL:-"3"}
DB_DUMP_BEGIN=${DB_DUMP_BEGIN:-+0} DB_DUMP_BEGIN=${DB_DUMP_BEGIN:-+0}
DB_DUMP_FREQ=${DB_DUMP_FREQ:-1440} DB_DUMP_FREQ=${DB_DUMP_FREQ:-1440}
DB_DUMP_TARGET=${DB_DUMP_TARGET:-/backup} DB_DUMP_TARGET=${DB_DUMP_TARGET:-/backup}
DBHOST=${DB_HOST} dbhost=${DB_HOST}
DBNAME=${DB_NAME} dbname=${DB_NAME}
DBPASS=${DB_PASS} dbpass=${DB_PASS}
DBTYPE=${DB_TYPE} dbuser=${DB_USER}
DBUSER=${DB_USER}
MD5=${MD5:-TRUE} MD5=${MD5:-TRUE}
PARALLEL_COMPRESSION=${PARALLEL_COMPRESSION:-TRUE} PARALLEL_COMPRESSION=${PARALLEL_COMPRESSION:-TRUE}
SIZE_VALUE=${SIZE_VALUE:-"bytes"} SIZE_VALUE=${SIZE_VALUE:-"bytes"}
SPLIT_DB=${SPLIT_DB:-FALSE} SPLIT_DB=${SPLIT_DB:-FALSE}
TMPDIR=/tmp/backups tmpdir=/tmp/backups
if [ "BACKUP_TYPE" = "S3" ] || [ "BACKUP_TYPE" = "s3" ] || [ "BACKUP_TYPE" = "MINIO" ] || [ "BACKUP_TYPE" = "minio" ] ; then if [ "$BACKUP_TYPE" = "S3" ] || [ "$BACKUP_TYPE" = "s3" ] || [ "$BACKUP_TYPE" = "MINIO" ] || [ "$BACKUP_TYPE" = "minio" ] ; then
S3_PROTOCOL=${S3_PROTOCOL:-"https"} S3_PROTOCOL=${S3_PROTOCOL:-"https"}
sanity_var S3_HOST "S3 Host" sanity_var S3_HOST "S3 Host"
sanity_var S3_BUCKET "S3 Bucket" sanity_var S3_BUCKET "S3 Bucket"
@@ -53,139 +92,138 @@ if [ "$1" = "NOW" ]; then
fi fi
### Set Compression Options ### Set Compression Options
if var_true $PARALLEL_COMPRESSION ; then if var_true "$PARALLEL_COMPRESSION" ; then
BZIP="pbzip2 -${COMPRESSION_LEVEL}" bzip="pbzip2 -${COMPRESSION_LEVEL}"
GZIP="pigz -${COMPRESSION_LEVEL}" gzip="pigz -${COMPRESSION_LEVEL}"
XZIP="pixz -${COMPRESSION_LEVEL}" xzip="pixz -${COMPRESSION_LEVEL}"
ZSTD="zstd --rm -${COMPRESSION_LEVEL}" zstd="zstd --rm -${COMPRESSION_LEVEL}"
else else
BZIP="bzip2 -${COMPRESSION_LEVEL}" bzip="bzip2 -${COMPRESSION_LEVEL}"
GZIP="gzip -${COMPRESSION_LEVEL}" gzip="gzip -${COMPRESSION_LEVEL}"
XZIP="xz -${COMPRESSION_LEVEL} " xzip="xz -${COMPRESSION_LEVEL} "
ZSTD="zstd --rm -${COMPRESSION_LEVEL}" zstd="zstd --rm -${COMPRESSION_LEVEL}"
fi fi
### Set the Database Authentication Details
### Set the Database Type case "$dbtype" in
case "$DBTYPE" in "mongo" )
"couch" | "couchdb" | "COUCH" | "COUCHDB" ) [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${dbuser}"
DBTYPE=couch [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${dbpass}"
DBPORT=${DB_PORT:-5984} [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${dbname}"
;; ;;
"influx" | "influxdb" | "INFLUX" | "INFLUXDB" ) "mysql" )
DBTYPE=influx [[ ( -n "${DB_PASS}" ) ]] && export MYSQL_PWD=${dbpass}
DBPORT=${DB_PORT:-8088}
;; ;;
"mongo" | "mongodb" | "MONGO" | "MONGODB" ) "postgres" )
DBTYPE=mongo [[ ( -n "${DB_PASS}" ) ]] && POSTGRES_PASS_STR="PGPASSWORD=${dbpass}"
DBPORT=${DB_PORT:-27017}
[[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DBUSER}"
[[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DBPASS}"
[[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DBNAME}"
;; ;;
"mysql" | "MYSQL" | "mariadb" | "MARIADB") "redis" )
DBTYPE=mysql [[ ( -n "${DB_PASS}" ) ]] && REDIS_PASS_STR=" -a ${dbpass}"
DBPORT=${DB_PORT:-3306}
[[ ( -n "${DB_PASS}" ) ]] && export MYSQL_PWD=${DBPASS}
;; ;;
"postgres" | "postgresql" | "pgsql" | "POSTGRES" | "POSTGRESQL" | "PGSQL" ) esac
DBTYPE=pgsql
DBPORT=${DB_PORT:-5432}
[[ ( -n "${DB_PASS}" ) ]] && POSTGRES_PASS_STR="PGPASSWORD=${DBPASS}"
;;
"redis" | "REDIS" )
DBTYPE=redis
DBPORT=${DB_PORT:-6379}
[[ ( -n "${DB_PASS}" ) ]] && REDIS_PASS_STR=" -a ${DBPASS}"
;;
"rethink" | "RETHINK" )
DBTYPE=rethink
DBPORT=${DB_PORT:-28015}
[[ ( -n "${DB_PASS}" ) ]] && echo $DB_PASS>/tmp/.rethink.auth; RETHINK_PASS_STR=" --password-file /tmp/.rethink.auth"
[[ ( -n "${DB_NAME}" ) ]] && RETHINK_DB_STR=" -e ${DBNAME}"
;;
esac
### Functions ### Functions
function backup_couch() { backup_couch() {
TARGET=couch_${DBNAME}_${DBHOST}_${now}.txt target=couch_${dbname}_${dbhost}_${now}.txt
curl -X GET http://${DBHOST}:${DBPORT}/${DBNAME}/_all_docs?include_docs=true >${TMPDIR}/${TARGET}
generate_md5
compression compression
curl -X GET http://${dbhost}:${dbport}/${dbname}/_all_docs?include_docs=true ${dumpoutput} | $dumpoutput > ${tmpdir}/${target}
exit_code=$?
generate_md5
move_backup move_backup
} }
function backup_mysql() { backup_influx() {
if var_true $SPLIT_DB ; then if [ "${COMPRESSION}" = "NONE" ] || [ "${COMPRESSION}" = "none" ] || [ "${COMPRESSION}" = "FALSE" ] || [ "${COMPRESSION}" = "false" ] ; then
DATABASES=`mysql -h ${DBHOST} -P $DBPORT -u$DBUSER --batch -e "SHOW DATABASES;" | grep -v Database|grep -v schema` :
else
print_notice "Compressing InfluxDB backup with gzip"
influx_compression="-portable"
fi
for DB in $DB_NAME; do
target=influx_${DB}_${dbhost}_${now}
influxd backup ${influx_compression} -database $DB -host ${dbhost}:${dbport} ${tmpdir}/${target}
exit_code=$?
generate_md5
move_backup
done
}
backup_mongo() {
if [ "${COMPRESSION}" = "NONE" ] || [ "${COMPRESSION}" = "none" ] || [ "${COMPRESSION}" = "FALSE" ] || [ "${COMPRESSION}" = "false" ] ; then
target=${dbtype}_${dbname}_${dbhost}_${now}.archive
else
print_notice "Compressing MongoDB backup with gzip"
target=${dbtype}_${dbname}_${dbhost}_${now}.archivegz
mongo_compression="--gzip"
fi
mongodump --archive=${tmpdir}/${target} ${mongo_compression} --host ${dbhost} --port ${dbport} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_DB_STR} ${EXTRA_OPTS}
exit_code=$?
cd ${tmpdir}
generate_md5
move_backup
}
backup_mssql() {
target=mssql_${dbname}_${dbhost}_${now}.bak
/opt/mssql-tools/bin/sqlcmd -E -C -S ${dbhost}\,${dbport} -U ${dbuser} -P ${dbpass} Q "BACKUP DATABASE \[${dbname}\] TO DISK = N'${tmpdir}/${target}' WITH NOFORMAT, NOINIT, NAME = '${dbname}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10"
}
backup_mysql() {
if var_true "$SPLIT_DB" ; then
DATABASES=$(mysql -h ${dbhost} -P $dbport -u$dbuser --batch -e "SHOW DATABASES;" | grep -v Database|grep -v schema)
for db in $DATABASES; do for db in $DATABASES; do
if [[ "$db" != "information_schema" ]] && [[ "$db" != _* ]] ; then if [[ "$db" != "information_schema" ]] && [[ "$db" != _* ]] ; then
echo "** [db-backup] Dumping database: $db" print_notice "Dumping MariaDB database: $db"
TARGET=mysql_${db}_${DBHOST}_${now}.sql target=mysql_${db}_${dbhost}_${now}.sql
mysqldump --max-allowed-packet=512M -h $DBHOST -P $DBPORT -u$DBUSER ${EXTRA_OPTS} --databases $db > ${TMPDIR}/${TARGET}
generate_md5
compression compression
mysqldump --max-allowed-packet=512M -h $dbhost -P $dbport -u$dbuser ${EXTRA_OPTS} --databases $db | $dumpoutput > ${tmpdir}/${target}
exit_code=$?
generate_md5
move_backup move_backup
fi fi
done done
else else
mysqldump --max-allowed-packet=512M -A -h $DBHOST -P $DBPORT -u$DBUSER ${EXTRA_OPTS} > ${TMPDIR}/${TARGET}
generate_md5
compression compression
mysqldump --max-allowed-packet=512M -A -h $dbhost -P $dbport -u$dbuser ${EXTRA_OPTS} | $dumpoutput > ${tmpdir}/${target}
exit_code=$?
generate_md5
move_backup move_backup
fi fi
} }
function backup_influx() { backup_pgsql() {
for DB in $DB_NAME; do
influxd backup -database $DB -host ${DBHOST}:${DBPORT} ${TMPDIR}/${TARGET}
generate_md5
compression
move_backup
done
}
function backup_mongo() {
mongodump --out ${TMPDIR}/${TARGET} --host ${DBHOST} --port ${DBPORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_DB_STR} ${EXTRA_OPTS}
cd ${TMPDIR}
tar cf ${TARGET}.tar ${TARGET}/*
TARGET=${TARGET}.tar
generate_md5
compression
move_backup
}
function backup_pgsql() {
if var_true $SPLIT_DB ; then if var_true $SPLIT_DB ; then
export PGPASSWORD=${DBPASS} export PGPASSWORD=${dbpass}
DATABASES=`psql -h $DBHOST -U $DBUSER -p ${DBPORT} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' ` DATABASES=$(psql -h $dbhost -U $dbuser -p ${dbport} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
for db in $DATABASES; do for db in $DATABASES; do
print_info "Dumping database: $db" print_info "Dumping database: $db"
TARGET=pgsql_${db}_${DBHOST}_${now}.sql target=pgsql_${db}_${dbhost}_${now}.sql
pg_dump -h ${DBHOST} -p ${DBPORT} -U ${DBUSER} $db ${EXTRA_OPTS}> ${TMPDIR}/${TARGET}
generate_md5
compression compression
pg_dump -h ${dbhost} -p ${dbport} -U ${dbuser} $db ${EXTRA_OPTS} | $dumpoutput > ${tmpdir}/${target}
exit_code=$?
generate_md5
move_backup move_backup
done done
else else
export PGPASSWORD=${DBPASS} export PGPASSWORD=${dbpass}
pg_dump -h ${DBHOST} -U ${DBUSER} -p ${DBPORT} ${DBNAME} ${EXTRA_OPTS}> ${TMPDIR}/${TARGET}
generate_md5
compression compression
pg_dump -h ${dbhost} -U ${dbuser} -p ${dbport} ${dbname} ${EXTRA_OPTS} | $dumpoutput > ${tmpdir}/${target}
exit_code=$?
generate_md5
move_backup move_backup
fi fi
} }
function backup_redis() { backup_redis() {
TARGET=redis_${db}_${DBHOST}_${now}.rdb target=redis_${db}_${dbhost}_${now}.rdb
echo bgsave | redis-cli -h ${DBHOST} -p ${DBPORT} ${REDIS_PASS_STR} --rdb ${TMPDIR}/${TARGET} ${EXTRA_OPTS} echo bgsave | redis-cli -h ${dbhost} -p ${dbport} ${REDIS_PASS_STR} --rdb ${tmpdir}/${target} ${EXTRA_OPTS}
print_info "Dumping Redis - Flushing Redis Cache First" print_info "Dumping Redis - Flushing Redis Cache First"
sleep 10 sleep 10
try=5 try=5
while [ $try -gt 0 ] ; do while [ $try -gt 0 ] ; do
saved=$(echo 'info Persistence' | redis-cli -h ${DBHOST} -p ${DBPORT} ${REDIS_PASS_STR} | awk '/rdb_bgsave_in_progress:0/{print "saved"}') saved=$(echo 'info Persistence' | redis-cli -h ${dbhost} -p ${dbport} ${REDIS_PASS_STR} | awk '/rdb_bgsave_in_progress:0/{print "saved"}')
ok=$(echo 'info Persistence' | redis-cli -h ${DBHOST} -p ${DBPORT} ${REDIS_PASS_STR} | awk '/rdb_last_bgsave_status:ok/{print "ok"}') ok=$(echo 'info Persistence' | redis-cli -h ${dbhost} -p ${dbport} ${REDIS_PASS_STR} | awk '/rdb_last_bgsave_status:ok/{print "ok"}')
if [[ "$saved" = "saved" ]] && [[ "$ok" = "ok" ]]; then if [[ "$saved" = "saved" ]] && [[ "$ok" = "ok" ]]; then
print_info "Redis Backup Complete" print_info "Redis Backup Complete"
fi fi
@@ -198,120 +236,119 @@ function backup_redis() {
move_backup move_backup
} }
function backup_rethink() { check_availability() {
TARGET=rethink_${db}_${DBHOST}_${now}.tar.gz
print_info "Dumping rethink Database: $db"
rethinkdb dump -f ${TMPDIR}/${TARGET} -c ${DBHOST}:${DBPORT} ${RETHINK_PASS_STR} ${RETHINK_DB_STR} ${EXTRA_OPTS}
move_backup
}
function check_availability() {
### Set the Database Type ### Set the Database Type
case "$DBTYPE" in case "$dbtype" in
"couch" ) "couch" )
COUNTER=0 COUNTER=0
while ! (nc -z ${DBHOST} ${DBPORT}) ; do while ! (nc -z ${dbhost} ${dbport}) ; do
sleep 5 sleep 5
let COUNTER+=5 (( COUNTER+=5 ))
print_warn "CouchDB Host '"$DBHOST"' is not accessible, retrying.. ($COUNTER seconds so far)" print_warn "CouchDB Host '${dbhost}' is not accessible, retrying.. ($COUNTER seconds so far)"
done done
;; ;;
"influx" ) "influx" )
COUNTER=0 COUNTER=0
while ! (nc -z ${DBHOST} ${DBPORT}) ; do while ! (nc -z ${dbhost} ${dbport}) ; do
sleep 5 sleep 5
let COUNTER+=5 (( COUNTER+=5 ))
print_warn "InfluxDB Host '"$DBHOST"' is not accessible, retrying.. ($COUNTER seconds so far)" print_warn "InfluxDB Host '${dbhost}' is not accessible, retrying.. ($COUNTER seconds so far)"
done done
;; ;;
"mongo" ) "mongo" )
COUNTER=0 COUNTER=0
while ! (nc -z ${DBHOST} ${DBPORT}) ; do while ! (nc -z ${dbhost} ${dbport}) ; do
sleep 5 sleep 5
let COUNTER+=5 (( COUNTER+=5 ))
print_warn "Mongo Host '"$DBHOST"' is not accessible, retrying.. ($COUNTER seconds so far)" print_warn "Mongo Host '${dbhost}' is not accessible, retrying.. ($COUNTER seconds so far)"
done done
;; ;;
"mysql" ) "mysql" )
COUNTER=0 COUNTER=0
while true; do while true; do
mysqlcmd='mysql -u'${DBUSER}' -P '${DBPORT}' -h '${DBHOST}' -p'${DBPASS} mysqlcmd='mysql -u'${dbuser}' -P '${dbport}' -h '${dbhost}' -p'${dbpass}
out="`$mysqlcmd -e "SELECT COUNT(*) FROM information_schema.FILES;" 2>&1`" out="$($mysqlcmd -e "SELECT COUNT(*) FROM information_schema.FILES;" 2>&1)"
echo "$out" | grep -E "COUNT|Enter" 2>&1 > /dev/null echo "$out" | grep -E "COUNT|Enter" 2>&1 > /dev/null
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
: :
break break
fi fi
print_warn "MySQL/MariaDB Server "$DBHOST" is not accessible, retrying.. ($COUNTER seconds so far)" print_warn "MySQL/MariaDB Server '${dbhost}' is not accessible, retrying.. ($COUNTER seconds so far)"
sleep 5 sleep 5
let COUNTER+=5 (( COUNTER+=5 ))
done
;;
"mssql" )
COUNTER=0
while ! (nc -z ${dbhost} ${dbport}) ; do
sleep 5
(( COUNTER+=5 ))
print_warn "MSSQL Host '${dbhost}' is not accessible, retrying.. ($COUNTER seconds so far)"
done done
;; ;;
"pgsql" ) "pgsql" )
# Wait until mongo logs that it's ready (or timeout after 60s)
COUNTER=0 COUNTER=0
export PGPASSWORD=${DBPASS} export PGPASSWORD=${dbpass}
until pg_isready --dbname=${DBNAME} --host=${DBHOST} --port=${DBPORT} --username=${DBUSER} -q until pg_isready --dbname=${dbname} --host=${dbhost} --port=${dbport} --username=${dbuser} -q
do do
sleep 5 sleep 5
let COUNTER+=5 (( COUNTER+=5 ))
print_warn "Postgres Host '"$DBHOST"' is not accessible, retrying.. ($COUNTER seconds so far)" print_warn "Postgres Host '${dbhost}' is not accessible, retrying.. ($COUNTER seconds so far)"
done done
;; ;;
"redis" ) "redis" )
COUNTER=0 COUNTER=0
while ! (nc -z ${DBHOST} ${DBPORT}) ; do while ! (nc -z "${dbhost}" "${dbport}") ; do
sleep 5 sleep 5
let COUNTER+=5 (( COUNTER+=5 ))
print_warn "Redis Host '"$DBHOST"' is not accessible, retrying.. ($COUNTER seconds so far)" print_warn "Redis Host '${dbhost}' is not accessible, retrying.. ($COUNTER seconds so far)"
done
;;
"rethink" )
COUNTER=0
while ! (nc -z ${DBHOST} ${DBPORT}) ; do
sleep 5
let COUNTER+=5
print_warn "RethinkDB Host '"$DBHOST"' is not accessible, retrying.. ($COUNTER seconds so far)"
done done
;; ;;
esac esac
} }
function compression() { compression() {
case "$COMPRESSION" in case "$COMPRESSION" in
"GZ" | "gz" | "gzip" | "GZIP") "GZ" | "gz" | "gzip" | "GZIP")
$GZIP ${TMPDIR}/${TARGET} print_notice "Compressing backup with gzip"
TARGET=${TARGET}.gz target=${target}.gz
dumpoutput="$gzip "
;; ;;
"BZ" | "bz" | "bzip2" | "BZIP2" | "bzip" | "BZIP" | "bz2" | "BZ2") "BZ" | "bz" | "bzip2" | "BZIP2" | "bzip" | "BZIP" | "bz2" | "BZ2")
$BZIP ${TMPDIR}/${TARGET} print_notice "Compressing backup with bzip2"
TARGET=${TARGET}.bz2 target=${target}.bz2
dumpoutput="$bzip "
;; ;;
"XZ" | "xz" | "XZIP" | "xzip" ) "XZ" | "xz" | "XZIP" | "xzip" )
$XZIP ${TMPDIR}/${TARGET} print_notice "Compressing backup with xzip"
TARGET=${TARGET}.xz target=${target}.xz
dumpoutput="$xzip "
;; ;;
"ZSTD" | "zstd" | "ZST" | "zst" ) "ZSTD" | "zstd" | "ZST" | "zst" )
$ZSTD ${TMPDIR}/${TARGET} print_notice "Compressing backup with zstd"
TARGET=${TARGET}.zst target=${target}.zst
dumpoutput="$zstd "
;; ;;
"NONE" | "none" | "FALSE" | "false") "NONE" | "none" | "FALSE" | "false")
dumpoutput="cat "
;; ;;
esac esac
} }
function generate_md5() { generate_md5() {
if var_true $MD5 ; then if var_true "$MD5" ; then
cd $TMPDIR print_notice "Generating MD5 for ${target}"
md5sum ${TARGET} > ${TARGET}.md5 cd $tmpdir
MD5VALUE=$(md5sum ${TARGET} | awk '{ print $1}') md5sum "${target}" > "${target}".md5
MD5VALUE=$(md5sum "${target}" | awk '{ print $1}')
fi fi
} }
function move_backup() { move_backup() {
case "$SIZE_VALUE" in case "$SIZE_VALUE" in
"b" | "bytes" ) "b" | "bytes" )
SIZE_VALUE=1 SIZE_VALUE=1
;; ;;
"[kK]" | "[kK][bB]" | "kilobytes" | "[mM]" | "[mM][bB]" | "megabytes" ) "[kK]" | "[kK][bB]" | "kilobytes" | "[mM]" | "[mM][bB]" | "megabytes" )
SIZE_VALUE="-h" SIZE_VALUE="-h"
@@ -321,31 +358,33 @@ function move_backup() {
;; ;;
esac esac
if [ "$SIZE_VALUE" = "1" ] ; then if [ "$SIZE_VALUE" = "1" ] ; then
FILESIZE=$(stat -c%s "${DB_DUMP_TARGET}/${TARGET}") FILESIZE=$(stat -c%s "${tmpdir}/${target}")
print_notice "Backup of ${target} created with the size of ${FILESIZE} bytes"
else else
FILESIZE=$(du -h "${DB_DUMP_TARGET}/${TARGET}" | awk '{ print $1}') FILESIZE=$(du -h "${tmpdir}/${target}" | awk '{ print $1}')
print_notice "Backup of ${target} created with the size of ${FILESIZE}"
fi fi
case "${BACKUP_LOCATION}" in case "${BACKUP_LOCATION}" in
"FILE" | "file" | "filesystem" | "FILESYSTEM" ) "FILE" | "file" | "filesystem" | "FILESYSTEM" )
mkdir -p ${DB_DUMP_TARGET} mkdir -p "${DB_DUMP_TARGET}"
mv ${TMPDIR}/*.md5 ${DB_DUMP_TARGET}/ mv ${tmpdir}/*.md5 "${DB_DUMP_TARGET}"/
mv ${TMPDIR}/${TARGET} ${DB_DUMP_TARGET}/${TARGET} mv ${tmpdir}/"${target}" "${DB_DUMP_TARGET}"/"${target}"
;; ;;
"S3" | "s3" | "MINIO" | "minio" ) "S3" | "s3" | "MINIO" | "minio" )
s3_content_type="application/octet-stream" s3_content_type="application/octet-stream"
if [ "$S3_URI_STYLE" = "VIRTUALHOST" ] || [ "$S3_URI_STYLE" = "VHOST" ] [ "$S3_URI_STYLE" = "virtualhost" ] [ "$S3_URI_STYLE" = "vhost" ] ; then if [ "$S3_URI_STYLE" = "VIRTUALHOST" ] || [ "$S3_URI_STYLE" = "VHOST" ] || [ "$S3_URI_STYLE" = "virtualhost" ] || [ "$S3_URI_STYLE" = "vhost" ] ; then
s3_url="${S3_BUCKET}.${S3_HOST}" s3_url="${S3_BUCKET}.${S3_HOST}"
else else
s3_url="${S3_HOST}/${S3_BUCKET}" s3_url="${S3_HOST}/${S3_BUCKET}"
fi fi
if var_true $MD5 ; then if var_true "$MD5" ; then
s3_date="$(LC_ALL=C date -u +"%a, %d %b %Y %X %z")" s3_date="$(LC_ALL=C date -u +"%a, %d %b %Y %X %z")"
s3_md5="$(libressl md5 -binary < "${TMPDIR}/${TARGET}.md5" | base64)" s3_md5="$(libressl md5 -binary < "${tmpdir}/${target}.md5" | base64)"
sig="$(printf "PUT\n$s3_md5\n${s3_content_type}\n$s3_date\n/$S3_BUCKET/$S3_PATH/${TARGET}.md5" | libressl sha1 -binary -hmac "${S3_KEY_SECRET}" | base64)" sig="$(printf "PUT\n$s3_md5\n${s3_content_type}\n$s3_date\n/$S3_BUCKET/$S3_PATH/${target}.md5" | libressl sha1 -binary -hmac "${S3_KEY_SECRET}" | base64)"
print_debug "Uploading ${TARGET}.md5 to S3" print_debug "Uploading ${target}.md5 to S3"
curl -T "${TMPDIR}/${TARGET}.md5" ${S3_PROTOCOL}://${s3_url}/${S3_PATH}/${TARGET}.md5 \ curl -T "${tmpdir}/${target}.md5" "${S3_PROTOCOL}"://"${s3_url}"/"${S3_PATH}"/"${target}".md5 \
-H "Date: $date" \ -H "Date: $date" \
-H "Authorization: AWS ${S3_KEY_ID}:$sig" \ -H "Authorization: AWS ${S3_KEY_ID}:$sig" \
-H "Content-Type: ${s3_content_type}" \ -H "Content-Type: ${s3_content_type}" \
@@ -353,26 +392,27 @@ function move_backup() {
fi fi
s3_date="$(LC_ALL=C date -u +"%a, %d %b %Y %X %z")" s3_date="$(LC_ALL=C date -u +"%a, %d %b %Y %X %z")"
s3_md5="$(libressl md5 -binary < "${TMPDIR}/${TARGET}" | base64)" s3_md5="$(libressl md5 -binary < "${tmpdir}/${target}" | base64)"
sig="$(printf "PUT\n$s3_md5\n${s3_content_type}\n$s3_date\n/$S3_BUCKET/$S3_PATH/${TARGET}" | libressl sha1 -binary -hmac "${S3_KEY_SECRET}" | base64)" sig="$(printf "PUT\n$s3_md5\n${s3_content_type}\n$s3_date\n/$S3_BUCKET/$S3_PATH/${target}" | libressl sha1 -binary -hmac "${S3_KEY_SECRET}" | base64)"
print_debug "Uploading ${TARGET} to S3" print_debug "Uploading ${target} to S3"
curl -T ${TMPDIR}/${TARGET} ${S3_PROTOCOL}://${s3_url}/${S3_PATH}/${TARGET} \ curl -T ${tmpdir}/"${target}" "${S3_PROTOCOL}"://"${s3_url}"/"${S3_PATH}"/"${target}" \
-H "Date: $s3_date" \ -H "Date: $s3_date" \
-H "Authorization: AWS ${S3_KEY_ID}:$sig" \ -H "Authorization: AWS ${S3_KEY_ID}:$sig" \
-H "Content-Type: ${s3_content_type}" \ -H "Content-Type: ${s3_content_type}" \
-H "Content-MD5: ${s3_md5}" -H "Content-MD5: ${s3_md5}"
rm -rf ${TMPDIR}/*.md5 rm -rf ${tmpdir}/*.md5
rm -rf ${TMPDIR}/${TARGET} rm -rf ${tmpdir}/"${target}"
;; ;;
esac esac
} }
### Container Startup ### Container Startup
print_info "Initialized on `date`" print_debug "Backup routines Initialized on $(date)"
### Wait for Next time to start backup ### Wait for Next time to start backup
if [ "$1" != "NOW" ]; then
current_time=$(date +"%s") current_time=$(date +"%s")
today=$(date +"%Y%m%d") today=$(date +"%Y%m%d")
@@ -386,22 +426,24 @@ print_info "Initialized on `date`"
waittime=$(($target_time - $current_time)) waittime=$(($target_time - $current_time))
fi fi
print_notice "Next Backup at $(date -d @${target_time} +"%Y-%m-%d %T %Z")"
sleep $waittime sleep $waittime
fi
### Commence Backup ### Commence Backup
while true; do while true; do
# make sure the directory exists # make sure the directory exists
mkdir -p $TMPDIR mkdir -p $tmpdir
### Define Target name ### Define Target name
now=$(date +"%Y%m%d-%H%M%S") now=$(date +"%Y%m%d-%H%M%S")
now_time=$(date +"%H:%M:%S") now_time=$(date +"%H:%M:%S")
now_date=$(date +"%Y-%m-%d") now_date=$(date +"%Y-%m-%d")
TARGET=${DBTYPE}_${DBNAME}_${DBHOST}_${now}.sql target=${dbtype}_${dbname}_${dbhost}_${now}.sql
### Take a Dump ### Take a Dump
case "$DBTYPE" in case "$dbtype" in
"couch" ) "couch" )
check_availability check_availability
backup_couch backup_couch
@@ -410,6 +452,10 @@ print_info "Initialized on `date`"
check_availability check_availability
backup_influx backup_influx
;; ;;
"mssql" )
check_availability
backup_mssql
;;
"mysql" ) "mysql" )
check_availability check_availability
backup_mysql backup_mysql
@@ -426,31 +472,34 @@ print_info "Initialized on `date`"
check_availability check_availability
backup_redis backup_redis
;; ;;
"rethink" )
check_availability
backup_rethink
;;
esac esac
### Zabbix ### Zabbix
if var_true $ENABLE_ZABBIX ; then if var_true "$ENABLE_ZABBIX" ; then
silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.size -o `stat -c%s ${DB_DUMP_TARGET}/${TARGET}` print_notice "Sending Backup Statistics to Zabbix"
silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.datetime -o `date -r ${DB_DUMP_TARGET}/${TARGET} +'%s'` silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.size -o "$(stat -c%s "${DB_DUMP_TARGET}"/"${target}")"
silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.datetime -o "$(date -r "${DB_DUMP_TARGET}"/"${target}" +'%s')"
fi fi
### Automatic Cleanup ### Automatic Cleanup
if [[ -n "$DB_CLEANUP_TIME" ]]; then if [[ -n "$DB_CLEANUP_TIME" ]]; then
find $DB_DUMP_TARGET/ -mmin +$DB_CLEANUP_TIME -iname "*" -exec rm {} \; print_notice "Cleaning up old backups"
find "${DB_DUMP_TARGET}"/ -mmin +"${DB_CLEANUP_TIME}" -iname "*" -exec rm {} \;
fi
if [ -n "$POST_SCRIPT" ] ; then
print_notice "Found POST_SCRIPT environment variable. Executing"
eval "${POST_SCRIPT}"
fi fi
### Post Backup Custom Script Support ### Post Backup Custom Script Support
if [ -d /assets/custom-scripts/ ] ; then if [ -d /assets/custom-scripts/ ] ; then
print_info "Found Custom Scripts to Execute" print_notice "Found Custom Filesystem Scripts to Execute"
for f in $(find /assets/custom-scripts/ -name \*.sh -type f); do for f in $(find /assets/custom-scripts/ -name \*.sh -type f); do
print_info "Running Script ${f}" print_notice "Running Script ${f}"
## script DB_TYPE DB_HOST DB_NAME DATE BACKUP_FILENAME FILESIZE MD5_VALUE ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME DATE BACKUP_FILENAME FILESIZE MD5_VALUE
chmod +x ${f} chmod +x "${f}"
${f} "${DBTYPE}" "${DBHOST}" "${DBNAME}" "${now_date}" "${now_time}" "${TARGET}" "${FILESIZE}" "${MD5VALUE}" ${f} "${exit_code}" "${dbtype}" "${dbhost}" "${dbname}" "${now_date}" "${now_time}" "${target}" "${FILESIZE}" "${MD5VALUE}"
done done
fi fi