Release 4.1.2 - See CHANGELOG.md

Resolve multiple issues using Azure blobs for remote storage

.github/workflows/main.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   build:
-    uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main
+    #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main
     #uses: tiredofit/github_actions/.github/workflows/default_amd64.yml@main
-    #uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main
+    uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main
     secrets: inherit

.github/workflows/manual.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   build:
-    uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main
+    #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main
     #uses: tiredofit/github_actions/.github/workflows/default_amd64.yml@main
-    #uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main
+    uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main
     secrets: inherit

CHANGELOG.md

@@ -1,3 +1,225 @@
+## 4.1.2 2024-07-02 <effectivelywild@github>
+
+   ### Added
+      - Add support for Azure Blob containers
+      - Fix timestamps when comparing previous backups
+      - Resolve unnecessary read operations in Azure
+      - Resolve issues with backup cleanup operations in Azure
+
+
+## 4.1.1 2024-06-19 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix issue where postgresql globals when backing up ALL not being deleted (#352)
+
+
+## 4.1.0 2024-05-25 <dave at tiredofit dot ca>
+
+    Note that arm/v7 builds have been removed from this release going forward
+
+   ### Added
+      - Introduce DEFAULT/DBXX_MYSQL_CLIENT option to use mariadb or mysql for client dumping to solve incompatibility issues
+      - Alpine 3.20 Base
+      - MariaDB 10.11.8 Client
+      - AWS Client 1.32.113
+      - MySQL Client 8.4.0
+
+
+## 4.0.35 2024-01-14 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix issue with emaail notifications and not being able to add from statement
+
+
+## 4.0.34 2024-01-02 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Change the way architectures are detected to re-enable backups with MSSQL and Influx2
+
+
+## 4.0.33 2023-12-18 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Allow _OPTS variabls to contain spaces
+      - Switch references of _DUMP_OPTS to _BACKUP_OPTS
+
+
+## 4.0.32 2023-12-15 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix issue with directories not properly being backed up (InfluxDB)
+
+
+## 4.0.31 2023-12-12 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Support backing up databases with spaces in them
+
+
+## 4.0.30 2023-12-11 <dave at tiredofit dot ca>
+
+   ### Added
+      - Seperate each job with its own temporary folder for isolation and to better cleanup jobs that backup as a directory instead of a flat file
+
+
+## 4.0.29 2023-12-04 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Skip blobxfer if either account or key is not present
+
+
+## 4.0.28 2023-12-04 <dave at tiredofit dot ca>
+
+   ### Changed
+      - AWS Cli 1.31.5
+      - Switch to using PIP for installing AWS-Cli to remove deprecation warnings
+
+
+## 4.0.27 2023-12-04 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Switch to using actual crontab for cron expressions
+
+
+## 4.0.26 2023-11-30 <dave at tiredofit dot ca>
+
+   ### Added
+      - AWS CLI 1.31.4
+
+
+## 4.0.25 2023-11-29 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix #297 - Add parameters to blobxfer to restore functionality
+
+
+## 4.0.24 2023-11-28 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix issue with cron parsing and 0 being a value getting clobbered by sort command
+
+
+## 4.0.23 2023-11-28 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Resolve issue with custom notification scripts not executing
+
+
+## 4.0.22 2023-11-25 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Move cleanup_old_data routines to happen within backup_ function to properly accomodate for globals, and ALL DB_NAME use cases
+
+
+## 4.0.21 2023-11-22 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix for SQLite backups not being cleaned up properly due to a malformed base
+
+
+## 4.0.20 2023-11-21 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Update base image to support S6 Overlay 3.1.6.2 to solve shutdown issues specifically with MODE=MANUAL and MANUAL_RUN_FOREVER=TRUE
+      - Add some safety nets for Manual scheduling
+
+
+## 4.0.19 2023-11-20 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Make adjustments to cron scheduling feature to be able to handle whitespace properly"
+
+
+## 4.0.18 2023-11-18 <joergmschulz@github>
+
+   ### Changed
+      - Fix loading msmtp configuration
+
+
+## 4.0.17 2023-11-17 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Provide more details when notifying via instant messages
+
+
+## 4.0.16 2023-11-17 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Switch to using msmtp instead of s-mail for notify()
+
+
+## 4.0.15 2023-11-16 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix cleanup of old backups
+
+
+## 4.0.14 2023-11-13 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Bugfix when PRE/POST scripts found not giving legacy warning
+      - Run pre / post scripts as root
+
+
+## 4.0.13 2023-11-12 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Check for any quotes if using MONGO_CUSTOM_URI and remove
+
+
+## 4.0.12 2023-11-12 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Allow creating schedulers if _MONGO_CUSTOM_URI is set and _DB_HOST blank
+
+
+## 4.0.11 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Resolve issue with backing up ALL databases with PGSQL and MySQL
+
+
+## 4.0.10 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Change environment variable parsing routines to properly accomodate for Passwords containing '=='
+
+
+## 4.0.9 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix issue with quotes being wrapped around _PASS variables
+
+
+## 4.0.8 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Tidy up file_encryption() routines
+      - Change environment variable _ENCRYPT_PUBKEY to _ENCRYPT_PUBLIC_KEY
+      - Add new environment variable _ENCRYPT_PRIVATE_KEY
+
+
+## 4.0.7 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Added
+      - Add seperate permissions for _FILESYSTEM_PATH
+
+   ### Changed
+      - More output and debugging additions
+      - SQLite3 now backs up without running into file permission/access problems
+      - Cleanup old sqlite backups from temp directory
+      - Handle multiple SQLite3 backups concurrently
+
+
+## 4.0.6 2023-11-10 <dave at tiredofit dot ca>
+
+   ### Added
+      - Add additional DEBUG_ statements
+
+   ### Changed
+      - Fix issue with Influx DB not properly detecting the correct version
+
+
 ## 4.0.5 2023-11-10 <dave at tiredofit dot ca>
 
    ### Added

Dockerfile

@@ -1,5 +1,5 @@
 ARG DISTRO=alpine
-ARG DISTRO_VARIANT=edge
+ARG DISTRO_VARIANT=3.20
 
 FROM docker.io/tiredofit/${DISTRO}:${DISTRO_VARIANT}
 LABEL maintainer="Dave Conroy (github.com/tiredofit)"
@@ -9,8 +9,10 @@ ENV INFLUX1_CLIENT_VERSION=1.8.0 \
     INFLUX2_CLIENT_VERSION=2.7.3 \
     MSODBC_VERSION=18.3.2.1-1 \
     MSSQL_VERSION=18.3.1.1-1 \
-    AWS_CLI_VERSION=1.29.78 \
-    CONTAINER_ENABLE_MESSAGING=FALSE \
+    MYSQL_VERSION=mysql-8.4.0 \
+    MYSQL_REPO_URL=https://github.com/mysql/mysql-server \
+    AWS_CLI_VERSION=1.32.113 \
+    CONTAINER_ENABLE_MESSAGING=TRUE \
     CONTAINER_ENABLE_MONITORING=TRUE \
     IMAGE_NAME="tiredofit/db-backup" \
     IMAGE_REPO_URL="https://github.com/tiredofit/docker-db-backup/"
@@ -27,11 +29,13 @@ RUN source /assets/functions/00-container && \
                     build-base \
                     bzip2-dev \
                     cargo \
+                    cmake \
                     git \
                     go \
                     libarchive-dev \
                     openssl-dev \
                     libffi-dev \
+                    ncurses-dev \
                     python3-dev \
                     py3-pip \
                     xz-dev \
@@ -47,8 +51,10 @@ RUN source /assets/functions/00-container && \
                     mariadb-client \
                     mariadb-connector-c \
                     mongodb-tools \
+                    ncurses \
                     openssl \
                     pigz \
+                    pixz \
                     postgresql16 \
                     postgresql16-client \
                     pv \
@@ -72,34 +78,33 @@ RUN source /assets/functions/00-container && \
     apkArch="$(uname -m)"; \
     case "$apkArch" in \
 	    x86_64) mssql=true ; mssql_arch=amd64; influx2=true ; influx_arch=amd64; ;; \
-        arm64 ) mssql=true ; mssql_arch=amd64; influx2=true ; influx_arch=arm64 ;; \
+        arm64 | aarch64 ) mssql=true ; mssql_arch=amd64; influx2=true ; influx_arch=arm64 ;; \
         *) sleep 0.1 ;; \
     esac; \
     \
-    if [ $mssql = "true" ] ; then curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk ; curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; ls -l ; echo y | apk add --allow-untrusted msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; else echo >&2 "Detected non x86_64 or ARM64 build variant, skipping MSSQL installation" ; fi; \
-    if [ $influx2 = "true" ] ; then curl -sSL https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX2_CLIENT_VERSION}-linux-${influx_arch}.tar.gz | tar xvfz - --strip=1 -C /usr/src/ ; chmod +x /usr/src/influx ; mv /usr/src/influx /usr/sbin/ ; else echo >&2 "Unable to build Influx 2 on this system" ; fi ; \
-    clone_git_repo https://github.com/aws/aws-cli "${AWS_CLI_VERSION}" && \
-    python3 setup.py install --prefix=/usr && \
+    if [[ $mssql = "true" ]] ; then curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk ; curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; echo y | apk add --allow-untrusted msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; else echo >&2 "Detected non x86_64 or ARM64 build variant, skipping MSSQL installation" ; fi; \
+    if [[ $influx2 = "true" ]] ; then curl -sSL https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX2_CLIENT_VERSION}-linux-${influx_arch}.tar.gz | tar xvfz - --strip=1 -C /usr/src/ ; chmod +x /usr/src/influx ; mv /usr/src/influx /usr/sbin/ ; else echo >&2 "Unable to build Influx 2 on this system" ; fi ; \
     clone_git_repo https://github.com/influxdata/influxdb "${INFLUX1_CLIENT_VERSION}" && \
     go build -o /usr/sbin/influxd ./cmd/influxd && \
     strip /usr/sbin/influxd && \
+    \
+    clone_git_repo "${MYSQL_REPO_URL}" "${MYSQL_VERSION}" && \
+    cmake \
+        -DCMAKE_BUILD_TYPE=MinSizeRel \
+        -DCMAKE_INSTALL_PREFIX=/opt/mysql \
+        -DFORCE_INSOURCE_BUILD=1 \
+        -DWITHOUT_SERVER:BOOL=ON \
+        && \
+    make -j$(nproc) install && \
+    \
+    pip3 install --break-system-packages awscli==${AWS_CLI_VERSION} && \
+    pip3 install --break-system-packages blobxfer && \
+    \
     mkdir -p /usr/src/pbzip2 && \
     curl -sSL https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz | tar xvfz - --strip=1 -C /usr/src/pbzip2 && \
     cd /usr/src/pbzip2 && \
     make && \
     make install && \
-    mkdir -p /usr/src/pixz && \
-    curl -sSL https://github.com/vasi/pixz/releases/download/v1.0.7/pixz-1.0.7.tar.xz | tar xvfJ - --strip 1 -C /usr/src/pixz && \
-    cd /usr/src/pixz && \
-    ./configure \
-        --prefix=/usr \
-        --sysconfdir=/etc \
-        --localstatedir=/var \
-        && \
-    make && \
-    make install && \
-    \
-    pip3 install --break-system-packages blobxfer && \
     \
     package remove .db-backup-build-deps && \
     package cleanup && \

README.md

@@ -214,12 +214,13 @@ If these are set and no other defaults or variables are set explicitly, they wil
 
 Encryption occurs after compression and the encrypted filename will have a `.gpg` suffix
 
-| Variable                     | Description                                 | Default |
-| ---------------------------- | ------------------------------------------- | ------- |
-| `DEFAULT_ENCRYPT`            | Encrypt file after backing up with GPG      | `FALSE` |
-| `DEFAULT_ENCRYPT_PASSPHRASE` | Passphrase to encrypt file with GPG         |         |
-| *or*                         |                                             |         |
-| `DEFAULT_ENCRYPT_PUBKEY`     | Path of public key to encrypt file with GPG |         |
+| Variable                      | Description                                  | Default | `_FILE` |
+| ----------------------------- | -------------------------------------------- | ------- | ------- |
+| `DEFAULT_ENCRYPT`             | Encrypt file after backing up with GPG       | `FALSE` |         |
+| `DEFAULT_ENCRYPT_PASSPHRASE`  | Passphrase to encrypt file with GPG          |         | x       |
+| *or*                          |                                              |         |         |
+| `DEFAULT_ENCRYPT_PUBLIC_KEY`  | Path of public key to encrypt file with GPG  |         | x       |
+| `DEFAULT_ENCRYPT_PRIVATE_KEY` | Path of private key to encrypt file with GPG |         | x       |
 
 ##### Scheduling Options
 
@@ -266,6 +267,7 @@ Encryption occurs after compression and the encrypted filename will have a `.gpg
 | `DEFAULT_EXTRA_BACKUP_OPTS`        | Pass extra arguments to the backup command only, add them here e.g. `--extra-command`                     |                           |         |
 | `DEFAULT_EXTRA_ENUMERATION_OPTS`   | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command`       |                           |         |
 | `DEFAULT_EXTRA_OPTS`               | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` |                           |         |
+| `DEFAULT_MYSQL_CLIENT`             | Choose between `mariadb` or `mysql` client to perform dump operations for compatibility purposes          | `mariadb`                 |         |
 | `DEFAULT_MYSQL_EVENTS`             | Backup Events                                                                                             | `TRUE`                    |         |
 | `DEFAULT_MYSQL_MAX_ALLOWED_PACKET` | Max allowed packet                                                                                        | `512M`                    |         |
 | `DEFAULT_MYSQL_SINGLE_TRANSACTION` | Backup in a single transaction                                                                            | `TRUE`                    |         |
@@ -323,11 +325,12 @@ Options that are related to the value of `DEFAULT_BACKUP_LOCATION`
 If `DEFAULT_BACKUP_LOCTION` = `FILESYSTEM` then the following options are used:
 
 | Variable                             | Description                                                                                           | Default                               |
-| --------------------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------- |
-| `DEFAULT_CREATE_LATEST_SYMLINK`   | Create a symbolic link pointing to last backup in this format: `latest-(DB_TYPE)-(DB_NAME)-(DB_HOST)` | `TRUE`                                |
+| ------------------------------------ | ----------------------------------------------------------------------------------------------------- | ------------------------------------- |
+| `DEFAULT_CREATE_LATEST_SYMLINK`      | Create a symbolic link pointing to last backup in this format: `latest-(DB_TYPE)_(DB_NAME)_(DB_HOST)` | `TRUE`                                |
 | `DEFAULT_FILESYSTEM_PATH`            | Directory where the database dumps are kept.                                                          | `/backup`                             |
+| `DEFAULT_FILESYSTEM_PATH_PERMISSION` | Permissions to apply to backup directory                                                              | `700`                                 |
 | `DEFAULT_FILESYSTEM_ARCHIVE_PATH`    | Optional Directory where the database dumps archives are kept                                         | `${DEFAULT_FILESYSTEM_PATH}/archive/` |
-| `DEFAULT_FILESYSTEM_PERMISSION`   | Directory and File permissions to apply to files.                                                     | `600`                                 |
+| `DEFAULT_FILESYSTEM_PERMISSION`      | Permissions to apply to files.                                                                        | `600`                                 |
 
 ###### S3
 
@@ -354,10 +357,13 @@ If `DEFAULT_BACKUP_LOCATION` = `S3` then the following options are used:
 If `DEFAULT_BACKUP_LOCATION` = `blobxfer` then the following options are used:.
 
 | Parameter                              | Description                                                         | Default             | `_FILE` |
-| -------------------------------------- | ------------------------------------------- | ------------------- | ------- |
+| -------------------------------------- | ------------------------------------------------------------------- | ------------------- | ------- |
 | `DEFAULT_BLOBXFER_STORAGE_ACCOUNT`     | Microsoft Azure Cloud storage account name.                         |                     | x       |
 | `DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key.                          |                     | x       |
 | `DEFAULT_BLOBXFER_REMOTE_PATH`         | Remote Azure path                                                   | `/docker-db-backup` | x       |
+| `DEFAULT_BLOBXFER_REMOTE_MODE`         | Azure Storage mode e.g. `auto`, `file`, `append`, `block` or `page` | `auto`              | x       |
+
+- When `DEFAULT_BLOBXFER_REMOTE_MODE` is set to auto it will use blob containers by default. If the `DEFAULT_BLOBXFER_REMOTE_PATH` path does not exist a blob container with that name will be created. 
 
 > This service uploads files from backup targed directory `DEFAULT_FILESYSTEM_PATH`.
 > If the a cleanup configuration in `DEFAULT_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically.
@@ -475,12 +481,14 @@ Otherwise, override them per backup job. Additional backup jobs can be scheduled
 
 Encryption will occur after compression and the resulting filename will have a `.gpg` suffix
 
-| Variable                  | Description                                 | Default |
-| ------------------------- | ------------------------------------------- | ------- |
-| `DB01_ENCRYPT`            | Encrypt file after backing up with GPG      | `FALSE` |
-| `DB01_ENCRYPT_PASSPHRASE` | Passphrase to encrypt file with GPG         |         |
-| *or*                      |                                             |         |
-| `DB01_ENCRYPT_PUBKEY`     | Path of public key to encrypt file with GPG |         |
+
+| Variable                   | Description                                  | Default | `_FILE` |
+| -------------------------- | -------------------------------------------- | ------- | ------- |
+| `DB01_ENCRYPT`             | Encrypt file after backing up with GPG       | `FALSE` |         |
+| `DB01_ENCRYPT_PASSPHRASE`  | Passphrase to encrypt file with GPG          |         | x       |
+| *or*                       |                                              |         |         |
+| `DB01_ENCRYPT_PUBLIC_KEY`  | Path of public key to encrypt file with GPG  |         | x       |
+| `DB01_ENCRYPT_PRIVATE_KEY` | Path of private key to encrypt file with GPG |         | x       |
 
 ##### Scheduling Options
 
@@ -532,6 +540,7 @@ Encryption will occur after compression and the resulting filename will have a `
 | `DB01_NAME`                     | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to.                      |                           |         |
 |                                 | Backup multiple by separating with commas eg `db1,db2`                                                    |                           | x       |
 | `DB01_NAME_EXCLUDE`             | If using `ALL` - use this as to exclude databases separated via commas from being backed up               |                           | x       |
+| `DB01_SPLIT_DB`                 | If using `ALL` - use this to split each database into its own file as opposed to one singular file        | `FALSE`                   |         |
 | `DB01_PORT`                     | MySQL / MariaDB Port                                                                                      | `3306`                    | x       |
 | `DB01_MYSQL_EVENTS`             | Backup Events for                                                                                         | `TRUE`                    |         |
 | `DB01_MYSQL_MAX_ALLOWED_PACKET` | Max allowed packet                                                                                        | `512M`                    |         |
@@ -573,6 +582,7 @@ Encryption will occur after compression and the resulting filename will have a `
 | `DB01_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command`       |         |         |
 | `DB01_NAME`                   | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to.                      |         |         |
 |                               | Backup multiple by separating with commas eg `db1,db2`                                                    |         | x       |
+| `DB01_SPLIT_DB`               | If using `ALL` - use this to split each database into its own file as opposed to one singular file        | `FALSE` |         |
 | `DB01_PORT`                   | PostgreSQL Port                                                                                           | `5432`  | x       |
 
 ###### Redis
@@ -598,10 +608,11 @@ Options that are related to the value of `DB01_BACKUP_LOCATION`
 If `DB01_BACKUP_LOCTION` = `FILESYSTEM` then the following options are used:
 
 | Variable                          | Description                                                                                           | Default                            |
-| ------------------------------ | ----------------------------------------------------------------------------------------------------- | --------------------------------- |
+| --------------------------------- | ----------------------------------------------------------------------------------------------------- | ---------------------------------- |
 | `DB01_CREATE_LATEST_SYMLINK`      | Create a symbolic link pointing to last backup in this format: `latest-(DB_TYPE)-(DB_NAME)-(DB_HOST)` | `TRUE`                             |
 | `DB01_FILESYSTEM_PATH`            | Directory where the database dumps are kept.                                                          | `/backup`                          |
-| `DB01_FILESYSTEM_ARCHIVE_PATH` | Optional Directory where the database dumps archives are kept                                         | `${DB01_FILESYSTEM_PATH/archive/` |
+| `DB01_FILESYSTEM_PATH_PERMISSION` | Permissions to apply to backup directory                                                              | `700`                              |
+| `DB01_FILESYSTEM_ARCHIVE_PATH`    | Optional Directory where the database dumps archives are kept                                         | `${DB01_FILESYSTEM_PATH}/archive/` |
 | `DB01_FILESYSTEM_PERMISSION`      | Directory and File permissions to apply to files.                                                     | `600`                              |
 
 ###### S3
@@ -629,10 +640,13 @@ If `DB01_BACKUP_LOCATION` = `S3` then the following options are used:
 If `DB01_BACKUP_LOCATION` = `blobxfer` then the following options are used:.
 
 | Parameter                              | Description                                                         | Default             | `_FILE` |
-| ----------------------------------- | ------------------------------------------- | ------------------- | ------- |
+| -------------------------------------- | ------------------------------------------------------------------- | ------------------- | ------- |
 | `DB01_BLOBXFER_STORAGE_ACCOUNT`        | Microsoft Azure Cloud storage account name.                         |                     | x       |
 | `DB01_BLOBXFER_STORAGE_ACCOUNT_KEY`    | Microsoft Azure Cloud storage account key.                          |                     | x       |
 | `DB01_BLOBXFER_REMOTE_PATH`            | Remote Azure path                                                   | `/docker-db-backup` | x       |
+| `DB01_BLOBXFER_REMOTE_MODE`            | Azure Storage mode e.g. `auto`, `file`, `append`, `block` or `page` | `auto`              | x       |
+
+- When `DEFAULT_BLOBXFER_REMOTE_MODE` is set to auto it will use blob containers by default. If the `DEFAULT_BLOBXFER_REMOTE_PATH` path does not exist a blob container with that name will be created. 
 
 > This service uploads files from backup directory `DB01_BACKUP_FILESYSTEM_PATH`.
 > If the a cleanup configuration in `DB01_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically.
@@ -737,6 +751,9 @@ $5 body/error message
 
 
 ##### Email Notifications
+
+See more details in the base image listed above for more mail environment variables.
+
 | Parameter   | Description                                                                               | Default | `_FILE` |
 | ----------- | ----------------------------------------------------------------------------------------- | ------- | ------- |
 | `MAIL_FROM` | What email address to send mail from for errors                                           |         |         |

examples/compose.yml

@@ -31,8 +31,8 @@ services:
       - BACKUP_JOB_CONCURRENCY=1     # Only run one job at a time
       - DEFAULT_CHECKSUM=NONE        # Don't create checksums
       - DEFAULT_COMPRESSION=ZSTD     # Compress all with ZSTD
-      - DEFAULT_DUMP_INTERVAL=1440   # Backup every 1440 minutes
-      - DEFAULT_DUMP_BEGIN=0000      # Start backing up at midnight
+      - DEFAULT_BACKUP_INTERVAL=1440   # Backup every 1440 minutes
+      - DEFAULT_BACKUP_BEGIN=0000      # Start backing up at midnight
       - DEFAULT_CLEANUP_TIME=8640    # Cleanup backups after a week
 
       - DB01_TYPE=mariadb
@@ -40,8 +40,8 @@ services:
       - DB01_NAME=example
       - DB01_USER=example
       - DB01_PASS=examplepassword
-      - DB01_DUMP_INTERVAL=30        # (override) Backup every 30 minutes
-      - DB01_DUMP_BEGIN=+1           # (override) Backup starts immediately
+      - DB01_BACKUP_INTERVAL=30        # (override) Backup every 30 minutes
+      - DB01_BACKUP_BEGIN=+1           # (override) Backup starts immediately
       - DB01_CLEANUP_TIME=180        # (override) Cleanup backups they are older than 180 minutes
       - DB01_CHECKSUM=SHA1           # (override) Create a SHA1 checksum
       - DB01_COMPRESSION=GZ          # (override) Compress with GZIP
@@ -51,8 +51,8 @@ services:
       #- DB02_NAME=example
       #- DB02_USER=example
       #- DB02_PASS=examplepassword
-      #- DB02_DUMP_INTERVAL=60         # (override) Backup every 60 minutes
-      #- DB02_DUMP_BEGIN=+10           # (override) Backup starts in ten minutes
+      #- DB02_BACKUP_INTERVAL=60         # (override) Backup every 60 minutes
+      #- DB02_BACKUP_BEGIN=+10           # (override) Backup starts in ten minutes
       #- DB02_CLEANUP_TIME=240         # (override) Cleanup backups they are older than 240 minutes
       #- DB02_CHECKSUM=MD5             # (override) Create a SHA1 checksum
       #- DB02_COMPRESSION=BZ           # (override) Compress with BZIP

examples/mssql-blobxfer/compose.yml

@@ -45,7 +45,7 @@ services:
       - DB01_NAME=test1             # Create this database
       - DB01_USER=sa
       - DB01_PASS=5hQa0utRFBpIY3yhoIyE
-      - DB01_DUMP_INTERVAL=5        # backup every 5 minute
+      - DB01_BACKUP_INTERVAL=5        # backup every 5 minute
       # - DB01_DUMP_BEGIN=0000      # backup starts at midnight vs not set immediately
       - DB01_CLEANUP_TIME=60         # clean backups they are older than 60 minutes
       - DB01_CHECKSUM=SHA1          # Set Checksum to be SHA1
@@ -57,7 +57,7 @@ services:
       # Add here azure storage account
       - DB01_BLOBXFER_STORAGE_ACCOUNT={TODO Add Storage Name}
       # Add here azure storage account key
-      - SB01_BLOBXFER_STORAGE_ACCOUNT_KEY={TODO Add Key}
+      - DB01_BLOBXFER_STORAGE_ACCOUNT_KEY={TODO Add Key}
       - DB01_BLOBXFER_REMOTE_PATH=docker-db-backup
     restart: always
     networks:

examples/mssql/compose.yml

@@ -46,7 +46,7 @@ services:
       - DB01_NAME=test1
       - DB01_USER=sa
       - DB01_PASS=5hQa0utRFBpIY3yhoIyE
-      - DB01_DUMP_INTERVAL=1            # backup every minute
+      - DB01_BACKUP_INTERVAL=1            # backup every minute
       # - DB01_DUMP_BEGIN=0000      # backup starts at midnight vs unset immediately
       - DB01_CLEANUP_TIME=5         # clean backups they are older than 5 minute
       - DB01_CHECKSUM=NONE

install/assets/dbbackup/template-dbbackup/run

@@ -8,12 +8,15 @@ source /assets/functions/10-db-backup
 source /assets/defaults/10-db-backup
 bootstrap_variables backup_init {{BACKUP_NUMBER}}
 bootstrap_variables parse_variables {{BACKUP_NUMBER}}
-PROCESS_NAME="{{BACKUP_NUMBER}}-${backup_job_db_host}__${backup_job_db_name}"
-
+if [ -z "${backup_job_db_name}" ]; then
+    PROCESS_NAME="{{BACKUP_NUMBER}}${backup_job_db_host//\//_}"
+else
+    PROCESS_NAME="{{BACKUP_NUMBER}}-${backup_job_db_host//\//_}__${backup_job_db_name}"
+fi
 
 trap ctrl_c INT
 
-if [[ "${MODE,,}" =~ "standalone" ]] || [ "${1,,}" = "manual" ] || [ "${1,,}" = "now" ]; then
+if [[ "${MODE,,}" =~ "standalone" ]] || [ "${MODE,,}" = "manual" ] || [ "${1,,}" = "manual" ] || [ "${1,,}" = "now" ]; then
     print_debug "Detected Manual Mode"
     persist=false
     backup_job_backup_begin=+0
@@ -21,7 +24,6 @@ else
     silent sleep {{BACKUP_NUMBER}}
     time_last_run=0
     time_current=$(date +'%s')
-
     if [[ "${backup_job_backup_begin}" =~ ^\+(.*)$ ]]; then
         print_debug "BACKUP_BEGIN is a jump of minute starting with +"
         timer plusvalue
@@ -31,10 +33,21 @@ else
     elif [[ "${backup_job_backup_begin}" =~ ([0-9]{4})-([0-9]{2})-([0-9]{2})[[:space:]]([0-9]{2}):([0-9]{2}):([0-9]{2}) ]]; then
         print_debug "BACKUP_BEGIN is a full date timestamp"
         timer datetime
+    #elif echo "${backup_job_backup_begin//\*/#}" | grep -qP "^(.*((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|#) ?){5}$" ; then # Allow slashes, yet not supporting advanced cron yet
     elif echo "${backup_job_backup_begin//\*/#}" | grep -qP "^(((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|#) ?){5}$" ; then
         print_debug "BACKUP_BEGIN is a cron expression"
+        if var_false "${CRON_ALTERNATE}"; then
             time_last_run=$(date +"%s")
+
+            backup_job_backup_begin=${backup_job_backup_begin//\"/}
+            backup_job_backup_begin=${backup_job_backup_begin//\'/}
             timer cron "${backup_job_backup_begin}" "${time_current}" "${time_last_run}"
+        else
+            echo "${backup_job_backup_begin} /var/run/s6/legacy-services/dbbackup-{{BACKUP_NUMBER}}/run now" > /tmp/.container/cron/{{BACKUP_NUMBER}}-backup
+            crontab -l | { cat; echo "${backup_job_backup_begin} /var/run/s6/legacy-services/dbbackup-{{BACKUP_NUMBER}}/run now"; } | crontab -
+            s6-svc -d /var/run/s6/legacy-services/dbbackup-{{BACKUP_NUMBER}}
+            exit 0
+        fi
     else
         print_error "_BACKUP_BEGIN is invalid - Unable to perform scheduling"
         cat <<EOF
@@ -94,8 +107,8 @@ while true; do
             s6-svc -d /var/run/s6/legacy-services/dbbackup-{{BACKUP_NUMBER}}
         else
             if [ ! "${time_cron}" = "true" ]; then
-                print_notice "Sleeping for another $(($backup_job_backup_interval*60-backup_job_total_time)) seconds. Waking up at $(date -d@"$(( $(date +%s)+$(($backup_job_backup_interval*60-backup_job_total_time))))" +'%Y-%m-%d %T %Z') "
-                silent sleep $(($backup_job_backup_interval*60-backup_job_total_time))
+                print_notice "Sleeping for another $((backup_job_backup_interval*60-backup_job_total_time)) seconds. Waking up at $(date -d@"$(( $(date +%s)+$((backup_job_backup_interval*60-backup_job_total_time))))" +'%Y-%m-%d %T %Z') "
+                silent sleep $((backup_job_backup_interval*60-backup_job_total_time))
             else
                 time_last_run=$(date +"%s")
                 timer cron "${backup_job_backup_begin}" "${time_current}" "${time_last_run}"

install/assets/defaults/10-db-backup

@@ -1,13 +1,14 @@
 #!/command/with-contenv bash
 
 BACKUP_JOB_CONCURRENCY=${BACKUP_JOB_CONCURRENCY:-"1"}
+CRON_ALTERNATE=${CRON_ALTERNATE:-"TRUE"}
 DBBACKUP_USER=${DBBACKUP_USER:-"dbbackup"}
-DBBACKUP_GROUP=${DBBACKUP_USER:-"${DBBACKUP_USER}"} # Must go after DBBACKUP_USER
+DBBACKUP_GROUP=${DBBACKUP_GROUP:-"${DBBACKUP_USER}"} # Must go after DBBACKUP_USER
 DEFAULT_BACKUP_BEGIN=${DEFAULT_BACKUP_BEGIN:-+0}
 DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440}
-DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440}
 DEFAULT_BACKUP_LOCATION=${DEFAULT_BACKUP_LOCATION:-"FILESYSTEM"}
 DEFAULT_BLOBXFER_REMOTE_PATH=${DEFAULT_BLOBXFER_REMOTE_PATH:-"/docker-db-backup"}
+DEFAULT_BLOBXFER_MODE=${DEFAULT_BLOBXFER_MODE:-"auto"}
 DEFAULT_CHECKSUM=${DEFAULT_CHECKSUM:-"MD5"}
 DEFAULT_COMPRESSION=${DEFAULT_COMPRESSION:-"ZSTD"}
 DEFAULT_COMPRESSION_LEVEL=${DEFAULT_COMPRESSION_LEVEL:-"3"}
@@ -15,9 +16,11 @@ DEFAULT_CREATE_LATEST_SYMLINK=${DEFAULT_CREATE_LATEST_SYMLINK:-"TRUE"}
 DEFAULT_ENABLE_PARALLEL_COMPRESSION=${DEFAULT_ENABLE_PARALLEL_COMPRESSION:-"TRUE"}
 DEFAULT_ENCRYPT=${DEFAULT_ENCRYPT:-"FALSE"}
 DEFAULT_FILESYSTEM_PATH=${DEFAULT_FILESYSTEM_PATH:-"/backup"}
+DEFAULT_FILESYSTEM_PATH_PERMISSION=${DEFAULT_FILESYSTEM_PATH_PERMISSION:-"700"}
 DEFAULT_FILESYSTEM_PERMISSION=${DEFAULT_FILESYSTEM_PERMISSION:-"600"}
 DEFAULT_FILESYSTEM_ARCHIVE_PATH=${DEFAULT_FILESYSTEM_ARCHIVE_PATH:-"${DEFAULT_FILESYSTEM_PATH}/archive/"}
 DEFAULT_LOG_LEVEL=${DEFAULT_LOG_LEVEL:-"notice"}
+DEFAULT_MYSQL_CLIENT=${DEFAULT_MYSQL_CLIENT:-"mariadb"}
 DEFAULT_MYSQL_ENABLE_TLS=${DEFAULT_MYSQL_ENABLE_TLS:-"FALSE"}
 DEFAULT_MYSQL_EVENTS=${DEFAULT_MYSQL_EVENTS:-"TRUE"}
 DEFAULT_MYSQL_MAX_ALLOWED_PACKET=${DEFAULT_MYSQL_MAX_ALLOWED_PACKET:-"512M"}