Release 2.3.0 - See CHANGELOG.md

Fix Docker Secrets injection from DB_USER_FILE/DB_PASS_FILE

CHANGELOG.md

@@ -1,3 +1,27 @@
+## 2.3.0 2020-10-15 <dave at tiredofit dot ca>
+
+   ### Added
+      - Microsoft SQL Server support (experimental)
+   ### Changed
+      - Compiled Postgresql 13 from source to backup psql/13 hosts
+
+## 2.2.2 2020-09-22 <tpansino@github>
+
+   ### Fixed
+      - Patch for 2.2.0 release fixing Docker Secrets Support. Was skipping password check.
+
+## 2.2.1 2020-09-17 <alwynpan@github>
+
+   ### Fixed
+      - Ondemand/Manual backup with `backup-now` was throwing errors not being able to find a proper date
+
+## 2.2.0 2020-09-14 <alwynpan@github>
+
+   ### Fixed
+      - Allow to use MariaDB and MongoDBs with no username and password while still allowing Docker Secrets
+      - Changed source of Alpine package repositories
+
+
 ## 2.1.1 2020-09-01 <zicklag@github>
 
    ### Fixed

Dockerfile

@@ -1,15 +1,112 @@
 FROM tiredofit/alpine:edge
-LABEL maintainer="Dave Conroy (dave at tiredofit dot ca)"
 
 ### Set Environment Variables
-ENV ENABLE_CRON=FALSE \
+ENV MSSQL_VERSION=17.5.2.1-1 \
+    ENABLE_CRON=FALSE \
     ENABLE_SMTP=FALSE \
     ENABLE_ZABBIX=TRUE \
     ZABBIX_HOSTNAME=db-backup
 
-### Dependencies
+ENV LANG=en_US.utf8 \
+    PG_MAJOR=13 \
+    PG_VERSION=13.0 \
+    PGDATA=/var/lib/postgresql/data
+
+
+### Create User Accounts
 RUN set -ex && \
-    echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
+    addgroup -g 70 postgres && \
+    adduser -S -D -H -h /var/lib/postgresql -s /bin/sh -G postgres -u 70 postgres && \
+    mkdir -p /var/lib/postgresql && \
+    chown -R postgres:postgres /var/lib/postgresql && \
+    \
+### Install Dependencies
+   apk update && \
+   apk upgrade && \
+   apk add \
+       openssl \
+       && \
+   \
+   apk add --no-cache --virtual .postgres-build-deps \
+	   bison \
+	   build-base \
+	   coreutils \
+	   dpkg-dev \
+	   dpkg \
+	   flex \
+	   gcc \
+	   icu-dev \
+	   libc-dev \
+   	   libedit-dev \
+	   libxml2-dev \
+	   libxslt-dev \
+	   linux-headers \
+	   make \
+	   openssl-dev \
+	   perl-utils \
+	   perl-ipc-run \
+	   util-linux-dev \
+	   zlib-dev \
+       && \
+   \
+### Build Postgresql
+   mkdir -p /usr/src/postgresql && \
+   curl -sSL "https://ftp.postgresql.org/pub/source/v$PG_VERSION/postgresql-$PG_VERSION.tar.bz2" | tar xvfj - --strip 1 -C /usr/src/postgresql && \
+   cd /usr/src/postgresql && \
+# update "DEFAULT_PGSOCKET_DIR" to "/var/run/postgresql" (matching Debian)
+# see https://anonscm.debian.org/git/pkg-postgresql/postgresql.git/tree/debian/patches/51-default-sockets-in-var.patch?id=8b539fcb3e093a521c095e70bdfa76887217b89f
+   awk '$1 == "#define" && $2 == "DEFAULT_PGSOCKET_DIR" && $3 == "\"/tmp\"" { $3 = "\"/var/run/postgresql\""; print; next } { print }' src/include/pg_config_manual.h > src/include/pg_config_manual.h.new && \
+   grep '/var/run/postgresql' src/include/pg_config_manual.h.new && \
+   mv src/include/pg_config_manual.h.new src/include/pg_config_manual.h && \
+   gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" && \
+# explicitly update autoconf config.guess and config.sub so they support more arches/libcs
+   wget -O config/config.guess 'https://git.savannah.gnu.org/cgit/config.git/plain/config.guess?id=7d3d27baf8107b630586c962c057e22149653deb' && \
+   wget -O config/config.sub 'https://git.savannah.gnu.org/cgit/config.git/plain/config.sub?id=7d3d27baf8107b630586c962c057e22149653deb' && \
+   ./configure \
+		--build="$gnuArch" \
+		--enable-integer-datetimes \
+		--enable-thread-safety \
+		--enable-tap-tests \
+		--disable-rpath \
+		--with-uuid=e2fs \
+		--with-gnu-ld \
+		--with-pgport=5432 \
+		--with-system-tzdata=/usr/share/zoneinfo \
+		--prefix=/usr/local \
+		--with-includes=/usr/local/include \
+		--with-libraries=/usr/local/lib \
+		--with-openssl \
+		--with-libxml \
+		--with-libxslt \
+		--with-icu \
+	&& \
+    \
+    make -j "$(nproc)" world && \
+    make install-world && \
+    make -C contrib install && \
+    runDeps="$( \
+		scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \
+			| tr ',' '\n' \
+			| sort -u \
+			| awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \
+	)" && \
+	apk add -t .postgres-additional-deps \
+	           $runDeps \
+	           && \
+	\
+### Cleanup
+    apk del .postgres-build-deps && \
+    cd / && \
+    rm -rf \
+	/usr/src/postgresql \
+	/usr/local/share/doc \
+	/usr/local/share/man && \
+    find /usr/local -name '*.a' -delete && \
+    rm -rf /var/cache/apk/* && \
+    \
+    ### Dependencies
+    set -ex && \
+    echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
     apk update && \
     apk upgrade && \
     apk add -t .db-backup-build-deps \
@@ -19,23 +116,28 @@ RUN set -ex && \
                xz-dev \
                && \
     \
-    apk add -t .db-backup-run-deps \
+    apk add --no-cache -t .db-backup-run-deps \
       	       bzip2 \
                influxdb \
                mariadb-client \
                mongodb-tools \
                libressl \
                pigz \
-               postgresql \
-               postgresql-client \
+               #postgresql \
+               #postgresql-client \
                redis \
                xz \
                zstd \
                && \
     \
-    apk add \
-            pixz@testing \
-            && \
+    apk add --no-cache \
+               pixz@testing \
+               && \
+    \
+    cd /usr/src && \
+    curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_${MSSQL_VERSION}_amd64.apk && \
+    curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_${MSSQL_VERSION}_amd64.apk && \
+    echo y | apk add --allow-untrusted msodbcsql17_${MSSQL_VERSION}_amd64.apk mssql-tools_${MSSQL_VERSION}_amd64.apk && \
     \
     mkdir -p /usr/src/pbzip2 && \
     curl -ssL https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz | tar xvfz - --strip=1 -C /usr/src/pbzip2 && \

README.md

@@ -32,28 +32,29 @@ Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis servers.
 
 ## Table of Contents
 
-- [Introduction](#introduction)
-- [Authors](#authors)
-- [Table of Contents](#table-of-contents)
-- [Prerequisites](#prerequisites)
-- [Installation](#installation)
-  - [Quick Start](#quick-start)
-- [Configuration](#configuration)
-  - [Data-Volumes](#data-volumes)
-  - [Environment Variables](#environment-variables)
-- [Maintenance](#maintenance)
-  - [Shell Access](#shell-access)
-    - [Custom Scripts](#custom-scripts)
-- [Example Post Script](#example-post-script)
-- [$1=EXIT_CODE (After running backup routine)](#1exit_code-after-running-backup-routine)
-- [$2=DB_TYPE (Type of Backup)](#2db_type-type-of-backup)
-- [$3=DB_HOST (Backup Host)](#3db_host-backup-host)
-- [#4=DB_NAME (Name of Database backed up](#4db_name-name-of-database-backed-up)
-- [$5=DATE (Date of Backup)](#5date-date-of-backup)
-- [$6=TIME (Time of Backup)](#6time-time-of-backup)
-- [$7=BACKUP_FILENAME (Filename of Backup)](#7backup_filename-filename-of-backup)
-- [$8=FILESIZE (Filesize of backup)](#8filesize-filesize-of-backup)
-- [$9=MD5_RESULT (MD5Sum if enabled)](#9md5_result-md5sum-if-enabled)
+- [hub.docker.com/r/tiredofit/db-backup](#hubdockercomrtiredofitdb-backup)
+  - [Introduction](#introduction)
+  - [Authors](#authors)
+  - [Table of Contents](#table-of-contents)
+  - [Prerequisites](#prerequisites)
+  - [Installation](#installation)
+    - [Quick Start](#quick-start)
+  - [Configuration](#configuration)
+    - [Data-Volumes](#data-volumes)
+    - [Environment Variables](#environment-variables)
+  - [Maintenance](#maintenance)
+    - [Shell Access](#shell-access)
+      - [Custom Scripts](#custom-scripts)
+  - [Example Post Script](#example-post-script)
+  - [$1=EXIT_CODE (After running backup routine)](#1exit_code-after-running-backup-routine)
+  - [$2=DB_TYPE (Type of Backup)](#2db_type-type-of-backup)
+  - [$3=DB_HOST (Backup Host)](#3db_host-backup-host)
+  - [#4=DB_NAME (Name of Database backed up](#4db_name-name-of-database-backed-up)
+  - [$5=DATE (Date of Backup)](#5date-date-of-backup)
+  - [$6=TIME (Time of Backup)](#6time-time-of-backup)
+  - [$7=BACKUP_FILENAME (Filename of Backup)](#7backup_filename-filename-of-backup)
+  - [$8=FILESIZE (Filesize of backup)](#8filesize-filesize-of-backup)
+  - [$9=MD5_RESULT (MD5Sum if enabled)](#9md5_result-md5sum-if-enabled)
 
 ## Prerequisites
 
@@ -90,8 +91,6 @@ The following directories are used for configuration and can be mapped for persi
 
 ### Environment Variables
 
-*If you are trying to backup a database that doesn't have a user or a password (you should!) make sure you set `CONTAINER_ENABLE_DOCKER_SECRETS=FALSE`*
-
 Along with the Environment Variables from the [Base image](https://hub.docker.com/r/tiredofit/alpine), below is the complete list of available options that can be used to customize your installation.
 
 | Parameter              | Description                                                                                                                                                                                        |

install/etc/services.available/10-db-backup/run

@@ -14,8 +14,48 @@ fi
 sanity_var DB_TYPE "Database Type"
 sanity_var DB_HOST "Database Host"
 
-file_env 'DB_USER'
-file_env 'DB_PASS'
+### Set the Database Type
+dbtype=${DB_TYPE}
+
+case "$dbtype" in
+    "couch" | "couchdb" | "COUCH" | "COUCHDB"  )
+        dbtype=couch
+        dbport=${DB_PORT:-5984}
+        file_env 'DB_USER'
+        file_env 'DB_PASS'
+    ;;
+    "influx" | "influxdb" | "INFLUX" | "INFLUXDB"  )
+        dbtype=influx
+        dbport=${DB_PORT:-8088}
+        file_env 'DB_USER'
+        file_env 'DB_PASS'
+    ;;
+    "mongo" | "mongodb" | "MONGO" | "MONGODB" )
+        dbtype=mongo
+        dbport=${DB_PORT:-27017}
+        [[ ( -n "${DB_USER}" ) || ( -n "${DB_USER_FILE}" ) ]] && file_env 'DB_USER'
+        [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS'
+    ;;
+    "mysql" | "MYSQL" | "mariadb" | "MARIADB")
+        dbtype=mysql
+        dbport=${DB_PORT:-3306}
+        [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS'
+    ;;
+    "mssql" | "MSSQL" | "microsoftsql" | "MICROSOFTSQL")
+        dbtype=mssql
+        dbport=${DB_PORT:-1433}
+    ;;
+    "postgres" | "postgresql" | "pgsql" | "POSTGRES" | "POSTGRESQL" | "PGSQL" )
+        dbtype=pgsql
+        dbport=${DB_PORT:-5432}
+        [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS'
+    ;;
+    "redis" | "REDIS"   )
+        dbtype=redis
+        dbport=${DB_PORT:-6379}
+        [[ ( -n "${DB_PASS}" || ( -n "${DB_PASS_FILE}" ) ) ]] && file_env 'DB_PASS'
+    ;;
+esac
 
 ### Set Defaults
 BACKUP_LOCATION=${BACKUP_LOCATION:-"FILESYSTEM"}
@@ -27,7 +67,6 @@ DB_DUMP_TARGET=${DB_DUMP_TARGET:-/backup}
 dbhost=${DB_HOST}
 dbname=${DB_NAME}
 dbpass=${DB_PASS}
-dbtype=${DB_TYPE}
 dbuser=${DB_USER}
 MD5=${MD5:-TRUE}
 PARALLEL_COMPRESSION=${PARALLEL_COMPRESSION:-TRUE}
@@ -65,39 +104,23 @@ else
     zstd="zstd --rm -${COMPRESSION_LEVEL}"
 fi
 
-### Set the Database Type
-       case "$dbtype" in
-            "couch" | "couchdb" | "COUCH" | "COUCHDB"  )
-                dbtype=couch
-                dbport=${DB_PORT:-5984}
-            ;;
-            "influx" | "influxdb" | "INFLUX" | "INFLUXDB"  )
-                dbtype=influx
-                dbport=${DB_PORT:-8088}
-            ;;
-            "mongo" | "mongodb" | "MONGO" | "MONGODB" )
-            dbtype=mongo
-            dbport=${DB_PORT:-27017}
-                [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${dbuser}"
-                [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${dbpass}"
-                [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${dbname}"
-            ;;
-            "mysql" | "MYSQL" | "mariadb" | "MARIADB")
-                dbtype=mysql
-                dbport=${DB_PORT:-3306}
-                [[ ( -n "${DB_PASS}" ) ]] && export MYSQL_PWD=${dbpass}
-            ;;
-            "postgres" | "postgresql" | "pgsql" | "POSTGRES" | "POSTGRESQL" | "PGSQL" )
-                dbtype=pgsql
-                dbport=${DB_PORT:-5432}
-                [[ ( -n "${DB_PASS}" ) ]] && POSTGRES_PASS_STR="PGPASSWORD=${dbpass}"
-            ;;
-            "redis" | "REDIS"   )
-                dbtype=redis
-                dbport=${DB_PORT:-6379}
-                [[ ( -n "${DB_PASS}" ) ]] && REDIS_PASS_STR=" -a ${dbpass}"
-            ;;
-        esac
+### Set the Database Authentication Details
+case "$dbtype" in
+    "mongo" )
+        [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${dbuser}"
+        [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${dbpass}"
+        [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${dbname}"
+    ;;
+    "mysql" )
+        [[ ( -n "${DB_PASS}" ) ]] && export MYSQL_PWD=${dbpass}
+    ;;
+    "postgres" )
+        [[ ( -n "${DB_PASS}" ) ]] && POSTGRES_PASS_STR="PGPASSWORD=${dbpass}"
+    ;;
+    "redis" )
+        [[ ( -n "${DB_PASS}" ) ]] && REDIS_PASS_STR=" -a ${dbpass}"
+    ;;
+esac
 
 ### Functions
 backup_couch() {
@@ -109,30 +132,6 @@ backup_couch() {
     move_backup
 }
 
-backup_mysql() {
-    if var_true "$SPLIT_DB" ; then
-        DATABASES=$(mysql -h ${dbhost} -P $dbport -u$dbuser --batch -e "SHOW DATABASES;" | grep -v Database|grep -v schema)
-
-        for db in $DATABASES; do
-                if [[ "$db" != "information_schema" ]] && [[ "$db" != _* ]] ; then
-                    print_notice "Dumping MariaDB database: $db"
-                    target=mysql_${db}_${dbhost}_${now}.sql
-                    compression
-                    mysqldump --max-allowed-packet=512M -h $dbhost -P $dbport -u$dbuser  ${EXTRA_OPTS} --databases $db | $dumpoutput > ${tmpdir}/${target}
-                    exit_code=$?
-                    generate_md5
-                    move_backup
-                fi
-        done
-    else
-        compression
-        mysqldump --max-allowed-packet=512M -A -h $dbhost -P $dbport -u$dbuser  ${EXTRA_OPTS} | $dumpoutput > ${tmpdir}/${target}
-        exit_code=$?
-        generate_md5
-        move_backup
-    fi
-}
-
 backup_influx() {
     if [ "${COMPRESSION}" = "NONE" ] || [ "${COMPRESSION}" = "none" ] || [ "${COMPRESSION}" = "FALSE" ] || [ "${COMPRESSION}" = "false" ] ; then
         :
@@ -163,6 +162,35 @@ backup_mongo() {
     move_backup
 }
 
+backup_mssql() {
+    target=mssql_${dbname}_${dbhost}_${now}.bak
+    /opt/mssql-tools/bin/sqlcmd -E -C -S ${dbhost}\,${dbport} -U ${dbuser} -P ${dbpass} –Q "BACKUP DATABASE \[${dbname}\] TO DISK = N'${tmpdir}/${target}' WITH NOFORMAT, NOINIT, NAME = '${dbname}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10"
+}
+
+backup_mysql() {
+    if var_true "$SPLIT_DB" ; then
+        DATABASES=$(mysql -h ${dbhost} -P $dbport -u$dbuser --batch -e "SHOW DATABASES;" | grep -v Database|grep -v schema)
+
+        for db in $DATABASES; do
+                if [[ "$db" != "information_schema" ]] && [[ "$db" != _* ]] ; then
+                    print_notice "Dumping MariaDB database: $db"
+                    target=mysql_${db}_${dbhost}_${now}.sql
+                    compression
+                    mysqldump --max-allowed-packet=512M -h $dbhost -P $dbport -u$dbuser  ${EXTRA_OPTS} --databases $db | $dumpoutput > ${tmpdir}/${target}
+                    exit_code=$?
+                    generate_md5
+                    move_backup
+                fi
+        done
+    else
+        compression
+        mysqldump --max-allowed-packet=512M -A -h $dbhost -P $dbport -u$dbuser  ${EXTRA_OPTS} | $dumpoutput > ${tmpdir}/${target}
+        exit_code=$?
+        generate_md5
+        move_backup
+    fi
+}
+
 backup_pgsql() {
   if var_true $SPLIT_DB ; then
       export PGPASSWORD=${dbpass}
@@ -249,6 +277,14 @@ check_availability() {
                 (( COUNTER+=5 ))
             done
         ;;
+        "mssql" )
+            COUNTER=0
+            while ! (nc -z ${dbhost} ${dbport}) ; do
+                sleep 5
+                (( COUNTER+=5 ))
+                print_warn "MSSQL Host '${dbhost}' is not accessible, retrying.. ($COUNTER seconds so far)"
+            done
+        ;;
         "pgsql" )
             COUNTER=0
             export PGPASSWORD=${dbpass}
@@ -375,21 +411,23 @@ move_backup() {
 print_debug "Backup routines Initialized on $(date)"
 
 ### Wait for Next time to start backup
-  current_time=$(date +"%s")
-  today=$(date +"%Y%m%d")
+  if [ "$1" != "NOW" ]; then
+    current_time=$(date +"%s")
+    today=$(date +"%Y%m%d")
 
-  if [[ $DB_DUMP_BEGIN =~ ^\+(.*)$ ]]; then
-        waittime=$(( ${BASH_REMATCH[1]} * 60 ))
-  else
-        target_time=$(date --date="${today}${DB_DUMP_BEGIN}" +"%s")
-    if [[ "$target_time" < "$current_time" ]]; then
-        target_time=$(($target_time + 24*60*60))
+    if [[ $DB_DUMP_BEGIN =~ ^\+(.*)$ ]]; then
+            waittime=$(( ${BASH_REMATCH[1]} * 60 ))
+    else
+            target_time=$(date --date="${today}${DB_DUMP_BEGIN}" +"%s")
+        if [[ "$target_time" < "$current_time" ]]; then
+            target_time=$(($target_time + 24*60*60))
+        fi
+        waittime=$(($target_time - $current_time))
     fi
-  waittime=$(($target_time - $current_time))
-  fi
 
-  print_notice "Next Backup at $(date -d @${target_time} +"%Y-%m-%d %T %Z")"
-  sleep $waittime
+        print_notice "Next Backup at $(date -d @${target_time} +"%Y-%m-%d %T %Z")"
+        sleep $waittime
+  fi
 
 
 ### Commence Backup
@@ -413,6 +451,10 @@ print_debug "Backup routines Initialized on $(date)"
             check_availability
             backup_influx
         ;;
+        "mssql" )
+            check_availability
+            backup_mssql
+        ;;
         "mysql" )
             check_availability
             backup_mysql