Merge branch 'master' of https://github.com/tiredofit/docker-db-backup

Add sqlite support

.github/workflows/main.yml

@@ -0,0 +1,110 @@
+### Application Level Image CI
+### Dave Conroy <dave at tiredofit dot ca>
+
+name: 'Build Images'
+
+on:
+  push:
+    paths:
+    - '**'
+    - '!README.md'
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=${GITHUB_REPOSITORY/docker-/}
+          set -x
+          if [[ $GITHUB_REF == refs/heads/* ]]; then
+             if [[ $GITHUB_REF == refs/heads/*/* ]] ; then
+               BRANCH="${DOCKER_IMAGE}:$(echo $GITHUB_REF | sed "s|refs/heads/||g" | sed "s|/|-|g")"
+             else
+               BRANCH=${GITHUB_REF#refs/heads/}
+             fi
+
+            case ${BRANCH} in
+              "main" | "master" )
+                  BRANCHTAG="${DOCKER_IMAGE}:latest"
+                ;;
+              "develop" )
+                  BRANCHTAG="${DOCKER_IMAGE}:develop"
+                ;;
+              * )
+                  if [ -n "${{ secrets.LATEST }}" ] ; then
+                    if [ "${BRANCHTAG}" = "${{ secrets.LATEST }}" ]; then
+                      BRANCHTAG="${DOCKER_IMAGE}:${BRANCH},${DOCKER_IMAGE}:${BRANCH}-latest,${DOCKER_IMAGE}:latest"
+                    else
+                      BRANCHTAG="${DOCKER_IMAGE}:${BRANCH},${DOCKER_IMAGE}:${BRANCH}-latest"
+                    fi
+                  else
+                    BRANCHTAG="${DOCKER_IMAGE}:${BRANCH},${DOCKER_IMAGE}:${BRANCH}-latest"
+                  fi
+                ;;
+            esac
+          fi
+
+
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+               GITTAG="${DOCKER_IMAGE}:$(echo $GITHUB_REF | sed 's|refs/tags/||g')"
+          fi
+
+          if [ -n "${BRANCHTAG}" ] && [ -n "${GITTAG}" ]; then
+            TAGS=${BRANCHTAG},${GITTAG}
+          else
+            TAGS="${BRANCHTAG}${GITTAG}"
+          fi
+
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=docker_image::${DOCKER_IMAGE}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Label
+        id: Label
+        run: |
+          if [ -f "Dockerfile" ] ; then
+            sed -i "/FROM .*/a LABEL tiredofit.image.git_repository=\"https://github.com/${GITHUB_REPOSITORY}\"" Dockerfile
+            sed -i "/FROM .*/a LABEL tiredofit.image.git_commit=\"${GITHUB_SHA}\"" Dockerfile
+            sed -i "/FROM .*/a LABEL tiredofit.image.git_committed_by=\"${GITHUB_ACTOR}\"" Dockerfile
+            sed -i "/FROM .*/a LABEL tiredofit.image.image_build_date=\"$(date +'%Y-%m-%d %H:%M:%S')\"" Dockerfile
+            if [ -f "CHANGELOG.md" ] ; then
+              sed -i "/FROM .*/a LABEL tiredofit.image.git_changelog_version=\"$(head -n1 ./CHANGELOG.md | awk '{print $2}')\"" Dockerfile
+            fi
+
+            if [[ $GITHUB_REF == refs/tags/* ]]; then
+              sed -i "/FROM .*/a LABEL tiredofit.image.git_tag=\"${GITHUB_REF#refs/tags/v}\"" Dockerfile
+            fi
+
+            if [[ $GITHUB_REF == refs/heads/* ]]; then
+              sed -i "/FROM .*/a LABEL tiredofit.image.git_branch=\"${GITHUB_REF#refs/heads/}\"" Dockerfile
+            fi
+          fi
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64
+          push: true
+          tags: ${{ steps.prep.outputs.tags }}

CHANGELOG.md

@@ -1,3 +1,50 @@
+## 2.6.0 2021-02-19 <tpansino@github>
+
+   ### Added
+      - SQLite support
+
+
+## 2.5.1 2021-02-14 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix xz backups with `PARALLEL_COMPRESION=TRUE`
+
+
+## 2.5.0 2021-01-25 <dave at tiredofit dot ca>
+
+   ### Added
+      - Multi Platform Build Variants (ARMv7 AMD64 AArch64)
+
+   ### Changed
+      - Alpine 3.13 Base
+      - Compile Pixz as opposed to relying on testing repository
+      - MSSQL Support only available under AMD64. Container exits if any other platform detected when MSSQL set to be backed up.
+
+## 2.4.0 2020-12-07 <dave at tiredofit dot ca>
+
+   ### Added
+      - Switch back to packges for Postgresql (now 13.1)
+
+
+## 2.3.2 2020-11-14 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Reapply S6-Overlay into filesystem as Postgresql build is removing S6 files due to edge containing S6 overlay
+
+
+## 2.3.1 2020-11-11 <bambi73@github>
+
+   ### Fixed
+      - Multiple Influx DB's not being backed up correctly
+
+## 2.3.0 2020-10-15 <dave at tiredofit dot ca>
+
+   ### Added
+      - Microsoft SQL Server support (experimental)
+
+   ### Changed
+      - Compiled Postgresql 13 from source to backup psql/13 hosts
+
 ## 2.2.2 2020-09-22 <tpansino@github>
 
    ### Fixed

Dockerfile

@@ -1,27 +1,28 @@
-FROM tiredofit/alpine:edge
-LABEL maintainer="Dave Conroy (dave at tiredofit dot ca)"
+FROM tiredofit/alpine:3.13
 
 ### Set Environment Variables
-ENV ENABLE_CRON=FALSE \
+ENV MSSQL_VERSION=17.5.2.1-1 \
+    ENABLE_CRON=FALSE \
     ENABLE_SMTP=FALSE \
     ENABLE_ZABBIX=TRUE \
     ZABBIX_HOSTNAME=db-backup
 
     ### Dependencies
 RUN set -ex && \
-    echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
     apk update && \
     apk upgrade && \
     apk add -t .db-backup-build-deps \
                build-base \
                bzip2-dev \
                git \
+               libarchive-dev \
                xz-dev \
                && \
     \
     apk add --no-cache -t .db-backup-run-deps \
       	       bzip2 \
                influxdb \
+               libarchive \
                mariadb-client \
                mongodb-tools \
                libressl \
@@ -29,19 +30,33 @@ RUN set -ex && \
                postgresql \
                postgresql-client \
                redis \
+               sqlite \
                xz \
                zstd \
                && \
     \
-    apk add --no-cache \
-               pixz@testing \
-               && \
+    cd /usr/src && \
     \
+    apkArch="$(apk --print-arch)"; \
+    case "$apkArch" in \
+	x86_64) mssql=true ; curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_${MSSQL_VERSION}_amd64.apk ; curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_${MSSQL_VERSION}_amd64.apk ; echo y | apk add --allow-untrusted msodbcsql17_${MSSQL_VERSION}_amd64.apk mssql-tools_${MSSQL_VERSION}_amd64.apk ;; \
+	*) echo >&2 "Detected non x86_64 build variant, skipping MSSQL installation" ;; \
+    esac; \
     mkdir -p /usr/src/pbzip2 && \
-    curl -ssL https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz | tar xvfz - --strip=1 -C /usr/src/pbzip2 && \
+    curl -sSL https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz | tar xvfz - --strip=1 -C /usr/src/pbzip2 && \
     cd /usr/src/pbzip2 && \
     make && \
     make install && \
+    mkdir -p /usr/src/pixz && \
+    curl -sSL https://github.com/vasi/pixz/releases/download/v1.0.7/pixz-1.0.7.tar.xz | tar xvfJ - --strip 1 -C /usr/src/pixz && \
+    cd /usr/src/pixz && \
+    ./configure \
+        --prefix=/usr \
+        --sysconfdir=/etc \
+        --localstatedir=/var \
+        && \
+     make && \
+     make install && \
      \
 ### Cleanup
     apk del .db-backup-build-deps && \

README.md

@@ -98,8 +98,8 @@ Along with the Environment Variables from the [Base image](https://hub.docker.co
 | `BACKUP_LOCATION`      | Backup to `FILESYSTEM` or `S3` compatible services like S3, Minio, Wasabi - Default `FILESYSTEM`                                                                                                   |
 | `COMPRESSION`          | Use either Gzip `GZ`, Bzip2 `BZ`, XZip `XZ`, ZSTD `ZSTD` or none `NONE` - Default `GZ`                                                                                                             |
 | `COMPRESSION_LEVEL`    | Numberical value of what level of compression to use, most allow `1` to `9` except for `ZSTD` which allows for `1` to `19` - Default `3`                                                           |
-| `DB_TYPE`              | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis`                                                                                                                       |
-| `DB_HOST`              | Server Hostname e.g. `mariadb`                                                                                                                                                                     |
+| `DB_TYPE`              | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` `sqlite3`                                                                                                             |
+| `DB_HOST`              | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3`                                                                                                      |
 | `DB_NAME`              | Schema Name e.g. `database`                                                                                                                                                                        |
 | `DB_USER`              | username for the database - use `root` to backup all MySQL of them.                                                                                                                                |
 | `DB_PASS`              | (optional if DB doesn't require it) password for the database                                                                                                                                      |

install/etc/services.available/10-db-backup/run

@@ -41,6 +41,15 @@ case "$dbtype" in
         dbport=${DB_PORT:-3306}
         [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS'
     ;;
+    "mssql" | "MSSQL" | "microsoftsql" | "MICROSOFTSQL")
+        apkArch="$(apk --print-arch)"; \
+	    case "$apkArch" in
+		    x86_64) mssql=true ;;
+		    *) print_error "MSSQL cannot operate on $apkArch processor!" ; exit 1 ;;
+        esac
+        dbtype=mssql
+        dbport=${DB_PORT:-1433}
+    ;;
     "postgres" | "postgresql" | "pgsql" | "POSTGRES" | "POSTGRESQL" | "PGSQL" )
         dbtype=pgsql
         dbport=${DB_PORT:-5432}
@@ -51,6 +60,9 @@ case "$dbtype" in
         dbport=${DB_PORT:-6379}
         [[ ( -n "${DB_PASS}" || ( -n "${DB_PASS_FILE}" ) ) ]] && file_env 'DB_PASS'
     ;;
+    "sqlite" | "sqlite3" | "SQLITE" | "SQLITE3" )
+        dbtype=sqlite3
+    ;;
 esac
 
 ### Set Defaults
@@ -128,6 +140,42 @@ backup_couch() {
     move_backup
 }
 
+backup_influx() {
+    if [ "${COMPRESSION}" = "NONE" ] || [ "${COMPRESSION}" = "none" ] || [ "${COMPRESSION}" = "FALSE" ] || [ "${COMPRESSION}" = "false" ] ; then
+        :
+    else
+        print_notice "Compressing InfluxDB backup with gzip"
+        influx_compression="-portable"
+    fi
+    for DB in $DB_NAME; do
+        target=influx_${DB}_${dbhost}_${now}
+        influxd backup ${influx_compression} -database $DB -host ${dbhost}:${dbport} ${tmpdir}/${target}
+        exit_code=$?
+        generate_md5
+        move_backup
+    done
+}
+
+backup_mongo() {
+    if [ "${COMPRESSION}" = "NONE" ] || [ "${COMPRESSION}" = "none" ] || [ "${COMPRESSION}" = "FALSE" ] || [ "${COMPRESSION}" = "false" ] ; then
+        target=${dbtype}_${dbname}_${dbhost}_${now}.archive
+    else
+        print_notice "Compressing MongoDB backup with gzip"
+        target=${dbtype}_${dbname}_${dbhost}_${now}.archivegz
+        mongo_compression="--gzip"
+    fi
+    mongodump --archive=${tmpdir}/${target} ${mongo_compression} --host ${dbhost} --port ${dbport} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_DB_STR} ${EXTRA_OPTS}
+    exit_code=$?
+    cd ${tmpdir}
+    generate_md5
+    move_backup
+}
+
+backup_mssql() {
+    target=mssql_${dbname}_${dbhost}_${now}.bak
+    /opt/mssql-tools/bin/sqlcmd -E -C -S ${dbhost}\,${dbport} -U ${dbuser} -P ${dbpass} –Q "BACKUP DATABASE \[${dbname}\] TO DISK = N'${tmpdir}/${target}' WITH NOFORMAT, NOINIT, NAME = '${dbname}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10"
+}
+
 backup_mysql() {
     if var_true "$SPLIT_DB" ; then
         DATABASES=$(mysql -h ${dbhost} -P $dbport -u$dbuser --batch -e "SHOW DATABASES;" | grep -v Database|grep -v schema)
@@ -152,36 +200,6 @@ backup_mysql() {
     fi
 }
 
-backup_influx() {
-    if [ "${COMPRESSION}" = "NONE" ] || [ "${COMPRESSION}" = "none" ] || [ "${COMPRESSION}" = "FALSE" ] || [ "${COMPRESSION}" = "false" ] ; then
-        :
-    else
-        print_notice "Compressing InfluxDB backup with gzip"
-        influx_compression="-portable"
-    fi
-    for DB in $DB_NAME; do
-        influxd backup ${influx_compression} -database $DB -host ${dbhost}:${dbport} ${tmpdir}/${target}
-        exit_code=$?
-        generate_md5
-        move_backup
-    done
-}
-
-backup_mongo() {
-    if [ "${COMPRESSION}" = "NONE" ] || [ "${COMPRESSION}" = "none" ] || [ "${COMPRESSION}" = "FALSE" ] || [ "${COMPRESSION}" = "false" ] ; then
-        target=${dbtype}_${dbname}_${dbhost}_${now}.archive
-    else
-        print_notice "Compressing MongoDB backup with gzip"
-        target=${dbtype}_${dbname}_${dbhost}_${now}.archivegz
-        mongo_compression="--gzip"
-    fi
-    mongodump --archive=${tmpdir}/${target} ${mongo_compression} --host ${dbhost} --port ${dbport} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_DB_STR} ${EXTRA_OPTS}
-    exit_code=$?
-    cd ${tmpdir}
-    generate_md5
-    move_backup
-}
-
 backup_pgsql() {
   if var_true $SPLIT_DB ; then
       export PGPASSWORD=${dbpass}
@@ -226,6 +244,22 @@ backup_redis() {
     move_backup
 }
 
+backup_sqlite3() {
+    db=$(basename "$dbhost")
+    db="${db%.*}"
+    target=sqlite3_${db}_${now}.sqlite3
+    compression
+
+    print_info "Dumping sqlite3 database: ${dbhost}"
+    sqlite3 "${dbhost}" ".backup '${tmpdir}/backup.sqlite3'"
+    exit_code=$?
+
+    cat "${tmpdir}/backup.sqlite3" | $dumpoutput > "${tmpdir}/${target}"
+
+    generate_md5
+    move_backup
+}
+
 check_availability() {
 ### Set the Database Type
    case "$dbtype" in
@@ -268,6 +302,14 @@ check_availability() {
                 (( COUNTER+=5 ))
             done
         ;;
+        "mssql" )
+            COUNTER=0
+            while ! (nc -z ${dbhost} ${dbport}) ; do
+                sleep 5
+                (( COUNTER+=5 ))
+                print_warn "MSSQL Host '${dbhost}' is not accessible, retrying.. ($COUNTER seconds so far)"
+            done
+        ;;
         "pgsql" )
             COUNTER=0
             export PGPASSWORD=${dbpass}
@@ -286,6 +328,21 @@ check_availability() {
                 print_warn "Redis Host '${dbhost}' is not accessible, retrying.. ($COUNTER seconds so far)"
             done
         ;;
+        "sqlite3" )
+            if [[ ! -e "${dbhost}" ]]; then
+                print_error "File '${dbhost}' does not exist."
+                exit_code=2
+                exit $exit_code
+            elif [[ ! -f "${dbhost}" ]]; then
+                print_error "File '${dbhost}' is not a file."
+                exit_code=2
+                exit $exit_code
+            elif [[ ! -r "${dbhost}" ]]; then
+                print_error "File '${dbhost}' is not readable."
+                exit_code=2
+                exit $exit_code
+            fi
+        ;;
     esac
 }
 
@@ -434,6 +491,10 @@ print_debug "Backup routines Initialized on $(date)"
             check_availability
             backup_influx
         ;;
+        "mssql" )
+            check_availability
+            backup_mssql
+        ;;
         "mysql" )
             check_availability
             backup_mysql
@@ -450,6 +511,10 @@ print_debug "Backup routines Initialized on $(date)"
             check_availability
             backup_redis
         ;;
+        "sqlite3" )
+            check_availability
+            backup_sqlite3
+        ;;
         esac
 
 ### Zabbix