#55 - Fix xz parallel compression

#51 Fix backup of multiple InfluxDB databases failure

.github/workflows/main.yml

@@ -0,0 +1,110 @@
+### Application Level Image CI
+### Dave Conroy <dave at tiredofit dot ca>
+
+name: 'Build Images'
+
+on:
+  push:
+    paths:
+    - '**'
+    - '!README.md'
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=${GITHUB_REPOSITORY/docker-/}
+          set -x
+          if [[ $GITHUB_REF == refs/heads/* ]]; then
+             if [[ $GITHUB_REF == refs/heads/*/* ]] ; then
+               BRANCH="${DOCKER_IMAGE}:$(echo $GITHUB_REF | sed "s|refs/heads/||g" | sed "s|/|-|g")"
+             else
+               BRANCH=${GITHUB_REF#refs/heads/}
+             fi
+
+            case ${BRANCH} in
+              "main" | "master" )
+                  BRANCHTAG="${DOCKER_IMAGE}:latest"
+                ;;
+              "develop" )
+                  BRANCHTAG="${DOCKER_IMAGE}:develop"
+                ;;
+              * )
+                  if [ -n "${{ secrets.LATEST }}" ] ; then
+                    if [ "${BRANCHTAG}" = "${{ secrets.LATEST }}" ]; then
+                      BRANCHTAG="${DOCKER_IMAGE}:${BRANCH},${DOCKER_IMAGE}:${BRANCH}-latest,${DOCKER_IMAGE}:latest"
+                    else
+                      BRANCHTAG="${DOCKER_IMAGE}:${BRANCH},${DOCKER_IMAGE}:${BRANCH}-latest"
+                    fi
+                  else
+                    BRANCHTAG="${DOCKER_IMAGE}:${BRANCH},${DOCKER_IMAGE}:${BRANCH}-latest"
+                  fi
+                ;;
+            esac
+          fi
+
+
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+               GITTAG="${DOCKER_IMAGE}:$(echo $GITHUB_REF | sed 's|refs/tags/||g')"
+          fi
+
+          if [ -n "${BRANCHTAG}" ] && [ -n "${GITTAG}" ]; then
+            TAGS=${BRANCHTAG},${GITTAG}
+          else
+            TAGS="${BRANCHTAG}${GITTAG}"
+          fi
+
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=docker_image::${DOCKER_IMAGE}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Label
+        id: Label
+        run: |
+          if [ -f "Dockerfile" ] ; then
+            sed -i "/FROM .*/a LABEL tiredofit.image.git_repository=\"https://github.com/${GITHUB_REPOSITORY}\"" Dockerfile
+            sed -i "/FROM .*/a LABEL tiredofit.image.git_commit=\"${GITHUB_SHA}\"" Dockerfile
+            sed -i "/FROM .*/a LABEL tiredofit.image.git_committed_by=\"${GITHUB_ACTOR}\"" Dockerfile
+            sed -i "/FROM .*/a LABEL tiredofit.image.image_build_date=\"$(date +'%Y-%m-%d %H:%M:%S')\"" Dockerfile
+            if [ -f "CHANGELOG.md" ] ; then
+              sed -i "/FROM .*/a LABEL tiredofit.image.git_changelog_version=\"$(head -n1 ./CHANGELOG.md | awk '{print $2}')\"" Dockerfile
+            fi
+
+            if [[ $GITHUB_REF == refs/tags/* ]]; then
+              sed -i "/FROM .*/a LABEL tiredofit.image.git_tag=\"${GITHUB_REF#refs/tags/v}\"" Dockerfile
+            fi
+
+            if [[ $GITHUB_REF == refs/heads/* ]]; then
+              sed -i "/FROM .*/a LABEL tiredofit.image.git_branch=\"${GITHUB_REF#refs/heads/}\"" Dockerfile
+            fi
+          fi
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          builder: ${{ steps.buildx.outputs.name }}
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64
+          push: true
+          tags: ${{ steps.prep.outputs.tags }}

CHANGELOG.md

@@ -1,7 +1,41 @@
+## 2.5.1 2021-02-14 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix xz backups with `PARALLEL_COMPRESION=TRUE`
+
+
+## 2.5.0 2021-01-25 <dave at tiredofit dot ca>
+
+   ### Added
+      - Multi Platform Build Variants (ARMv7 AMD64 AArch64)
+
+   ### Changed
+      - Alpine 3.13 Base
+      - Compile Pixz as opposed to relying on testing repository
+      - MSSQL Support only available under AMD64. Container exits if any other platform detected when MSSQL set to be backed up.
+
+## 2.4.0 2020-12-07 <dave at tiredofit dot ca>
+
+   ### Added
+      - Switch back to packges for Postgresql (now 13.1)
+
+
+## 2.3.2 2020-11-14 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Reapply S6-Overlay into filesystem as Postgresql build is removing S6 files due to edge containing S6 overlay
+
+
+## 2.3.1 2020-11-11 <bambi73@github>
+
+   ### Fixed
+      - Multiple Influx DB's not being backed up correctly
+
 ## 2.3.0 2020-10-15 <dave at tiredofit dot ca>
 
    ### Added
       - Microsoft SQL Server support (experimental)
+
    ### Changed
       - Compiled Postgresql 13 from source to backup psql/13 hosts
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM tiredofit/alpine:edge
+FROM tiredofit/alpine:3.13
 
 ### Set Environment Variables
 ENV MSSQL_VERSION=17.5.2.1-1 \
@@ -7,144 +7,56 @@ ENV MSSQL_VERSION=17.5.2.1-1 \
     ENABLE_ZABBIX=TRUE \
     ZABBIX_HOSTNAME=db-backup
 
-ENV LANG=en_US.utf8 \
-    PG_MAJOR=13 \
-    PG_VERSION=13.0 \
-    PGDATA=/var/lib/postgresql/data
-
-
-### Create User Accounts
-RUN set -ex && \
-    addgroup -g 70 postgres && \
-    adduser -S -D -H -h /var/lib/postgresql -s /bin/sh -G postgres -u 70 postgres && \
-    mkdir -p /var/lib/postgresql && \
-    chown -R postgres:postgres /var/lib/postgresql && \
-    \
-### Install Dependencies
-   apk update && \
-   apk upgrade && \
-   apk add \
-       openssl \
-       && \
-   \
-   apk add --no-cache --virtual .postgres-build-deps \
-	   bison \
-	   build-base \
-	   coreutils \
-	   dpkg-dev \
-	   dpkg \
-	   flex \
-	   gcc \
-	   icu-dev \
-	   libc-dev \
-   	   libedit-dev \
-	   libxml2-dev \
-	   libxslt-dev \
-	   linux-headers \
-	   make \
-	   openssl-dev \
-	   perl-utils \
-	   perl-ipc-run \
-	   util-linux-dev \
-	   zlib-dev \
-       && \
-   \
-### Build Postgresql
-   mkdir -p /usr/src/postgresql && \
-   curl -sSL "https://ftp.postgresql.org/pub/source/v$PG_VERSION/postgresql-$PG_VERSION.tar.bz2" | tar xvfj - --strip 1 -C /usr/src/postgresql && \
-   cd /usr/src/postgresql && \
-# update "DEFAULT_PGSOCKET_DIR" to "/var/run/postgresql" (matching Debian)
-# see https://anonscm.debian.org/git/pkg-postgresql/postgresql.git/tree/debian/patches/51-default-sockets-in-var.patch?id=8b539fcb3e093a521c095e70bdfa76887217b89f
-   awk '$1 == "#define" && $2 == "DEFAULT_PGSOCKET_DIR" && $3 == "\"/tmp\"" { $3 = "\"/var/run/postgresql\""; print; next } { print }' src/include/pg_config_manual.h > src/include/pg_config_manual.h.new && \
-   grep '/var/run/postgresql' src/include/pg_config_manual.h.new && \
-   mv src/include/pg_config_manual.h.new src/include/pg_config_manual.h && \
-   gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" && \
-# explicitly update autoconf config.guess and config.sub so they support more arches/libcs
-   wget -O config/config.guess 'https://git.savannah.gnu.org/cgit/config.git/plain/config.guess?id=7d3d27baf8107b630586c962c057e22149653deb' && \
-   wget -O config/config.sub 'https://git.savannah.gnu.org/cgit/config.git/plain/config.sub?id=7d3d27baf8107b630586c962c057e22149653deb' && \
-   ./configure \
-		--build="$gnuArch" \
-		--enable-integer-datetimes \
-		--enable-thread-safety \
-		--enable-tap-tests \
-		--disable-rpath \
-		--with-uuid=e2fs \
-		--with-gnu-ld \
-		--with-pgport=5432 \
-		--with-system-tzdata=/usr/share/zoneinfo \
-		--prefix=/usr/local \
-		--with-includes=/usr/local/include \
-		--with-libraries=/usr/local/lib \
-		--with-openssl \
-		--with-libxml \
-		--with-libxslt \
-		--with-icu \
-	&& \
-    \
-    make -j "$(nproc)" world && \
-    make install-world && \
-    make -C contrib install && \
-    runDeps="$( \
-		scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \
-			| tr ',' '\n' \
-			| sort -u \
-			| awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \
-	)" && \
-	apk add -t .postgres-additional-deps \
-	           $runDeps \
-	           && \
-	\
-### Cleanup
-    apk del .postgres-build-deps && \
-    cd / && \
-    rm -rf \
-	/usr/src/postgresql \
-	/usr/local/share/doc \
-	/usr/local/share/man && \
-    find /usr/local -name '*.a' -delete && \
-    rm -rf /var/cache/apk/* && \
-    \
     ### Dependencies
-    set -ex && \
+RUN set -ex && \
-    echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
     apk update && \
     apk upgrade && \
     apk add -t .db-backup-build-deps \
                build-base \
                bzip2-dev \
                git \
+               libarchive-dev \
                xz-dev \
                && \
     \
     apk add --no-cache -t .db-backup-run-deps \
       	       bzip2 \
                influxdb \
+               libarchive \
                mariadb-client \
                mongodb-tools \
                libressl \
                pigz \
-               #postgresql \
+               postgresql \
-               #postgresql-client \
+               postgresql-client \
                redis \
                xz \
                zstd \
                && \
     \
-    apk add --no-cache \
-               pixz@testing \
-               && \
-    \
     cd /usr/src && \
-    curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_${MSSQL_VERSION}_amd64.apk && \
-    curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_${MSSQL_VERSION}_amd64.apk && \
-    echo y | apk add --allow-untrusted msodbcsql17_${MSSQL_VERSION}_amd64.apk mssql-tools_${MSSQL_VERSION}_amd64.apk && \
     \
+    apkArch="$(apk --print-arch)"; \
+    case "$apkArch" in \
+	x86_64) mssql=true ; curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_${MSSQL_VERSION}_amd64.apk ; curl -O https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_${MSSQL_VERSION}_amd64.apk ; echo y | apk add --allow-untrusted msodbcsql17_${MSSQL_VERSION}_amd64.apk mssql-tools_${MSSQL_VERSION}_amd64.apk ;; \
+	*) echo >&2 "Detected non x86_64 build variant, skipping MSSQL installation" ;; \
+    esac; \
     mkdir -p /usr/src/pbzip2 && \
-    curl -ssL https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz | tar xvfz - --strip=1 -C /usr/src/pbzip2 && \
+    curl -sSL https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz | tar xvfz - --strip=1 -C /usr/src/pbzip2 && \
     cd /usr/src/pbzip2 && \
     make && \
     make install && \
-    \
+    mkdir -p /usr/src/pixz && \
+    curl -sSL https://github.com/vasi/pixz/releases/download/v1.0.7/pixz-1.0.7.tar.xz | tar xvfJ - --strip 1 -C /usr/src/pixz && \
+    cd /usr/src/pixz && \
+    ./configure \
+        --prefix=/usr \
+        --sysconfdir=/etc \
+        --localstatedir=/var \
+        && \
+     make && \
+     make install && \
+     \
 ### Cleanup
     apk del .db-backup-build-deps && \
     rm -rf /usr/src/* && \

install/etc/services.available/10-db-backup/run

@@ -42,6 +42,11 @@ case "$dbtype" in
         [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS'
     ;;
     "mssql" | "MSSQL" | "microsoftsql" | "MICROSOFTSQL")
+        apkArch="$(apk --print-arch)"; \
+	    case "$apkArch" in
+		    x86_64) mssql=true ;;
+		    *) print_error "MSSQL cannot operate on $apkArch processor!" ; exit 1 ;;
+        esac
         dbtype=mssql
         dbport=${DB_PORT:-1433}
     ;;
@@ -140,6 +145,7 @@ backup_influx() {
         influx_compression="-portable"
     fi
     for DB in $DB_NAME; do
+        target=influx_${DB}_${dbhost}_${now}
         influxd backup ${influx_compression} -database $DB -host ${dbhost}:${dbport} ${tmpdir}/${target}
         exit_code=$?
         generate_md5