Release 3.6.1 - See CHANGELOG.md

Bugfix: Generating S3 prefix to store new backups

.github/workflows/main.yml

@@ -19,7 +19,6 @@ jobs:
         id: prep
         run: |
           DOCKER_IMAGE=${GITHUB_REPOSITORY/docker-/}
-          set -x
           if [[ $GITHUB_REF == refs/heads/* ]]; then
              if [[ $GITHUB_REF == refs/heads/*/* ]] ; then
                BRANCH="${DOCKER_IMAGE}:$(echo $GITHUB_REF | sed "s|refs/heads/||g" | sed "s|/|-|g")"

.github/workflows/manual.yml

@@ -19,7 +19,6 @@ jobs:
         id: prep
         run: |
           DOCKER_IMAGE=${GITHUB_REPOSITORY/docker-/}
-          set -x
           if [[ $GITHUB_REF == refs/heads/* ]]; then
              if [[ $GITHUB_REF == refs/heads/*/* ]] ; then
                BRANCH="${DOCKER_IMAGE}:$(echo $GITHUB_REF | sed "s|refs/heads/||g" | sed "s|/|-|g")"

CHANGELOG.md

@@ -1,3 +1,53 @@
+## 3.6.1 2022-11-23 <dave at tiredofit dot ca>
+
+   ### Added
+      - Switch to Alpine 3.17 base
+      - Switch to OpenSSL instead of LibreSSL
+
+
+## 3.6.0 2022-11-21 <dave at tiredofit dot ca>
+
+   ### Added
+      - Postgresql 15 Support
+
+
+## 3.5.6 2022-11-15 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Add failure if DB_TYPE empty or malformed
+
+
+## 3.5.5 2022-10-18 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix for S3 backups and trailing slashes (@greena13)
+
+
+## 3.5.4 2022-10-13 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix for Influx DB 1 backups when compression enabled
+
+
+## 3.5.3 2022-10-12 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Remove build dependencies for blobxfer making image size smaller
+      - Remove silencing commands limiting Postgres backups from working without DEBUG_MODE=TRUE
+
+
+## 3.5.2 2022-10-11 <dave at tiredofit dot ca>
+
+   ### Added
+      - Update Influxdb client to 2.4.0 (jauderho@github)
+
+
+## 3.5.1 2022-10-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Tighten up cleanup routines to not call blobxfer unless absolutely necessary
+
+
 ## 3.5.0 2022-10-10 <dave at tiredofit dot ca>
 
    ### Added

Dockerfile

@@ -1,9 +1,8 @@
-FROM docker.io/tiredofit/alpine:3.16
+FROM docker.io/tiredofit/alpine:3.17
 LABEL maintainer="Dave Conroy (github.com/tiredofit)"
 
 ### Set Environment Variables
-
+ENV INFLUX2_VERSION=2.4.0 \
-ENV INFLUX2_VERSION=2.2.1 \
     MSSQL_VERSION=18.0.1.1-1 \
     CONTAINER_ENABLE_MESSAGING=FALSE \
     CONTAINER_ENABLE_MONITORING=TRUE \
@@ -11,8 +10,105 @@ ENV INFLUX2_VERSION=2.2.1 \
     IMAGE_NAME="tiredofit/db-backup" \
     IMAGE_REPO_URL="https://github.com/tiredofit/docker-db-backup/"
 
-### Dependencies
+ENV LANG=en_US.utf8 \
+    PG_MAJOR=15 \
+    PG_VERSION=15.1 \
+    PGDATA=/var/lib/postgresql/data
+
+### Create User Accounts
 RUN set -ex && \
+    addgroup -g 70 postgres && \
+    adduser -S -D -H -h /var/lib/postgresql -s /bin/sh -G postgres -u 70 postgres && \
+    mkdir -p /var/lib/postgresql && \
+    chown -R postgres:postgres /var/lib/postgresql && \
+    \
+### Install Dependencies
+   apk update && \
+   apk upgrade && \
+   apk add \
+       openssl \
+       && \
+   \
+   apk add --no-cache --virtual .postgres-build-deps \
+	   bison \
+	   build-base \
+	   coreutils \
+	   dpkg-dev \
+	   dpkg \
+	   flex \
+	   gcc \
+	   icu-dev \
+	   libc-dev \
+   	   libedit-dev \
+	   libxml2-dev \
+	   libxslt-dev \
+	   linux-headers \
+	   make \
+	   openssl-dev \
+	   perl-utils \
+	   perl-ipc-run \
+	   util-linux-dev \
+           wget \
+	   zlib-dev \
+       && \
+   \
+### Build Postgresql
+   mkdir -p /usr/src/postgresql && \
+   curl -sSL "https://ftp.postgresql.org/pub/source/v$PG_VERSION/postgresql-$PG_VERSION.tar.bz2" | tar xvfj - --strip 1 -C /usr/src/postgresql && \
+   cd /usr/src/postgresql && \
+# update "DEFAULT_PGSOCKET_DIR" to "/var/run/postgresql" (matching Debian)
+# see https://anonscm.debian.org/git/pkg-postgresql/postgresql.git/tree/debian/patches/51-default-sockets-in-var.patch?id=8b539fcb3e093a521c095e70bdfa76887217b89f
+   awk '$1 == "#define" && $2 == "DEFAULT_PGSOCKET_DIR" && $3 == "\"/tmp\"" { $3 = "\"/var/run/postgresql\""; print; next } { print }' src/include/pg_config_manual.h > src/include/pg_config_manual.h.new && \
+   grep '/var/run/postgresql' src/include/pg_config_manual.h.new && \
+   mv src/include/pg_config_manual.h.new src/include/pg_config_manual.h && \
+   gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)" && \
+# explicitly update autoconf config.guess and config.sub so they support more arches/libcs
+   wget --inet4-only -O config/config.guess 'https://git.savannah.gnu.org/cgit/config.git/plain/config.guess?id=7d3d27baf8107b630586c962c057e22149653deb' && \
+   wget --inet4-only -O config/config.sub 'https://git.savannah.gnu.org/cgit/config.git/plain/config.sub?id=7d3d27baf8107b630586c962c057e22149653deb' && \
+   ./configure \
+		--build="$gnuArch" \
+		--enable-integer-datetimes \
+		--enable-thread-safety \
+		--enable-tap-tests \
+		--disable-rpath \
+		--with-uuid=e2fs \
+		--with-gnu-ld \
+		--with-pgport=5432 \
+		--with-system-tzdata=/usr/share/zoneinfo \
+		--prefix=/usr/local \
+		--with-includes=/usr/local/include \
+		--with-libraries=/usr/local/lib \
+		--with-openssl \
+		--with-libxml \
+		--with-libxslt \
+		--with-icu \
+	&& \
+    \
+    make -j "$(nproc)" world && \
+    make install-world && \
+    make -C contrib install && \
+    runDeps="$( \
+		scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \
+			| tr ',' '\n' \
+			| sort -u \
+			| awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \
+	)" && \
+	apk add -t .postgres-additional-deps \
+	           $runDeps \
+	           && \
+	\
+### Cleanup
+    apk del .postgres-build-deps && \
+    cd / && \
+    rm -rf \
+	/usr/src/postgresql \
+	/usr/local/share/doc \
+	/usr/local/share/man && \
+    find /usr/local -name '*.a' -delete && \
+    rm -rf /var/cache/apk/* && \
+    \
+### Dependencies
+    set -ex && \
     apk update && \
     apk upgrade && \
     apk add -t .db-backup-build-deps \
@@ -20,6 +116,10 @@ RUN set -ex && \
                bzip2-dev \
                git \
                libarchive-dev \
+               openssl-dev \
+               libffi-dev \
+               python3-dev \
+               py3-pip \
                xz-dev \
                && \
     \
@@ -31,10 +131,10 @@ RUN set -ex && \
                mariadb-client \
                mariadb-connector-c \
                mongodb-tools \
-               libressl \
+               openssl \
                pigz \
-               postgresql \
+               #postgresql \
-               postgresql-client \
+               #postgresql-client \
                pv \
                py3-cryptography \
                redis \
@@ -69,7 +169,6 @@ RUN set -ex && \
     make && \
     make install && \
     \
-    apk add gcc build-base libressl-dev libffi-dev python3-dev py3-pip && \
     pip3 install blobxfer && \
     \
 ### Cleanup
@@ -80,4 +179,4 @@ RUN set -ex && \
     rm -rf /root/.cache /tmp/* /var/cache/apk/*
 
 ### S6 Setup
-ADD install  /
+COPY install  /

README.md

@@ -189,19 +189,19 @@ Your Organization will be mapped to `DB_USER` and your root token will need to b
 
 If `BACKUP_LOCATION` = `S3` then the following options are used.
 
-| Parameter             | Description                                                                               | Default |
+| Parameter             | Description                                                                              | Default |
-|-----------------------|-------------------------------------------------------------------------------------------|---------|
+|-----------------------|------------------------------------------------------------------------------------------|---------|
-| `S3_BUCKET`           | S3 Bucket name e.g. `mybucket`                                                            |         |
+| `S3_BUCKET`           | S3 Bucket name e.g. `mybucket`                                                           |         |
-| `S3_KEY_ID`           | S3 Key ID                                                                                 |         |
+| `S3_KEY_ID`           | S3 Key ID                                                                                |         |
-| `S3_KEY_SECRET`       | S3 Key Secret                                                                             |         |
+| `S3_KEY_SECRET`       | S3 Key Secret                                                                            |         |
-| `S3_PATH`             | S3 Pathname to save to (must end in a trailing slash e.g. '`backup/`')                    |         |
+| `S3_PATH`             | S3 Pathname to save to (must NOT end in a trailing slash e.g. '`backup`')                |         |
-| `S3_REGION`           | Define region in which bucket is defined. Example: `ap-northeast-2`                       |         |
+| `S3_REGION`           | Define region in which bucket is defined. Example: `ap-northeast-2`                      |         |
-| `S3_HOST`             | Hostname (and port) of S3-compatible service, e.g. `minio:8080`. Defaults to AWS.         |         |
+| `S3_HOST`             | Hostname (and port) of S3-compatible service, e.g. `minio:8080`. Defaults to AWS.        |         |
-| `S3_PROTOCOL`         | Protocol to connect to `S3_HOST`. Either `http` or `https`. Defaults to `https`.          | `https` |
+| `S3_PROTOCOL`         | Protocol to connect to `S3_HOST`. Either `http` or `https`. Defaults to `https`.         | `https` |
-| `S3_EXTRA_OPTS`       | Add any extra options to the end of the `aws-cli` process execution                       |         |
+| `S3_EXTRA_OPTS`       | Add any extra options to the end of the `aws-cli` process execution                      |         |
 | `S3_CERT_CA_FILE`     | Map a volume and point to your custom CA Bundle for verification e.g. `/certs/bundle.pem` |         |
-| _*OR*_                |                                                                                           |         |
+| _*OR*_                |                                                                                          |         |
-| `S3_CERT_SKIP_VERIFY` | Skip verifying self signed certificates when connecting                                   | `TRUE`  |
+| `S3_CERT_SKIP_VERIFY` | Skip verifying self signed certificates when connecting                                  | `TRUE`  |
 
 #### Upload to a Azure storage account by `blobxfer`
 

install/assets/functions/10-db-backup

@@ -1,6 +1,7 @@
 #!/command/with-contenv bash
 
 bootstrap_variables() {
+    sanity_var DB_TYPE "Set appropriate DB_TYPE"
     case "${DB_TYPE,,}" in
         couch* )
             dbtype=couch
@@ -74,6 +75,10 @@ bootstrap_variables() {
         sqlite* )
             dbtype=sqlite3
         ;;
+        * )
+            print_error "I don't recognize 'DB_TYPE=${DB_TYPE}' - Exitting.."
+            exit 99
+        ;;
     esac
 
     if [ "${BACKUP_LOCATION,,}" = "s3" ] || [ "${BACKUP_LOCATION,,}" = "minio" ] ; then
@@ -116,7 +121,7 @@ backup_influx() {
                 influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS} "${TEMP_LOCATION}"/"${target_dir}"
                 exit_code=$?
                 check_exit_code $target_dir
-                print_notice "Creating archive file of '${target_dir}' with tar ${compresion_string}"
+                print_notice "Creating archive file of '${target_dir}' with tar ${compression_string}"
                 tar cf - "${TEMP_LOCATION}"/"${target_dir}" | $dir_compress_cmd > "${TEMP_LOCATION}"/"${target_dir}".tar"${extension}"
                 target=influx_${db}_${DB_HOST#*//}_${now}.tar${extension}
                 generate_checksum
@@ -214,7 +219,7 @@ backup_mysql() {
                 compression
                 pre_dbbackup $db
                 print_notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}"
-                silent mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${EXTRA_OPTS} --databases $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
+                mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${EXTRA_OPTS} --databases $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
                 exit_code=$?
                 check_exit_code $target
                 generate_checksum
@@ -228,7 +233,7 @@ backup_mysql() {
         compression
         pre_dbbackup all
         print_notice "Dumping all MySQL / MariaDB databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}"
-        silent mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${EXTRA_OPTS} --databases $(echo ${db_names} | xargs) | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
+        mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${EXTRA_OPTS} --databases $(echo ${db_names} | xargs) | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
         exit_code=$?
         check_exit_code $target
         generate_checksum
@@ -263,7 +268,7 @@ backup_pgsql() {
                 compression
                 pre_dbbackup $db
                 print_notice "Dumping PostgresSQL database: '${db}' ${compression_string}"
-                silent pg_dump -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} $db ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target}
+                pg_dump -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} $db ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target}
                 exit_code=$?
                 check_exit_code $target
                 generate_checksum
@@ -285,7 +290,7 @@ backup_pgsql() {
         for x_db_name in ${tmp_db_names} ; do
             pgexclude_arg=$(echo ${pgexclude_arg} --exclude-database=${x_db_name})
         done
-        silent pg_dumpall -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} ${pgexclude_arg} ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target}
+        pg_dumpall -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} ${pgexclude_arg} ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target}
         exit_code=$?
         check_exit_code $target
         generate_checksum
@@ -460,17 +465,18 @@ cleanup_old_data() {
     if [ -n "${DB_CLEANUP_TIME}" ]; then
         if [ "${master_exit_code}" != 1 ]; then
             case "${BACKUP_LOCATION,,}" in
-                "file" | "filesystem" | "blobxfer" )
+                "file" | "filesystem" )
-                    print_info "Cleaning up old backups"
+                    print_info "Cleaning up old backups on filesystem"
                     mkdir -p "${DB_DUMP_TARGET}"
                     find "${DB_DUMP_TARGET}"/  -mmin +"${DB_CLEANUP_TIME}" -iname "*" -exec rm {} \;
-
+                    if [ "${BACKUP_LOCATION,,}" = "blobxfer" ] ; then
-                    print_info "Cleaning up old backups on S3 storage with blobxfer"
+                        print_info "Syncing changes via blobxfer"
-                    blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET} --delete --delete-only
+                        silent blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET} --delete --delete-only
+                    fi
                 ;;
                 "s3" | "minio" )
-                    print_info "Cleaning up old backups"
+                    print_info "Cleaning up old backups on S3 storage"
-                    aws ${PARAM_AWS_ENDPOINT_URL} s3 ls s3://${S3_BUCKET}/${S3_PATH} ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} | grep " DIR " -v | grep " PRE " -v | while read -r s3_file; do
+                    aws ${PARAM_AWS_ENDPOINT_URL} s3 ls s3://${S3_BUCKET}/${S3_PATH}/ ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} | grep " DIR " -v | grep " PRE " -v | while read -r s3_file; do
                         s3_createdate=$(echo $s3_file | awk {'print $1" "$2'})
                         s3_createdate=$(date -d "$s3_createdate" "+%s")
                         s3_olderthan=$(echo $(( $(date +%s)-${DB_CLEANUP_TIME}*60 )))
@@ -478,7 +484,7 @@ cleanup_old_data() {
                             s3_filename=$(echo $s3_file | awk {'print $4'})
                             if [ "$s3_filename" != "" ] ; then
                                 print_debug "Deleting $s3_filename"
-                                silent aws ${PARAM_AWS_ENDPOINT_URL} s3 rm s3://${S3_BUCKET}/${S3_PATH}${s3_filename} ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS}
+                                silent aws ${PARAM_AWS_ENDPOINT_URL} s3 rm s3://${S3_BUCKET}/${S3_PATH}/${s3_filename} ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS}
                             fi
                         fi
 
@@ -652,7 +658,7 @@ move_dbbackup() {
                 mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/
                 mv "${TEMP_LOCATION}"/"${target}" "${DB_DUMP_TARGET}"/"${target}"
 
-                blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET}
+                silent blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET}
 
                 rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}"
                 rm -rf "${TEMP_LOCATION}"/"${target}"