Release 3.3.3 - See CHANGELOG.md

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/main.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Prepare
         id: prep
@@ -63,17 +63,17 @@ jobs:
           echo ::set-output name=docker_image::${DOCKER_IMAGE}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
         with:
           platforms: all
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -102,7 +102,7 @@ jobs:
           fi
 
       - name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           builder: ${{ steps.buildx.outputs.name }}
           context: .

.github/workflows/manual.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Prepare
         id: prep
@@ -63,17 +63,17 @@ jobs:
           echo ::set-output name=docker_image::${DOCKER_IMAGE}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
         with:
           platforms: all
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
@@ -102,7 +102,7 @@ jobs:
           fi
 
       - name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           builder: ${{ steps.buildx.outputs.name }}
           context: .

CHANGELOG.md

@@ -1,3 +1,35 @@
+## 3.3.3 2022-05-24 <dave at tiredofit dot ca>
+
+   ### Added
+      - Alpine 3.16 base
+
+
+## 3.3.2 2022-05-02 <dave at tiredofit dot ca>
+
+   ### Added
+      - Add POST_SCRIPT_SKIP_X_VERIFY environment variables to allow for more host compatibility for post scripts
+
+
+## 3.3.1 2022-04-30 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Compressing silently was causing 0 byte backups
+
+
+## 3.3.0 2022-04-30 <dave at tiredofit dot ca>
+
+   ### Added
+      - Ability to auto clean old S3 / Minio Hosts like what occurs on filesysten
+      - Alert user how to turn off Zabbix Monitoring if fails
+      - Allow Zabbix Monitoring to work with S3
+      - Silence some more compression statements
+   ### Changed
+      - Fix for Redis not backing up properly
+      - Start sending checksums for S3 Outputs
+      - Cleanup some code functions
+      - FIx Container Log Level always in DEBUG
+
+
 ## 3.2.4 2022-04-21 <dave at tiredofit dot ca>
 
    ### Changed

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/tiredofit/alpine:3.15
+FROM docker.io/tiredofit/alpine:3.16
 LABEL maintainer="Dave Conroy (github.com/tiredofit)"
 
 ### Set Environment Variables

README.md

@@ -261,6 +261,9 @@ Outputs the following on the console:
 
 If you wish to change the size value from bytes to megabytes set environment variable `SIZE_VALUE=megabytes`
 
+You must make your scripts executible otherwise there is an internal check that will skip trying to run it otherwise.
+If for some reason your filesystem or host is not detecting it right, use the environment variable `POST_SCRIPT_SKIP_X_VERIFY=TRUE` to bypass.
+
 ## Support
 
 These images were built to serve a specific need in a production environment and gradually have had more functionality added based on requests from the community.

install/assets/functions/10-db-backup

@@ -279,7 +279,7 @@ backup_redis() {
     pre_dbbackup
     print_notice "Dumping Redis - Flushing Redis Cache First"
     target=redis_all_${DB_HOST,,}_${now}.rdb
-    echo bgsave | redis-cli -h ${DB_HOST} -p ${DB_PORT} ${REDIS_PASS_STR} --rdb ${TEMP_LOCATION}/${target} ${EXTRA_OPTS}
+    echo bgsave | silent redis-cli -h ${DB_HOST} -p ${DB_PORT} ${REDIS_PASS_STR} --rdb ${TEMP_LOCATION}/${target} ${EXTRA_OPTS}
     sleep 10
     try=5
     while [ $try -gt 0 ] ; do
@@ -287,6 +287,7 @@ backup_redis() {
         ok=$(echo 'info Persistence' | redis-cli -h ${DB_HOST} -p ${DB_PORT} ${REDIS_PASS_STR} | awk '/rdb_last_bgsave_status:ok/{print "ok"}')
         if [[ "$saved" = "saved" ]] && [[ "$ok" = "ok" ]]; then
             print_notice "Redis Backup Complete"
+            exit_code=0
             break
         fi
         try=$((try - 1))
@@ -296,6 +297,7 @@ backup_redis() {
     target_original=${target}
     compression
     $compress_cmd "${TEMP_LOCATION}/${target_original}"
+    check_exit_code $target
     generate_checksum
     move_dbbackup
     post_dbbackup all
@@ -430,11 +432,31 @@ check_exit_code() {
 cleanup_old_data() {
     if [ -n "${DB_CLEANUP_TIME}" ]; then
         if [ "${master_exit_code}" != 1 ]; then
-            print_info "Cleaning up old backups"
+            case "${BACKUP_LOCATION,,}" in
-            mkdir -p "${DB_DUMP_TARGET}"
+                "file" | "filesystem" )
-            find "${DB_DUMP_TARGET}"/  -mmin +"${DB_CLEANUP_TIME}" -iname "*" -exec rm {} \;
+                    print_info "Cleaning up old backups"
+                    mkdir -p "${DB_DUMP_TARGET}"
+                    find "${DB_DUMP_TARGET}"/  -mmin +"${DB_CLEANUP_TIME}" -iname "*" -exec rm {} \;
+                ;;
+                "s3" | "minio" )
+                    print_info "Cleaning up old backups"
+                    aws ${PARAM_AWS_ENDPOINT_URL} s3 ls s3://${S3_BUCKET}/${S3_PATH} ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} | grep " DIR " -v | while read -r s3_file; do
+                        s3_createdate=$(echo $s3_file | awk {'print $1" "$2'})
+                        s3_createdate=$(date -d "$s3_createdate" "+%s")
+                        s3_olderthan=$(echo $(( $(date +%s)-${DB_CLEANUP_TIME}*60 )))
+                        if [[ $s3_createdate -le $s3_olderthan ]] ; then
+                            s3_filename=$(echo $s3_file | awk {'print $4'})
+                            if [ $s3_filename != "" ] ; then
+                                print_debug "Deleting $s3_filename"
+                                silent aws ${PARAM_AWS_ENDPOINT_URL} s3 rm s3://${S3_BUCKET}/${S3_PATH}/${s3_filename} ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS}
+                            fi
+                        fi
+
+                    done
+                ;;
+            esac
         else
-            print_info "Skipping Cleaning up old backups because there were errors in backing up"
+            print_error "Skipping Cleaning up old backups because there were errors in backing up"
         fi
     fi
 }
@@ -446,7 +468,7 @@ compression() {
 
    case "${COMPRESSION,,}" in
         gz* )
-            compress_cmd="pigz -${COMPRESSION_LEVEL} -p ${PARALLEL_COMPRESSION_THREADS} "
+            compress_cmd="pigz -q -${COMPRESSION_LEVEL} -p ${PARALLEL_COMPRESSION_THREADS} "
             compression_type="gzip"
             extension=".gz"
             dir_compress_cmd=${compress_cmd}
@@ -454,7 +476,7 @@ compression() {
             target=${target}.gz
         ;;
         bz* )
-            compress_cmd="pbzip2 -${COMPRESSION_LEVEL} -p${PARALLEL_COMPRESSION_THREADS} "
+            compress_cmd="pbzip2 -q -${COMPRESSION_LEVEL} -p${PARALLEL_COMPRESSION_THREADS} "
             compression_type="bzip2"
             dir_compress_cmd=${compress_cmd}
             extension=".bz2"
@@ -470,7 +492,7 @@ compression() {
             target=${target}.xz
         ;;
         zst* )
-            compress_cmd="zstd --rm -${COMPRESSION_LEVEL} -T${PARALLEL_COMPRESSION_THREADS} "
+            compress_cmd="zstd -q -q --rm -${COMPRESSION_LEVEL} -T${PARALLEL_COMPRESSION_THREADS} "
             compression_type="zstd"
             dir_compress_cmd=${compress_cmd}
             extension=".zst"
@@ -506,12 +528,12 @@ create_archive() {
         print_notice "Creating archive file of '${target_dir}' with tar ${compresion_string}"
         tar cf - "${TEMP_LOCATION}"/"${target_dir}" | $dir_compress_cmd > "${TEMP_LOCATION}"/"${target_dir}".tar"${extension}"
     else
-        print_warn "Skipping creating archive file because backup did not complete successfully"
+        print_error "Skipping creating archive file because backup did not complete successfully"
     fi
 }
 
 generate_checksum() {
-    if var_true "${ENABLE_CHECKSUM}" ;then
+    if var_true "${ENABLE_CHECKSUM}" ; then
         if [ "${exit_code}" = "0" ] ; then
             case "${CHECKSUM,,}" in
                 "md5" )
@@ -530,13 +552,16 @@ generate_checksum() {
             checksum_value=$(${checksum_command} "${target}" | awk ' { print $1}')
             print_debug "${checksum_extension^^}: ${checksum_value} - ${target}"
         else
-            print_warn "Skipping Checksum creation because backup did not complete successfully"
+            print_error "Skipping Checksum creation because backup did not complete successfully"
         fi
     fi
 }
 
 move_dbbackup() {
     if [ "${exit_code}" = "0" ] ; then
+        dbbackup_size="$(stat -c%s "${TEMP_LOCATION}"/"${target}")"
+        dbbackup_date="$(date -r  "${TEMP_LOCATION}"/"${target}" +'%s')"
+
         case "${SIZE_VALUE,,}" in
             "b" | "bytes" )
                 SIZE_VALUE=1
@@ -579,15 +604,17 @@ move_dbbackup() {
 
                 [[ ( -n "${S3_HOST}" ) ]] && PARAM_AWS_ENDPOINT_URL=" --endpoint-url ${S3_PROTOCOL}://${S3_HOST}"
 
-                aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_LOCATION}/${target} s3://${S3_BUCKET}/${S3_PATH}/${target} ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS}
+                silent aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_LOCATION}/${target} s3://${S3_BUCKET}/${S3_PATH}/${target} ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS}
-                unset s3_ssl
+                if var_true "${ENABLE_CHECKSUM}" ; then
-                unset s3_ca_cert
+                    silent aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_LOCATION}/*.${checksum_extension} s3://${S3_BUCKET}/${S3_PATH}/ ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS}
+                fi
+
                 rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}"
                 rm -rf "${TEMP_LOCATION}"/"${target}"
             ;;
         esac
     else
-        print_warn "Skipping moving DB Backup to final location because backup did not complete successfully"
+        print_error "Skipping moving DB Backup to final location because backup did not complete successfully"
     fi
 
     rm -rf "${TEMP_LOCATION}"/*
@@ -607,36 +634,47 @@ post_dbbackup() {
 
     if var_true "${CONTAINER_ENABLE_MONITORING}" ; then
         print_notice "Sending Backup Statistics to Zabbix"
-        silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.size -o "$(stat -c%s "${DB_DUMP_TARGET}"/"${target}")"
+        silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.size -o "${dbbackup_size}"
-        silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.datetime -o "$(date -r  "${DB_DUMP_TARGET}"/"${target}" +'%s')"
+        silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.datetime -o "${dbbackup_date}"
         silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.status -o "${exit_code}"
         silent zabbix_sender -c /etc/zabbix/zabbix_agentd.conf -k dbbackup.backup_duration -o "$(echo $((dbbackup_finish_time-dbbackup_start_time)))"
+        if [ "$?" != "0" ] ; then print_error "Error sending statistics, consider disabling with 'CONTAINER_ENABLE_MONITORING=FALSE'" ; fi
     fi
 
     ### Post Script Support
     if [ -n "${POST_SCRIPT}" ] ; then
-        if [ -x "${POST_SCRIPT}" ] ; then
+        if var_true "${POST_SCRIPT_SKIP_X_VERIFY}" ; then
-            print_notice "Found POST_SCRIPT environment variable. Executing '${POST_SCRIPT}"
             eval "${POST_SCRIPT}" "${exit_code}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${target}" "${FILESIZE}" "${checksum_value}"
         else
-            print_error "Can't execute POST_SCRIPT environment variable '${POST_SCRIPT}' as its filesystem bit is not executible!"
+            if [ -x "${POST_SCRIPT}" ] ; then
+                print_notice "Found POST_SCRIPT environment variable. Executing '${POST_SCRIPT}"
+                eval "${POST_SCRIPT}" "${exit_code}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${target}" "${FILESIZE}" "${checksum_value}"
+            else
+                print_error "Can't execute POST_SCRIPT environment variable '${POST_SCRIPT}' as its filesystem bit is not executible!"
+            fi
         fi
     fi
 
     ### Post Backup Custom Script Support
     if [ -d "/assets/custom-scripts/" ] ; then
         for f in $(find /assets/custom-scripts/ -name \*.sh -type f); do
-            if [ -x "${f}" ] ; then
+            if var_true "${POST_SCRIPT_SKIP_X_VERIFY}" ; then
-                print_notice "Executing post backup custom script : '${f}'"
-                ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME STARTEPOCH FINISHEPOCH DURATIONEPOCH BACKUP_FILENAME FILESIZE CHECKSUMVALUE
                 ${f} "${exit_code}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${target}" "${FILESIZE}" "${checksum_value}"
             else
-                print_error "Can't run post backup custom script: '${f}' as its filesystem bit is not executible!"
+                if [ -x "${f}" ] ; then
+                    print_notice "Executing post backup custom script : '${f}'"
+                    ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME STARTEPOCH FINISHEPOCH DURATIONEPOCH BACKUP_FILENAME FILESIZE CHECKSUMVALUE
+                    ${f} "${exit_code}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${target}" "${FILESIZE}" "${checksum_value}"
+                else
+                    print_error "Can't run post backup custom script: '${f}' as its filesystem bit is not executible!"
+                fi
             fi
         done
     fi
 
     print_notice "DB Backup for '${1}' time taken: $(echo ${dbbackup_total_time} | awk '{printf "Hours: %d Minutes: %02d Seconds: %02d", $1/3600, ($1/60)%60, $1%60}')"
+    unset s3_ssl
+    unset s3_ca_cert
 }
 
 sanity_test() {

install/etc/services.available/10-db-backup/run

@@ -4,7 +4,6 @@ source /assets/functions/00-container
 source /assets/functions/10-db-backup
 source /assets/defaults/10-db-backup
 PROCESS_NAME="db-backup"
-CONTAINER_LOG_LEVEL=DEBUG
 
 bootstrap_variables