Release 4.0.29 - See CHANGELOG.md

CHANGELOG.md

@@ -1,3 +1,16 @@
+## 4.0.29 2023-12-04 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Skip blobxfer if either account or key is not present
+
+
+## 4.0.28 2023-12-04 <dave at tiredofit dot ca>
+
+   ### Changed
+      - AWS Cli 1.31.5
+      - Switch to using PIP for installing AWS-Cli to remove deprecation warnings
+
+
 ## 4.0.27 2023-12-04 <dave at tiredofit dot ca>
 
    ### Changed

Dockerfile

@@ -9,7 +9,7 @@ ENV INFLUX1_CLIENT_VERSION=1.8.0 \
     INFLUX2_CLIENT_VERSION=2.7.3 \
     MSODBC_VERSION=18.3.2.1-1 \
     MSSQL_VERSION=18.3.1.1-1 \
-    AWS_CLI_VERSION=1.31.4 \
+    AWS_CLI_VERSION=1.31.5 \
     CONTAINER_ENABLE_MESSAGING=TRUE \
     CONTAINER_ENABLE_MONITORING=TRUE \
     IMAGE_NAME="tiredofit/db-backup" \
@@ -78,8 +78,6 @@ RUN source /assets/functions/00-container && \
     \
     if [ $mssql = "true" ] ; then curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk ; curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; echo y | apk add --allow-untrusted msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; else echo >&2 "Detected non x86_64 or ARM64 build variant, skipping MSSQL installation" ; fi; \
     if [ $influx2 = "true" ] ; then curl -sSL https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX2_CLIENT_VERSION}-linux-${influx_arch}.tar.gz | tar xvfz - --strip=1 -C /usr/src/ ; chmod +x /usr/src/influx ; mv /usr/src/influx /usr/sbin/ ; else echo >&2 "Unable to build Influx 2 on this system" ; fi ; \
-    clone_git_repo https://github.com/aws/aws-cli "${AWS_CLI_VERSION}" && \
-    python3 setup.py install --prefix=/usr && \
     clone_git_repo https://github.com/influxdata/influxdb "${INFLUX1_CLIENT_VERSION}" && \
     go build -o /usr/sbin/influxd ./cmd/influxd && \
     strip /usr/sbin/influxd && \
@@ -99,6 +97,7 @@ RUN source /assets/functions/00-container && \
     make && \
     make install && \
     \
+    pip3 install --break-system-packages awscli==${AWS_CLI_VERSION} && \
     pip3 install --break-system-packages blobxfer && \
     \
     package remove .db-backup-build-deps && \

examples/mssql-blobxfer/compose.yml

@@ -57,7 +57,7 @@ services:
       # Add here azure storage account
       - DB01_BLOBXFER_STORAGE_ACCOUNT={TODO Add Storage Name}
       # Add here azure storage account key
-      - SB01_BLOBXFER_STORAGE_ACCOUNT_KEY={TODO Add Key}
+      - DB01_BLOBXFER_STORAGE_ACCOUNT_KEY={TODO Add Key}
       - DB01_BLOBXFER_REMOTE_PATH=docker-db-backup
     restart: always
     networks:

install/assets/functions/10-db-backup

@@ -577,7 +577,7 @@ backup_mongo() {
     fi
     if var_true "${DEBUG_BACKUP_MONGO}" ; then debug off; fi
     pre_dbbackup "${backup_job_db_name}"
-    write_log notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}"
+    write_log notice "Dumping MongoDB database: '${backup_job_db_name}' ${compression_string}"
     if var_true "${DEBUG_BACKUP_MONGO}" ; then debug on; fi
     silent run_as_user ${play_fair} mongodump --archive=${TEMP_PATH}/${backup_job_filename} ${mongo_compression} ${mongo_backup_parameter}
     exit_code=$?
@@ -600,7 +600,7 @@ backup_mssql() {
             backup_job_filename=mssql_${backup_job_db_name,,}_${backup_job_db_host,,}_${now}.bak
             backup_job_filename_base=mssql_${backup_job_db_name,,}_${backup_job_db_host,,}
             pre_dbbackup "${backup_job_db_name}"
-            write_log notice "Dumping MSSQL database: '${DB_NAME}'"
+            write_log notice "Dumping MSSQL database: '${backup_job_db_name}'"
             if var_true "${DEBUG_BACKUP_MSSQL}" ; then debug on; fi
             silent run_as_user ${play_fair} /opt/mssql-tools18/bin/sqlcmd -C -S ${backup_job_db_host}\,${backup_job_db_port} -U ${backup_job_db_user} -P ${backup_job_db_pass} -Q "BACKUP DATABASE [${backup_job_db_name}] TO DISK = N'${TEMP_PATH}/${backup_job_filename}' WITH NOFORMAT, NOINIT, NAME = '${backup_job_db_name}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10"
             exit_code=$?
@@ -625,7 +625,7 @@ backup_mssql() {
             backup_job_filename=mssql_${backup_job_db_name,,}_${backup_job_db_host,,}_${now}.trn
             backup_job_filename_base=mssql_${backup_job_db_name,,}_trn_${backup_job_db_host,,}
             pre_dbbackup "${backup_job_db_name}"
-            write_log notice "Dumping MSSQL database: '${DB_NAME}'"
+            write_log notice "Dumping MSSQL database: '${backup_job_db_name}'"
             if var_true "${DEBUG_BACKUP_MSSQL}" ; then debug on; fi
             silent run_as_user ${play_fair} /opt/mssql-tools18/bin/sqlcmd -C -S ${backup_job_db_host}\,${backup_job_db_port} -U ${backup_job_db_user} -P ${backup_job_db_pass} -Q "BACKUP LOG [${backup_job_db_name}] TO DISK = N'${TEMP_PATH}/${backup_job_filename}' WITH NOFORMAT, NOINIT, NAME = '${backup_job_db_name}-log', SKIP, NOREWIND, NOUNLOAD, STATS = 10"
             exit_code=$?
@@ -634,7 +634,7 @@ backup_mssql() {
             compression
             pre_dbbackup all
             run_as_user ${compress_cmd} "${TEMP_PATH}/${backup_job_filename_original}"
-            check_exit_code backup "${backup_job_filename}"
+
             file_encryption
             timer backup finish
             generate_checksum
@@ -1043,8 +1043,12 @@ cleanup_old_data() {
                     write_log info "Cleaning up old backups on filesystem"
                     run_as_user mkdir -p "${backup_job_filesystem_path}"
                     find "${backup_job_filesystem_path}"/  -type f -mmin +"${backup_job_cleanup_time}" -iname "${backup_job_filename_base}*" -exec rm -f {} \;
-                    write_log info "Syncing changes via blobxfer"
+                    if  [ -z "${backup_job_blobxfer_storage_account}" ] || [ -z "${backup_job_blobxfer_storage_account_key}" ]; then
-                    silent run_as_user blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete --delete-only
+                        write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions"
+                    else
+                        write_log info "Syncing changes via blobxfer"
+                        silent run_as_user blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete --delete-only
+                    fi
                 ;;
                 "file" | "filesystem" )
                     write_log info "Cleaning up old backups on filesystem"
@@ -1598,20 +1602,24 @@ EOF
                 run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}"
             ;;
             "blobxfer" )
-                write_log info "Synchronize local storage from S3 Bucket with blobxfer"
+                if  [ -z "${backup_job_blobxfer_storage_account}" ] || [ -z "${backup_job_blobxfer_storage_account_key}" ]; then
-                ${play_fair} blobxfer download --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete
+                    write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions"
+                else
+                    write_log info "Synchronize local storage from S3 Bucket with blobxfer"
+                    ${play_fair} blobxfer download --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete
 
-                write_log info "Moving backup to external storage with blobxfer"
+                    write_log info "Moving backup to external storage with blobxfer"
-                mkdir -p "${backup_job_filesystem_path}"
+                    mkdir -p "${backup_job_filesystem_path}"
-                if [ "${backup_job_checksum}" != "none" ] ; then run_as_user mv "${TEMP_PATH}"/*."${checksum_extension}" "${backup_job_filesystem_path}"/; fi
+                    if [ "${backup_job_checksum}" != "none" ] ; then run_as_user mv "${TEMP_PATH}"/*."${checksum_extension}" "${backup_job_filesystem_path}"/; fi
 
-                run_as_user mv "${TEMP_PATH}"/"${backup_job_filename}" "${backup_job_filesystem_path}"/"${backup_job_filename}"
+                    run_as_user mv "${TEMP_PATH}"/"${backup_job_filename}" "${backup_job_filesystem_path}"/"${backup_job_filename}"
 
-                silent run_as_user ${play_fair} blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path}
+                    silent run_as_user ${play_fair} blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path}
-                move_exit_code=$?
+                    move_exit_code=$?
 
-                if [ "${backup_job_checksum}" != "none" ] ; then run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}"."${checksum_extension}" ; fi
+                    if [ "${backup_job_checksum}" != "none" ] ; then run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}"."${checksum_extension}" ; fi
-                run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}"
+                    run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}"
+                fi
             ;;
         esac
     else
@@ -2000,13 +2008,11 @@ timer() {
                 fi
                 cron_parsed=0
             done
-set -x
             local cron_future=$(date --date="${cron_next_year}-$(printf "%02d" ${cron_next_month})-$(printf "%02d" ${cron_next_day_of_month})T$(printf "%02d" ${cron_next_hour}):$(printf "%02d" ${cron_next_minute}):00" "+%s")
             local cron_future_difference=$(( cron_future - cron_compare_seconds ))
             time_cron=true
             time_wait="${cron_future_difference}"
             time_future="${cron_future}"
-set +x
         ;;
         datetime)
             time_begin=$(date -d "${backup_job_backup_begin}" +%s)