Release 4.0.29 - See CHANGELOG.md

CHANGELOG.md

@@ -1,3 +1,40 @@
+## 4.0.29 2023-12-04 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Skip blobxfer if either account or key is not present
+
+
+## 4.0.28 2023-12-04 <dave at tiredofit dot ca>
+
+   ### Changed
+      - AWS Cli 1.31.5
+      - Switch to using PIP for installing AWS-Cli to remove deprecation warnings
+
+
+## 4.0.27 2023-12-04 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Switch to using actual crontab for cron expressions
+
+
+## 4.0.26 2023-11-30 <dave at tiredofit dot ca>
+
+   ### Added
+      - AWS CLI 1.31.4
+
+
+## 4.0.25 2023-11-29 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix #297 - Add parameters to blobxfer to restore functionality
+
+
+## 4.0.24 2023-11-28 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix issue with cron parsing and 0 being a value getting clobbered by sort command
+
+
 ## 4.0.23 2023-11-28 <dave at tiredofit dot ca>
 
    ### Changed

Dockerfile

@@ -9,7 +9,7 @@ ENV INFLUX1_CLIENT_VERSION=1.8.0 \
     INFLUX2_CLIENT_VERSION=2.7.3 \
     MSODBC_VERSION=18.3.2.1-1 \
     MSSQL_VERSION=18.3.1.1-1 \
-    AWS_CLI_VERSION=1.29.78 \
+    AWS_CLI_VERSION=1.31.5 \
     CONTAINER_ENABLE_MESSAGING=TRUE \
     CONTAINER_ENABLE_MONITORING=TRUE \
     IMAGE_NAME="tiredofit/db-backup" \
@@ -78,8 +78,6 @@ RUN source /assets/functions/00-container && \
     \
     if [ $mssql = "true" ] ; then curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk ; curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; echo y | apk add --allow-untrusted msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; else echo >&2 "Detected non x86_64 or ARM64 build variant, skipping MSSQL installation" ; fi; \
     if [ $influx2 = "true" ] ; then curl -sSL https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX2_CLIENT_VERSION}-linux-${influx_arch}.tar.gz | tar xvfz - --strip=1 -C /usr/src/ ; chmod +x /usr/src/influx ; mv /usr/src/influx /usr/sbin/ ; else echo >&2 "Unable to build Influx 2 on this system" ; fi ; \
-    clone_git_repo https://github.com/aws/aws-cli "${AWS_CLI_VERSION}" && \
-    python3 setup.py install --prefix=/usr && \
     clone_git_repo https://github.com/influxdata/influxdb "${INFLUX1_CLIENT_VERSION}" && \
     go build -o /usr/sbin/influxd ./cmd/influxd && \
     strip /usr/sbin/influxd && \
@@ -99,6 +97,7 @@ RUN source /assets/functions/00-container && \
     make && \
     make install && \
     \
+    pip3 install --break-system-packages awscli==${AWS_CLI_VERSION} && \
     pip3 install --break-system-packages blobxfer && \
     \
     package remove .db-backup-build-deps && \

README.md

@@ -536,6 +536,7 @@ Encryption will occur after compression and the resulting filename will have a `
 | `DB01_NAME`                     | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to.                      |                           |         |
 |                                 | Backup multiple by separating with commas eg `db1,db2`                                                    |                           | x       |
 | `DB01_NAME_EXCLUDE`             | If using `ALL` - use this as to exclude databases separated via commas from being backed up               |                           | x       |
+| `DB01_SPLIT_DB`                 | If using `ALL` - use this to split each database into its own file as opposed to one singular file        | `FALSE`                   |         |
 | `DB01_PORT`                     | MySQL / MariaDB Port                                                                                      | `3306`                    | x       |
 | `DB01_MYSQL_EVENTS`             | Backup Events for                                                                                         | `TRUE`                    |         |
 | `DB01_MYSQL_MAX_ALLOWED_PACKET` | Max allowed packet                                                                                        | `512M`                    |         |
@@ -577,6 +578,7 @@ Encryption will occur after compression and the resulting filename will have a `
 | `DB01_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command`       |         |         |
 | `DB01_NAME`                   | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to.                      |         |         |
 |                               | Backup multiple by separating with commas eg `db1,db2`                                                    |         | x       |
+| `DB01_SPLIT_DB`               | If using `ALL` - use this to split each database into its own file as opposed to one singular file        | `FALSE` |         |
 | `DB01_PORT`                   | PostgreSQL Port                                                                                           | `5432`  | x       |
 
 ###### Redis

examples/mssql-blobxfer/compose.yml

@@ -57,7 +57,7 @@ services:
       # Add here azure storage account
       - DB01_BLOBXFER_STORAGE_ACCOUNT={TODO Add Storage Name}
       # Add here azure storage account key
-      - SB01_BLOBXFER_STORAGE_ACCOUNT_KEY={TODO Add Key}
+      - DB01_BLOBXFER_STORAGE_ACCOUNT_KEY={TODO Add Key}
       - DB01_BLOBXFER_REMOTE_PATH=docker-db-backup
     restart: always
     networks:

install/assets/dbbackup/template-dbbackup/run

@@ -24,7 +24,6 @@ else
     silent sleep {{BACKUP_NUMBER}}
     time_last_run=0
     time_current=$(date +'%s')
-
     if [[ "${backup_job_backup_begin}" =~ ^\+(.*)$ ]]; then
         print_debug "BACKUP_BEGIN is a jump of minute starting with +"
         timer plusvalue
@@ -37,10 +36,18 @@ else
     #elif echo "${backup_job_backup_begin//\*/#}" | grep -qP "^(.*((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|#) ?){5}$" ; then # Allow slashes, yet not supporting advanced cron yet
     elif echo "${backup_job_backup_begin//\*/#}" | grep -qP "^(((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|#) ?){5}$" ; then
         print_debug "BACKUP_BEGIN is a cron expression"
-        time_last_run=$(date +"%s")
-        backup_job_backup_begin=${backup_job_backup_begin//\"/}
-        backup_job_backup_begin=${backup_job_backup_begin//\'/}
-        timer cron "${backup_job_backup_begin}" "${time_current}" "${time_last_run}"
+        if var_false "${CRON_ALTERNATE}"; then
+            time_last_run=$(date +"%s")
+
+            backup_job_backup_begin=${backup_job_backup_begin//\"/}
+            backup_job_backup_begin=${backup_job_backup_begin//\'/}
+            timer cron "${backup_job_backup_begin}" "${time_current}" "${time_last_run}"
+        else
+            echo "${backup_job_backup_begin} /var/run/s6/legacy-services/dbbackup-{{BACKUP_NUMBER}}/run now" > /tmp/.container/cron/{{BACKUP_NUMBER}}-backup
+            crontab -l | { cat; echo "${backup_job_backup_begin} /var/run/s6/legacy-services/dbbackup-{{BACKUP_NUMBER}}/run now"; } | crontab -
+            s6-svc -d /var/run/s6/legacy-services/dbbackup-{{BACKUP_NUMBER}}
+            exit 0
+        fi
     else
         print_error "_BACKUP_BEGIN is invalid - Unable to perform scheduling"
         cat <<EOF

install/assets/defaults/10-db-backup

@@ -1,6 +1,7 @@
 #!/command/with-contenv bash
 
 BACKUP_JOB_CONCURRENCY=${BACKUP_JOB_CONCURRENCY:-"1"}
+CRON_ALTERNATE=${CRON_ALTERNATE:-"TRUE"}
 DBBACKUP_USER=${DBBACKUP_USER:-"dbbackup"}
 DBBACKUP_GROUP=${DBBACKUP_GROUP:-"${DBBACKUP_USER}"} # Must go after DBBACKUP_USER
 DEFAULT_BACKUP_BEGIN=${DEFAULT_BACKUP_BEGIN:-+0}

install/assets/functions/10-db-backup

@@ -94,7 +94,7 @@ bootstrap_variables() {
                             DB"${backup_instance_number}"_BLOBXFER_STORAGE_ACCOUNT_KEY \
                             DB"${backup_instance_number}"_BLOBXFER_REMOTE_PATH \
                             BLOBXFER_STORAGE_ACCOUNT \
-                            BLOBXFER_STORAGE_KEY \
+                            BLOBXFER_STORAGE_ACCOUNT_KEY \
                             DB_HOST \
                             DB_NAME \
                             DB_PORT \
@@ -188,7 +188,7 @@ bootstrap_variables() {
         transform_backup_instance_variable "${backup_instance_number}" BLACKOUT_END backup_job_snapshot_blackout_finish
         transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_REMOTE_PATH backup_job_blobxfer_remote_path
         transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_ACCOUNT backup_job_blobxfer_storage_account
-        transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_KEY backup_job_blobxfer_storage_key
+        transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_ACCOUNT_KEY backup_job_blobxfer_storage_account_key
         transform_backup_instance_variable "${backup_instance_number}" CHECKSUM backup_job_checksum
         transform_backup_instance_variable "${backup_instance_number}" CLEANUP_TIME backup_job_cleanup_time
         transform_backup_instance_variable "${backup_instance_number}" COMPRESSION backup_job_compression
@@ -577,7 +577,7 @@ backup_mongo() {
     fi
     if var_true "${DEBUG_BACKUP_MONGO}" ; then debug off; fi
     pre_dbbackup "${backup_job_db_name}"
-    write_log notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}"
+    write_log notice "Dumping MongoDB database: '${backup_job_db_name}' ${compression_string}"
     if var_true "${DEBUG_BACKUP_MONGO}" ; then debug on; fi
     silent run_as_user ${play_fair} mongodump --archive=${TEMP_PATH}/${backup_job_filename} ${mongo_compression} ${mongo_backup_parameter}
     exit_code=$?
@@ -600,7 +600,7 @@ backup_mssql() {
             backup_job_filename=mssql_${backup_job_db_name,,}_${backup_job_db_host,,}_${now}.bak
             backup_job_filename_base=mssql_${backup_job_db_name,,}_${backup_job_db_host,,}
             pre_dbbackup "${backup_job_db_name}"
-            write_log notice "Dumping MSSQL database: '${DB_NAME}'"
+            write_log notice "Dumping MSSQL database: '${backup_job_db_name}'"
             if var_true "${DEBUG_BACKUP_MSSQL}" ; then debug on; fi
             silent run_as_user ${play_fair} /opt/mssql-tools18/bin/sqlcmd -C -S ${backup_job_db_host}\,${backup_job_db_port} -U ${backup_job_db_user} -P ${backup_job_db_pass} -Q "BACKUP DATABASE [${backup_job_db_name}] TO DISK = N'${TEMP_PATH}/${backup_job_filename}' WITH NOFORMAT, NOINIT, NAME = '${backup_job_db_name}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10"
             exit_code=$?
@@ -625,7 +625,7 @@ backup_mssql() {
             backup_job_filename=mssql_${backup_job_db_name,,}_${backup_job_db_host,,}_${now}.trn
             backup_job_filename_base=mssql_${backup_job_db_name,,}_trn_${backup_job_db_host,,}
             pre_dbbackup "${backup_job_db_name}"
-            write_log notice "Dumping MSSQL database: '${DB_NAME}'"
+            write_log notice "Dumping MSSQL database: '${backup_job_db_name}'"
             if var_true "${DEBUG_BACKUP_MSSQL}" ; then debug on; fi
             silent run_as_user ${play_fair} /opt/mssql-tools18/bin/sqlcmd -C -S ${backup_job_db_host}\,${backup_job_db_port} -U ${backup_job_db_user} -P ${backup_job_db_pass} -Q "BACKUP LOG [${backup_job_db_name}] TO DISK = N'${TEMP_PATH}/${backup_job_filename}' WITH NOFORMAT, NOINIT, NAME = '${backup_job_db_name}-log', SKIP, NOREWIND, NOUNLOAD, STATS = 10"
             exit_code=$?
@@ -634,7 +634,7 @@ backup_mssql() {
             compression
             pre_dbbackup all
             run_as_user ${compress_cmd} "${TEMP_PATH}/${backup_job_filename_original}"
-            check_exit_code backup "${backup_job_filename}"
+
             file_encryption
             timer backup finish
             generate_checksum
@@ -1043,8 +1043,12 @@ cleanup_old_data() {
                     write_log info "Cleaning up old backups on filesystem"
                     run_as_user mkdir -p "${backup_job_filesystem_path}"
                     find "${backup_job_filesystem_path}"/  -type f -mmin +"${backup_job_cleanup_time}" -iname "${backup_job_filename_base}*" -exec rm -f {} \;
-                    write_log info "Syncing changes via blobxfer"
-                    silent run_as_user blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --local-path ${backup_job_filesystem_path} --delete --delete-only
+                    if  [ -z "${backup_job_blobxfer_storage_account}" ] || [ -z "${backup_job_blobxfer_storage_account_key}" ]; then
+                        write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions"
+                    else
+                        write_log info "Syncing changes via blobxfer"
+                        silent run_as_user blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete --delete-only
+                    fi
                 ;;
                 "file" | "filesystem" )
                     write_log info "Cleaning up old backups on filesystem"
@@ -1598,20 +1602,24 @@ EOF
                 run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}"
             ;;
             "blobxfer" )
-                write_log info "Synchronize local storage from S3 Bucket with blobxfer"
-                ${play_fair} blobxfer download --mode file --remote-path ${backup_job_blobxfer_remote_path} --local-path ${backup_job_filesystem_path} --delete
+                if  [ -z "${backup_job_blobxfer_storage_account}" ] || [ -z "${backup_job_blobxfer_storage_account_key}" ]; then
+                    write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions"
+                else
+                    write_log info "Synchronize local storage from S3 Bucket with blobxfer"
+                    ${play_fair} blobxfer download --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete
 
-                write_log info "Moving backup to external storage with blobxfer"
-                mkdir -p "${backup_job_filesystem_path}"
-                if [ "${backup_job_checksum}" != "none" ] ; then run_as_user mv "${TEMP_PATH}"/*."${checksum_extension}" "${backup_job_filesystem_path}"/; fi
+                    write_log info "Moving backup to external storage with blobxfer"
+                    mkdir -p "${backup_job_filesystem_path}"
+                    if [ "${backup_job_checksum}" != "none" ] ; then run_as_user mv "${TEMP_PATH}"/*."${checksum_extension}" "${backup_job_filesystem_path}"/; fi
 
-                run_as_user mv "${TEMP_PATH}"/"${backup_job_filename}" "${backup_job_filesystem_path}"/"${backup_job_filename}"
+                    run_as_user mv "${TEMP_PATH}"/"${backup_job_filename}" "${backup_job_filesystem_path}"/"${backup_job_filename}"
 
-                silent run_as_user ${play_fair} blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --local-path ${backup_job_filesystem_path}
-                move_exit_code=$?
+                    silent run_as_user ${play_fair} blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path}
+                    move_exit_code=$?
 
-                if [ "${backup_job_checksum}" != "none" ] ; then run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}"."${checksum_extension}" ; fi
-                run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}"
+                    if [ "${backup_job_checksum}" != "none" ] ; then run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}"."${checksum_extension}" ; fi
+                    run_as_user rm -rf "${TEMP_PATH}"/"${backup_job_filename}"
+                fi
             ;;
         esac
     else
@@ -1865,7 +1873,7 @@ timer() {
                     fi
                 done
 
-                validate_all=$(echo "${validate_all}" | tr ' ' '\n' | sort -n -u | tr '\n' ' ')
+                validate_all=$(echo "${validate_all}" | tr ' ' '\n' | sort -g -u | tr '\n' ' ')
                 for entry in $validate_all; do
                     if [ ${entry} -ge ${3} ]; then
                       echo "${entry}"
@@ -1882,12 +1890,12 @@ timer() {
             local cron_compare_difference=$(( cron_compare  - ${4} ))
 
             if [ "${cron_compare_difference}" -lt 60 ]; then
-                cron_compare=$((${cron_compare} + $(( 60 - cron_compare_difference )) ))
+                cron_compare=$((cron_compare + $(( 60 - cron_compare_difference )) ))
             fi
 
             local cron_current_seconds="$(date --date=@"${cron_compare_seconds}" +"%-S")"
             if [ "${cron_current_seconds}" -ne 0 ]; then
-                cron_compare_seconds=$(( cron_compare_seconds - cron_current_seconds ))
+                cron_compare=$(( cron_compare_seconds - cron_current_seconds ))
             fi
 
             local cron_minute="$(echo -n "${2}" | awk '{print $1}')"
@@ -1926,6 +1934,7 @@ timer() {
                     fi
 
                     cron_next_hour="${cron_next}"
+                    cron_next_minute=0
                 fi
 
                 print_debug "[timer] [cron] Parse Day of Week"
@@ -1999,7 +2008,6 @@ timer() {
                 fi
                 cron_parsed=0
             done
-
             local cron_future=$(date --date="${cron_next_year}-$(printf "%02d" ${cron_next_month})-$(printf "%02d" ${cron_next_day_of_month})T$(printf "%02d" ${cron_next_hour}):$(printf "%02d" ${cron_next_minute}):00" "+%s")
             local cron_future_difference=$(( cron_future - cron_compare_seconds ))
             time_cron=true