Release 4.1.4 - See CHANGELOG.md

[docs] fixed blobxfer mode correct parameter name

CHANGELOG.md

@@ -1,3 +1,33 @@
+## 4.1.4 2024-08-13 <dave at tiredofit dot ca>
+
+Please note that if using encryption using a passphrase, you may be encountering issues with manual decryption. This release fixes that.
+If you try to manually decrypt and your passphrase fails. Try wrapping it in single (') or double (") quotes.
+
+   ### Changed
+      - Fix for stray quotes appearing inside of ENCRYPT_PASSPHRASE variables
+
+
+## 4.1.3 2024-07-05 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Rebuild to support tiredofit/alpine:7.10.0
+
+
+## 4.1.2 2024-07-02 <effectivelywild@github>
+
+   ### Added
+      - Add support for Azure Blob containers
+      - Fix timestamps when comparing previous backups
+      - Resolve unnecessary read operations in Azure
+      - Resolve issues with backup cleanup operations in Azure
+
+
+## 4.1.1 2024-06-19 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix issue where postgresql globals when backing up ALL not being deleted (#352)
+
+
 ## 4.1.0 2024-05-25 <dave at tiredofit dot ca>
 
     Note that arm/v7 builds have been removed from this release going forward

README.md

@@ -356,11 +356,14 @@ If `DEFAULT_BACKUP_LOCATION` = `S3` then the following options are used:
 
 If `DEFAULT_BACKUP_LOCATION` = `blobxfer` then the following options are used:.
 
-| Parameter                              | Description                                 | Default             | `_FILE` |
+| Parameter                              | Description                                                         | Default             | `_FILE` |
-| -------------------------------------- | ------------------------------------------- | ------------------- | ------- |
+| -------------------------------------- | ------------------------------------------------------------------- | ------------------- | ------- |
-| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT`     | Microsoft Azure Cloud storage account name. |                     | x       |
+| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT`     | Microsoft Azure Cloud storage account name.                         |                     | x       |
-| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key.  |                     | x       |
+| `DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key.                          |                     | x       |
-| `DEFAULT_BLOBXFER_REMOTE_PATH`         | Remote Azure path                           | `/docker-db-backup` | x       |
+| `DEFAULT_BLOBXFER_REMOTE_PATH`         | Remote Azure path                                                   | `/docker-db-backup` | x       |
+| `DEFAULT_BLOBXFER_MODE`                | Azure Storage mode e.g. `auto`, `file`, `append`, `block` or `page` | `auto`              | x       |
+
+- When `DEFAULT_BLOBXFER_MODE` is set to auto it will use blob containers by default. If the `DEFAULT_BLOBXFER_REMOTE_PATH` path does not exist a blob container with that name will be created. 
 
 > This service uploads files from backup targed directory `DEFAULT_FILESYSTEM_PATH`.
 > If the a cleanup configuration in `DEFAULT_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically.
@@ -636,11 +639,14 @@ If `DB01_BACKUP_LOCATION` = `S3` then the following options are used:
 
 If `DB01_BACKUP_LOCATION` = `blobxfer` then the following options are used:.
 
-| Parameter                           | Description                                 | Default             | `_FILE` |
+| Parameter                              | Description                                                         | Default             | `_FILE` |
-| ----------------------------------- | ------------------------------------------- | ------------------- | ------- |
+| -------------------------------------- | ------------------------------------------------------------------- | ------------------- | ------- |
-| `DB01_BLOBXFER_STORAGE_ACCOUNT`     | Microsoft Azure Cloud storage account name. |                     | x       |
+| `DB01_BLOBXFER_STORAGE_ACCOUNT`        | Microsoft Azure Cloud storage account name.                         |                     | x       |
-| `DB01_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key.  |                     | x       |
+| `DB01_BLOBXFER_STORAGE_ACCOUNT_KEY`    | Microsoft Azure Cloud storage account key.                          |                     | x       |
-| `DB01_BLOBXFER_REMOTE_PATH`         | Remote Azure path                           | `/docker-db-backup` | x       |
+| `DB01_BLOBXFER_REMOTE_PATH`            | Remote Azure path                                                   | `/docker-db-backup` | x       |
+| `DB01_BLOBXFER_REMOTE_MODE`            | Azure Storage mode e.g. `auto`, `file`, `append`, `block` or `page` | `auto`              | x       |
+
+- When `DEFAULT_BLOBXFER_MODE` is set to auto it will use blob containers by default. If the `DEFAULT_BLOBXFER_REMOTE_PATH` path does not exist a blob container with that name will be created. 
 
 > This service uploads files from backup directory `DB01_BACKUP_FILESYSTEM_PATH`.
 > If the a cleanup configuration in `DB01_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically.

install/assets/defaults/10-db-backup

@@ -8,6 +8,7 @@ DEFAULT_BACKUP_BEGIN=${DEFAULT_BACKUP_BEGIN:-+0}
 DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440}
 DEFAULT_BACKUP_LOCATION=${DEFAULT_BACKUP_LOCATION:-"FILESYSTEM"}
 DEFAULT_BLOBXFER_REMOTE_PATH=${DEFAULT_BLOBXFER_REMOTE_PATH:-"/docker-db-backup"}
+DEFAULT_BLOBXFER_MODE=${DEFAULT_BLOBXFER_MODE:-"auto"}
 DEFAULT_CHECKSUM=${DEFAULT_CHECKSUM:-"MD5"}
 DEFAULT_COMPRESSION=${DEFAULT_COMPRESSION:-"ZSTD"}
 DEFAULT_COMPRESSION_LEVEL=${DEFAULT_COMPRESSION_LEVEL:-"3"}

install/assets/functions/10-db-backup

@@ -66,6 +66,7 @@ bootstrap_variables() {
                             DEFAULT_BLOBXFER_STORAGE_ACCOUNT \
                             DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY \
                             DEFAULT_BLOBXFER_REMOTE_PATH \
+                            DEFAULT_BLOBXFER_MODE \
                             DB"${backup_instance_number}"_AUTH \
                             DB"${backup_instance_number}"_TYPE \
                             DB"${backup_instance_number}"_HOST \
@@ -93,6 +94,7 @@ bootstrap_variables() {
                             DB"${backup_instance_number}"_BLOBXFER_STORAGE_ACCOUNT \
                             DB"${backup_instance_number}"_BLOBXFER_STORAGE_ACCOUNT_KEY \
                             DB"${backup_instance_number}"_BLOBXFER_REMOTE_PATH \
+                            DB"${backup_instance_number}"_BLOBXFER_MODE \
                             BLOBXFER_STORAGE_ACCOUNT \
                             BLOBXFER_STORAGE_ACCOUNT_KEY \
                             DB_HOST \
@@ -163,6 +165,11 @@ bootstrap_variables() {
             sed -i "s|_PASS='\(.*\)'|_PASS=\1|g" "${backup_instance_vars}"
         fi
 
+        if grep -qo ".*_PASSPHRASE='.*'" "${backup_instance_vars}"; then
+            print_debug "[bootstrap_variables] [backup_init] Found _PASSPHRASE variable with quotes"
+            sed -i "s|_PASSPHRASE='\(.*\)'|_PASSPHRASE=\1|g" "${backup_instance_vars}"
+        fi
+
         if grep -qo "MONGO_CUSTOM_URI='.*'" "${backup_instance_vars}"; then
             print_debug "[bootstrap_variables] [backup_init] Found _MONGO_CUSTOM_URI variable with quotes"
             sed -i "s|MONGO_CUSTOM_URI='\(.*\)'|MONGO_CUSTOM_URI=\1|g" "${backup_instance_vars}"
@@ -199,6 +206,7 @@ bootstrap_variables() {
         transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_REMOTE_PATH backup_job_blobxfer_remote_path
         transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_ACCOUNT backup_job_blobxfer_storage_account
         transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_ACCOUNT_KEY backup_job_blobxfer_storage_account_key
+        transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_MODE backup_job_blobxfer_mode
         transform_backup_instance_variable "${backup_instance_number}" CHECKSUM backup_job_checksum
         transform_backup_instance_variable "${backup_instance_number}" CLEANUP_TIME backup_job_cleanup_time
         transform_backup_instance_variable "${backup_instance_number}" COMPRESSION backup_job_compression
@@ -525,11 +533,11 @@ backup_influx() {
             print_debug "[backup_influx] Influx DB Version 1 selected"
             for db in ${db_names}; do
                 prepare_dbbackup
-                 if var_true "${DEBUG_BACKUP_INFLUX}" ; then debug on; fi
+                if var_true "${DEBUG_BACKUP_INFLUX}" ; then debug on; fi
                 if [ "${db}" != "justbackupeverything" ] ; then bucket="-db ${db}" ; else db=all ; fi
                 backup_job_filename=influx_${db}_${backup_job_db_host#*//}_${now}
                 backup_job_filename_base=influx_${db}_${backup_job_db_host#*//}
-                 if var_true "${DEBUG_BACKUP_INFLUX}" ; then debug off; fi
+                if var_true "${DEBUG_BACKUP_INFLUX}" ; then debug off; fi
                 pre_dbbackup "${db}"
                 write_log notice "Dumping Influx database: '${db}'"
                 if var_true "${DEBUG_BACKUP_INFLUX}" ; then debug on; fi
@@ -654,7 +662,6 @@ backup_mssql() {
             compression
             pre_dbbackup all
             run_as_user ${compress_cmd} "${temporary_directory}/${backup_job_filename_original}"
-
             file_encryption
             timer backup finish
             generate_checksum
@@ -742,6 +749,7 @@ backup_pgsql() {
     backup_pgsql_globals() {
         prepare_dbbackup
         backup_job_filename=pgsql_globals_${backup_job_db_host,,}_${now}.sql
+        backup_job_global_base=pgsql_globals_${backup_job_db_host,,}
         compression
         pre_dbbackup "globals"
         print_notice "Dumping PostgresSQL globals: with 'pg_dumpall -g' ${compression_string}"
@@ -1063,17 +1071,24 @@ cleanup_old_data() {
                     write_log info "Cleaning up old backups on filesystem"
                     run_as_user mkdir -p "${backup_job_filesystem_path}"
                     find "${backup_job_filesystem_path}"/  -type f -mmin +"${backup_job_cleanup_time}" -iname "${backup_job_filename_base}*" -exec rm -f {} \;
+                    if var_true "${_postgres_backup_globals}"; then
+                        find "${backup_job_filesystem_path}"/  -type f -mmin +"${backup_job_cleanup_time}" -iname "${backup_job_global_base}*" -exec rm -f {} \;
+                    fi
+
                     if  [ -z "${backup_job_blobxfer_storage_account}" ] || [ -z "${backup_job_blobxfer_storage_account_key}" ]; then
                         write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions"
                     else
                         write_log info "Syncing changes via blobxfer"
-                        silent run_as_user blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete --delete-only
+                        silent run_as_user blobxfer upload --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete --delete-only
                     fi
                 ;;
                 "file" | "filesystem" )
                     write_log info "Cleaning up old backups on filesystem"
                     run_as_user mkdir -p "${backup_job_filesystem_path}"
                     run_as_user find "${backup_job_filesystem_path}"/  -type f -mmin +"${backup_job_cleanup_time}" -iname "${backup_job_filename_base}*" -exec rm -f {} \;
+                    if var_true "${_postgres_backup_globals}"; then
+                        run_as_user find "${backup_job_filesystem_path}"/  -type f -mmin +"${backup_job_cleanup_time}" -iname "${backup_job_global_base}*" -exec rm -f {} \;
+                    fi
                 ;;
                 "s3" | "minio" )
                     write_log info "Cleaning up old backups on S3 storage"
@@ -1627,8 +1642,8 @@ EOF
                 if  [ -z "${backup_job_blobxfer_storage_account}" ] || [ -z "${backup_job_blobxfer_storage_account_key}" ]; then
                     write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions"
                 else
-                    write_log info "Synchronize local storage from S3 Bucket with blobxfer"
+                    write_log info "Synchronize local storage from blob container with blobxfer"
-                    ${play_fair} blobxfer download --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete
+                    ${play_fair} blobxfer download --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --restore-file-lmt --delete
 
                     write_log info "Moving backup to external storage with blobxfer"
                     mkdir -p "${backup_job_filesystem_path}"
@@ -1636,7 +1651,7 @@ EOF
 
                     run_as_user mv "${temporary_directory}"/"${backup_job_filename}" "${backup_job_filesystem_path}"/"${backup_job_filename}"
 
-                    silent run_as_user ${play_fair} blobxfer upload --mode file --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path}
+                    silent run_as_user ${play_fair} blobxfer upload --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path}
                     move_exit_code=$?
 
                     if [ "${backup_job_checksum}" != "none" ] ; then run_as_user rm -rf "${temporary_directory}"/"${backup_job_filename}"."${checksum_extension}" ; fi