Release 4.0.16 - See CHANGELOG.md

warn instead of warning

CHANGELOG.md

@@ -1,9 +1,58 @@
+## 4.0.16 2023-11-17 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Switch to using msmtp instead of s-mail for notify()
+
+
+## 4.0.15 2023-11-16 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix cleanup of old backups
+
+
+## 4.0.14 2023-11-13 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Bugfix when PRE/POST scripts found not giving legacy warning
+      - Run pre / post scripts as root
+
+
+## 4.0.13 2023-11-12 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Check for any quotes if using MONGO_CUSTOM_URI and remove
+
+
+## 4.0.12 2023-11-12 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Allow creating schedulers if _MONGO_CUSTOM_URI is set and _DB_HOST blank
+
+
+## 4.0.11 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Resolve issue with backing up ALL databases with PGSQL and MySQL
+
+
+## 4.0.10 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Change environment variable parsing routines to properly accomodate for Passwords containing '=='
+
+
+## 4.0.9 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix issue with quotes being wrapped around _PASS variables
+
+
 ## 4.0.8 2023-11-11 <dave at tiredofit dot ca>
 
    ### Changed
       - Tidy up file_encryption() routines
-      - Change environment variable _ENCRYPTION_PUBKEY to _ENCRYPTION_PUBLIC_KEY
-      - Add new environment variable _ENCRYPTION_PRIVATE_KEY
+      - Change environment variable _ENCRYPT_PUBKEY to _ENCRYPT_PUBLIC_KEY
+      - Add new environment variable _ENCRYPT_PRIVATE_KEY
 
 
 ## 4.0.7 2023-11-11 <dave at tiredofit dot ca>

Dockerfile

@@ -10,7 +10,7 @@ ENV INFLUX1_CLIENT_VERSION=1.8.0 \
     MSODBC_VERSION=18.3.2.1-1 \
     MSSQL_VERSION=18.3.1.1-1 \
     AWS_CLI_VERSION=1.29.78 \
-    CONTAINER_ENABLE_MESSAGING=FALSE \
+    CONTAINER_ENABLE_MESSAGING=TRUE \
     CONTAINER_ENABLE_MONITORING=TRUE \
     IMAGE_NAME="tiredofit/db-backup" \
     IMAGE_REPO_URL="https://github.com/tiredofit/docker-db-backup/"

README.md

@@ -601,13 +601,13 @@ Options that are related to the value of `DB01_BACKUP_LOCATION`
 
 If `DB01_BACKUP_LOCTION` = `FILESYSTEM` then the following options are used:
 
-| Variable                          | Description                                                                                           | Default                           |
-| --------------------------------- | ----------------------------------------------------------------------------------------------------- | --------------------------------- |
-| `DB01_CREATE_LATEST_SYMLINK`      | Create a symbolic link pointing to last backup in this format: `latest-(DB_TYPE)-(DB_NAME)-(DB_HOST)` | `TRUE`                            |
-| `DB01_FILESYSTEM_PATH`            | Directory where the database dumps are kept.                                                          | `/backup`                         |
-| `DB01_FILESYSTEM_PATH_PERMISSION` | Permissions to apply to backup directory                                                              | `700`                             |
-| `DB01_FILESYSTEM_ARCHIVE_PATH`    | Optional Directory where the database dumps archives are kept                                         | `${DB01_FILESYSTEM_PATH/archive/` |
-| `DB01_FILESYSTEM_PERMISSION`      | Directory and File permissions to apply to files.                                                     | `600`                             |
+| Variable                          | Description                                                                                           | Default                            |
+| --------------------------------- | ----------------------------------------------------------------------------------------------------- | ---------------------------------- |
+| `DB01_CREATE_LATEST_SYMLINK`      | Create a symbolic link pointing to last backup in this format: `latest-(DB_TYPE)-(DB_NAME)-(DB_HOST)` | `TRUE`                             |
+| `DB01_FILESYSTEM_PATH`            | Directory where the database dumps are kept.                                                          | `/backup`                          |
+| `DB01_FILESYSTEM_PATH_PERMISSION` | Permissions to apply to backup directory                                                              | `700`                              |
+| `DB01_FILESYSTEM_ARCHIVE_PATH`    | Optional Directory where the database dumps archives are kept                                         | `${DB01_FILESYSTEM_PATH}/archive/` |
+| `DB01_FILESYSTEM_PERMISSION`      | Directory and File permissions to apply to files.                                                     | `600`                              |
 
 ###### S3
 
@@ -742,6 +742,9 @@ $5 body/error message
 
 
 ##### Email Notifications
+
+See more details in the base image listed above for more mail environment variables.
+
 | Parameter   | Description                                                                               | Default | `_FILE` |
 | ----------- | ----------------------------------------------------------------------------------------- | ------- | ------- |
 | `MAIL_FROM` | What email address to send mail from for errors                                           |         |         |

install/assets/dbbackup/template-dbbackup/run

@@ -87,6 +87,7 @@ while true; do
     fi
 
     symlink_log
+    cleanup_old_data
 
     if var_false "${persist}" ; then
         print_debug "Exiting due to manual mode"

install/assets/functions/10-db-backup

@@ -153,18 +153,28 @@ bootstrap_variables() {
         fi
         ##
 
+        if grep -qo ".*_PASS='.*'" "${backup_instance_vars}"; then
+            print_debug "[bootstrap_variables] [backup_init] Found _PASS variable with quotes"
+            sed -i "s|_PASS='\(.*\)'|_PASS=\1|g" "${backup_instance_vars}"
+        fi
+
+        if grep -qo "MONGO_CUSTOM_URI='.*'" "${backup_instance_vars}"; then
+            print_debug "[bootstrap_variables] [backup_init] Found _MONGO_CUSTOM_URI variable with quotes"
+            sed -i "s|MONGO_CUSTOM_URI='\(.*\)'|MONGO_CUSTOM_URI=\1|g" "${backup_instance_vars}"
+        fi
+
         transform_backup_instance_variable() {
             if grep -q "^DB${1}_${2}=" "${backup_instance_vars}" && [ "$(grep "^DB${1}_${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then
-                export "$3"="$(grep "^DB${1}_${2}=" "${backup_instance_vars}" | cut -d = -f2)"
+                export "$3"="$(grep "^DB${1}_${2}=" "${backup_instance_vars}" | cut -d = -f2-)"
             elif grep -q "^DB_${2}=" "${backup_instance_vars}" && [ "$(grep "^DB_${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then
                 # Allow old legacy work, perhaps remove old DB_ functionality in future? This should allow for seamless upgrades
                 #print_warn "Legacy Variable 'DB_${2}'' detected - Please upgrade your variables as they will be removed in version 4.3.0"
-                export "$3"="$(grep "^DB_${2}=" "${backup_instance_vars}" | cut -d = -f2)"
+                export "$3"="$(grep "^DB_${2}=" "${backup_instance_vars}" | cut -d = -f2-)"
             elif grep -q "^${2}=" "${backup_instance_vars}" && [ "$(grep "^${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then
                 print_warn "Legacy unsupported variable '${2}' detected - Please upgrade your variables as they will be removed in version 4.3.0"
-                export "$3"="$(grep "^${2}=" "${backup_instance_vars}" | cut -d = -f2)"
+                export "$3"="$(grep "^${2}=" "${backup_instance_vars}" | cut -d = -f2-)"
             elif grep -q "^DEFAULT_${2}=" "${backup_instance_vars}" && [ "$(grep "^DEFAULT_${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then
-                export "$3"="$(grep "^DEFAULT_${2}=" "${backup_instance_vars}" | cut -d = -f2)"
+                export "$3"="$(grep "^DEFAULT_${2}=" "${backup_instance_vars}" | cut -d = -f2-)"
             fi
         }
 
@@ -729,7 +739,7 @@ backup_pgsql() {
     fi
     if [ "${backup_job_db_name,,}" = "all" ] ; then
         write_log debug "Preparing to back up all databases"
-        db_names=$(run_as_user psql -h ${backup_job_db_host} -U ${backup_job_db_user} -p ${backup_job_db_port} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
+        db_names=$(psql -h ${backup_job_db_host} -U ${backup_job_db_user} -p ${backup_job_db_port} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
         if [ -n "${backup_job_db_name_exclude}" ] ; then
             db_names_exclusions=$(echo "${backup_job_db_name_exclude}" | tr ',' '\n')
             for db_exclude in ${db_names_exclusions} ; do
@@ -777,7 +787,7 @@ backup_pgsql() {
         pre_dbbackup all
         write_log notice "Dumping all PostgreSQL databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}"
         if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug on; fi
-        tmp_db_names=$(run_as_user psql -h ${backup_job_db_host} -p ${backup_job_db_port} -U ${backup_job_db_user} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
+        tmp_db_names=$(psql -h ${backup_job_db_host} -p ${backup_job_db_port} -U ${backup_job_db_user} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
         for r_db_name in $(echo $db_names | xargs); do
             tmp_db_names=$(echo "$tmp_db_names" | xargs | sed "s|${r_db_name}||g"  )
         done
@@ -1136,10 +1146,11 @@ create_archive() {
 
 create_schedulers() {
     if var_true "${DEBUG_CREATE_SCHEDULERS}" ; then debug on; fi
+
     backup() {
-        bootstrap_variables upgrade BACKUP
-        local backup_instances=$(printenv | sort | grep -c "^DB[0-9]._HOST")
+        local backup_instances=$(set -o posix ; set | grep -Pc "^(DB[0-9]._HOST=|.*MONGO_CUSTOM_URI=)")
         print_debug "[create_schedulers] Found '${backup_instances}' DB_HOST instances"
+
         if [ -n "${DB_HOST}" ] && [ "${backup_instances}" ]; then
           backup_instances=1;
           print_debug "[create_schedulers] Detected using old DB_ variables"
@@ -1183,7 +1194,7 @@ EOF
 }
 
 ctrl_c() {
-    sed -i "/^{{BACKUP_NUMBER}}/d" /tmp/.container/db-backup-backups
+    sed -i "/^${backup_instance_number}/d" /tmp/.container/db-backup-backups
     symlink_log
     print_warn "User aborted"
     exit
@@ -1352,13 +1363,15 @@ notify() {
         if [ -z "${SMTP_HOST}" ] ; then write_log error "[notifications] No SMTP_HOST variable set - Skipping sending Email notifications" ; skip_mail=true ; fi
         if [ -z "${SMTP_PORT}" ] ; then write_log error "[notifications] No SMTP_PORT variable set - Skipping sending Email notifications" ; skip_mail=true ; fi
         if var_nottrue "${skip_mail}" ; then
+            if ! grep -q ^from /etc/msmptrc ; then
+                echo "from ${MAIL_FROM}" >> /etc/msmtprc
+            fi
             mail_recipients=$(echo "${MAIL_TO}" | tr "," "\n")
             for mail_recipient in $mail_recipients ; do
-                cat <<EOF | s-nail -v \
-                                        -r "${MAIL_FROM}" \
-                                        -s "[db-backup] [${DOMAIN}] ${3}" \
-                                        -S smtp="${SMTP_HOST}":"${SMTP_PORT}" \
-                                        "${mail_recipient}"
+                cat <<EOF | msmtp -t "${mail_recipient}" -c /etc/msmtprc
+To: ${mail_recipient}
+Subject: [db-backup] [${DOMAIN}] ${3}
+
 Time: ${1}
 Log File: {2}
 Error Code: ${3}
@@ -1620,20 +1633,20 @@ pre_dbbackup() {
 
     ### Pre Backup Custom Script Support
     if [ -d "/assets/custom-scripts/pre" ] && dir_notempty "/assets/custom-scripts/pre" ; then
-        write_log warning "Found Custom Post Scripts in /assets/custom-scripts/pre - Automatically moving them to '${backup_job_script_location_pre}'"
-        run_as_user mkdir -p "${backup_job_script_location_pre}"
-        silent run_as_user cp /assets/custom-scripts/pre/* "${backup_job_script_location_pre}"
+        write_log warn "Found Custom Post Scripts in /assets/custom-scripts/pre - Automatically moving them to '${backup_job_script_location_pre}'"
+        mkdir -p "${backup_job_script_location_pre}"
+        silent cp -aR /assets/custom-scripts/pre/* "${backup_job_script_location_pre}"
     fi
 
     if [ -d "${backup_job_script_location_pre}" ] && dir_notempty "${backup_job_script_location_pre}" ; then
         for f in $(find ${backup_job_script_location_pre} -name \*.sh -type f); do
             if var_true "${backup_job_pre_script_x_verify}" ; then
-                run_as_user ${f} "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${backup_job_filename}"
+                ${f} "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${backup_job_filename}"
             else
                 if [ -x "${f}" ] ; then
                     write_log notice "Executing pre backup custom script : '${f}'"
                     ## script DB_TYPE DB_HOST DB_NAME STARTEPOCH BACKUP_FILENAME
-                    run_as_user ${f} "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${backup_job_filename}"
+                    ${f} "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${backup_job_filename}"
                 else
                     write_log error "Can't run pre backup custom script: '${f}' as its filesystem bit is not executible!"
                 fi
@@ -1676,11 +1689,11 @@ EOZP
     ### Post Script Support
     if [ -n "${backup_job_post_script}" ] ; then
         if var_true "${backup_job_post_script_x_verify}" ; then
-            run_as_user eval "${backup_job_post_script}" "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
+            eval "${backup_job_post_script}" "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
         else
             if [ -x "${backup_job_post_script}" ] ; then
                 write_log notice "Found POST_SCRIPT environment variable. Executing '${backup_job_post_script}"
-                run_as_user eval "${backup_job_post_script}" "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
+                eval "${backup_job_post_script}" "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
             else
                 write_log error "Can't execute POST_SCRIPT environment variable '${backup_job_post_script}' as its filesystem bit is not executible!"
             fi
@@ -1689,20 +1702,20 @@ EOZP
 
     ### Post Backup Custom Script Support
     if [ -d "/assets/custom-scripts/" ] && dir_notempty "/assets/custom-scripts" ; then
-        write_log warning "Found Custom Post Scripts in /assets/custom-scripts/ - Automatically moving them to '${backup_job_script_location_post}'"
-        run_as_user mkdir -p "${backup_job_script_location_post}"
-        silent run_as_user cp /assets/custom-scripts/* "${backup_job_script_location_post}"
+        write_log warn "Found Custom Post Scripts in /assets/custom-scripts/ - Automatically moving them to '${backup_job_script_location_post}'"
+        mkdir -p "${backup_job_script_location_post}"
+        cp -aR /assets/custom-scripts/* "${backup_job_script_location_post}"
     fi
 
     if [ -d "${backup_job_script_location_post}" ] && dir_notempty "${backup_job_script_location_post}" ; then
         for f in $(run_as_user find "${backup_job_script_location_post}" -name \*.sh -type f); do
             if var_true "${backup_job_post_script_x_verify}" ; then
-                run_as_user ${f} "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
+                ${f} "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
             else
                 if [ -x "${f}" ] ; then
                     write_log notice "Executing post backup custom script : '${f}'"
                     ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME STARTEPOCH FINISHEPOCH DURATIONEPOCH BACKUP_FILENAME FILESIZE CHECKSUMVALUE
-                    run_as_user ${f} "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
+                    ${f} "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
                 else
                     write_log error "Can't run post backup custom script: '${f}' as its filesystem bit is not executible!"
                 fi
@@ -1738,7 +1751,7 @@ process_limiter() {
 }
 
 run_as_user() {
-    sudo -u "${DBBACKUP_USER}" $@
+    sudo -Eu "${DBBACKUP_USER}" "$@"
 }
 
 setup_mode() {
@@ -2057,4 +2070,4 @@ write_log() {
     print_${_arg_log_level} "${_arg_log_message}"
     output_on
     if var_true "${DEBUG_WRITE_LOG}" ; then debug off; fi
-}
+}