Release 4.0.15 - See CHANGELOG.md

warn instead of warning

CHANGELOG.md

@@ -1,3 +1,40 @@
+## 4.0.15 2023-11-16 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix cleanup of old backups
+
+
+## 4.0.14 2023-11-13 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Bugfix when PRE/POST scripts found not giving legacy warning
+      - Run pre / post scripts as root
+
+
+## 4.0.13 2023-11-12 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Check for any quotes if using MONGO_CUSTOM_URI and remove
+
+
+## 4.0.12 2023-11-12 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Allow creating schedulers if _MONGO_CUSTOM_URI is set and _DB_HOST blank
+
+
+## 4.0.11 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Resolve issue with backing up ALL databases with PGSQL and MySQL
+
+
+## 4.0.10 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Change environment variable parsing routines to properly accomodate for Passwords containing '=='
+
+
 ## 4.0.9 2023-11-11 <dave at tiredofit dot ca>
 
    ### Changed

install/assets/dbbackup/template-dbbackup/run

@@ -87,6 +87,7 @@ while true; do
     fi
 
     symlink_log
+    cleanup_old_data
 
     if var_false "${persist}" ; then
         print_debug "Exiting due to manual mode"

install/assets/functions/10-db-backup

@@ -158,18 +158,23 @@ bootstrap_variables() {
             sed -i "s|_PASS='\(.*\)'|_PASS=\1|g" "${backup_instance_vars}"
         fi
 
+        if grep -qo "MONGO_CUSTOM_URI='.*'" "${backup_instance_vars}"; then
+            print_debug "[bootstrap_variables] [backup_init] Found _MONGO_CUSTOM_URI variable with quotes"
+            sed -i "s|MONGO_CUSTOM_URI='\(.*\)'|MONGO_CUSTOM_URI=\1|g" "${backup_instance_vars}"
+        fi
+
         transform_backup_instance_variable() {
             if grep -q "^DB${1}_${2}=" "${backup_instance_vars}" && [ "$(grep "^DB${1}_${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then
-                export "$3"="$(grep "^DB${1}_${2}=" "${backup_instance_vars}" | cut -d = -f2)"
+                export "$3"="$(grep "^DB${1}_${2}=" "${backup_instance_vars}" | cut -d = -f2-)"
             elif grep -q "^DB_${2}=" "${backup_instance_vars}" && [ "$(grep "^DB_${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then
                 # Allow old legacy work, perhaps remove old DB_ functionality in future? This should allow for seamless upgrades
                 #print_warn "Legacy Variable 'DB_${2}'' detected - Please upgrade your variables as they will be removed in version 4.3.0"
-                export "$3"="$(grep "^DB_${2}=" "${backup_instance_vars}" | cut -d = -f2)"
+                export "$3"="$(grep "^DB_${2}=" "${backup_instance_vars}" | cut -d = -f2-)"
             elif grep -q "^${2}=" "${backup_instance_vars}" && [ "$(grep "^${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then
                 print_warn "Legacy unsupported variable '${2}' detected - Please upgrade your variables as they will be removed in version 4.3.0"
-                export "$3"="$(grep "^${2}=" "${backup_instance_vars}" | cut -d = -f2)"
+                export "$3"="$(grep "^${2}=" "${backup_instance_vars}" | cut -d = -f2-)"
             elif grep -q "^DEFAULT_${2}=" "${backup_instance_vars}" && [ "$(grep "^DEFAULT_${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then
-                export "$3"="$(grep "^DEFAULT_${2}=" "${backup_instance_vars}" | cut -d = -f2)"
+                export "$3"="$(grep "^DEFAULT_${2}=" "${backup_instance_vars}" | cut -d = -f2-)"
             fi
         }
 
@@ -734,7 +739,7 @@ backup_pgsql() {
     fi
     if [ "${backup_job_db_name,,}" = "all" ] ; then
         write_log debug "Preparing to back up all databases"
-        db_names=$(run_as_user psql -h ${backup_job_db_host} -U ${backup_job_db_user} -p ${backup_job_db_port} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
+        db_names=$(psql -h ${backup_job_db_host} -U ${backup_job_db_user} -p ${backup_job_db_port} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
         if [ -n "${backup_job_db_name_exclude}" ] ; then
             db_names_exclusions=$(echo "${backup_job_db_name_exclude}" | tr ',' '\n')
             for db_exclude in ${db_names_exclusions} ; do
@@ -782,7 +787,7 @@ backup_pgsql() {
         pre_dbbackup all
         write_log notice "Dumping all PostgreSQL databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}"
         if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug on; fi
-        tmp_db_names=$(run_as_user psql -h ${backup_job_db_host} -p ${backup_job_db_port} -U ${backup_job_db_user} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
+        tmp_db_names=$(psql -h ${backup_job_db_host} -p ${backup_job_db_port} -U ${backup_job_db_user} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
         for r_db_name in $(echo $db_names | xargs); do
             tmp_db_names=$(echo "$tmp_db_names" | xargs | sed "s|${r_db_name}||g"  )
         done
@@ -1141,10 +1146,11 @@ create_archive() {
 
 create_schedulers() {
     if var_true "${DEBUG_CREATE_SCHEDULERS}" ; then debug on; fi
+
     backup() {
-        bootstrap_variables upgrade BACKUP
+        local backup_instances=$(set -o posix ; set | grep -Pc "^(DB[0-9]._HOST=|.*MONGO_CUSTOM_URI=)")
-        local backup_instances=$(printenv | sort | grep -c "^DB[0-9]._HOST")
         print_debug "[create_schedulers] Found '${backup_instances}' DB_HOST instances"
+
         if [ -n "${DB_HOST}" ] && [ "${backup_instances}" ]; then
           backup_instances=1;
           print_debug "[create_schedulers] Detected using old DB_ variables"
@@ -1625,20 +1631,20 @@ pre_dbbackup() {
 
     ### Pre Backup Custom Script Support
     if [ -d "/assets/custom-scripts/pre" ] && dir_notempty "/assets/custom-scripts/pre" ; then
-        write_log warning "Found Custom Post Scripts in /assets/custom-scripts/pre - Automatically moving them to '${backup_job_script_location_pre}'"
+        write_log warn "Found Custom Post Scripts in /assets/custom-scripts/pre - Automatically moving them to '${backup_job_script_location_pre}'"
-        run_as_user mkdir -p "${backup_job_script_location_pre}"
+        mkdir -p "${backup_job_script_location_pre}"
-        silent run_as_user cp /assets/custom-scripts/pre/* "${backup_job_script_location_pre}"
+        silent cp -aR /assets/custom-scripts/pre/* "${backup_job_script_location_pre}"
     fi
 
     if [ -d "${backup_job_script_location_pre}" ] && dir_notempty "${backup_job_script_location_pre}" ; then
         for f in $(find ${backup_job_script_location_pre} -name \*.sh -type f); do
             if var_true "${backup_job_pre_script_x_verify}" ; then
-                run_as_user ${f} "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${backup_job_filename}"
+                ${f} "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${backup_job_filename}"
             else
                 if [ -x "${f}" ] ; then
                     write_log notice "Executing pre backup custom script : '${f}'"
                     ## script DB_TYPE DB_HOST DB_NAME STARTEPOCH BACKUP_FILENAME
-                    run_as_user ${f} "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${backup_job_filename}"
+                    ${f} "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${backup_job_filename}"
                 else
                     write_log error "Can't run pre backup custom script: '${f}' as its filesystem bit is not executible!"
                 fi
@@ -1681,11 +1687,11 @@ EOZP
     ### Post Script Support
     if [ -n "${backup_job_post_script}" ] ; then
         if var_true "${backup_job_post_script_x_verify}" ; then
-            run_as_user eval "${backup_job_post_script}" "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
+            eval "${backup_job_post_script}" "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
         else
             if [ -x "${backup_job_post_script}" ] ; then
                 write_log notice "Found POST_SCRIPT environment variable. Executing '${backup_job_post_script}"
-                run_as_user eval "${backup_job_post_script}" "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
+                eval "${backup_job_post_script}" "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
             else
                 write_log error "Can't execute POST_SCRIPT environment variable '${backup_job_post_script}' as its filesystem bit is not executible!"
             fi
@@ -1694,20 +1700,20 @@ EOZP
 
     ### Post Backup Custom Script Support
     if [ -d "/assets/custom-scripts/" ] && dir_notempty "/assets/custom-scripts" ; then
-        write_log warning "Found Custom Post Scripts in /assets/custom-scripts/ - Automatically moving them to '${backup_job_script_location_post}'"
+        write_log warn "Found Custom Post Scripts in /assets/custom-scripts/ - Automatically moving them to '${backup_job_script_location_post}'"
-        run_as_user mkdir -p "${backup_job_script_location_post}"
+        mkdir -p "${backup_job_script_location_post}"
-        silent run_as_user cp /assets/custom-scripts/* "${backup_job_script_location_post}"
+        cp -aR /assets/custom-scripts/* "${backup_job_script_location_post}"
     fi
 
     if [ -d "${backup_job_script_location_post}" ] && dir_notempty "${backup_job_script_location_post}" ; then
         for f in $(run_as_user find "${backup_job_script_location_post}" -name \*.sh -type f); do
             if var_true "${backup_job_post_script_x_verify}" ; then
-                run_as_user ${f} "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
+                ${f} "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
             else
                 if [ -x "${f}" ] ; then
                     write_log notice "Executing post backup custom script : '${f}'"
                     ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME STARTEPOCH FINISHEPOCH DURATIONEPOCH BACKUP_FILENAME FILESIZE CHECKSUMVALUE
-                    run_as_user ${f} "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
+                    ${f} "${exit_code}" "${dbtype}" "${backup_job_db_host}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${backup_job_filename}" "${filesize}" "${checksum_value}" "${move_exit_code}"
                 else
                     write_log error "Can't run post backup custom script: '${f}' as its filesystem bit is not executible!"
                 fi
@@ -1743,7 +1749,7 @@ process_limiter() {
 }
 
 run_as_user() {
-    sudo -Eu "${DBBACKUP_USER}" $@
+    sudo -Eu "${DBBACKUP_USER}" "$@"
 }
 
 setup_mode() {