Release 4.0.22 - See CHANGELOG.md

Update 10-db-backup msmtp -C

CHANGELOG.md

@@ -1,3 +1,40 @@
+## 4.0.22 2023-11-25 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Move cleanup_old_data routines to happen within backup_ function to properly accomodate for globals, and ALL DB_NAME use cases
+
+
+## 4.0.21 2023-11-22 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix for SQLite backups not being cleaned up properly due to a malformed base
+
+
+## 4.0.20 2023-11-21 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Update base image to support S6 Overlay 3.1.6.2 to solve shutdown issues specifically with MODE=MANUAL and MANUAL_RUN_FOREVER=TRUE
+      - Add some safety nets for Manual scheduling
+
+
+## 4.0.19 2023-11-20 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Make adjustments to cron scheduling feature to be able to handle whitespace properly"
+
+
+## 4.0.18 2023-11-18 <joergmschulz@github>
+
+   ### Changed
+      - Fix loading msmtp configuration
+
+
+## 4.0.17 2023-11-17 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Provide more details when notifying via instant messages
+
+
 ## 4.0.16 2023-11-17 <dave at tiredofit dot ca>
 
    ### Changed

Dockerfile

@@ -76,7 +76,7 @@ RUN source /assets/functions/00-container && \
         *) sleep 0.1 ;; \
     esac; \
     \
-    if [ $mssql = "true" ] ; then curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk ; curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; ls -l ; echo y | apk add --allow-untrusted msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; else echo >&2 "Detected non x86_64 or ARM64 build variant, skipping MSSQL installation" ; fi; \
+    if [ $mssql = "true" ] ; then curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk ; curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; echo y | apk add --allow-untrusted msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; else echo >&2 "Detected non x86_64 or ARM64 build variant, skipping MSSQL installation" ; fi; \
     if [ $influx2 = "true" ] ; then curl -sSL https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX2_CLIENT_VERSION}-linux-${influx_arch}.tar.gz | tar xvfz - --strip=1 -C /usr/src/ ; chmod +x /usr/src/influx ; mv /usr/src/influx /usr/sbin/ ; else echo >&2 "Unable to build Influx 2 on this system" ; fi ; \
     clone_git_repo https://github.com/aws/aws-cli "${AWS_CLI_VERSION}" && \
     python3 setup.py install --prefix=/usr && \

install/assets/dbbackup/template-dbbackup/run

@@ -16,7 +16,7 @@ fi
 
 trap ctrl_c INT
 
-if [[ "${MODE,,}" =~ "standalone" ]] || [ "${1,,}" = "manual" ] || [ "${1,,}" = "now" ]; then
+if [[ "${MODE,,}" =~ "standalone" ]] || [ "${MODE,,}" = "manual" ] || [ "${1,,}" = "manual" ] || [ "${1,,}" = "now" ]; then
     print_debug "Detected Manual Mode"
     persist=false
     backup_job_backup_begin=+0
@@ -34,9 +34,12 @@ else
     elif [[ "${backup_job_backup_begin}" =~ ([0-9]{4})-([0-9]{2})-([0-9]{2})[[:space:]]([0-9]{2}):([0-9]{2}):([0-9]{2}) ]]; then
         print_debug "BACKUP_BEGIN is a full date timestamp"
         timer datetime
+    #elif echo "${backup_job_backup_begin//\*/#}" | grep -qP "^(.*((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|#) ?){5}$" ; then # Allow slashes, yet not supporting advanced cron yet
     elif echo "${backup_job_backup_begin//\*/#}" | grep -qP "^(((\d+,)+\d+|(\d+(\/|-)\d+)|\d+|#) ?){5}$" ; then
         print_debug "BACKUP_BEGIN is a cron expression"
         time_last_run=$(date +"%s")
+        backup_job_backup_begin=${backup_job_backup_begin//\"/}
+        backup_job_backup_begin=${backup_job_backup_begin//\'/}
         timer cron "${backup_job_backup_begin}" "${time_current}" "${time_last_run}"
     else
         print_error "_BACKUP_BEGIN is invalid - Unable to perform scheduling"
@@ -87,7 +90,6 @@ while true; do
     fi
 
     symlink_log
-    cleanup_old_data
 
     if var_false "${persist}" ; then
         print_debug "Exiting due to manual mode"

install/assets/functions/10-db-backup

@@ -482,6 +482,7 @@ backup_couch() {
     move_dbbackup
     check_exit_code move "${backup_job_filename}"
     post_dbbackup ${backup_job_db_name}
+    cleanup_old_data
 }
 
 backup_influx() {
@@ -522,6 +523,7 @@ backup_influx() {
                 move_dbbackup
                 check_exit_code move "${backup_job_filename_dir}"
                 post_dbbackup "${db}"
+                cleanup_old_data
             done
         ;;
         2 )
@@ -550,6 +552,7 @@ backup_influx() {
                 move_dbbackup
                 check_exit_code move "${backup_job_filename_dir}"
                 post_dbbackup "${db}"
+                cleanup_old_data
             done
         ;;
     esac
@@ -586,6 +589,7 @@ backup_mongo() {
     move_dbbackup
     check_exit_code move "${backup_job_filename}"
     post_dbbackup "${backup_job_db_name}"
+    cleanup_old_data
     if var_true "${DEBUG_BACKUP_MONGO}" ; then debug off; fi
 }
 
@@ -614,6 +618,7 @@ backup_mssql() {
             move_dbbackup
             check_exit_code move "${backup_job_filename}"
             post_dbbackup "${backup_job_db_name}"
+            cleanup_old_data
         ;;
         trn|transaction )
             prepare_dbbackup
@@ -636,6 +641,7 @@ backup_mssql() {
             move_dbbackup
             check_exit_code move "${backup_job_filename}"
             post_dbbackup "${backup_job_db_name}"
+            cleanup_old_data
         ;;
     esac
 }
@@ -687,6 +693,7 @@ backup_mysql() {
                 move_dbbackup
                 check_exit_code move "${backup_job_filename}"
                 post_dbbackup "${db}"
+                cleanup_old_data
         done
     else
         write_log debug "Not splitting database dumps into their own files"
@@ -707,6 +714,7 @@ backup_mysql() {
         move_dbbackup
         check_exit_code move "${backup_job_filename}"
         post_dbbackup all
+        cleanup_old_data
     fi
 }
 
@@ -728,6 +736,7 @@ backup_pgsql() {
         move_dbbackup
         check_exit_code move "${backup_job_filename}"
         post_dbbackup "globals"
+        cleanup_old_data
     }
 
     if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug on; fi
@@ -776,6 +785,7 @@ backup_pgsql() {
                 move_dbbackup
                 check_exit_code move "${backup_job_filename}"
                 post_dbbackup "${db}"
+                cleanup_old_data
         done
         if var_true "${_postgres_backup_globals}" ; then backup_pgsql_globals; fi
     else
@@ -805,6 +815,7 @@ backup_pgsql() {
         move_dbbackup
         check_exit_code move "${backup_job_filename}"
         post_dbbackup all
+        cleanup_old_data
         if var_true "${_postgres_backup_globals}" ; then backup_pgsql_globals; fi
     fi
 }
@@ -844,6 +855,7 @@ backup_redis() {
     move_dbbackup
     check_exit_code move "${backup_job_filename}"
     post_dbbackup all
+    cleanup_old_data
 }
 
 backup_sqlite3() {
@@ -851,7 +863,7 @@ backup_sqlite3() {
     db=$(basename "${backup_job_db_host}")
     db="${db%.*}"
     backup_job_filename=sqlite3_${db}_${now}.sqlite3
-    backup_job_filename_base=sqlite3_${db}.sqlite3
+    backup_job_filename_base=sqlite3_${db}
     pre_dbbackup "${db}"
     write_log notice "Dumping sqlite3 database: '${backup_job_db_host}' ${compression_string}"
     if var_true "${DEBUG_BACKUP_SQLITE3}" ; then debug on; fi
@@ -872,6 +884,7 @@ backup_sqlite3() {
     move_dbbackup
     check_exit_code move "${backup_job_filename}"
     post_dbbackup "${db}"
+    cleanup_old_data
 }
 
 check_availability() {
@@ -1160,13 +1173,14 @@ create_schedulers() {
             instance=$(printf "%02d" $instance)
             cp -R /assets/dbbackup/template-dbbackup /etc/services.available/dbbackup-"${instance}"
             sed -i "s|{{BACKUP_NUMBER}}|${instance}|g" /etc/services.available/dbbackup-"${instance}"/run
-
+            if [ "${MODE,,}" = "manual" ] ; then service_stop dbbackup-"${instance}" ; fi
             cat <<EOF >> /usr/bin/backup"${instance}"-now
 #!/bin/bash
 source /assets/functions/00-container
 PROCESS_NAME=db-backup${instance}
 print_info "Starting Manual Backup for db-backup${instance}"
-/var/run/s6/legacy-services/dbbackup-${instance}/run now
+#/var/run/s6/legacy-services/dbbackup-${instance}/run now
+/etc/services.available/dbbackup-${instance}/run now
 
 EOF
             chmod +x /usr/bin/backup"${instance}"-now
@@ -1176,11 +1190,11 @@ EOF
                 cat <<EOF > /usr/bin/backup-now
 #!/bin/bash
 
-/usr/bin/backup${instance}-now
+/usr/bin/backup${instance}-now now
 
 EOF
             else
-                echo "/usr/bin/backup${instance}-now" >> /usr/bin/backup-now
+                echo "/usr/bin/backup${instance}-now now" >> /usr/bin/backup-now
             fi
 
             instance=$(echo "${instance} +1" | bc)
@@ -1368,15 +1382,15 @@ notify() {
             fi
             mail_recipients=$(echo "${MAIL_TO}" | tr "," "\n")
             for mail_recipient in $mail_recipients ; do
-                cat <<EOF | msmtp -t "${mail_recipient}" -c /etc/msmtprc
+                cat <<EOF | msmtp -t "${mail_recipient}" -C /etc/msmtprc
 To: ${mail_recipient}
-Subject: [db-backup] [${DOMAIN}] ${3}
+Subject: [db-backup] ${4}
 
 Time: ${1}
 Log File: {2}
 Error Code: ${3}
 
-${4}
+${5}
 EOF
             done
         fi
@@ -1392,7 +1406,7 @@ EOF
         if [ -z "${MATTERMOST_WEBHOOK_URL}" ] ; then write_log error "[notifications] No MATTERMOST_WEBHOOK_URL variable set - Skipping sending Mattermost notifications" ; skip_mattermost=true ; fi
         if var_nottrue "${skip_mattermost}" ; then
         emoji=":bomb:"
-        message="*[db-backup] ${3}*\n${4}\n*Timestamp:* ${1}\n*Logfile:* ${2}\n*Error Code: ${3}"
+        message="*[db-backup] ${4}*\n${5}\n*Timestamp:* ${1}\n*Logfile:* ${2}\n*Error Code: ${3}"
         mattermost_recipients=$(echo "${MATTERMOST_RECIPIENT}" | tr "," "\n")
             for mattermost_recipient in $mattermost_recipients ; do
                 payload="payload={\"channel\": \"${mattermost_recipient//\"/\\\"}\", \"username\": \"${MATTERMOST_USERNAME//\"/\\\"}\", \"text\": \"${message//\"/\\\"}\", \"icon_emoji\": \"${emoji}\"}"
@@ -1417,7 +1431,7 @@ EOF
             for matrix_room in $matrix_rooms ; do
                 curl \
                         -XPOST \
-                        -d "{\"msgtype\":\"m.text\", \"body\":\"*[db-backup] ${3}*\n${4}\n*Timestamp:* ${1}\n*Logfile:* ${2}\n*Error Code: ${3}*\"}" \
+                        -d "{\"msgtype\":\"m.text\", \"body\":\"*[db-backup] ${4}*\n${5}\n*Timestamp:* ${1}\n*Logfile:* ${2}\n*Error Code: ${3}*\"}" \
                         "${MATRIX_HOST}/_matrix/client/r0/rooms/${matrix_room}/send/m.room.message?access_token=${MATRIX_ACCESS_TOKEN}"
             done
         fi
@@ -1433,7 +1447,7 @@ EOF
         if [ -z "${ROCKETCHAT_WEBHOOK_URL}" ] ; then write_log error "[notifications] No ROCKETCHAT_WEBHOOK_URL variable set - Skipping sending Rocket.Chat notifications" ; skip_rocketchat=true ; fi
         if var_nottrue "${skip_rocketchat}" ; then
             emoji=":bomb:"
-            message="*[db-backup] ${3}*\n${4}\n*Timestamp:* ${1}\n*Logfile:* ${2}\n*Error Code: ${3}"
+            message="*[db-backup] ${4}*\n${5}\n*Timestamp:* ${1}\n*Logfile:* ${2}\n*Error Code: ${3}"
             rocketchat_recipients=$(echo "${ROCKETCHAT_RECIPIENT}" | tr "," "\n")
             for rocketchat_recipient in $rocketchat_recipients ; do
                 payload="payload={\"channel\": \"${rocketchat_recipient//\"/\\\"}\", \"username\": \"${ROCKETCHAT_USERNAME//\"/\\\"}\", \"text\": \"${message//\"/\\\"}\", \"icon_emoji\": \"${emoji}\"}"
@@ -1843,7 +1857,7 @@ timer() {
                     if [ "${expression_step}" != "${expression}" ]; then
                         for step in ${validate_temp}; do
                             if [ $(( ( step - expression_start ) % expression_step )) -eq 0 ]; then
-                              validate_all="$validate_all ${step}"
+                              validate_all="${validate_all} ${step}"
                             fi
                         done
                     else
@@ -1853,13 +1867,14 @@ timer() {
 
                 validate_all=$(echo "${validate_all}" | tr ' ' '\n' | sort -n -u | tr '\n' ' ')
                 for entry in $validate_all; do
-                    if [ "${entry}" -ge "${3}" ]; then
+                    if [ ${entry} -ge ${3} ]; then
                       echo "${entry}"
                       return 0
                     fi
                 done
 
-                echo "${validate_all%% *}"
+                echo "${validate_all// /}"
+                #echo "${validate_all%% *}"
             }
 
             local cron_compare="${3}"
@@ -1878,7 +1893,10 @@ timer() {
             local cron_minute="$(echo -n "${2}" | awk '{print $1}')"
             local cron_hour="$(echo -n "${2}" | awk '{print $2}')"
             local cron_day_of_month="$(echo -n "${2}" | awk '{print $3}')"
-            local cron_month="$(echo -n "${2}" | awk '{print $4}')"Generating
+            local cron_month="$(echo -n "${2}" | awk '{print $4}')"
+            local cron_day_of_week="$(echo -n "${2}" | awk '{print $5}')"
+
+            local cron_next_minute="$(date --date=@"${cron_compare}" +"%-M")"
             local cron_next_hour="$(date --date=@"${cron_compare}" +"%-H")"
             local cron_next_day_of_month="$(date --date=@"${cron_compare}" +"%-d")"
             local cron_next_month="$(date --date=@"${cron_compare}" +"%-m")"
@@ -1887,8 +1905,10 @@ timer() {
             local cron_next_year="$(date --date=@"${cron_compare}" +"%-Y")"
 
             local cron_next=
+            local cron_parsed=1
 
-            while [ "$cron_parsed" != "0" ]; do
+            while [ "${cron_parsed}" != "0" ]; do
+                print_debug "[timer] [cron] Parse Minute"
                 cron_next=$(parse_expression "${cron_minute}" 59 "${cron_next_minute}")
                 if [ "${cron_next}" != "${cron_next_minute}" ]; then
                     if [ "${cron_next_minute}" -gt "${cron_next}" ]; then
@@ -1898,20 +1918,22 @@ timer() {
                     cron_next_minute="${cron_next}"
                 fi
 
+                print_debug "[timer] [cron] Parse Hour"
                 cron_next=$(parse_expression "${cron_hour}" 23 "${cron_next_hour}")
                 if [ "${cron_next}" != "${cron_next_hour}" ]; then
                     if [ "${cron_next_hour}" -gt "${cron_next}" ]; then
                         cron_next_day_of_month=$(( cron_next_day_of_month + 1 ))
                     fi
+
                     cron_next_hour="${cron_next}"
-                    #cron_next_minute=0
                 fi
 
+                print_debug "[timer] [cron] Parse Day of Week"
                 cron_next=$(parse_expression "${cron_day_of_week}" 6 "${cron_next_day_of_week}")
                 if [ "${cron_next}" != "${cron_next_day_of_week}" ]; then
-                    day_of_week_difference=$(( ${cron_next} - ${cron_next_day_of_week} ))
+                    day_of_week_difference=$(( cron_next - cron_next_day_of_week ))
 
-                    if [ "${day_of_week_difference}" -lt "0" ]; then
+                    if [ "${day_of_week_difference}" -lt 0 ]; then
                         day_of_week_difference=$(( day_of_week_difference + 7 ))
                     fi
 
@@ -1920,6 +1942,7 @@ timer() {
                     cron_next_minute=0
                 fi
 
+                print_debug "[timer] [cron] Parse day of month"
                 case "${cron_next_month}" in
                     1|3|5|7|8|10|12)
                         last_day_of_month="31"
@@ -1959,6 +1982,7 @@ timer() {
                     cron_next_day_of_month=$cron_next
                 fi
 
+                print_debug "[timer] [cron] Parse Next Month"
                 cron_next=$(parse_expression "${cron_month}" 12 "${cron_next_month}")
                 if [ "${cron_next}" != "${cron_next_month}" ]; then
                     if [ "${cron_next}" -gt "12" ]; then