Release 4.1.21 - See CHANGELOG.md

fix: wildcard case order

CHANGELOG.md

@@ -1,3 +1,55 @@
+## 4.1.21 2025-08-08 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix for 4.1.20 release inadvertently failing backups (credit tlex@github)
+
+
+## 4.1.20 2025-07-23 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix backup error checking routines #417 - credit alteriks@github
+
+
+## 4.1.19 2025-05-28 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Force overwrite manual scripts as opposed to append (#414)
+
+
+## 4.1.18 2025-05-12 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix MongoDB restore from not dropping DB each time before restore except explicitly told (credit logicoa@github)
+
+
+## 4.1.17 2025-04-17 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix issue with Postgres database cleanup when ALL databases being backed up as one file (SPLIT_DB=FALSE)
+
+
+## 4.1.16 2025-02-21 <dave at tiredofit dot ca>
+
+   ### Added
+      - Update to tiredofit/alpine:7.10.28
+      - Support TLS connectivity with restore script (credit fermion2020@github)
+
+
+## 4.1.15 2025-01-29 <dave at tiredofit dot ca>
+
+   ### Added
+      - Add support for username and password support when checking for connectivity to couchdb (credit: JvSomeren)
+
+   ### Changed
+      - Fix issue with couchdb compression routines
+
+
+## 4.1.14 2025-01-21 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Downgrade AWS Client to 1.36.40 due to incompatibilities with providers with 1.37x. for time being
+
+
 ## 4.1.13 2025-01-21 <dave at tiredofit dot ca>
 
    ### Added

Dockerfile

@@ -1,5 +1,5 @@
 ARG DISTRO=alpine
-ARG DISTRO_VARIANT=3.21-7.10.27
+ARG DISTRO_VARIANT=3.21-7.10.28
 
 FROM docker.io/tiredofit/${DISTRO}:${DISTRO_VARIANT}
 LABEL maintainer="Dave Conroy (github.com/tiredofit)"
@@ -10,7 +10,7 @@ ENV INFLUX1_CLIENT_VERSION=1.8.0 \
     MSSQL_VERSION=18.4.1.1-1 \
     MYSQL_VERSION=mysql-8.4.4 \
     MYSQL_REPO_URL=https://github.com/mysql/mysql-server \
-    AWS_CLI_VERSION=1.37.2 \
+    AWS_CLI_VERSION=1.36.40 \
     CONTAINER_ENABLE_MESSAGING=TRUE \
     CONTAINER_ENABLE_MONITORING=TRUE \
     IMAGE_NAME="tiredofit/db-backup" \

install/assets/functions/10-db-backup

@@ -532,7 +532,7 @@ backup_couch() {
     prepare_dbbackup
     backup_job_filename=couch_${backup_job_db_name}_${backup_job_db_host#*//}_${now}.txt
     backup_job_filename_base=couch_${backup_job_db_name}_${backup_job_db_host#*//}
-    compressionzyclonite
+    compression
     if var_true "${DEBUG_BACKUP_COUCH}" ; then debug off; fi
     check_exit_code backup "${backup_job_filename}"
     timer backup finish
@@ -737,7 +737,7 @@ backup_mysql() {
                 write_log notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}"
                 if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug on; fi
                 run_as_user ${play_fair} ${_mysql_prefix}${_mysql_bin_prefix}dump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${events} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_backup_opts} $db | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null
-                exit_code=$?
+                exit_code=$((PIPESTATUS[0] + PIPESTATUS[1] + PIPESTATUS[2]))
                 if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug off; fi
                 check_exit_code backup "${backup_job_filename}"
                 timer backup finish
@@ -758,7 +758,7 @@ backup_mysql() {
         write_log notice "Dumping all MySQL / MariaDB databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}"
         if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug on; fi
         run_as_user ${play_fair} ${_mysql_prefix}${_mysql_bin_prefix}dump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${events} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_backup_opts} --databases $(echo ${db_names} | xargs) | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null
-        exit_code=$?
+                exit_code=$((PIPESTATUS[0] + PIPESTATUS[1] + PIPESTATUS[2]))
         if var_true "${DEBUG_BACKUP_MYSQL}" ; then debug off; fi
         check_exit_code backup "${backup_job_filename}"
         timer backup finish
@@ -781,7 +781,7 @@ backup_pgsql() {
         print_notice "Dumping PostgresSQL globals: with 'pg_dumpall -g' ${compression_string}"
         if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug on; fi
         run_as_user ${play_fair} pg_dumpall -h "${backup_job_db_host}" -U "${backup_job_db_user}" -p "${backup_job_db_port}" -g ${backup_job_extra_opts} ${backup_job_extra_backup_opts} | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null
-        exit_code=$?
+                exit_code=$((PIPESTATUS[0] + PIPESTATUS[1] + PIPESTATUS[2]))
         if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug off; fi
         check_exit_code "${backup_job_filename}"
         timer backup finish
@@ -830,7 +830,7 @@ backup_pgsql() {
                 write_log notice "Dumping PostgresSQL database: '${db}' ${compression_string}"
                 if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug on; fi
                 run_as_user ${play_fair} pg_dump -h "${backup_job_db_host}" -p "${backup_job_db_port}" -U "${backup_job_db_user}" $db ${backup_job_extra_opts} ${backup_job_extra_backup_opts} | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null
-                exit_code=$?
+                exit_code=$((PIPESTATUS[0] + PIPESTATUS[1] + PIPESTATUS[2]))
                 if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug off; fi
                 check_exit_code backup "${backup_job_filename}"
                 timer backup finish
@@ -846,7 +846,11 @@ backup_pgsql() {
         write_log debug "Not splitting database dumps into their own files"
         prepare_dbbackup
         backup_job_filename=pgsql_all_${backup_job_db_host,,}_${now}.sql
-        backup_job_filename_base=pgsql_${db}_${backup_job_db_host,,}
+        if [ "${backup_job_db_name,,}" = "all" ] ; then
+            backup_job_filename_base=pgsql_all_${backup_job_db_host,,}
+        else
+            backup_job_filename_base=pgsql_${db}_${backup_job_db_host,,}
+        fi
         compression
         pre_dbbackup all
         write_log notice "Dumping all PostgreSQL databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}"
@@ -860,7 +864,7 @@ backup_pgsql() {
             pgexclude_arg=$(echo ${pgexclude_arg} --exclude-database=${x_db_name})
         done
         run_as_user ${play_fair} pg_dumpall -h ${backup_job_db_host} -p ${backup_job_db_port} -U ${backup_job_db_user} ${pgexclude_arg} ${backup_job_extra_opts} ${backup_job_extra_backup_opts} | ${compress_cmd} | run_as_user tee "${temporary_directory}"/"${backup_job_filename}" > /dev/null
-        exit_code=$?
+                exit_code=$((PIPESTATUS[0] + PIPESTATUS[1] + PIPESTATUS[2]))
         if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug off; fi
         check_exit_code backup "${backup_job_filename}"
         timer backup finish
@@ -943,14 +947,16 @@ backup_sqlite3() {
 
 check_availability() {
     if var_true "${DEBUG_CHECK_AVAILABILITY}" ; then debug on; fi
-    ### Set the Database Type
     if var_false "${backup_job_skip_availability_check}" ; then
         case "${dbtype}" in
             "couch" )
                 counter=0
                 code_received=0
+                if [ -n "${backup_job_db_user}" ] && [ -n ${backup_job_db_pass} ]; then
+                    _ca_couch_auth="-u ${backup_job_db_user}:${backup_job_db_pass}"
+                fi
                 while [ "${code_received}" != "200" ]; do
-                    code_received=$(run_as_user curl -XGET -sSL -o /dev/null -L -w ''%{http_code}'' ${backup_job_db_host}:${backup_job_db_port})
+                    code_received=$(run_as_user curl -XGET -sSL -o /dev/null -L -w ''%{http_code}'' ${_ca_couch_auth} ${backup_job_db_host}:${backup_job_db_port})
                     if [ "${code_received}" = "200" ] ; then break ; fi
                     sleep 5
                     (( counter+=5 ))
@@ -1241,7 +1247,7 @@ create_schedulers() {
             cp -R /assets/dbbackup/template-dbbackup /etc/services.available/dbbackup-"${instance}"
             sed -i "s|{{BACKUP_NUMBER}}|${instance}|g" /etc/services.available/dbbackup-"${instance}"/run
             if [ "${MODE,,}" = "manual" ] ; then service_stop dbbackup-"${instance}" ; fi
-            cat <<EOF >> /usr/bin/backup"${instance}"-now
+            cat <<EOF > /usr/bin/backup"${instance}"-now
 #!/bin/bash
 source /assets/functions/00-container
 PROCESS_NAME=db-backup${instance}
@@ -1261,7 +1267,7 @@ EOF
 
 EOF
             else
-                echo "/usr/bin/backup${instance}-now now" >> /usr/bin/backup-now
+                echo "/usr/bin/backup${instance}-now now" > /usr/bin/backup-now
             fi
 
             instance=$(echo "${instance} +1" | bc)
@@ -1854,7 +1860,7 @@ setup_mode() {
             cat <<EOF > /etc/services.d/99-run_forever/run
 #!/bin/bash
 while true; do
-	sleep 86400
+    sleep 86400
 done
 EOF
             chmod +x /etc/services.d/99-run_forever/run

install/usr/local/bin/restore

@@ -912,6 +912,83 @@ get_filename() {
     r_filename=${opt}
 }
 
+get_ssl() {
+    if grep -q "^DB${detected_host_num}_MYSQL_ENABLE_TLS=" "${restore_vars}" ; then
+        detected_ssl_value=$(grep "^DB${detected_host_num}_MYSQL_ENABLE_TLS=" "${restore_vars}" | head -n1 | cut -d '=' -f 2)
+    fi
+
+    if [[ -z "${detected_ssl_value}" ]]; then
+        print_debug "Parsed SSL Variant: 1 - No Env Variable Found"
+        default_ssl="false"  # Default if no env variable
+        q_ssl_variant=1
+        q_ssl_menu_opt_default="| (${cwh}N${cdgy}) * "
+        q_ssl_menu="" #No menu option
+    else
+        print_debug "Parsed SSL Variant: 2 - Env Variable DB${detected_host_num}_MYSQL_ENABLE_TLS = '${detected_ssl_value}'"
+        default_ssl="${detected_ssl_value,,}"
+        q_ssl_variant=2
+        q_ssl_menu="E ) Environment Variable DB${detected_host_num}_MYSQL_ENABLE_TLS: '${detected_ssl_value}'"
+        q_ssl_menu_opt_default="| (${cwh}E${cdgy}) * "
+    fi
+
+    cat <<EOF
+
+Do you wish to use SSL for the connection?
+${q_ssl_menu}
+
+    Y ) Yes
+    N ) No
+    Q ) Quit
+
+EOF
+
+    r_ssl=""
+    case "${q_ssl_variant}" in
+        1)  # No env variable, ask directly
+            while true; do
+                read -r -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}Y${cdgy}\) \| \(${cwh}N\*${cdgy}\)  : ${cwh}${coff}) " q_ssl
+                case "${q_ssl,,}" in 
+                    y*)
+                        r_ssl="true"
+                        break
+                        ;;
+                    n* | "")
+                        r_ssl="false"
+                        break
+                        ;;
+                    q*)
+                        print_info "Quitting Script"
+                        exit 1
+                        ;;
+                esac
+            done
+            ;;
+        2)  # Env variable exists, offer it as an option
+            while true; do
+                read -r -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}E\*${cdgy}\) \| \(${cwh}Y${cdgy}\) \| \(${cwh}N${cdgy}\) : ${cwh}${coff}) " q_ssl
+                case "${q_ssl,,}" in
+                    e* | "") # Default to env variable if just enter is pressed.
+                        r_ssl="${detected_ssl_value}"
+                        break
+                        ;;
+                    y*)
+                        r_ssl="true"
+                        break
+                        ;;
+                    n*)
+                        r_ssl="false"
+                        break
+                        ;;
+                    q*)
+                        print_info "Quitting Script"
+                        exit 1
+                        ;;
+                esac
+            done
+            ;;
+    esac
+}
+
 #### SCRIPT START
 trap control_c INT
 bootstrap_variables restore_init
@@ -984,6 +1061,20 @@ else
 fi
 print_debug "Database Port '${r_dbport}'"
 
+## Question SSL connection
+if [[ "${r_dbtype,,}" == "mariadb" || "${r_dbtype,,}" == "mysql" ]]; then
+    if [ -n "${8}" ]; then
+        r_ssl="${8}"
+    else
+        get_ssl
+    fi
+    print_debug "SSL enable: '${r_ssl}'"
+else
+    r_ssl="false"
+    print_debug "SSL disabled for ${r_dbtype}"
+fi
+
+
 ## Parse Extension
 case "${r_filename##*.}" in
           bz* )
@@ -1013,8 +1104,13 @@ esac
 ## Perform a restore
 case "${r_dbtype}" in
     mariadb | mysql )
+        if [[ "${r_ssl,,}" == "false" ]]; then
+            mysql_ssl_option="--disable-ssl"
+        else
+            mysql_ssl_option=""
+        fi
         print_info "Restoring '${r_filename}' into '${r_dbhost}'/'${r_dbname}'"
-        pv ${r_filename} | ${decompress_cmd}cat | mysql -u${r_dbuser} -p${r_dbpass} -P${r_dbport} -h${r_dbhost} ${r_dbname}
+        pv ${r_filename} | ${decompress_cmd}cat | mariadb -u${r_dbuser} -p${r_dbpass} -P${r_dbport} -h${r_dbhost} ${mysql_ssl_option} ${r_dbname}
         exit_code=$?
     ;;
     pgsql | postgres* )
@@ -1036,9 +1132,6 @@ EOF
         echo -e "${coff}"
         read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}Y${cdgy}\) \| \(${cwh}N${cdgy}\) \| \(${cwh}Q${cdgy}\)  : ${cwh}${coff})" q_menu_mongo_dropdb
         case "${q_menu_mongo_dropdb,,}" in
-            "y" | "yes" | * )
-                mongo_dropdb="--drop"
-            ;;
             "n" | "update" )
                 unset mongo_dropdb
             ;;
@@ -1046,6 +1139,9 @@ EOF
                 print_info "Quitting Script"
                 exit 1
             ;;
+            "y" | "yes" | * )
+                mongo_dropdb="--drop"
+            ;;
         esac
 
         print_info "Restoring '${r_filename}' into '${r_dbhost}'/'${r_dbname}'"