Release 4.0.11 - See CHANGELOG.md

CHANGELOG.md

@@ -1,3 +1,29 @@
+## 4.0.11 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Resolve issue with backing up ALL databases with PGSQL and MySQL
+
+
+## 4.0.10 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Change environment variable parsing routines to properly accomodate for Passwords containing '=='
+
+
+## 4.0.9 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Fix issue with quotes being wrapped around _PASS variables
+
+
+## 4.0.8 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Tidy up file_encryption() routines
+      - Change environment variable _ENCRYPT_PUBKEY to _ENCRYPT_PUBLIC_KEY
+      - Add new environment variable _ENCRYPT_PRIVATE_KEY
+
+
 ## 4.0.7 2023-11-11 <dave at tiredofit dot ca>
 
    ### Added

README.md

@@ -214,12 +214,13 @@ If these are set and no other defaults or variables are set explicitly, they wil
 
 Encryption occurs after compression and the encrypted filename will have a `.gpg` suffix
 
-| Variable                     | Description                                 | Default |
-| ---------------------------- | ------------------------------------------- | ------- |
-| `DEFAULT_ENCRYPT`            | Encrypt file after backing up with GPG      | `FALSE` |
-| `DEFAULT_ENCRYPT_PASSPHRASE` | Passphrase to encrypt file with GPG         |         |
-| *or*                         |                                             |         |
-| `DEFAULT_ENCRYPT_PUBKEY`     | Path of public key to encrypt file with GPG |         |
+| Variable                      | Description                                  | Default | `_FILE` |
+| ----------------------------- | -------------------------------------------- | ------- | ------- |
+| `DEFAULT_ENCRYPT`             | Encrypt file after backing up with GPG       | `FALSE` |         |
+| `DEFAULT_ENCRYPT_PASSPHRASE`  | Passphrase to encrypt file with GPG          |         | x       |
+| *or*                          |                                              |         |         |
+| `DEFAULT_ENCRYPT_PUBLIC_KEY`  | Path of public key to encrypt file with GPG  |         | x       |
+| `DEFAULT_ENCRYPT_PRIVATE_KEY` | Path of private key to encrypt file with GPG |         | x       |
 
 ##### Scheduling Options
 
@@ -476,12 +477,14 @@ Otherwise, override them per backup job. Additional backup jobs can be scheduled
 
 Encryption will occur after compression and the resulting filename will have a `.gpg` suffix
 
-| Variable                  | Description                                 | Default |
-| ------------------------- | ------------------------------------------- | ------- |
-| `DB01_ENCRYPT`            | Encrypt file after backing up with GPG      | `FALSE` |
-| `DB01_ENCRYPT_PASSPHRASE` | Passphrase to encrypt file with GPG         |         |
-| *or*                      |                                             |         |
-| `DB01_ENCRYPT_PUBKEY`     | Path of public key to encrypt file with GPG |         |
+
+| Variable                   | Description                                  | Default | `_FILE` |
+| -------------------------- | -------------------------------------------- | ------- | ------- |
+| `DB01_ENCRYPT`             | Encrypt file after backing up with GPG       | `FALSE` |         |
+| `DB01_ENCRYPT_PASSPHRASE`  | Passphrase to encrypt file with GPG          |         | x       |
+| *or*                       |                                              |         |         |
+| `DB01_ENCRYPT_PUBLIC_KEY`  | Path of public key to encrypt file with GPG  |         | x       |
+| `DB01_ENCRYPT_PRIVATE_KEY` | Path of private key to encrypt file with GPG |         | x       |
 
 ##### Scheduling Options
 

install/assets/functions/10-db-backup

@@ -48,7 +48,8 @@ bootstrap_variables() {
                             DEFAULT_USER \
                             DEFAULT_PASS \
                             DEFAULT_ENCRYPT_PASSPHRASE \
-                            DEFAULT_ENCRYPT_PUBKEY \
+                            DEFAULT_ENCRYPT_PUBLIC_KEY \
+                            DEFAULT_ENCRYPT_PRIVATE_KEY \
                             DEFAULT_MONGO_CUSTOM_URI \
                             DEFAULT_MYSQL_TLS_CA_FILE \
                             DEFAULT_MYSQL_TLS_CERT_FILE \
@@ -74,7 +75,8 @@ bootstrap_variables() {
                             DB"${backup_instance_number}"_USER \
                             DB"${backup_instance_number}"_PASS \
                             DB"${backup_instance_number}"_ENCRYPT_PASSPHRASE \
-                            DB"${backup_instance_number}"_ENCRYPT_PUBKEY \
+                            DB"${backup_instance_number}"_ENCRYPT_PUBLIC_KEY \
+                            DB"${backup_instance_number}"_ENCRYPT_PRIVATE_KEY \
                             DB"${backup_instance_number}"_MONGO_CUSTOM_URI \
                             DB"${backup_instance_number}"_MYSQL_TLS_CA_FILE \
                             DB"${backup_instance_number}"_MYSQL_TLS_CERT_FILE \
@@ -151,18 +153,23 @@ bootstrap_variables() {
         fi
         ##
 
+        if grep -qo ".*_PASS='.*'" "${backup_instance_vars}"; then
+            print_debug "[bootstrap_variables] [backup_init] Found _PASS variable with quotes"
+            sed -i "s|_PASS='\(.*\)'|_PASS=\1|g" "${backup_instance_vars}"
+        fi
+
         transform_backup_instance_variable() {
             if grep -q "^DB${1}_${2}=" "${backup_instance_vars}" && [ "$(grep "^DB${1}_${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then
-                export "$3"="$(grep "^DB${1}_${2}=" "${backup_instance_vars}" | cut -d = -f2)"
+                export "$3"="$(grep "^DB${1}_${2}=" "${backup_instance_vars}" | cut -d = -f2-)"
             elif grep -q "^DB_${2}=" "${backup_instance_vars}" && [ "$(grep "^DB_${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then
                 # Allow old legacy work, perhaps remove old DB_ functionality in future? This should allow for seamless upgrades
                 #print_warn "Legacy Variable 'DB_${2}'' detected - Please upgrade your variables as they will be removed in version 4.3.0"
-                export "$3"="$(grep "^DB_${2}=" "${backup_instance_vars}" | cut -d = -f2)"
+                export "$3"="$(grep "^DB_${2}=" "${backup_instance_vars}" | cut -d = -f2-)"
             elif grep -q "^${2}=" "${backup_instance_vars}" && [ "$(grep "^${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then
                 print_warn "Legacy unsupported variable '${2}' detected - Please upgrade your variables as they will be removed in version 4.3.0"
-                export "$3"="$(grep "^${2}=" "${backup_instance_vars}" | cut -d = -f2)"
+                export "$3"="$(grep "^${2}=" "${backup_instance_vars}" | cut -d = -f2-)"
             elif grep -q "^DEFAULT_${2}=" "${backup_instance_vars}" && [ "$(grep "^DEFAULT_${2}=" "${backup_instance_vars}" | cut -d = -f2)" != "unset" ]; then
-                export "$3"="$(grep "^DEFAULT_${2}=" "${backup_instance_vars}" | cut -d = -f2)"
+                export "$3"="$(grep "^DEFAULT_${2}=" "${backup_instance_vars}" | cut -d = -f2-)"
             fi
         }
 
@@ -185,7 +192,8 @@ bootstrap_variables() {
         transform_backup_instance_variable "${backup_instance_number}" ENABLE_PARALLEL_COMPRESSION backup_job_parallel_compression
         transform_backup_instance_variable "${backup_instance_number}" ENCRYPT backup_job_encrypt
         transform_backup_instance_variable "${backup_instance_number}" ENCRYPT_PASSPHRASE backup_job_encrypt_passphrase
-        transform_backup_instance_variable "${backup_instance_number}" ENCRYPT_PUBKEY backup_job_encrypt_pubkey
+        transform_backup_instance_variable "${backup_instance_number}" ENCRYPT_PRIVATE_KEY backup_job_encrypt_private_key
+        transform_backup_instance_variable "${backup_instance_number}" ENCRYPT_PUBLIC_KEY backup_job_encrypt_public_key
         transform_backup_instance_variable "${backup_instance_number}" EXTRA_DUMP_OPTS backup_job_extra_dump_opts
         transform_backup_instance_variable "${backup_instance_number}" EXTRA_ENUMERATION_OPTS backup_job_extra_enumeration_opts
         transform_backup_instance_variable "${backup_instance_number}" EXTRA_OPTS backup_job_extra_opts
@@ -726,7 +734,7 @@ backup_pgsql() {
     fi
     if [ "${backup_job_db_name,,}" = "all" ] ; then
         write_log debug "Preparing to back up all databases"
-        db_names=$(run_as_user psql -h ${backup_job_db_host} -U ${backup_job_db_user} -p ${backup_job_db_port} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
+        db_names=$(psql -h ${backup_job_db_host} -U ${backup_job_db_user} -p ${backup_job_db_port} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
         if [ -n "${backup_job_db_name_exclude}" ] ; then
             db_names_exclusions=$(echo "${backup_job_db_name_exclude}" | tr ',' '\n')
             for db_exclude in ${db_names_exclusions} ; do
@@ -774,7 +782,7 @@ backup_pgsql() {
         pre_dbbackup all
         write_log notice "Dumping all PostgreSQL databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}"
         if var_true "${DEBUG_BACKUP_PGSQL}" ; then debug on; fi
-        tmp_db_names=$(run_as_user psql -h ${backup_job_db_host} -p ${backup_job_db_port} -U ${backup_job_db_user} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
+        tmp_db_names=$(psql -h ${backup_job_db_host} -p ${backup_job_db_port} -U ${backup_job_db_user} -d ${authdb} -c 'COPY (SELECT datname FROM pg_database WHERE datistemplate = false) TO STDOUT;' )
         for r_db_name in $(echo $db_names | xargs); do
             tmp_db_names=$(echo "$tmp_db_names" | xargs | sed "s|${r_db_name}||g"  )
         done
@@ -1180,7 +1188,7 @@ EOF
 }
 
 ctrl_c() {
-    sed -i "/^{{BACKUP_NUMBER}}/d" /tmp/.container/db-backup-backups
+    sed -i "/^${backup_instance_number}/d" /tmp/.container/db-backup-backups
     symlink_log
     print_warn "User aborted"
     exit
@@ -1195,7 +1203,7 @@ db_backup_container_init() {
 debug() {
     case "${1}" in
         off)
-            backup_job_log_level=$_original_job_log_log_level}
+            backup_job_log_level=$_original_job_log_level}
             CONTAINER_LOG_LEVEL=${_original_container_log_level}
             DEBUG_MODE=${_original_debug_mode}
             SHOW_OUTPUT=${_original_show_output}
@@ -1219,6 +1227,9 @@ debug() {
             fi
             if [ -z "${_original_show_output}" ]; then
                 _original_show_output="${SHOW_OUTPUT}"
+                if ! [[ "${_original_show_output,,}" =~ true|false ]]; then
+                    __original_show_output="FALSE"
+                fi
             fi
             backup_job_log_level=DEBUG
             CONTAINER_LOG_LEVEL=DEBUG
@@ -1234,21 +1245,26 @@ file_encryption() {
         if [ "${exit_code}" = "0" ] ; then
             print_debug "[file_encryption] Encrypting"
             output_off
-            if [ -n "${backup_job_encrypt_passphrase}" ] && [ -n "${backup_job_encrypt_pubkey}" ]; then
+            if [ -n "${backup_job_encrypt_passphrase}" ] && [ -n "${backup_job_encrypt_public_key}" ]; then
                 print_error "Can't encrypt as both ENCRYPT_PASSPHRASE and ENCRYPT_PUBKEY exist!"
                 return
-            elif [ -n "${backup_job_encrypt_passphrase}" ] && [ -z "${backup_job_encrypt_pubkey}" ]; then
+            elif [ -n "${backup_job_encrypt_passphrase}" ] && [ -z "${backup_job_encrypt_public_key}" ]; then
                 print_notice "Encrypting with GPG Passphrase"
                 encrypt_routines_start_time=$(date +'%s')
                 encrypt_tmp_dir=$(run_as_user mktemp -d)
                 echo "${backup_job_encrypt_passphrase}" | silent run_as_user ${play_fair} gpg --batch --home ${encrypt_tmp_dir} --yes --passphrase-fd 0 -c "${TEMP_PATH}"/"${backup_job_filename}"
                 rm -rf "${encrypt_tmp_dir}"
-            elif [ -z "${backup_job_encrypt_passphrase}" ] && [ -n "${backup_job_encrypt_pubkey}" ]; then
-                if [ -f "${backup_job_encrypt_pubkey}" ]; then
+            elif [ -z "${backup_job_encrypt_passphrase}" ] && [ -n "${backup_job_encrypt_public_key}" ] && [ -n "${backup_job_encrypt_private_key}" ]; then
+                if [ -f "${backup_job_encrypt_private_key}" ]; then
                     encrypt_routines_start_time=$(date +'%s')
-                    print_notice "Encrypting with GPG Public Key"
+                    print_notice "Encrypting with GPG Private Key"
                     encrypt_tmp_dir=$(run_as_user mktemp -d)
-                    silent run_as_user ${play_fair} gpg --batch --yes --home "${encrypt_tmp_dir}" --recipient-file "${backup_job_encrypt_pubkey}" -c "${TEMP_PATH}"/"${backup_job_filename}"
+                    cat "${backup_job_encrypt_private_key}" | run_as_user tee "${encrypt_tmp_dir}"/private_key.asc > /dev/null
+                    print_debug "[file_encryption] [key] Importing Private Key"
+                    silent run_as_user gpg --home ${encrypt_tmp_dir} --batch --import "${encrypt_tmp_dir}"/private_key.asc
+                    print_debug "[file_encryption] [key] Encrypting to Public Key"
+                    cat "${backup_job_encrypt_public_key}" | run_as_user tee "${encrypt_tmp_dir}"/public_key.asc > /dev/null
+                    silent run_as_user ${play_fair} gpg --batch --yes --home "${encrypt_tmp_dir}" --encrypt --recipient-file "${encrypt_tmp_dir}"/public_key.asc "${TEMP_PATH}"/"${backup_job_filename}"
                     rm -rf "${encrypt_tmp_dir}"
                 fi
             fi
@@ -1263,6 +1279,9 @@ file_encryption() {
 - dbbackup.backup.encrypt.duration.[${backup_job_db_host}.${backup_job_db_name}] ${encrypt_routines_total_time}
 EOF
                 )
+            else
+                print_error "Encryption failed! Could not detect encrypted file"
+                return 99
             fi
         else
             write_log error "Skipping encryption because backup did not complete successfully"
@@ -1724,7 +1743,7 @@ process_limiter() {
 }
 
 run_as_user() {
-    sudo -u "${DBBACKUP_USER}" $@
+    sudo -Eu "${DBBACKUP_USER}" "$@"
 }
 
 setup_mode() {