Release 4.0.7 - See CHANGELOG.md

CHANGELOG.md

@@ -1,3 +1,15 @@
+## 4.0.7 2023-11-11 <dave at tiredofit dot ca>
+
+   ### Added
+      - Add seperate permissions for _FILESYSTEM_PATH
+
+   ### Changed
+      - More output and debugging additions
+      - SQLite3 now backs up without running into file permission/access problems
+      - Cleanup old sqlite backups from temp directory
+      - Handle multiple SQLite3 backups concurrently
+
+
 ## 4.0.6 2023-11-10 <dave at tiredofit dot ca>
 
    ### Added

README.md

@@ -322,12 +322,13 @@ Options that are related to the value of `DEFAULT_BACKUP_LOCATION`
 
 If `DEFAULT_BACKUP_LOCTION` = `FILESYSTEM` then the following options are used:
 
-| Variable                          | Description                                                                                           | Default                               |
+| Variable                             | Description                                                                                           | Default                               |
-| --------------------------------- | ----------------------------------------------------------------------------------------------------- | ------------------------------------- |
+| ------------------------------------ | ----------------------------------------------------------------------------------------------------- | ------------------------------------- |
-| `DEFAULT_CREATE_LATEST_SYMLINK`   | Create a symbolic link pointing to last backup in this format: `latest-(DB_TYPE)-(DB_NAME)-(DB_HOST)` | `TRUE`                                |
+| `DEFAULT_CREATE_LATEST_SYMLINK`      | Create a symbolic link pointing to last backup in this format: `latest-(DB_TYPE)-(DB_NAME)-(DB_HOST)` | `TRUE`                                |
-| `DEFAULT_FILESYSTEM_PATH`         | Directory where the database dumps are kept.                                                          | `/backup`                             |
+| `DEFAULT_FILESYSTEM_PATH`            | Directory where the database dumps are kept.                                                          | `/backup`                             |
-| `DEFAULT_FILESYSTEM_ARCHIVE_PATH` | Optional Directory where the database dumps archives are kept                                         | `${DEFAULT_FILESYSTEM_PATH}/archive/` |
+| `DEFAULT_FILESYSTEM_PATH_PERMISSION` | Permissions to apply to backup directory                                                              | `700`                                 |
-| `DEFAULT_FILESYSTEM_PERMISSION`   | Directory and File permissions to apply to files.                                                     | `600`                                 |
+| `DEFAULT_FILESYSTEM_ARCHIVE_PATH`    | Optional Directory where the database dumps archives are kept                                         | `${DEFAULT_FILESYSTEM_PATH}/archive/` |
+| `DEFAULT_FILESYSTEM_PERMISSION`      | Permissions to apply to files.                                                                        | `600`                                 |
 
 ###### S3
 
@@ -492,7 +493,7 @@ Encryption will occur after compression and the resulting filename will have a `
 |                              | Absolute HHMM, e.g. `2330` or `0415`                                                                                                           |         |
 |                              | Relative +MM, i.e. how many minutes after starting the container, e.g. `+0` (immediate), `+10` (in 10 minutes), or `+90` in an hour and a half |         |
 |                              | Full datestamp e.g. `2023-12-21 23:30:00`                                                                                                      |         |
-|                              | Cron expression e.g. `30 23 * * *`  [Understand the format](https://en.wikipedia.org/wiki/Cron) - *BACKUP_INTERVAL is ignored*              |         |
+|                              | Cron expression e.g. `30 23 * * *`  [Understand the format](https://en.wikipedia.org/wiki/Cron) - *BACKUP_INTERVAL is ignored*                 |         |
 | `DB01_CLEANUP_TIME`          | Value in minutes to delete old backups (only fired when backup interval executes)                                                              | `FALSE` |
 |                              | 1440 would delete anything above 1 day old. You don't need to set this variable if you want to hold onto everything.                           |         |
 | `DB01_ARCHIVE_TIME`          | Value in minutes to move all files files older than (x) from `DB01_BACKUP_FILESYSTEM_PATH`                                                     |         |
@@ -597,12 +598,13 @@ Options that are related to the value of `DB01_BACKUP_LOCATION`
 
 If `DB01_BACKUP_LOCTION` = `FILESYSTEM` then the following options are used:
 
-| Variable                       | Description                                                                                           | Default                           |
+| Variable                          | Description                                                                                           | Default                           |
-| ------------------------------ | ----------------------------------------------------------------------------------------------------- | --------------------------------- |
+| --------------------------------- | ----------------------------------------------------------------------------------------------------- | --------------------------------- |
-| `DB01_CREATE_LATEST_SYMLINK`   | Create a symbolic link pointing to last backup in this format: `latest-(DB_TYPE)-(DB_NAME)-(DB_HOST)` | `TRUE`                            |
+| `DB01_CREATE_LATEST_SYMLINK`      | Create a symbolic link pointing to last backup in this format: `latest-(DB_TYPE)-(DB_NAME)-(DB_HOST)` | `TRUE`                            |
-| `DB01_FILESYSTEM_PATH`         | Directory where the database dumps are kept.                                                          | `/backup`                         |
+| `DB01_FILESYSTEM_PATH`            | Directory where the database dumps are kept.                                                          | `/backup`                         |
-| `DB01_FILESYSTEM_ARCHIVE_PATH` | Optional Directory where the database dumps archives are kept                                         | `${DB01_FILESYSTEM_PATH/archive/` |
+| `DB01_FILESYSTEM_PATH_PERMISSION` | Permissions to apply to backup directory                                                              | `700`                             |
-| `DB01_FILESYSTEM_PERMISSION`   | Directory and File permissions to apply to files.                                                     | `600`                             |
+| `DB01_FILESYSTEM_ARCHIVE_PATH`    | Optional Directory where the database dumps archives are kept                                         | `${DB01_FILESYSTEM_PATH/archive/` |
+| `DB01_FILESYSTEM_PERMISSION`      | Directory and File permissions to apply to files.                                                     | `600`                             |
 
 ###### S3
 

install/assets/dbbackup/template-dbbackup/run

@@ -8,8 +8,11 @@ source /assets/functions/10-db-backup
 source /assets/defaults/10-db-backup
 bootstrap_variables backup_init {{BACKUP_NUMBER}}
 bootstrap_variables parse_variables {{BACKUP_NUMBER}}
-PROCESS_NAME="{{BACKUP_NUMBER}}-${backup_job_db_host}__${backup_job_db_name}"
+if [ -z "${backup_job_db_name}" ]; then
-
+    PROCESS_NAME="{{BACKUP_NUMBER}}${backup_job_db_host//\//_}"
+else
+    PROCESS_NAME="{{BACKUP_NUMBER}}-${backup_job_db_host//\//_}__${backup_job_db_name}"
+fi
 
 trap ctrl_c INT
 

install/assets/defaults/10-db-backup

@@ -2,7 +2,7 @@
 
 BACKUP_JOB_CONCURRENCY=${BACKUP_JOB_CONCURRENCY:-"1"}
 DBBACKUP_USER=${DBBACKUP_USER:-"dbbackup"}
-DBBACKUP_GROUP=${DBBACKUP_USER:-"${DBBACKUP_USER}"} # Must go after DBBACKUP_USER
+DBBACKUP_GROUP=${DBBACKUP_GROUP:-"${DBBACKUP_USER}"} # Must go after DBBACKUP_USER
 DEFAULT_BACKUP_BEGIN=${DEFAULT_BACKUP_BEGIN:-+0}
 DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440}
 DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440}
@@ -15,6 +15,7 @@ DEFAULT_CREATE_LATEST_SYMLINK=${DEFAULT_CREATE_LATEST_SYMLINK:-"TRUE"}
 DEFAULT_ENABLE_PARALLEL_COMPRESSION=${DEFAULT_ENABLE_PARALLEL_COMPRESSION:-"TRUE"}
 DEFAULT_ENCRYPT=${DEFAULT_ENCRYPT:-"FALSE"}
 DEFAULT_FILESYSTEM_PATH=${DEFAULT_FILESYSTEM_PATH:-"/backup"}
+DEFAULT_FILESYSTEM_PATH_PERMISSION=${DEFAULT_FILESYSTEM_PATH_PERMISSION:-"700"}
 DEFAULT_FILESYSTEM_PERMISSION=${DEFAULT_FILESYSTEM_PERMISSION:-"600"}
 DEFAULT_FILESYSTEM_ARCHIVE_PATH=${DEFAULT_FILESYSTEM_ARCHIVE_PATH:-"${DEFAULT_FILESYSTEM_PATH}/archive/"}
 DEFAULT_LOG_LEVEL=${DEFAULT_LOG_LEVEL:-"notice"}

install/assets/functions/10-db-backup

@@ -6,11 +6,11 @@ bootstrap_filesystem() {
         mkdir -p "${backup_job_filesystem_path}"
     fi
     if [ "$(stat -c %U "${backup_job_filesystem_path}")" != "${DBBACKUP_USER}" ] ; then chown -R "${DBBACKUP_USER}":"${DBBACKUP_GROUP}" "${backup_job_filesystem_path}" ; fi
-    if [ "$(stat -c %a "${backup_job_filesystem_path}")" != "${backup_job_filesystem_permission}" ] ; then chmod -R "${backup_job_filesystem_permission}" "${backup_job_filesystem_path}" ; fi
+    if [ "$(stat -c %a "${backup_job_filesystem_path}")" != "${backup_job_filesystem_path_permission}" ] ; then chmod "${backup_job_filesystem_path_permission}" "${backup_job_filesystem_path}" ; fi
 
     if [ -d "${backup_job_filesystem_archive_path}" ]; then
         if [ "$(stat -c %U "${backup_job_filesystem_archive_path}")" != "${DBBACKUP_USER}" ] ; then chown -R "${DBBACKUP_USER}":"${DBBACKUP_GROUP}" "${backup_job_filesystem_archive_path}" ; fi
-        if [ "$(stat -c %a "${backup_job_filesystem_archive_path}")" != "${backup_job_filesystem_permission}" ] ; then chmod -R "${backup_job_filesystem_permission}" "${backup_job_filesystem_archive_path}" ; fi
+        if [ "$(stat -c %a "${backup_job_filesystem_archive_path}")" != "${backup_job_filesystem_path_permission}" ] ; then chmod "${backup_job_filesystem_path_permission}" "${backup_job_filesystem_archive_path}" ; fi
     fi
 
     if [ ! -d "${LOG_PATH}" ]; then
@@ -191,6 +191,7 @@ bootstrap_variables() {
         transform_backup_instance_variable "${backup_instance_number}" EXTRA_OPTS backup_job_extra_opts
         transform_backup_instance_variable "${backup_instance_number}" FILESYSTEM_ARCHIVE_PATH backup_job_filesystem_archive_path
         transform_backup_instance_variable "${backup_instance_number}" FILESYSTEM_PATH backup_job_filesystem_path
+        transform_backup_instance_variable "${backup_instance_number}" FILESYSTEM_PATH_PERMISSION backup_job_filesystem_path_permission
         transform_backup_instance_variable "${backup_instance_number}" FILESYSTEM_PERMISSION backup_job_filesystem_permission
         transform_backup_instance_variable "${backup_instance_number}" GZ_RSYNCABLE backup_job_gz_rsyncable
         transform_backup_instance_variable "${backup_instance_number}" HOST backup_job_db_host
@@ -459,12 +460,7 @@ backup_couch() {
     prepare_dbbackup
     backup_job_filename=couch_${backup_job_db_name}_${backup_job_db_host#*//}_${now}.txt
     backup_job_filename_base=couch_${backup_job_db_name}_${backup_job_db_host#*//}
-    compression
+    compressionzyclonite
-    pre_dbbackup ${backup_job_db_name}
-    write_log notice "Dumping CouchDB database: '${backup_job_db_name}' ${compression_string}"
-    if var_true "${DEBUG_BACKUP_COUCH}" ; then debug on; fi
-    run_as_user curl -sSL -X GET ${backup_job_db_host}:${backup_job_db_port}/${backup_job_db_name}/_all_docs?include_docs=true | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${backup_job_filename}" > /dev/null
-    exit_code=$?
     if var_true "${DEBUG_BACKUP_COUCH}" ; then debug off; fi
     check_exit_code backup "${backup_job_filename}"
     timer backup finish
@@ -843,16 +839,21 @@ backup_sqlite3() {
     db="${db%.*}"
     backup_job_filename=sqlite3_${db}_${now}.sqlite3
     backup_job_filename_base=sqlite3_${db}.sqlite3
-    compression
     pre_dbbackup "${db}"
     write_log notice "Dumping sqlite3 database: '${backup_job_db_host}' ${compression_string}"
     if var_true "${DEBUG_BACKUP_SQLITE3}" ; then debug on; fi
-    silent run_as_user ${play_fair} sqlite3 "${backup_job_db_host}" ".backup '${TEMP_PATH}/backup.sqlite3'"
+    silent ${play_fair} sqlite3 "${backup_job_db_host}" ".backup '${TEMP_PATH}/backup_${now}.sqlite3'"
     exit_code=$?
     check_exit_code backup "${backup_job_filename}"
-    run_as_user ${play_fair} cat "${TEMP_PATH}"/backup.sqlite3 | ${dir_compress_cmd} | run_as_user tee "${TEMP_PATH}/${backup_job_filename}" > /dev/null
+    if [ ! -f "${TEMP_PATH}"/backup_${now}.sqlite3 ] ; then
-    timer backup finish
+        print_error "SQLite3 backup failed! Exitting"
+        return 1
+    fi
+    compression
+    run_as_user ${play_fair} cat "${TEMP_PATH}"/backup_${now}.sqlite3 | ${dir_compress_cmd} | run_as_user tee "${TEMP_PATH}/${backup_job_filename}" > /dev/null
+    rm -rf "${TEMP_PATH}"/backup_${now}.sqlite3
     if var_true "${DEBUG_BACKUP_SQLITE3}" ; then debug off; fi
+    timer backup finish
     file_encryption
     generate_checksum
     move_dbbackup
@@ -1194,7 +1195,11 @@ db_backup_container_init() {
 debug() {
     case "${1}" in
         off)
+            backup_job_log_level=$_original_job_log_log_level}
+            CONTAINER_LOG_LEVEL=${_original_container_log_level}
             DEBUG_MODE=${_original_debug_mode}
+            SHOW_OUTPUT=${_original_show_output}
+
             if var_true "${DEBUG_MODE}" ; then
                 set -x
             else
@@ -1202,9 +1207,22 @@ debug() {
             fi
         ;;
         on)
+            if [ -z "${_original_container_log_level}" ]; then
+                _original_container_log_level="${CONTAINER_LOG_LEVEL}"
+            fi
+            if [ -z "${_original_job_log_level}" ]; then
+                _original_job_log_level="${backup_job_log_level}"
+            fi
+
             if [ -z "${_original_debug_mode}" ]; then
                 _original_debug_mode="${DEBUG_MODE}"
             fi
+            if [ -z "${_original_show_output}" ]; then
+                _original_show_output="${SHOW_OUTPUT}"
+            fi
+            backup_job_log_level=DEBUG
+            CONTAINER_LOG_LEVEL=DEBUG
+            SHOW_OUTPUT=TRUE
             set -x
         ;;
     esac