Release 3.10.3 - See CHANGELOG.md

Sort filenames alphabetically when using the restore command

CHANGELOG.md

@@ -1,3 +1,52 @@
+## 3.10.3 2023-10-11 <thomas-negrault@github>
+
+   ### Changed
+      - Change sorting for restore script
+
+
+## 3.10.2 2023-09-14 <pimjansen@github>
+
+   ### Changed
+      - Update to wording when sending files to blobxfer
+      - Remove --databases flag when backing up a single mysql/mariadb backup which allows to omit the "USE <db_name>" statement in the backup allowing for better restores
+
+
+## 3.10.1 2023-09-13 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Bugfix to 3.10.0 with syntax error revolving around unbraced variable
+
+
+## 3.10.0 2023-09-13 <jcdirks@github>
+
+   ### Added
+      - Add EXTRA_DUMP_OPTS and EXTRA_ENUMERATION_OPTS to add different arguments when checking for databases, vs doing the actual backup
+
+
+## 3.9.12 2023-09-04 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Perform additional checks for ENABLE_CHECKSUM=FALSE and skip executing actions for S3/BlobXfer
+
+
+## 3.9.11 2023-08-24 <dave at tiredofit dot ca>
+
+   ### Changed
+      - AWS CLI 2.13.9
+
+
+## 3.9.10 2023-08-23 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Stop trying to move a non existent checksum file when ENABLE_CHECKSUM=FALSE
+
+
+## 3.9.9 2023-08-21 <dave at tiredofit dot ca>
+
+   ### Changed
+      - Start compiling aws-cli instead of from packages to continue to support arm/v7
+
+
 ## 3.9.8 2023-08-20 <ToshY@github>
 
    ### Changed

Dockerfile

@@ -8,6 +8,7 @@ LABEL maintainer="Dave Conroy (github.com/tiredofit)"
 ENV INFLUX_VERSION=1.8.0 \
     INFLUX2_VERSION=2.4.0 \
     MSSQL_VERSION=18.0.1.1-1 \
+    AWS_CLI_VERSION=1.25.97 \
     CONTAINER_ENABLE_MESSAGING=FALSE \
     CONTAINER_ENABLE_MONITORING=TRUE \
     CONTAINER_PROCESS_RUNAWAY_PROTECTOR=FALSE \
@@ -34,8 +35,8 @@ RUN source /assets/functions/00-container && \
                && \
     \
     package install .db-backup-run-deps \
-               aws-cli \
                bzip2 \
+               groff \
                libarchive \
                mariadb-client \
                mariadb-connector-c \
@@ -45,7 +46,16 @@ RUN source /assets/functions/00-container && \
                postgresql15 \
                postgresql15-client \
                pv \
+               py3-botocore \
+               py3-colorama \
                py3-cryptography \
+               py3-docutils \
+               py3-jmespath \
+               py3-rsa \
+               py3-setuptools \
+               py3-s3transfer \
+               py3-yaml \
+               python3 \
                redis \
                sqlite \
                xz \
@@ -62,6 +72,8 @@ RUN source /assets/functions/00-container && \
     \
     if [ $mssql = "true" ] ; then curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/msodbcsql18_${MSSQL_VERSION}_amd64.apk ; curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/mssql-tools18_${MSSQL_VERSION}_amd64.apk ; echo y | apk add --allow-untrusted msodbcsql18_${MSSQL_VERSION}_amd64.apk mssql-tools18_${MSSQL_VERSION}_amd64.apk ; else echo >&2 "Detected non x86_64 build variant, skipping MSSQL installation" ; fi; \
     if [ $influx2 = "true" ] ; then curl -sSL https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX2_VERSION}-linux-${influx_arch}.tar.gz | tar xvfz - --strip=1 -C /usr/src/ ; chmod +x /usr/src/influx ; mv /usr/src/influx /usr/sbin/ ; else echo >&2 "Unable to build Influx 2 on this system" ; fi ; \
+    clone_git_repo https://github.com/aws/aws-cli "${AWS_CLI_VERSION}" && \
+    python3 setup.py install --prefix=/usr && \
     clone_git_repo https://github.com/influxdata/influxdb "${INFLUX_VERSION}" && \
     go build -o /usr/sbin/influxd ./cmd/influxd && \
     strip /usr/sbin/influxd && \

README.md

@@ -179,7 +179,7 @@ Your Organization will be mapped to `DB_USER` and your root token will need to b
 
 ### Backup Options
 | Parameter                      | Description                                                                                                                  | Default                   | `_FILE` |
-| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------- | ------------------------- | ------- |
+|--------------------------------|------------------------------------------------------------------------------------------------------------------------------|---------------------------|---------|
 | `COMPRESSION`                  | Use either Gzip `GZ`, Bzip2 `BZ`, XZip `XZ`, ZSTD `ZSTD` or none `NONE`                                                      | `ZSTD`                    |         |
 | `COMPRESSION_LEVEL`            | Numberical value of what level of compression to use, most allow `1` to `9` except for `ZSTD` which allows for `1` to `19` - | `3`                       |         |
 | `ENABLE_PARALLEL_COMPRESSION`  | Use multiple cores when compressing backups `TRUE` or `FALSE`                                                                | `TRUE`                    |         |
@@ -187,7 +187,9 @@ Your Organization will be mapped to `DB_USER` and your root token will need to b
 | `GZ_RSYNCABLE`                 | Use `--rsyncable` (gzip only) for faster rsync transfers and incremental backup deduplication. e.g. `TRUE`                   | `FALSE`                   |         |
 | `ENABLE_CHECKSUM`              | Generate either a MD5 or SHA1 in Directory, `TRUE` or `FALSE`                                                                | `TRUE`                    |         |
 | `CHECKSUM`                     | Either `MD5` or `SHA1`                                                                                                       | `MD5`                     |         |
-| `EXTRA_OPTS`                   | If you need to pass extra arguments to the backup command, add them here e.g. `--extra-command`                              |                           |         |
+| `EXTRA_OPTS`                   | If you need to pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command`     |                           |         |
+| `EXTRA_DUMP_OPTS`              | If you need to pass extra arguments to the backup command only, add them here e.g. `--extra-command`                         |                           |         |
+| `EXTRA_ENUMERATION_OPTS`       | If you need to pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command`           |                           |         |
 | `MYSQL_MAX_ALLOWED_PACKET`     | Max allowed packet if backing up MySQL / MariaDB                                                                             | `512M`                    |         |
 | `MYSQL_SINGLE_TRANSACTION`     | Backup in a single transaction with MySQL / MariaDB                                                                          | `TRUE`                    |         |
 | `MYSQL_STORED_PROCEDURES`      | Backup stored procedures with MySQL / MariaDB                                                                                | `TRUE`                    |         |

install/assets/functions/10-db-backup

@@ -159,7 +159,7 @@ backup_influx() {
                 compression
                 pre_dbbackup $db
                 print_notice "Dumping Influx database: '${db}'"
-                influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS} "${TEMP_LOCATION}"/"${target_dir}"
+                influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} "${TEMP_LOCATION}"/"${target_dir}"
                 exit_code=$?
                 check_exit_code backup $target_dir
                 print_notice "Creating archive file of '${target_dir}' with tar ${compression_string}"
@@ -181,7 +181,7 @@ backup_influx() {
                 compression
                 pre_dbbackup $db
                 print_notice "Dumping Influx2 database: '${db}'"
-                influx backup --org ${DB_USER} ${bucket} --host ${DB_HOST}:${DB_PORT} --token ${DB_PASS} ${EXTRA_OPTS} --compression none "${TEMP_LOCATION}"/"${target_dir}"
+                influx backup --org ${DB_USER} ${bucket} --host ${DB_HOST}:${DB_PORT} --token ${DB_PASS} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} --compression none "${TEMP_LOCATION}"/"${target_dir}"
                 exit_code=$?
                 check_exit_code backup $target_dir
                 create_archive
@@ -208,9 +208,9 @@ backup_mongo() {
         compression_string="and compressing with gzip"
     fi
     if [ -n "${MONGO_CUSTOM_URI}" ] ; then
-        mongo_backup_parameter="--uri=${MONGO_CUSTOM_URI} ${EXTRA_OPTS}"
+        mongo_backup_parameter="--uri=${MONGO_CUSTOM_URI} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS}"
     else
-        mongo_backup_parameter="--host ${DB_HOST} --port ${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS}"
+        mongo_backup_parameter="--host ${DB_HOST} --port ${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS}"
     fi
     pre_dbbackup "${DB_NAME}"
     print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}"
@@ -249,7 +249,7 @@ backup_mysql() {
 
     if [ "${DB_NAME,,}" = "all" ] ; then
         print_debug "Preparing to back up everything except for information_schema and _* prefixes"
-        db_names=$(mysql -h ${DB_HOST} -P $DB_PORT -u$DB_USER ${mysql_tls_args} ${EXTRA_OPTS} --batch -e "SHOW DATABASES;" | grep -v Database | grep -v schema )
+        db_names=$(mysql -h ${DB_HOST} -P $DB_PORT -u$DB_USER ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_ENUMERATION_OPTS} --batch -e "SHOW DATABASES;" | grep -v Database | grep -v schema )
         if [ -n "${DB_NAME_EXCLUDE}" ] ; then
             db_names_exclusions=$(echo "${DB_NAME_EXCLUDE}" | tr ',' '\n')
             for db_exclude in ${db_names_exclusions} ; do
@@ -271,7 +271,7 @@ backup_mysql() {
                 compression
                 pre_dbbackup $db
                 print_notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}"
-                mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} --databases $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
+                mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
                 exit_code=$?
                 check_exit_code backup $target
                 generate_checksum
@@ -287,7 +287,7 @@ backup_mysql() {
         compression
         pre_dbbackup all
         print_notice "Dumping all MySQL / MariaDB databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}"
-        mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} --databases $(echo ${db_names} | xargs) | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
+        mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} --databases $(echo ${db_names} | xargs) | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
         exit_code=$?
         check_exit_code backup $target
         generate_checksum
@@ -324,7 +324,7 @@ backup_pgsql() {
                 compression
                 pre_dbbackup $db
                 print_notice "Dumping PostgresSQL database: '${db}' ${compression_string}"
-                pg_dump -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} $db ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target}
+                pg_dump -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} $db ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target}
                 exit_code=$?
                 check_exit_code backup $target
                 generate_checksum
@@ -348,7 +348,7 @@ backup_pgsql() {
         for x_db_name in ${tmp_db_names} ; do
             pgexclude_arg=$(echo ${pgexclude_arg} --exclude-database=${x_db_name})
         done
-        pg_dumpall -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} ${pgexclude_arg} ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target}
+        pg_dumpall -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} ${pgexclude_arg} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target}
         exit_code=$?
         check_exit_code backup $target
         generate_checksum
@@ -363,7 +363,7 @@ backup_redis() {
     print_notice "Dumping Redis - Flushing Redis Cache First"
     target=redis_all_${DB_HOST,,}_${now}.rdb
     ltarget=redis_${DB_HOST,,}
-    echo bgsave | silent redis-cli -h ${DB_HOST} -p ${DB_PORT} ${REDIS_PASS_STR} --rdb ${TEMP_LOCATION}/${target} ${EXTRA_OPTS}
+    echo bgsave | silent redis-cli -h ${DB_HOST} -p ${DB_PORT} ${REDIS_PASS_STR} --rdb ${TEMP_LOCATION}/${target} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS}
     sleep 10
     try=5
     while [ $try -gt 0 ] ; do
@@ -707,7 +707,7 @@ move_dbbackup() {
             "file" | "filesystem" )
                 print_debug "Moving backup to filesystem"
                 mkdir -p "${DB_DUMP_TARGET}"
-                mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/
+                 if var_true "${ENABLE_CHECKSUM}" ; then mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/ ; fi
                 mv "${TEMP_LOCATION}"/"${target}" "${DB_DUMP_TARGET}"/"${target}"
                 move_exit_code=$?
                 if var_true "${CREATE_LATEST_SYMLINK}" ; then
@@ -744,20 +744,21 @@ move_dbbackup() {
                     silent aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_LOCATION}/*.${checksum_extension} s3://${S3_BUCKET}/${S3_PATH}/ ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS}
                 fi
 
-                rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}"
+                if var_true "${ENABLE_CHECKSUM}" ; then rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}"; fi
                 rm -rf "${TEMP_LOCATION}"/"${target}"
             ;;
             "blobxfer" )
-                print_info "Moving backup to S3 Bucket with blobxfer"
+                print_info "Moving backup to external storage with blobxfer"
 
                 mkdir -p "${DB_DUMP_TARGET}"
-                mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/
+                if var_true "${ENABLE_CHECKSUM}" ; then mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/; fi
+
                 mv "${TEMP_LOCATION}"/"${target}" "${DB_DUMP_TARGET}"/"${target}"
 
                 silent blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET}
                 move_exit_code=$?
 
-                rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}"
+                if var_true "${ENABLE_CHECKSUM}" ; then rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}" ; fi
                 rm -rf "${TEMP_LOCATION}"/"${target}"
             ;;
         esac

install/usr/local/bin/restore

@@ -78,7 +78,7 @@ fi
 get_filename() {
     COLUMNS=12
     prompt="Please select a file to restore:"
-    options=( $(find "${DB_DUMP_TARGET}" -type f -maxdepth 1  -not -name '*.md5' -not -name '*.sha1' -print0 | xargs -0) )
+    options=( $(find "${DB_DUMP_TARGET}" -type f -maxdepth 1  -not -name '*.md5' -not -name '*.sha1' -print0 | sort -z | xargs -0) )
     PS3="$prompt "
     select opt in "${options[@]}" "Custom" "Quit" ; do
         if (( REPLY == 2 + ${#options[@]} )) ; then