From e71334564ffeb163fdd02f0c8e181d78447a7e1b Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Sun, 11 Jun 2023 11:42:13 -0700 Subject: [PATCH 01/17] Drop auto builds for armv7 --- .github/workflows/main.yml | 4 ++-- .github/workflows/manual.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f1f341..aff358f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,8 +8,8 @@ on: jobs: build: - uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main + #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main #uses: tiredofit/github_actions/.github/workflows/default_amd64.yml@main #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main - #uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main + uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main secrets: inherit diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index 34cb11b..53fe788 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -9,8 +9,8 @@ on: jobs: build: - uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main + #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main #uses: tiredofit/github_actions/.github/workflows/default_amd64.yml@main #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main - #uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main + uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main secrets: inherit From 1e4699681203353982fb3ca2c68e8301f2b986b2 Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Tue, 13 Jun 2023 10:16:04 -0700 Subject: [PATCH 02/17] Release 3.9.4 - See CHANGELOG.md --- CHANGELOG.md | 6 ++++++ install/assets/functions/10-db-backup | 17 +++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29c321a..4633605 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 3.9.4 2023-06-13 + + ### Added + - Add abliity to use --rsyncable argument to zstd archives + + ## 3.9.3 2023-06-05 ### Added diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 77bbeaf..f9f8326 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -565,15 +565,16 @@ cleanup_old_data() { compression() { - if var_false "${ENABLE_PARALLEL_COMPRESSION}" ; then - PARALLEL_COMPRESSION_THREADS=1 - fi + if var_false "${ENABLE_PARALLEL_COMPRESSION}" ; then + PARALLEL_COMPRESSION_THREADS=1 + fi - case "${COMPRESSION,,}" in + if var_true "${GZ_RSYNCABLE}" ; then + gz_rsyncable=--rsyncable + fi + + case "${COMPRESSION,,}" in gz* ) - if var_true "${GZ_RSYNCABLE}" ; then - gz_rsyncable=--rsyncable - fi compress_cmd="pigz -q -${COMPRESSION_LEVEL} -p ${PARALLEL_COMPRESSION_THREADS} ${gz_rsyncable}" compression_type="gzip" extension=".gz" @@ -598,7 +599,7 @@ compression() { target=${target}.xz ;; zst* ) - compress_cmd="zstd -q -q --rm -${COMPRESSION_LEVEL} -T${PARALLEL_COMPRESSION_THREADS} " + compress_cmd="zstd -q -q --rm -${COMPRESSION_LEVEL} -T${PARALLEL_COMPRESSION_THREADS} ${gz_rsyncable}" compression_type="zstd" dir_compress_cmd=${compress_cmd} extension=".zst" From 425383639a64f8ea4b04766d2aeef9157d0a607a Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Tue, 13 Jun 2023 16:24:17 -0700 Subject: [PATCH 03/17] Release 3.9.5 - See CHANGELOG.md --- CHANGELOG.md | 6 ++++++ Dockerfile | 10 +++++++--- examples/mysql/docker-compose.yml | 4 +++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4633605..3bf3f86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 3.9.5 2023-06-13 + + ### Changed + - Start building Influx DB v1 manually due to being removed from Alpine repositories + + ## 3.9.4 2023-06-13 ### Added diff --git a/Dockerfile b/Dockerfile index c64fb2f..503c139 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,8 @@ FROM docker.io/tiredofit/${DISTRO}:${DISTRO_VARIANT} LABEL maintainer="Dave Conroy (github.com/tiredofit)" ### Set Environment Variables -ENV INFLUX2_VERSION=2.4.0 \ +ENV INFLUX_VERSION=1.8.0 \ + INFLUX2_VERSION=2.4.0 \ MSSQL_VERSION=18.0.1.1-1 \ CONTAINER_ENABLE_MESSAGING=FALSE \ CONTAINER_ENABLE_MONITORING=TRUE \ @@ -22,6 +23,7 @@ RUN source /assets/functions/00-container && \ build-base \ bzip2-dev \ git \ + go \ libarchive-dev \ openssl-dev \ libffi-dev \ @@ -33,7 +35,6 @@ RUN source /assets/functions/00-container && \ package install .db-backup-run-deps \ aws-cli \ bzip2 \ - influxdb \ libarchive \ mariadb-client \ mariadb-connector-c \ @@ -60,7 +61,9 @@ RUN source /assets/functions/00-container && \ \ if [ $mssql = "true" ] ; then curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/msodbcsql18_${MSSQL_VERSION}_amd64.apk ; curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/mssql-tools18_${MSSQL_VERSION}_amd64.apk ; echo y | apk add --allow-untrusted msodbcsql18_${MSSQL_VERSION}_amd64.apk mssql-tools18_${MSSQL_VERSION}_amd64.apk ; else echo >&2 "Detected non x86_64 build variant, skipping MSSQL installation" ; fi; \ if [ $influx2 = "true" ] ; then curl -sSL https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX2_VERSION}-linux-${influx_arch}.tar.gz | tar xvfz - --strip=1 -C /usr/src/ ; chmod +x /usr/src/influx ; mv /usr/src/influx /usr/sbin/ ; else echo >&2 "Unable to build Influx 2 on this system" ; fi ; \ - \ + clone_git_repo https://github.com/influxdata/influxdb "${INFLUX_VERSION}" && \ + go build -o /usr/sbin/influxd ./cmd/influxd && \ + strip /usr/sbin/influxd && \ mkdir -p /usr/src/pbzip2 && \ curl -sSL https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz | tar xvfz - --strip=1 -C /usr/src/pbzip2 && \ cd /usr/src/pbzip2 && \ @@ -85,6 +88,7 @@ RUN source /assets/functions/00-container && \ /*.apk \ /etc/logrotate.d/* \ /root/.cache \ + /root/go \ /tmp/* \ /usr/src/* diff --git a/examples/mysql/docker-compose.yml b/examples/mysql/docker-compose.yml index f65b18b..f306aae 100644 --- a/examples/mysql/docker-compose.yml +++ b/examples/mysql/docker-compose.yml @@ -31,6 +31,8 @@ services: - ./backups:/backup #- ./post-script.sh:/assets/custom-scripts/post-script.sh environment: + - TIMEZONE=America/Vancouver + - CONTAINER_ENABLE_MONITORING=FALSE # - DEBUG_MODE=TRUE - DB_TYPE=mariadb - DB_HOST=example-db-host @@ -43,7 +45,7 @@ services: - CHECKSUM=SHA1 - COMPRESSION=GZ - SPLIT_DB=FALSE - - CONTAINER_ENABLE_MONITORING=FALSE + restart: always networks: - example-db-network From 94e9881b7b47bd09f6314f9b65bf20621653e564 Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Fri, 16 Jun 2023 09:50:16 -0700 Subject: [PATCH 04/17] Release 3.9.6 - See CHANGELOG.md --- CHANGELOG.md | 6 ++ README.md | 5 +- install/assets/functions/10-db-backup | 79 ++++++++++++++++++--------- 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bf3f86..bb11a28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 3.9.6 2023-06-16 + + ### Changed + - Resolve issues introduced with 3.9.3. Split exit codes to be specific for backing up and moving. Uses paremter $11 for post backup scripts + + ## 3.9.5 2023-06-13 ### Changed diff --git a/README.md b/README.md index 8f16788..aaf62dd 100644 --- a/README.md +++ b/README.md @@ -323,17 +323,18 @@ $ cat post-script.sh # #### $8=BACKUP FILENAME (Filename) # #### $9=BACKUP FILESIZE # #### $10=HASH (If CHECKSUM enabled) +# #### $11=MOVE_EXIT_CODE echo "${1} ${2} Backup Completed on ${3} for ${4} on ${5} ending ${6} for a duration of ${7} seconds. Filename: ${8} Size: ${9} bytes MD5: ${10}" ```` ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME STARTEPOCH FINISHEPOCH DURATIONEPOCH BACKUP_FILENAME FILESIZE CHECKSUMVALUE - ${f} "${exit_code}" "${dbtype}" "${dbhost}" "${dbname}" "${backup_start_timme}" "${backup_finish_time}" "${backup_total_time}" "${target}" "${FILESIZE}" "${checksum_value}" + ${f} "${exit_code}" "${dbtype}" "${dbhost}" "${dbname}" "${backup_start_timme}" "${backup_finish_time}" "${backup_total_time}" "${target}" "${FILESIZE}" "${checksum_value}" "${move_exit_code} Outputs the following on the console: -`0 mysql Backup Completed on example-db for example on 1647370800 ending 1647370920 for a duration of 120 seconds. Filename: mysql_example_example-db_202200315-000000.sql.bz2 Size: 7795 bytes Hash: 952fbaafa30437494fdf3989a662cd40` +`0 mysql Backup Completed on example-db for example on 1647370800 ending 1647370920 for a duration of 120 seconds. Filename: mysql_example_example-db_202200315-000000.sql.bz2 Size: 7795 bytes Hash: 952fbaafa30437494fdf3989a662cd40 0` If you wish to change the size value from bytes to megabytes set environment variable `SIZE_VALUE=megabytes` diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index f9f8326..12494a2 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -134,10 +134,10 @@ backup_couch() { print_notice "Dumping CouchDB database: '${DB_NAME}' ${compression_string}" curl -sSL -X GET ${DB_HOST}:${DB_PORT}/${DB_NAME}/_all_docs?include_docs=true ${compress_cmd} | $compress_cmd > "${TEMP_LOCATION}"/"${target}" exit_code=$? - check_exit_code $target + check_exit_code backup $target generate_checksum move_dbbackup - check_exit_code "move backup file" + check_exit_code move $target post_dbbackup ${DB_NAME} } @@ -161,14 +161,14 @@ backup_influx() { print_notice "Dumping Influx database: '${db}'" influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS} "${TEMP_LOCATION}"/"${target_dir}" exit_code=$? - check_exit_code $target_dir + check_exit_code backup $target_dir print_notice "Creating archive file of '${target_dir}' with tar ${compression_string}" tar cf - "${TEMP_LOCATION}"/"${target_dir}" | $dir_compress_cmd > "${TEMP_LOCATION}"/"${target_dir}".tar"${extension}" target=influx_${db}_${DB_HOST#*//}_${now}.tar${extension} ltarget=influx_${db}_${DB_HOST#*//} generate_checksum move_dbbackup - check_exit_code "move backup file" + check_exit_code move $target_dir post_dbbackup $db done ;; @@ -183,13 +183,13 @@ backup_influx() { print_notice "Dumping Influx2 database: '${db}'" influx backup --org ${DB_USER} ${bucket} --host ${DB_HOST}:${DB_PORT} --token ${DB_PASS} ${EXTRA_OPTS} --compression none "${TEMP_LOCATION}"/"${target_dir}" exit_code=$? - check_exit_code $target_dir + check_exit_code backup $target_dir create_archive target=influx2_${db}_${DB_HOST#*//}_${now}.tar${extension} ltarget=influx2_${db}_${DB_HOST#*//} generate_checksum move_dbbackup - check_exit_code "move backup file" + check_exit_code move $target_dir post_dbbackup $db done ;; @@ -219,7 +219,7 @@ backup_mongo() { check_exit_code $target generate_checksum move_dbbackup - check_exit_code "move backup file" + check_exit_code move $target post_dbbackup "${DB_NAME}" } @@ -232,10 +232,10 @@ backup_mssql() { print_notice "Dumping MSSQL database: '${DB_NAME}'" silent /opt/mssql-tools18/bin/sqlcmd -C -S ${DB_HOST}\,${DB_PORT} -U ${DB_USER} -P ${DB_PASS} -Q "BACKUP DATABASE [${DB_NAME}] TO DISK = N'${TEMP_LOCATION}/${target}' WITH NOFORMAT, NOINIT, NAME = '${DB_NAME}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10" exit_code=$? - check_exit_code $target + check_exit_code backup $target generate_checksum move_dbbackup - check_exit_code "move backup file" + check_exit_code move $target post_dbbackup $DB_NAME } @@ -273,10 +273,10 @@ backup_mysql() { print_notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}" mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} --databases $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}" exit_code=$? - check_exit_code $target + check_exit_code backup $target generate_checksum move_dbbackup - check_exit_code "move backup file" + check_exit_code move $target post_dbbackup $db done else @@ -292,7 +292,7 @@ backup_mysql() { check_exit_code $target generate_checksum move_dbbackup - check_exit_code "move backup file" + check_exit_code move $target post_dbbackup all fi } @@ -326,10 +326,10 @@ backup_pgsql() { print_notice "Dumping PostgresSQL database: '${db}' ${compression_string}" pg_dump -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} $db ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} exit_code=$? - check_exit_code $target + check_exit_code backup $target generate_checksum move_dbbackup - check_exit_code "move backup file" + check_exit_code move $target post_dbbackup $db done else @@ -350,10 +350,10 @@ backup_pgsql() { done pg_dumpall -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} ${pgexclude_arg} ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} exit_code=$? - check_exit_code $target + check_exit_code backup $target generate_checksum move_dbbackup - check_exit_code "move backup file" + check_exit_code move $target post_dbbackup all fi } @@ -382,10 +382,10 @@ backup_redis() { compression pre_dbbackup all $compress_cmd "${TEMP_LOCATION}/${target_original}" - check_exit_code $target + check_exit_code backup $target generate_checksum move_dbbackup - check_exit_code "move backup file" + check_exit_code move $target post_dbbackup all } @@ -400,11 +400,11 @@ backup_sqlite3() { print_notice "Dumping sqlite3 database: '${DB_HOST}' ${compression_string}" silent sqlite3 "${DB_HOST}" ".backup '${TEMP_LOCATION}/backup.sqlite3'" exit_code=$? - check_exit_code $target + check_exit_code backup $target cat "${TEMP_LOCATION}"/backup.sqlite3 | ${dir_compress_cmd} > "${TEMP_LOCATION}/${target}" generate_checksum move_dbbackup - check_exit_code "move backup file" + check_exit_code move $target post_dbbackup $db } @@ -513,6 +513,32 @@ check_availability() { check_exit_code() { print_debug "DB Backup Exit Code is ${exit_code}" + case "${1}" in + backup ) + case "${exit_code}" in + 0 ) + print_info "DB Backup of '${2}' completed successfully" + ;; + * ) + print_error "DB Backup of '${2}' reported errors" + master_exit_code=1 + ;; + esac + ;; + move ) + case "${move_exit_code}" in + 0 ) + print_debug "Moving of backup '${2}' completed successfully" + ;; + * ) + print_error "Moving of backup '${1}' reported errors" + master_exit_code=1 + ;; + esac + ;; + esac + + case "${exit_code}" in 0 ) print_info "DB Backup of '${1}' completed successfully" @@ -694,6 +720,7 @@ move_dbbackup() { mkdir -p "${DB_DUMP_TARGET}" mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/ mv "${TEMP_LOCATION}"/"${target}" "${DB_DUMP_TARGET}"/"${target}" + move_exit_code=$? if var_true "${CREATE_LATEST_SYMLINK}" ; then ln -sf "${DB_DUMP_TARGET}"/"${target}" "${DB_DUMP_TARGET}"/latest-"${ltarget}" fi @@ -723,7 +750,7 @@ move_dbbackup() { [[ ( -n "${S3_HOST}" ) ]] && PARAM_AWS_ENDPOINT_URL=" --endpoint-url ${S3_PROTOCOL}://${S3_HOST}" silent aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_LOCATION}/${target} s3://${S3_BUCKET}/${S3_PATH}/${target} ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} - exit_code=$? + move_exit_code=$? if var_true "${ENABLE_CHECKSUM}" ; then silent aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_LOCATION}/*.${checksum_extension} s3://${S3_BUCKET}/${S3_PATH}/ ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} fi @@ -739,7 +766,7 @@ move_dbbackup() { mv "${TEMP_LOCATION}"/"${target}" "${DB_DUMP_TARGET}"/"${target}" silent blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET} - exit_code=$? + move_exit_code=$? rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}" rm -rf "${TEMP_LOCATION}"/"${target}" @@ -816,11 +843,11 @@ post_dbbackup() { ### Post Script Support if [ -n "${POST_SCRIPT}" ] ; then if var_true "${POST_SCRIPT_SKIP_X_VERIFY}" ; then - eval "${POST_SCRIPT}" "${exit_code}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${target}" "${filesize}" "${checksum_value}" + eval "${POST_SCRIPT}" "${exit_code}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${target}" "${filesize}" "${checksum_value}" "${move_exit_code}" else if [ -x "${POST_SCRIPT}" ] ; then print_notice "Found POST_SCRIPT environment variable. Executing '${POST_SCRIPT}" - eval "${POST_SCRIPT}" "${exit_code}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${target}" "${filesize}" "${checksum_value}" + eval "${POST_SCRIPT}" "${exit_code}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${target}" "${filesize}" "${checksum_value}" "${move_exit_code}" else print_error "Can't execute POST_SCRIPT environment variable '${POST_SCRIPT}' as its filesystem bit is not executible!" fi @@ -837,12 +864,12 @@ post_dbbackup() { if [ -d "${SCRIPT_LOCATION_POST}" ] && dir_notempty "${SCRIPT_LOCATION_POST}" ; then for f in $(find ${SCRIPT_LOCATION_POST} -name \*.sh -type f); do if var_true "${POST_SCRIPT_SKIP_X_VERIFY}" ; then - ${f} "${exit_code}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${target}" "${filesize}" "${checksum_value}" + ${f} "${exit_code}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${target}" "${filesize}" "${checksum_value}" "${move_exit_code}" else if [ -x "${f}" ] ; then print_notice "Executing post backup custom script : '${f}'" ## script EXIT_CODE DB_TYPE DB_HOST DB_NAME STARTEPOCH FINISHEPOCH DURATIONEPOCH BACKUP_FILENAME FILESIZE CHECKSUMVALUE - ${f} "${exit_code}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${target}" "${filesize}" "${checksum_value}" + ${f} "${exit_code}" "${dbtype}" "${DB_HOST}" "${1}" "${dbbackup_start_time}" "${dbbackup_finish_time}" "${dbbackup_total_time}" "${target}" "${filesize}" "${checksum_value}" "${move_exit_code}" else print_error "Can't run post backup custom script: '${f}' as its filesystem bit is not executible!" fi From 10e7debc6549f46eeedf96db7e5c29027ba3f868 Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Tue, 18 Jul 2023 07:26:59 -0700 Subject: [PATCH 05/17] Release 3.9.7 - See CHANGELOG.md --- CHANGELOG.md | 6 ++++++ install/assets/functions/10-db-backup | 17 +++-------------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb11a28..0f69a5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 3.9.7 2023-07-18 + + ### Changed + - Cleanup check_exit_code parameter and reduce duplicate output + + ## 3.9.6 2023-06-16 ### Changed diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 12494a2..c1e1d06 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -216,7 +216,7 @@ backup_mongo() { print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} ${mongo_backup_parameter} exit_code=$? - check_exit_code $target + check_exit_code backup $target generate_checksum move_dbbackup check_exit_code move $target @@ -289,7 +289,7 @@ backup_mysql() { print_notice "Dumping all MySQL / MariaDB databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}" mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} --databases $(echo ${db_names} | xargs) | $compress_cmd > "${TEMP_LOCATION}"/"${target}" exit_code=$? - check_exit_code $target + check_exit_code backup $target generate_checksum move_dbbackup check_exit_code move $target @@ -531,23 +531,12 @@ check_exit_code() { print_debug "Moving of backup '${2}' completed successfully" ;; * ) - print_error "Moving of backup '${1}' reported errors" + print_error "Moving of backup '${2}' reported errors" master_exit_code=1 ;; esac ;; esac - - - case "${exit_code}" in - 0 ) - print_info "DB Backup of '${1}' completed successfully" - ;; - * ) - print_error "DB Backup of '${1}' reported errors" - master_exit_code=1 - ;; - esac } cleanup_old_data() { From 2265a6acf543464d844e5da67a011b10e198e7a5 Mon Sep 17 00:00:00 2001 From: ToshY <31921460+ToshY@users.noreply.github.com> Date: Sat, 5 Aug 2023 14:39:13 +0200 Subject: [PATCH 06/17] Add cargo to build dependencies --- .github/workflows/main.yml | 5 ++--- .github/workflows/manual.yml | 5 ++--- Dockerfile | 1 + 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aff358f..bb3b61f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,8 +8,7 @@ on: jobs: build: - #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main + uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main #uses: tiredofit/github_actions/.github/workflows/default_amd64.yml@main - #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main - uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main + #uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main secrets: inherit diff --git a/.github/workflows/manual.yml b/.github/workflows/manual.yml index 53fe788..9463b9f 100644 --- a/.github/workflows/manual.yml +++ b/.github/workflows/manual.yml @@ -9,8 +9,7 @@ on: jobs: build: - #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main + uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main #uses: tiredofit/github_actions/.github/workflows/default_amd64.yml@main - #uses: tiredofit/github_actions/.github/workflows/default_amd64_armv7_arm64.yml@main - uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main + #uses: tiredofit/github_actions/.github/workflows/default_amd64_arm64.yml@main secrets: inherit diff --git a/Dockerfile b/Dockerfile index 503c139..7bc4ea5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,7 @@ RUN source /assets/functions/00-container && \ package install .db-backup-build-deps \ build-base \ bzip2-dev \ + cargo \ git \ go \ libarchive-dev \ From 6d1ef8704218b992c6ab5c25dd84982857268c48 Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Sun, 20 Aug 2023 08:18:04 -0700 Subject: [PATCH 07/17] Release 3.9.8 - See CHANGELOG.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f69a5e..2176e72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 3.9.8 2023-08-20 + + ### Changed + - Restore armv7 and aarch64 builds + + ## 3.9.7 2023-07-18 ### Changed From fb3b65b33a610cb3cf0cae24d61b6add465bec56 Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Mon, 21 Aug 2023 15:38:51 -0700 Subject: [PATCH 08/17] Release 3.9.9 - See CHANGELOG.md --- CHANGELOG.md | 6 ++++++ Dockerfile | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2176e72..568ea56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 3.9.9 2023-08-21 + + ### Changed + - Start compiling aws-cli instead of from packages to continue to support arm/v7 + + ## 3.9.8 2023-08-20 ### Changed diff --git a/Dockerfile b/Dockerfile index 7bc4ea5..36180c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ LABEL maintainer="Dave Conroy (github.com/tiredofit)" ENV INFLUX_VERSION=1.8.0 \ INFLUX2_VERSION=2.4.0 \ MSSQL_VERSION=18.0.1.1-1 \ + AWS_CLI_VERSION=1.25.97 \ CONTAINER_ENABLE_MESSAGING=FALSE \ CONTAINER_ENABLE_MONITORING=TRUE \ CONTAINER_PROCESS_RUNAWAY_PROTECTOR=FALSE \ @@ -29,13 +30,14 @@ RUN source /assets/functions/00-container && \ openssl-dev \ libffi-dev \ python3-dev \ + py3-setuptools \ py3-pip \ xz-dev \ && \ \ package install .db-backup-run-deps \ - aws-cli \ bzip2 \ + groff \ libarchive \ mariadb-client \ mariadb-connector-c \ @@ -45,7 +47,15 @@ RUN source /assets/functions/00-container && \ postgresql15 \ postgresql15-client \ pv \ + py3-botocore \ + py3-colorama \ py3-cryptography \ + py3-docutils \ + py3-jmespath \ + py3-rsa \ + py3-s3transfer \ + py3-yaml \ + python3 \ redis \ sqlite \ xz \ @@ -62,6 +72,8 @@ RUN source /assets/functions/00-container && \ \ if [ $mssql = "true" ] ; then curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/msodbcsql18_${MSSQL_VERSION}_amd64.apk ; curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/mssql-tools18_${MSSQL_VERSION}_amd64.apk ; echo y | apk add --allow-untrusted msodbcsql18_${MSSQL_VERSION}_amd64.apk mssql-tools18_${MSSQL_VERSION}_amd64.apk ; else echo >&2 "Detected non x86_64 build variant, skipping MSSQL installation" ; fi; \ if [ $influx2 = "true" ] ; then curl -sSL https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX2_VERSION}-linux-${influx_arch}.tar.gz | tar xvfz - --strip=1 -C /usr/src/ ; chmod +x /usr/src/influx ; mv /usr/src/influx /usr/sbin/ ; else echo >&2 "Unable to build Influx 2 on this system" ; fi ; \ + clone_git_repo https://github.com/aws/aws-cli "${AWS_CLI_VERSION}" && \ + python3 setup.py install --prefix=/usr && \ clone_git_repo https://github.com/influxdata/influxdb "${INFLUX_VERSION}" && \ go build -o /usr/sbin/influxd ./cmd/influxd && \ strip /usr/sbin/influxd && \ From bf646381cb6bb6c65397e24444627108b0fead5d Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Wed, 23 Aug 2023 15:31:41 -0700 Subject: [PATCH 09/17] Release 3.9.10 - See CHANGELOG.md --- CHANGELOG.md | 6 ++++++ install/assets/functions/10-db-backup | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 568ea56..4261291 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 3.9.10 2023-08-23 + + ### Changed + - Stop trying to move a non existent checksum file when ENABLE_CHECKSUM=FALSE + + ## 3.9.9 2023-08-21 ### Changed diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index c1e1d06..9990e93 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -707,7 +707,7 @@ move_dbbackup() { "file" | "filesystem" ) print_debug "Moving backup to filesystem" mkdir -p "${DB_DUMP_TARGET}" - mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/ + if var_true "${ENABLE_CHECKSUM}" ; then mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/ ; fi mv "${TEMP_LOCATION}"/"${target}" "${DB_DUMP_TARGET}"/"${target}" move_exit_code=$? if var_true "${CREATE_LATEST_SYMLINK}" ; then From 36506091bee954dea27f79bb6f13433c545ab995 Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Thu, 24 Aug 2023 12:16:10 -0700 Subject: [PATCH 10/17] Release 3.9.11 - See CHANGELOG.md --- CHANGELOG.md | 6 ++++++ Dockerfile | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4261291..6e26b8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 3.9.11 2023-08-24 + + ### Changed + - AWS CLI 2.13.9 + + ## 3.9.10 2023-08-23 ### Changed diff --git a/Dockerfile b/Dockerfile index 36180c8..979c7b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,6 @@ RUN source /assets/functions/00-container && \ openssl-dev \ libffi-dev \ python3-dev \ - py3-setuptools \ py3-pip \ xz-dev \ && \ @@ -53,6 +52,7 @@ RUN source /assets/functions/00-container && \ py3-docutils \ py3-jmespath \ py3-rsa \ + py3-setuptools \ py3-s3transfer \ py3-yaml \ python3 \ From 663667dbff7af1db343e348342ef7f2784dcf6d0 Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Mon, 4 Sep 2023 08:32:05 -0700 Subject: [PATCH 11/17] Release 3.9.12 - See CHANGELOG.md --- CHANGELOG.md | 6 ++++++ install/assets/functions/10-db-backup | 7 ++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e26b8a..09c67f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 3.9.12 2023-09-04 + + ### Changed + - Perform additional checks for ENABLE_CHECKSUM=FALSE and skip executing actions for S3/BlobXfer + + ## 3.9.11 2023-08-24 ### Changed diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 9990e93..86c5c5c 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -744,20 +744,21 @@ move_dbbackup() { silent aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_LOCATION}/*.${checksum_extension} s3://${S3_BUCKET}/${S3_PATH}/ ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} fi - rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}" + if var_true "${ENABLE_CHECKSUM}" ; then rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}"; fi rm -rf "${TEMP_LOCATION}"/"${target}" ;; "blobxfer" ) print_info "Moving backup to S3 Bucket with blobxfer" mkdir -p "${DB_DUMP_TARGET}" - mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/ + if var_true "${ENABLE_CHECKSUM}" ; then mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/; fi + mv "${TEMP_LOCATION}"/"${target}" "${DB_DUMP_TARGET}"/"${target}" silent blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET} move_exit_code=$? - rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}" + if var_true "${ENABLE_CHECKSUM}" ; then rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}" ; fi rm -rf "${TEMP_LOCATION}"/"${target}" ;; esac From cd1899d849c3dbfe1044236975c64677fc48f1ce Mon Sep 17 00:00:00 2001 From: Jan-Claas Dirks Date: Wed, 13 Sep 2023 10:00:41 +0200 Subject: [PATCH 12/17] add env variables EXTRA_DUMP_OPTS and EXTRA_ENUMERATION_OPTS --- README.md | 6 ++++-- install/assets/functions/10-db-backup | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index aaf62dd..e62263c 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ Your Organization will be mapped to `DB_USER` and your root token will need to b ### Backup Options | Parameter | Description | Default | `_FILE` | -| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------- | ------------------------- | ------- | +|--------------------------------|------------------------------------------------------------------------------------------------------------------------------|---------------------------|---------| | `COMPRESSION` | Use either Gzip `GZ`, Bzip2 `BZ`, XZip `XZ`, ZSTD `ZSTD` or none `NONE` | `ZSTD` | | | `COMPRESSION_LEVEL` | Numberical value of what level of compression to use, most allow `1` to `9` except for `ZSTD` which allows for `1` to `19` - | `3` | | | `ENABLE_PARALLEL_COMPRESSION` | Use multiple cores when compressing backups `TRUE` or `FALSE` | `TRUE` | | @@ -187,7 +187,9 @@ Your Organization will be mapped to `DB_USER` and your root token will need to b | `GZ_RSYNCABLE` | Use `--rsyncable` (gzip only) for faster rsync transfers and incremental backup deduplication. e.g. `TRUE` | `FALSE` | | | `ENABLE_CHECKSUM` | Generate either a MD5 or SHA1 in Directory, `TRUE` or `FALSE` | `TRUE` | | | `CHECKSUM` | Either `MD5` or `SHA1` | `MD5` | | -| `EXTRA_OPTS` | If you need to pass extra arguments to the backup command, add them here e.g. `--extra-command` | | | +| `EXTRA_OPTS` | If you need to pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | | +| `EXTRA_DUMP_OPTS` | If you need to pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | | +| `EXTRA_ENUMERATION_OPTS` | If you need to pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | | | `MYSQL_MAX_ALLOWED_PACKET` | Max allowed packet if backing up MySQL / MariaDB | `512M` | | | `MYSQL_SINGLE_TRANSACTION` | Backup in a single transaction with MySQL / MariaDB | `TRUE` | | | `MYSQL_STORED_PROCEDURES` | Backup stored procedures with MySQL / MariaDB | `TRUE` | | diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 86c5c5c..5979539 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -159,7 +159,7 @@ backup_influx() { compression pre_dbbackup $db print_notice "Dumping Influx database: '${db}'" - influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS} "${TEMP_LOCATION}"/"${target_dir}" + influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS ${EXTRA_DUMP_OPTS} "${TEMP_LOCATION}"/"${target_dir}" exit_code=$? check_exit_code backup $target_dir print_notice "Creating archive file of '${target_dir}' with tar ${compression_string}" @@ -181,7 +181,7 @@ backup_influx() { compression pre_dbbackup $db print_notice "Dumping Influx2 database: '${db}'" - influx backup --org ${DB_USER} ${bucket} --host ${DB_HOST}:${DB_PORT} --token ${DB_PASS} ${EXTRA_OPTS} --compression none "${TEMP_LOCATION}"/"${target_dir}" + influx backup --org ${DB_USER} ${bucket} --host ${DB_HOST}:${DB_PORT} --token ${DB_PASS} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} --compression none "${TEMP_LOCATION}"/"${target_dir}" exit_code=$? check_exit_code backup $target_dir create_archive @@ -208,9 +208,9 @@ backup_mongo() { compression_string="and compressing with gzip" fi if [ -n "${MONGO_CUSTOM_URI}" ] ; then - mongo_backup_parameter="--uri=${MONGO_CUSTOM_URI} ${EXTRA_OPTS}" + mongo_backup_parameter="--uri=${MONGO_CUSTOM_URI} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS}" else - mongo_backup_parameter="--host ${DB_HOST} --port ${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS}" + mongo_backup_parameter="--host ${DB_HOST} --port ${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS}" fi pre_dbbackup "${DB_NAME}" print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}" @@ -249,7 +249,7 @@ backup_mysql() { if [ "${DB_NAME,,}" = "all" ] ; then print_debug "Preparing to back up everything except for information_schema and _* prefixes" - db_names=$(mysql -h ${DB_HOST} -P $DB_PORT -u$DB_USER ${mysql_tls_args} ${EXTRA_OPTS} --batch -e "SHOW DATABASES;" | grep -v Database | grep -v schema ) + db_names=$(mysql -h ${DB_HOST} -P $DB_PORT -u$DB_USER ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_ENUMERATION_OPTS} --batch -e "SHOW DATABASES;" | grep -v Database | grep -v schema ) if [ -n "${DB_NAME_EXCLUDE}" ] ; then db_names_exclusions=$(echo "${DB_NAME_EXCLUDE}" | tr ',' '\n') for db_exclude in ${db_names_exclusions} ; do @@ -271,7 +271,7 @@ backup_mysql() { compression pre_dbbackup $db print_notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}" - mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} --databases $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}" + mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} --databases $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}" exit_code=$? check_exit_code backup $target generate_checksum @@ -287,7 +287,7 @@ backup_mysql() { compression pre_dbbackup all print_notice "Dumping all MySQL / MariaDB databases: '$(echo ${db_names} | xargs | tr ' ' ',')' ${compression_string}" - mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} --databases $(echo ${db_names} | xargs) | $compress_cmd > "${TEMP_LOCATION}"/"${target}" + mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} --databases $(echo ${db_names} | xargs) | $compress_cmd > "${TEMP_LOCATION}"/"${target}" exit_code=$? check_exit_code backup $target generate_checksum @@ -324,7 +324,7 @@ backup_pgsql() { compression pre_dbbackup $db print_notice "Dumping PostgresSQL database: '${db}' ${compression_string}" - pg_dump -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} $db ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} + pg_dump -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} $db ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} exit_code=$? check_exit_code backup $target generate_checksum @@ -348,7 +348,7 @@ backup_pgsql() { for x_db_name in ${tmp_db_names} ; do pgexclude_arg=$(echo ${pgexclude_arg} --exclude-database=${x_db_name}) done - pg_dumpall -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} ${pgexclude_arg} ${EXTRA_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} + pg_dumpall -h ${DB_HOST} -U ${DB_USER} -p ${DB_PORT} ${pgexclude_arg} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} | $compress_cmd > ${TEMP_LOCATION}/${target} exit_code=$? check_exit_code backup $target generate_checksum @@ -363,7 +363,7 @@ backup_redis() { print_notice "Dumping Redis - Flushing Redis Cache First" target=redis_all_${DB_HOST,,}_${now}.rdb ltarget=redis_${DB_HOST,,} - echo bgsave | silent redis-cli -h ${DB_HOST} -p ${DB_PORT} ${REDIS_PASS_STR} --rdb ${TEMP_LOCATION}/${target} ${EXTRA_OPTS} + echo bgsave | silent redis-cli -h ${DB_HOST} -p ${DB_PORT} ${REDIS_PASS_STR} --rdb ${TEMP_LOCATION}/${target} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} sleep 10 try=5 while [ $try -gt 0 ] ; do From 8b1308ffd1e5c7308826c91fe3275716e1759650 Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Wed, 13 Sep 2023 08:32:22 -0700 Subject: [PATCH 13/17] Release 3.10.0 - See CHANGELOG.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09c67f6..50c999e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 3.10.0 2023-09-13 + + ### Added + - Add EXTRA_DUMP_OPTS and EXTRA_ENUMERATION_OPTS to add different arguments when checking for databases, vs doing the actual backup + + ## 3.9.12 2023-09-04 ### Changed From 48a1ff8bbeb5dc4a43a60f6063ea4b5b48be8dfc Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Wed, 13 Sep 2023 22:37:21 -0700 Subject: [PATCH 14/17] Release 3.10.1 - See CHANGELOG.md --- CHANGELOG.md | 6 ++++++ install/assets/functions/10-db-backup | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50c999e..6953eb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 3.10.1 2023-09-13 + + ### Changed + - Bugfix to 3.10.0 with syntax error revolving around unbraced variable + + ## 3.10.0 2023-09-13 ### Added diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 5979539..982cfb8 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -159,7 +159,7 @@ backup_influx() { compression pre_dbbackup $db print_notice "Dumping Influx database: '${db}'" - influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS ${EXTRA_DUMP_OPTS} "${TEMP_LOCATION}"/"${target_dir}" + influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} "${TEMP_LOCATION}"/"${target_dir}" exit_code=$? check_exit_code backup $target_dir print_notice "Creating archive file of '${target_dir}' with tar ${compression_string}" From cb5b49b90b0eb810bcbd886186ddd76910ef5f00 Mon Sep 17 00:00:00 2001 From: Pim Jansen Date: Thu, 14 Sep 2023 10:36:55 +0200 Subject: [PATCH 15/17] Remove the --database flag for a single db dump which ensures there is no `use` statement in the dump --- install/assets/functions/10-db-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 982cfb8..372d942 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -271,7 +271,7 @@ backup_mysql() { compression pre_dbbackup $db print_notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}" - mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} --databases $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}" + mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}" exit_code=$? check_exit_code backup $target generate_checksum From db808d25c7e3238c8d9db40c69d044c87e750736 Mon Sep 17 00:00:00 2001 From: Pim Jansen Date: Thu, 14 Sep 2023 10:39:34 +0200 Subject: [PATCH 16/17] Updated name where it is not writing to s3 --- install/assets/functions/10-db-backup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 982cfb8..2289fb2 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -748,7 +748,7 @@ move_dbbackup() { rm -rf "${TEMP_LOCATION}"/"${target}" ;; "blobxfer" ) - print_info "Moving backup to S3 Bucket with blobxfer" + print_info "Moving backup to external storage with blobxfer" mkdir -p "${DB_DUMP_TARGET}" if var_true "${ENABLE_CHECKSUM}" ; then mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/; fi From e3faab5c3660b4fe38d009e41cdab0e846a4c94b Mon Sep 17 00:00:00 2001 From: "dave@tiredofit.ca" Date: Thu, 14 Sep 2023 08:13:56 -0700 Subject: [PATCH 17/17] Release 3.10.2 - See CHANGELOG.md --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6953eb4..9523f2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 3.10.2 2023-09-14 + + ### Changed + - Update to wording when sending files to blobxfer + - Remove --databases flag when backing up a single mysql/mariadb backup which allows to omit the "USE " statement in the backup allowing for better restores + + ## 3.10.1 2023-09-13 ### Changed