Release 3.11.1 - See CHANGELOG.md

Use authentification database in mongorestore

CHANGELOG.md

@@ -1,3 +1,49 @@
+## 3.11.1 2023-10-23 <dave at tiredofit dot ca>
+
+   ### Added
+      - Switch to tiredofit/alpine:edge for newer packages
+      - Postgresql 16 support
+
+   ### Changed
+      - Add --break-system-packages flag to pip when installing blobxfer
+
+
+## 3.11.0 2023-10-12 <dave at tiredofit dot ca>
+
+   ### Added
+      - Introduce aarch64 (ARMv8) support for Microsoft SQL Server backups
+      - Microsoft ODBC Driver 18.3.2.1-1
+      - Microsoft SQL Client 18.3.1.1-1
+
+
+## 3.10.5 2023-10-11 <dave at tiredofit dot ca>
+
+   ### Added
+      - Add option to drop exsiting data from MongoDB restore
+
+   ### Changed
+      - Fix some capabilities of not being able to select mongodb manually to restore
+
+
+## 3.10.4 2023-10-11 <thomas-negrault@github>
+
+   ### Changed
+      - Use authentication database for MongoDB restores
+
+
+## 3.10.3 2023-10-11 <thomas-negrault@github>
+
+   ### Changed
+      - Change sorting for restore script
+
+
+## 3.10.2 2023-09-14 <pimjansen@github>
+
+   ### Changed
+      - Update to wording when sending files to blobxfer
+      - Remove --databases flag when backing up a single mysql/mariadb backup which allows to omit the "USE <db_name>" statement in the backup allowing for better restores
+
+
 ## 3.10.1 2023-09-13 <dave at tiredofit dot ca>
 
    ### Changed

Dockerfile

@@ -1,5 +1,5 @@
 ARG DISTRO=alpine
-ARG DISTRO_VARIANT=3.18
+ARG DISTRO_VARIANT=edge
 
 FROM docker.io/tiredofit/${DISTRO}:${DISTRO_VARIANT}
 LABEL maintainer="Dave Conroy (github.com/tiredofit)"
@@ -7,7 +7,8 @@ LABEL maintainer="Dave Conroy (github.com/tiredofit)"
 ### Set Environment Variables
 ENV INFLUX_VERSION=1.8.0 \
     INFLUX2_VERSION=2.4.0 \
-    MSSQL_VERSION=18.0.1.1-1 \
+    MSODBC_VERSION=18.3.2.1-1 \
+    MSSQL_VERSION=18.3.1.1-1 \
     AWS_CLI_VERSION=1.25.97 \
     CONTAINER_ENABLE_MESSAGING=FALSE \
     CONTAINER_ENABLE_MONITORING=TRUE \
@@ -43,8 +44,8 @@ RUN source /assets/functions/00-container && \
                mongodb-tools \
                openssl \
                pigz \
-               postgresql15 \
-               postgresql15-client \
+               postgresql16 \
+               postgresql16-client \
                pv \
                py3-botocore \
                py3-colorama \
@@ -63,14 +64,14 @@ RUN source /assets/functions/00-container && \
                zstd \
                && \
     \
-    apkArch="$(apk --print-arch)"; \
+    apkArch="$(uname -m)"; \
     case "$apkArch" in \
-	x86_64) mssql=true ; influx2=true ; influx_arch=amd64; ;; \
-    aarch64 ) influx2=true ; influx_arch=arm64 ;; \
-    *) sleep 0.1 ;; \
+	x86_64) mssql=true ; mssql_arch=amd64; influx2=true ; influx_arch=amd64; ;; \
+        arm64 ) mssql=true ; mssql_arch=amd64; influx2=true ; influx_arch=arm64 ;; \
+        *) sleep 0.1 ;; \
     esac; \
     \
-    if [ $mssql = "true" ] ; then curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/msodbcsql18_${MSSQL_VERSION}_amd64.apk ; curl -O https://download.microsoft.com/download/b/9/f/b9f3cce4-3925-46d4-9f46-da08869c6486/mssql-tools18_${MSSQL_VERSION}_amd64.apk ; echo y | apk add --allow-untrusted msodbcsql18_${MSSQL_VERSION}_amd64.apk mssql-tools18_${MSSQL_VERSION}_amd64.apk ; else echo >&2 "Detected non x86_64 build variant, skipping MSSQL installation" ; fi; \
+    if [ $mssql = "true" ] ; then curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk ; curl -O https://download.microsoft.com/download/3/5/5/355d7943-a338-41a7-858d-53b259ea33f5/mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; ls -l ; echo y | apk add --allow-untrusted msodbcsql18_${MSODBC_VERSION}_${mssql_arch}.apk mssql-tools18_${MSSQL_VERSION}_${mssql_arch}.apk ; else echo >&2 "Detected non x86_64 or ARM64 build variant, skipping MSSQL installation" ; fi; \
     if [ $influx2 = "true" ] ; then curl -sSL https://dl.influxdata.com/influxdb/releases/influxdb2-client-${INFLUX2_VERSION}-linux-${influx_arch}.tar.gz | tar xvfz - --strip=1 -C /usr/src/ ; chmod +x /usr/src/influx ; mv /usr/src/influx /usr/sbin/ ; else echo >&2 "Unable to build Influx 2 on this system" ; fi ; \
     clone_git_repo https://github.com/aws/aws-cli "${AWS_CLI_VERSION}" && \
     python3 setup.py install --prefix=/usr && \
@@ -93,7 +94,7 @@ RUN source /assets/functions/00-container && \
     make && \
     make install && \
     \
-    pip3 install blobxfer && \
+    pip3 install --break-system-packages blobxfer && \
     \
     package remove .db-backup-build-deps && \
     package cleanup && \

README.md

@@ -12,7 +12,7 @@
 
 This will build a container for backing up multiple types of DB Servers
 
-Currently backs up CouchDB, InfluxDB, MySQL, MongoDB, Postgres, Redis servers.
+Currently backs up CouchDB, InfluxDB, MySQL, Microsoft SQL, MongoDB, Postgres, Redis servers.
 
 * dump to local filesystem or backup to S3 Compatible services, and Azure.
 * select database user and password
@@ -149,7 +149,7 @@ Be sure to view the following repositories to understand all the customizable op
 | Parameter          | Description                                                                                                                                                                          | Default | `_FILE` |
 | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | ------- |
 | `DB_AUTH`          | (Mongo Only - Optional) Authentication Database                                                                                                                                      |         |         |
-| `DB_TYPE`          | Type of DB Server to backup `couch` `influx` `mysql` `pgsql` `mongo` `redis` `sqlite3`                                                                                               |         |         |
+| `DB_TYPE`          | Type of DB Server to backup `couch` `influx` `mysql` `mssql` `pgsql` `mongo` `redis` `sqlite3`                                                                                       |         |         |
 | `DB_HOST`          | Server Hostname e.g. `mariadb`. For `sqlite3`, full path to DB file e.g. `/backup/db.sqlite3`                                                                                        |         | x       |
 | `DB_NAME`          | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. Backup multiple by seperating with commas eg `db1,db2`                                          |         | x       |
 | `DB_NAME_EXCLUDE`  | If using `ALL` - use this as to exclude databases seperated via commas from being backed up                                                                                          |         | x       |

install/assets/functions/10-db-backup

@@ -271,7 +271,7 @@ backup_mysql() {
                 compression
                 pre_dbbackup $db
                 print_notice "Dumping MySQL/MariaDB database: '${db}' ${compression_string}"
-                mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} --databases $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
+                mysqldump --max-allowed-packet=${MYSQL_MAX_ALLOWED_PACKET} -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${EXTRA_OPTS} ${EXTRA_DUMP_OPTS} $db | $compress_cmd > "${TEMP_LOCATION}"/"${target}"
                 exit_code=$?
                 check_exit_code backup $target
                 generate_checksum
@@ -748,7 +748,7 @@ move_dbbackup() {
                 rm -rf "${TEMP_LOCATION}"/"${target}"
             ;;
             "blobxfer" )
-                print_info "Moving backup to S3 Bucket with blobxfer"
+                print_info "Moving backup to external storage with blobxfer"
 
                 mkdir -p "${DB_DUMP_TARGET}"
                 if var_true "${ENABLE_CHECKSUM}" ; then mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/; fi

install/usr/local/bin/restore

@@ -78,7 +78,7 @@ fi
 get_filename() {
     COLUMNS=12
     prompt="Please select a file to restore:"
-    options=( $(find "${DB_DUMP_TARGET}" -type f -maxdepth 1  -not -name '*.md5' -not -name '*.sha1' -print0 | xargs -0) )
+    options=( $(find "${DB_DUMP_TARGET}" -type f -maxdepth 1  -not -name '*.md5' -not -name '*.sha1' -print0 | sort -z | xargs -0) )
     PS3="$prompt "
     select opt in "${options[@]}" "Custom" "Quit" ; do
         if (( REPLY == 2 + ${#options[@]} )) ; then
@@ -131,7 +131,7 @@ EOF
     fi
 
     if [ -z "${DB_HOST}" ] && [ -n "${parsed_host}" ]; then
-        print_debug "Parsed DBHostpho Variant: 3 - No Env, Parsed Filename"
+        print_debug "Parsed DBHost Variant: 3 - No Env, Parsed Filename"
         q_dbhost_variant=3
         q_dbhost_menu=$(cat <<EOF
 
@@ -335,7 +335,7 @@ EOF
     case "${q_dbtype_variant}" in
         1 )
             while true; do
-                read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}M${cdgy}\) | \(${cwh}P${cdgy}\) : ${cwh}${coff}) " q_dbtype
+                read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}M${cdgy}\) \| \(${cwh}O${cdgy}\) \| \(${cwh}P${cdgy}\) : ${cwh}${coff}) " q_dbtype
                 case "${q_dbtype,,}" in
                     m* )
                         r_dbtype=mysql
@@ -358,7 +358,7 @@ EOF
         ;;
         2 )
             while true; do
-                read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}E${cdgy}\) \| \(${cwh}M${cdgy}\) \| \(${cwh}P${cdgy}\) : ${cwh}${coff}) " q_dbtype
+                read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}E${cdgy}\) \| \(${cwh}M${cdgy}\) \| \(${cwh}O${cdgy}\) \| \(${cwh}P${cdgy}\) : ${cwh}${coff}) " q_dbtype
                 case "${q_dbtype,,}" in
                     e* | "" )
                         r_dbtype=${DB_TYPE}
@@ -385,7 +385,7 @@ EOF
         ;;
         3 )
             while true; do
-                read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}F${cdgy}\) \| \(${cwh}M${cdgy}\) \| \(${cwh}P${cdgy}\) : ${cwh}${coff}) " q_dbtype
+                read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}F${cdgy}\) \| \(${cwh}M${cdgy}\) \| \(${cwh}O${cdgy}\) \| \(${cwh}P${cdgy}\) : ${cwh}${coff}) " q_dbtype
                 case "${q_dbtype,,}" in
                     f* | "" )
                         r_dbtype=${p_dbtype}
@@ -413,7 +413,7 @@ EOF
 
         4 )
             while true; do
-                read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}E${cdgy}\) \| \(${cwh}F${cdgy}\) \| \(${cwh}M${cdgy}\) \| \(${cwh}P${cdgy}\) : ${cwh}${coff}) " q_dbtype
+                read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}E${cdgy}\) \| \(${cwh}F${cdgy}\) \| \(${cwh}M${cdgy}\) \| \(${cwh}O${cdgy}\) \| \(${cwh}P${cdgy}\) : ${cwh}${coff}) " q_dbtype
                 case "${q_dbtype,,}" in
                     e* | "" )
                         r_dbtype=${DB_TYPE}
@@ -427,6 +427,10 @@ EOF
                         r_dbtype=mysql
                         break
                     ;;
+                    o* )
+                        r_dbtype=mongo
+                        break
+                    ;;
                     p* )
                         r_dbtype=postgresql
                         break
@@ -915,6 +919,30 @@ case "${r_dbtype}" in
         exit_code=$?
     ;;
     mongo )
+        cat << EOF
+
+Do you wish to drop any existing data before restoring?
+    Y ) Yes
+    N ) No
+    Q ) Quit
+
+EOF
+
+        echo -e "${coff}"
+        read -p "$(echo -e ${clg}** ${cdgy}Enter Value \(${cwh}Y${cdgy}\) \| \(${cwh}N${cdgy}\) \| \(${cwh}Q${cdgy}\)  : ${cwh}${coff})" q_menu_mongo_dropdb
+        case "${q_menu_mongo_dropdb,,}" in
+            "y" | "yes" | * )
+                mongo_dropdb="--drop"
+            ;;
+            "n" | "update" )
+                unset mongo_dropdb
+            ;;
+            "q" | "exit" )
+                print_info "Quitting Script"
+                exit 1
+            ;;
+        esac
+
         print_info "Restoring '${r_filename}' into '${r_dbhost}'/'${r_dbname}'"
         if [ "${ENABLE_COMPRESSION,,}" != "none" ] && [ "${ENABLE_COMPRESSION,,}" != "false" ] ; then
             mongo_compression="--gzip"
@@ -925,7 +953,11 @@ case "${r_dbtype}" in
         if [ -n "${r_dbpass}" ] ; then
             mongo_pass="-p=${r_dbpass}"
         fi
-        mongorestore ${mongo_compression} -d=${r_dbname} -h=${r_dbhost} --port=${r_dbport} ${mongo_user} ${mongo_pass} --archive=${r_filename}
+        if [ -n "${DB_AUTH}" ] ; then
+            mongo_auth_database="--authenticationDatabase=${DB_AUTH}"
+        fi
+
+        mongorestore ${mongo_compression} -d=${r_dbname} -h=${r_dbhost} --port=${r_dbport} ${mongo_dropdb} ${mongo_user} ${mongo_pass} --archive=${r_filename} ${mongo_auth_database}
         exit_code=$?
     ;;
     * )
@@ -939,4 +971,4 @@ if [ "${exit_code}" = 0 ] ; then
     print_info "Restore complete!"
 else
     print_error "Restore reported errors"
-fi
+fi