Merge branch 'master' of github.com:tiredofit/docker-db-backup into bugfix/s3_backup_prefixes

# Conflicts:
#	install/assets/functions/10-db-backup

install/assets/defaults/10-db-backup

@@ -1,6 +1,7 @@
 #!/command/with-contenv bash
 
 BACKUP_LOCATION=${BACKUP_LOCATION:-"FILESYSTEM"}
+BLOBXFER_REMOTE_PATH=${BLOBXFER_REMOTE_PATH:-"/docker-db-backup"}
 CHECKSUM=${CHECKSUM:-"MD5"}
 COMPRESSION=${COMPRESSION:-"ZSTD"}
 COMPRESSION_LEVEL=${COMPRESSION_LEVEL:-"3"}
@@ -20,5 +21,6 @@ S3_PROTOCOL=${S3_PROTOCOL:-"https"}
 SCRIPT_LOCATION_PRE=${SCRIPT_LOCATION_PRE:-"/assets/scripts/pre/"}
 SCRIPT_LOCATION_POST=${SCRIPT_LOCATION_POST:-"/assets/scripts/post/"}
 SIZE_VALUE=${SIZE_VALUE:-"bytes"}
+SKIP_AVAILABILITY_CHECK=${SKIP_AVAILABILITY_CHECK:-"FALSE"}
 SPLIT_DB=${SPLIT_DB:-"TRUE"}
-TEMP_LOCATION=${TEMP_LOCATION:-"/tmp/backups"}
+TEMP_LOCATION=${TEMP_LOCATION:-"/tmp/backups"}

install/assets/functions/10-db-backup

@@ -20,14 +20,33 @@ bootstrap_variables() {
         ;;
         mongo* )
             dbtype=mongo
-            DB_PORT=${DB_PORT:-27017}
-            [[ ( -n "${DB_USER}" ) || ( -n "${DB_USER_FILE}" ) ]] && file_env 'DB_USER'
-            [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS'
+            if [ -n "${MONGO_CUSTOM_URI}" ] ; then
+                mongo_uri_proto=$(echo ${MONGO_CUSTOM_URI} | grep :// | sed -e's,^\(.*://\).*,\1,g')
+                mongo_uri_scratch="${MONGO_CUSTOM_URI/${mongo_uri_proto}/}"
+                mongo_uri_username_password=$(echo ${mongo_uri_scratch} | grep @ | rev | cut -d@ -f2- | rev)
+                if [ -n "${mongo_uri_username_password}" ]; then mongo_uri_scratch=$(echo ${mongo_uri_scratch} | rev | cut -d@ -f1 | rev) ; fi
+                mongo_uri_port=$(echo ${mongo_uri_scratch} | grep : | rev | cut -d: -f2- | rev)
+                if [ -n "${mongo_uri_port}" ]; then mongo_uri_port=$(echo ${mongo_uri_scratch} | rev | cut -d: -f1 | cut -d/ -f2 | rev) ; fi
+                mongo_uri_hostname=$(echo ${mongo_uri_scratch} | cut -d/ -f1 | cut -d: -f1 )
+                mongo_uri_database=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f1 )
+                mongo_uri_options=$(echo ${mongo_uri_scratch} | cut -d/ -f2 | cut -d? -f2 )
+                DB_NAME=${DB_NAME:-"${mongo_uri_database,,}"}
+                DB_HOST=${DB_HOST:-"${mongo_uri_hostname,,}"}
+            else
+                DB_PORT=${DB_PORT:-27017}
+                [[ ( -n "${DB_USER}" ) || ( -n "${DB_USER_FILE}" ) ]] && file_env 'DB_USER'
+                [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS'
+                [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}"
+                [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}"
+                [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}"
+                [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}"
+            fi
         ;;
         "mysql" | "mariadb" )
             dbtype=mysql
             DB_PORT=${DB_PORT:-3306}
             [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS'
+            [[ ( -n "${DB_PASS}" ) ]] && export MYSQL_PWD=${DB_PASS}
             sanity_var DB_NAME "Database Name to backup. Multiple seperated by commas"
         ;;
         "mssql" | "microsoftsql" )
@@ -43,12 +62,14 @@ bootstrap_variables() {
             dbtype=pgsql
             DB_PORT=${DB_PORT:-5432}
             [[ ( -n "${DB_PASS}" ) || ( -n "${DB_PASS_FILE}" ) ]] && file_env 'DB_PASS'
+            [[ ( -n "${DB_PASS}" ) ]] && POSTGRES_PASS_STR="PGPASSWORD=${DB_PASS}"
             sanity_var DB_NAME "Database Name to backup. Multiple seperated by commas"
         ;;
         "redis" )
             dbtype=redis
             DB_PORT=${DB_PORT:-6379}
             [[ ( -n "${DB_PASS}" || ( -n "${DB_PASS_FILE}" ) ) ]] && file_env 'DB_PASS'
+            [[ ( -n "${DB_PASS}" ) ]] && REDIS_PASS_STR=" -a ${DB_PASS}"
         ;;
         sqlite* )
             dbtype=sqlite3
@@ -59,25 +80,6 @@ bootstrap_variables() {
         file_env 'S3_KEY_ID'
         file_env 'S3_KEY_SECRET'
     fi
-
-    ### Set the Database Authentication Details
-    case "$dbtype" in
-        "mongo" )
-            [[ ( -n "${DB_USER}" ) ]] && MONGO_USER_STR=" --username ${DB_USER}"
-            [[ ( -n "${DB_PASS}" ) ]] && MONGO_PASS_STR=" --password ${DB_PASS}"
-            [[ ( -n "${DB_NAME}" ) ]] && MONGO_DB_STR=" --db ${DB_NAME}"
-            [[ ( -n "${DB_AUTH}" ) ]] && MONGO_AUTH_STR=" --authenticationDatabase ${DB_AUTH}"
-        ;;
-        "mysql" )
-            [[ ( -n "${DB_PASS}" ) ]] && export MYSQL_PWD=${DB_PASS}
-        ;;
-        "postgres" )
-            [[ ( -n "${DB_PASS}" ) ]] && POSTGRES_PASS_STR="PGPASSWORD=${DB_PASS}"
-        ;;
-        "redis" )
-            [[ ( -n "${DB_PASS}" ) ]] && REDIS_PASS_STR=" -a ${DB_PASS}"
-        ;;
-    esac
 }
 
 backup_couch() {
@@ -114,7 +116,7 @@ backup_influx() {
                 influxd backup ${influx_compression} ${bucket} -portable -host ${DB_HOST}:${DB_PORT} ${EXTRA_OPTS} "${TEMP_LOCATION}"/"${target_dir}"
                 exit_code=$?
                 check_exit_code $target_dir
-                print_notice "Creating archive file of '${target_dir}' with tar ${compresion_string}"
+                print_notice "Creating archive file of '${target_dir}' with tar ${compression_string}"
                 tar cf - "${TEMP_LOCATION}"/"${target_dir}" | $dir_compress_cmd > "${TEMP_LOCATION}"/"${target_dir}".tar"${extension}"
                 target=influx_${db}_${DB_HOST#*//}_${now}.tar${extension}
                 generate_checksum
@@ -152,19 +154,19 @@ backup_mongo() {
         mongo_compression="--gzip"
         compression_string="and compressing with gzip"
     fi
-    if [ "${MONGO_HOST_TYPE,,}" = "atlas" ] ; then
-        MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb+srv://"}
+    if [ -n "${MONGO_CUSTOM_URI}" ] ; then
+        mongo_backup_parameter="--uri=${MONGO_CUSTOM_URI} ${EXTRA_OPTS}"
     else
-        MONGO_URI_PREFIX=${MONGO_URI_PREFIX:-"mongodb://"}
+        mongo_backup_parameter="--host ${DB_HOST} --port ${DB_PORT} ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS}"
     fi
     pre_dbbackup "${DB_NAME}"
     print_notice "Dumping MongoDB database: '${DB_NAME}' ${compression_string}"
-    mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} --uri="${MONGO_URI_PREFIX}${DB_HOST}:${DB_PORT}" ${MONGO_USER_STR}${MONGO_PASS_STR}${MONGO_AUTH_STR}${MONGO_DB_STR} ${EXTRA_OPTS}
+    silent mongodump --archive=${TEMP_LOCATION}/${target} ${mongo_compression} ${mongo_backup_parameter}
     exit_code=$?
     check_exit_code $target
     generate_checksum
     move_dbbackup
-    post_dbbackup
+    post_dbbackup "${DB_NAME}"
 }
 
 backup_mssql() {
@@ -173,7 +175,7 @@ backup_mssql() {
     compression
     pre_dbbackup "${DB_NAME}"
     print_notice "Dumping MSSQL database: '${DB_NAME}'"
-    /opt/mssql-tools/bin/sqlcmd -E -C -S ${DB_HOST}\,${DB_PORT} -U ${DB_USER} -P ${DB_PASS} –Q "BACKUP DATABASE \[${DB_NAME}\] TO DISK = N'${TEMP_LOCATION}/${target}' WITH NOFORMAT, NOINIT, NAME = '${DB_NAME}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10"
+    silent /opt/mssql-tools18/bin/sqlcmd -C -S ${DB_HOST}\,${DB_PORT} -U ${DB_USER} -P ${DB_PASS} -Q "BACKUP DATABASE [${DB_NAME}] TO DISK = N'${TEMP_LOCATION}/${target}' WITH NOFORMAT, NOINIT, NAME = '${DB_NAME}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10"
     exit_code=$?
     check_exit_code $target
     generate_checksum
@@ -329,7 +331,7 @@ backup_sqlite3() {
     compression
     pre_dbbackup $db
     print_notice "Dumping sqlite3 database: '${DB_HOST}' ${compression_string}"
-    sqlite3 "${DB_HOST}" ".backup '${TEMP_LOCATION}/backup.sqlite3'"
+    silent sqlite3 "${DB_HOST}" ".backup '${TEMP_LOCATION}/backup.sqlite3'"
     exit_code=$?
     check_exit_code $target
     cat "${TEMP_LOCATION}"/backup.sqlite3 | ${dir_compress_cmd} > "${TEMP_LOCATION}/${target}"
@@ -340,101 +342,105 @@ backup_sqlite3() {
 
 check_availability() {
 ### Set the Database Type
-   case "$dbtype" in
-        "couch" )
-            counter=0
-            code_received=0
-            while [ "${code_received}" != "200" ]; do
-                code_received=$(curl -XGET -sSL -o /dev/null -L -w ''%{http_code}'' ${DB_HOST}:${DB_PORT})
-                if [ "${code_received}" = "200" ] ; then break ; fi
-                sleep 5
-                (( counter+=5 ))
-                print_warn "CouchDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)"
-            done
-        ;;
-        "influx" )
-            counter=0
-            case "${INFLUX_VERSION,,}" in
-                1 )
-                    while ! (nc -z ${DB_HOST#*//} ${DB_PORT}) ; do
+    if var_false "${SKIP_AVAILABILITY_CHECK}" ; then
+        case "$dbtype" in
+            "couch" )
+                counter=0
+                code_received=0
+                while [ "${code_received}" != "200" ]; do
+                    code_received=$(curl -XGET -sSL -o /dev/null -L -w ''%{http_code}'' ${DB_HOST}:${DB_PORT})
+                    if [ "${code_received}" = "200" ] ; then break ; fi
+                    sleep 5
+                    (( counter+=5 ))
+                    print_warn "CouchDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)"
+                done
+            ;;
+            "influx" )
+                counter=0
+                case "${INFLUX_VERSION,,}" in
+                    1 )
+                        while ! (nc -z ${DB_HOST#*//} ${DB_PORT}) ; do
+                            sleep 5
+                            (( counter+=5 ))
+                            print_warn "InfluxDB Host '${DB_HOST#*//}' is not accessible, retrying.. ($counter seconds so far)"
+                        done
+                    ;;
+                    2 )
+                        code_received=0
+                        while [ "${code_received}" != "200" ]; do
+                            code_received=$(curl -XGET -sSL -o /dev/null -w ''%{http_code}'' ${DB_HOST}:${DB_PORT}/health)
+                            if [ "${code_received}" = "200" ] ; then break ; fi
+                            sleep 5
+                            (( counter+=5 ))
+                            print_warn "InfluxDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)"
+                        done
+                    ;;
+                esac
+            ;;
+            "mongo" )
+                if [ -n "${MONGO_CUSTOM_URI}" ] ; then
+                    print_debug "Skipping Connectivity Check"
+                else
+                    counter=0
+                    while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do
                         sleep 5
                         (( counter+=5 ))
-                        print_warn "InfluxDB Host '${DB_HOST#*//}' is not accessible, retrying.. ($counter seconds so far)"
+                        print_warn "Mongo Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)"
                     done
-                ;;
-                2 )
-                    code_received=0
-                    while [ "${code_received}" != "200" ]; do
-                        code_received=$(curl -XGET -sSL -o /dev/null -w ''%{http_code}'' ${DB_HOST}:${DB_PORT}/health)
-                        if [ "${code_received}" = "200" ] ; then break ; fi
-                        sleep 5
-                        (( counter+=5 ))
-                        print_warn "InfluxDB Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)"
-                    done
-                ;;
-            esac
-        ;;
-        "mongo" )
-            if [ "${MONGO_HOST_TYPE,,}" != "atlas" ] ; then
+                fi
+            ;;
+            "mysql" )
+                counter=0
+                export MYSQL_PWD=${DB_PASS}
+                while ! (mysqladmin -u"${DB_USER}" -P"${DB_PORT}" -h"${DB_HOST}" status > /dev/null 2>&1) ; do
+                    sleep 5
+                    (( counter+=5 ))
+                    print_warn "MySQL/MariaDB Server '${DB_HOST}' is not accessible, retrying.. (${counter} seconds so far)"
+                done
+            ;;
+            "mssql" )
                 counter=0
                 while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do
                     sleep 5
                     (( counter+=5 ))
-                    print_warn "Mongo Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)"
+                    print_warn "MSSQL Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)"
                 done
-            fi
-        ;;
-        "mysql" )
-            counter=0
-            export MYSQL_PWD=${DB_PASS}
-            while ! (mysqladmin -u"${DB_USER}" -P"${DB_PORT}" -h"${DB_HOST}" status > /dev/null 2>&1) ; do
-              sleep 5
-              (( counter+=5 ))
-              print_warn "MySQL/MariaDB Server '${DB_HOST}' is not accessible, retrying.. (${counter} seconds so far)"
-            done
-        ;;
-        "mssql" )
-            counter=0
-            while ! (nc -z ${DB_HOST} ${DB_PORT}) ; do
-                sleep 5
-                (( counter+=5 ))
-                print_warn "MSSQL Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)"
-            done
-        ;;
-        "pgsql" )
-            counter=0
-            export PGPASSWORD=${DB_PASS}
-            until pg_isready --dbname=${DB_NAME} --host=${DB_HOST} --port=${DB_PORT} --username=${DB_USER} -q
-            do
-                sleep 5
-                (( counter+=5 ))
-                print_warn "Postgres Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)"
-            done
-        ;;
-        "redis" )
-            counter=0
-            while ! (nc -z "${DB_HOST}" "${DB_PORT}") ; do
-                sleep 5
-                (( counter+=5 ))
-                print_warn "Redis Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)"
-            done
-        ;;
-        "sqlite3" )
-            if [[ ! -e "${DB_HOST}" ]]; then
-                print_error "File '${DB_HOST}' does not exist."
-                exit_code=2
-                exit $exit_code
-            elif [[ ! -f "${DB_HOST}" ]]; then
-                print_error "File '${DB_HOST}' is not a file."
-                exit_code=2
-                exit $exit_code
-            elif [[ ! -r "${DB_HOST}" ]]; then
-                print_error "File '${DB_HOST}' is not readable."
-                exit_code=2
-                exit $exit_code
-            fi
-        ;;
-    esac
+            ;;
+            "pgsql" )
+                counter=0
+                export PGPASSWORD=${DB_PASS}
+                until pg_isready --dbname=${DB_NAME} --host=${DB_HOST} --port=${DB_PORT} --username=${DB_USER} -q
+                do
+                    sleep 5
+                    (( counter+=5 ))
+                    print_warn "Postgres Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)"
+                done
+            ;;
+            "redis" )
+                counter=0
+                while ! (nc -z "${DB_HOST}" "${DB_PORT}") ; do
+                    sleep 5
+                    (( counter+=5 ))
+                    print_warn "Redis Host '${DB_HOST}' is not accessible, retrying.. ($counter seconds so far)"
+                done
+            ;;
+            "sqlite3" )
+                if [[ ! -e "${DB_HOST}" ]]; then
+                    print_error "File '${DB_HOST}' does not exist."
+                    exit_code=2
+                    exit $exit_code
+                elif [[ ! -f "${DB_HOST}" ]]; then
+                    print_error "File '${DB_HOST}' is not a file."
+                    exit_code=2
+                    exit $exit_code
+                elif [[ ! -r "${DB_HOST}" ]]; then
+                    print_error "File '${DB_HOST}' is not readable."
+                    exit_code=2
+                    exit $exit_code
+                fi
+            ;;
+        esac
+    fi
 }
 
 check_exit_code() {
@@ -455,12 +461,16 @@ cleanup_old_data() {
         if [ "${master_exit_code}" != 1 ]; then
             case "${BACKUP_LOCATION,,}" in
                 "file" | "filesystem" )
-                    print_info "Cleaning up old backups"
+                    print_info "Cleaning up old backups on filesystem"
                     mkdir -p "${DB_DUMP_TARGET}"
                     find "${DB_DUMP_TARGET}"/  -mmin +"${DB_CLEANUP_TIME}" -iname "*" -exec rm {} \;
+                    if [ "${BACKUP_LOCATION,,}" = "blobxfer" ] ; then
+                        print_info "Syncing changes via blobxfer"
+                        silent blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET} --delete --delete-only
+                    fi
                 ;;
                 "s3" | "minio" )
-                    print_info "Cleaning up old backups"
+                    print_info "Cleaning up old backups on S3 storage"
                     aws ${PARAM_AWS_ENDPOINT_URL} s3 ls s3://${S3_BUCKET}/${S3_PATH}/ ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS} | grep " DIR " -v | grep " PRE " -v | while read -r s3_file; do
                         s3_createdate=$(echo $s3_file | awk {'print $1" "$2'})
                         s3_createdate=$(date -d "$s3_createdate" "+%s")
@@ -633,6 +643,18 @@ move_dbbackup() {
                     silent aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_LOCATION}/*.${checksum_extension} s3://${S3_BUCKET}/${S3_PATH}/ ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS}
                 fi
 
+                rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}"
+                rm -rf "${TEMP_LOCATION}"/"${target}"
+            ;;
+            "blobxfer" )
+                print_info "Moving backup to S3 Bucket with blobxfer"
+
+                mkdir -p "${DB_DUMP_TARGET}"
+                mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/
+                mv "${TEMP_LOCATION}"/"${target}" "${DB_DUMP_TARGET}"/"${target}"
+
+                silent blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET}
+
                 rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}"
                 rm -rf "${TEMP_LOCATION}"/"${target}"
             ;;

install/etc/cont-init.d/10-db-backup

@@ -6,6 +6,7 @@ prepare_service 03-monitoring
 PROCESS_NAME="db-backup"
 output_off
 
+bootstrap_variables
 sanity_test
 setup_mode
 create_zabbix dbbackup