From 5dafa69439a8f50ab959dd2d376e3e7a1a329d95 Mon Sep 17 00:00:00 2001 From: Dave Conroy Date: Fri, 3 Nov 2023 19:46:46 -0700 Subject: [PATCH] feat - add file encryption --- Dockerfile | 1 + README.md | 41 ++++++++++++---- install/assets/defaults/10-db-backup | 1 + install/assets/functions/10-db-backup | 71 ++++++++++++++++++++++++++- zabbix_templates/db_backup.json | 19 +++++++ 5 files changed, 121 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index a83ad3f..098bc10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,6 +42,7 @@ RUN source /assets/functions/00-container && \ bzip2 \ coreutils \ gpg \ + gpg-agent \ groff \ libarchive \ mariadb-client \ diff --git a/README.md b/README.md index 777d024..4709758 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Backs up CouchDB, InfluxDB, MySQL, Microsoft SQL, MongoDB, Postgres, Redis serve - backup all to separate files or one singular file - checksum support choose to have an MD5 or SHA1 hash generated after backup for verification - compression support (none, gz, bz, xz, zstd) +- encryption support (passphrase and public key) - Zabbix Metrics support - Hooks to execute pre and post backup job for customization purposes - Companion script to aid in restores @@ -52,6 +53,7 @@ Backs up CouchDB, InfluxDB, MySQL, Microsoft SQL, MongoDB, Postgres, Redis serve - [Container Options](#container-options) - [Job Defaults](#job-defaults) - [Compression Options](#compression-options) + - [Encryption Options](#encryption-options) - [Scheduling Options](#scheduling-options) - [Default Database Options](#default-database-options) - [CouchDB](#couchdb) @@ -71,6 +73,7 @@ Backs up CouchDB, InfluxDB, MySQL, Microsoft SQL, MongoDB, Postgres, Redis serve - [Post backup](#post-backup) - [Job Backup Options](#job-backup-options) - [Compression Options](#compression-options-1) + - [Encryption Options](#encryption-options-1) - [Scheduling Options](#scheduling-options-1) - [Specific Database Options](#specific-database-options) - [CouchDB](#couchdb-1) @@ -192,7 +195,7 @@ If these are set and no other defaults or variables are set explicitly, they wil | `DEFAULT_BACKUP_LOCATION` | Backup to `FILESYSTEM`, `blobxfer` or `S3` compatible services like S3, Minio, Wasabi | `FILESYSTEM` | | `DEFAULT_CHECKSUM` | Either `MD5` or `SHA1` or `NONE` | `MD5` | | `DEFAULT_LOG_LEVEL` | Log output on screen and in files `INFO` `NOTICE` `ERROR` `WARN` `DEBUG` | `notice` | -| `DEFAULT_RESOURCE_OPTIMIZED` | Perform operations at a lower priority to the CPU scheduler | `FALSE` | +| `DEFAULT_RESOURCE_OPTIMIZED` | Perform operations at a lower priority to the CPU scheduler | `FALSE` | | `DEFAULT_SKIP_AVAILABILITY_CHECK` | Before backing up - skip connectivity check | `FALSE` | ##### Compression Options @@ -206,6 +209,15 @@ If these are set and no other defaults or variables are set explicitly, they wil | `DEFAULT_ENABLE_PARALLEL_COMPRESSION` | Use multiple cores when compressing backups `TRUE` or `FALSE` | `TRUE` | | `DEFAULT_PARALLEL_COMPRESSION_THREADS` | Maximum amount of threads to use when compressing - Integer value e.g. `8` | `autodetected` | +##### Encryption Options + +| Variable | Description | Default | +| ---------------------------- | ------------------------------------------- | ------- | +| `DEFAULT_ENCRYPT` | Encrypt file after backing up with GPG | `FALSE` | +| `DEFAULT_ENCRYPT_PASSPHRASE` | Passphrase to encrypt file with GPG | | +| *or* | | | +| `DEFAULT_ENCRYPT_PUBKEY` | Path of public key to encrypt file with GPG | | + ##### Scheduling Options | Variable | Description | Default | @@ -438,7 +450,7 @@ Otherwise, override them per backup job. Additional backup jobs can be scheduled | `DB01_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | | `DB01_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | | `DB01_LOG_LEVEL` | Log output on screen and in files `INFO` `NOTICE` `ERROR` `WARN` `DEBUG` | `debug` | -| `DB01_RESOURCE_OPTIMIZED` | Perform operations at a lower priority to the CPU scheduler | `FALSE` | +| `DB01_RESOURCE_OPTIMIZED` | Perform operations at a lower priority to the CPU scheduler | `FALSE` | | `DB01_SKIP_AVAILABILITY_CHECK` | Before backing up - skip connectivity check | `FALSE` | ##### Compression Options @@ -452,6 +464,15 @@ Otherwise, override them per backup job. Additional backup jobs can be scheduled | `DB01_ENABLE_PARALLEL_COMPRESSION` | Use multiple cores when compressing backups `TRUE` or `FALSE` | `TRUE` | | `DB01_PARALLEL_COMPRESSION_THREADS` | Maximum amount of threads to use when compressing - Integer value e.g. `8` | `autodetected` | +##### Encryption Options + +| Variable | Description | Default | +| ------------------------- | ------------------------------------------- | ------- | +| `DB01_ENCRYPT` | Encrypt file after backing up with GPG | `FALSE` | +| `DB01_ENCRYPT_PASSPHRASE` | Passphrase to encrypt file with GPG | | +| *or* | | | +| `DB01_ENCRYPT_PUBKEY` | Path of public key to encrypt file with GPG | | + ##### Scheduling Options | Variable | Description | Default | @@ -494,9 +515,9 @@ Otherwise, override them per backup job. Additional backup jobs can be scheduled | Variable | Description | Default | `_FILE` | | ------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------- | ------- | -| `DB01_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | -| `DB01_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | -| `DB01_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | +| `DB01_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | || +| `DB01_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | || +| `DB01_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | || | `DB01_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. | | | | | Backup multiple by separating with commas eg `db1,db2` | | x | | `DB01_NAME_EXCLUDE` | If using `ALL` - use this as to exclude databases separated via commas from being backed up | | x | @@ -534,9 +555,9 @@ Otherwise, override them per backup job. Additional backup jobs can be scheduled | Variable | Description | Default | `_FILE` | | ----------------------------- | --------------------------------------------------------------------------------------------------------- | ------- | ------- | | `DB01_AUTH` | (Optional) Authentication Database | | | -| `DB01_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | -| `DB01_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | -| `DB01_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | +| `DB01_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | | +| `DB01_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | | +| `DB01_EXTRA_ENUMERATION_OPTS` | Pass extra arguments to the database enumeration command only, add them here e.g. `--extra-command` | | | | `DB01_NAME` | Schema Name e.g. `database` or `ALL` to backup all databases the user has access to. | | | | | Backup multiple by separating with commas eg `db1,db2` | | x | | `DB01_PORT` | PostgreSQL Port | `5432` | x | @@ -545,8 +566,8 @@ Otherwise, override them per backup job. Additional backup jobs can be scheduled | Variable | Description | Default | `_FILE` | | ------------------------ | --------------------------------------------------------------------------------------------------------- | ------- | ------- | -| `DB01_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | | -| `DB01_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | | +| `DB01_EXTRA_OPTS` | Pass extra arguments to the backup and database enumeration command, add them here e.g. `--extra-command` | || +| `DB01_EXTRA_BACKUP_OPTS` | Pass extra arguments to the backup command only, add them here e.g. `--extra-command` | || | `DB01_PORT` | Redis Port | `6379` | x | ###### SQLite diff --git a/install/assets/defaults/10-db-backup b/install/assets/defaults/10-db-backup index 2b28d67..86bb12e 100644 --- a/install/assets/defaults/10-db-backup +++ b/install/assets/defaults/10-db-backup @@ -11,6 +11,7 @@ DEFAULT_COMPRESSION=${DEFAULT_COMPRESSION:-"ZSTD"} DEFAULT_COMPRESSION_LEVEL=${DEFAULT_COMPRESSION_LEVEL:-"3"} DEFAULT_CREATE_LATEST_SYMLINK=${DEFAULT_CREATE_LATEST_SYMLINK:-"TRUE"} DEFAULT_ENABLE_PARALLEL_COMPRESSION=${DEFAULT_ENABLE_PARALLEL_COMPRESSION:-"TRUE"} +DEFAULT_ENCRYPT=${DEFAULT_ENCRYPT:-"FALSE"} DEFAULT_FILESYSTEM_PATH=${DEFAULT_FILESYSTEM_PATH:-"/backup"} DEFAULT_FILESYSTEM_PERMISSION=${DEFAULT_FILESYSTEM_PERMISSION:-"700"} DEFAULT_FILESYSTEM_ARCHIVE_PATH=${DEFAULT_FILESYSTEM_ARCHIVE_PATH:-"${DEFAULT_FILESYSTEM_PATH}/archive/"} diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 9086a85..e1dfe2a 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -44,6 +44,8 @@ bootstrap_variables() { DEFAULT_NAME_EXCLUDE \ DEFAULT_USER \ DEFAULT_PASS \ + DEFAULT_ENCRYPT_PASSPHRASE \ + DEFAULT_ENCRYPT_PUBKEY \ DEFAULT_MONGO_CUSTOM_URI \ DEFAULT_MYSQL_TLS_CA_FILE \ DEFAULT_MYSQL_TLS_CERT_FILE \ @@ -64,6 +66,14 @@ bootstrap_variables() { DB"${backup_instance_number}"_TYPE \ DB"${backup_instance_number}"_HOST \ DB"${backup_instance_number}"_PORT \ + DB"${backup_instance_number}"_NAME \ + DB"${backup_instance_number}"_NAME_EXCLUDE \ + DB"${backup_instance_number}"_USER \ + DB"${backup_instance_number}"_PASS \ + DB"${backup_instance_number}"_ENCRYPT_PASSPHRASE \ + DB"${backup_instance_number}"_ENCRYPT_PUBKEY \ + DB"${backup_instance_number}"_MONGO_CUSTOM_URI \ + DB"${backup_instance_number}"_MYSQL_TLS_CA_FILE \ DB"${backup_instance_number}"_MYSQL_TLS_CERT_FILE \ DB"${backup_instance_number}"_MYSQL_TLS_KEY_FILE \ DB"${backup_instance_number}"_S3_BUCKET \ @@ -163,6 +173,9 @@ bootstrap_variables() { transform_backup_instance_variable "${backup_instance_number}" COMPRESSION_LEVEL backup_job_compression_level transform_backup_instance_variable "${backup_instance_number}" CREATE_LATEST_SYMLINK backup_job_create_latest_symlink transform_backup_instance_variable "${backup_instance_number}" ENABLE_PARALLEL_COMPRESSION backup_job_parallel_compression + transform_backup_instance_variable "${backup_instance_number}" ENCRYPT backup_job_encrypt + transform_backup_instance_variable "${backup_instance_number}" ENCRYPT_PASSPHRASE backup_job_encrypt_passphrase + transform_backup_instance_variable "${backup_instance_number}" ENCRYPT_PUBKEY backup_job_encrypt_pubkey transform_backup_instance_variable "${backup_instance_number}" EXTRA_DUMP_OPTS backup_job_extra_dump_opts transform_backup_instance_variable "${backup_instance_number}" EXTRA_ENUMERATION_OPTS backup_job_extra_enumeration_opts transform_backup_instance_variable "${backup_instance_number}" EXTRA_OPTS backup_job_extra_opts @@ -359,6 +372,7 @@ backup_couch() { run_as_user curl -sSL -X GET ${backup_job_db_host}:${backup_job_db_port}/${backup_job_db_name}/_all_docs?include_docs=true | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${target}" > /dev/null exit_code=$? check_exit_code backup $target + file_encryption generate_checksum move_dbbackup check_exit_code move $target @@ -390,6 +404,7 @@ backup_influx() { run_as_user tar cf - "${TEMP_PATH}"/"${target_dir}" | ${dir_compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${target_dir}".tar"${extension}" > /dev/null target=influx_${db}_${backup_job_db_host#*//}_${now}.tar${extension} ltarget=influx_${db}_${backup_job_db_host#*//} + file_encryption generate_checksum move_dbbackup check_exit_code move $target_dir @@ -411,6 +426,7 @@ backup_influx() { create_archive target=influx2_${db}_${backup_job_db_host#*//}_${now}.tar${extension} ltarget=influx2_${db}_${backup_job_db_host#*//} + file_encryption generate_checksum move_dbbackup check_exit_code move $target_dir @@ -442,6 +458,7 @@ backup_mongo() { silent run_as_user ${nice} mongodump --archive=${TEMP_PATH}/${target} ${mongo_compression} ${mongo_backup_parameter} exit_code=$? check_exit_code backup $target + file_encryption generate_checksum move_dbbackup check_exit_code move $target @@ -460,6 +477,7 @@ backup_mssql() { silent run_as_user ${nice} /opt/mssql-tools18/bin/sqlcmd -C -S ${backup_job_db_host}\,${backup_job_db_port} -U ${backup_job_db_user} -P ${backup_job_db_pass} -Q "BACKUP DATABASE [${backup_job_db_name}] TO DISK = N'${TEMP_PATH}/${target}' WITH NOFORMAT, NOINIT, NAME = '${backup_job_db_name}-full', SKIP, NOREWIND, NOUNLOAD, STATS = 10" exit_code=$? check_exit_code backup $target + file_encryption generate_checksum move_dbbackup check_exit_code move $target @@ -475,12 +493,13 @@ backup_mssql() { silent run_as_user ${nice} /opt/mssql-tools18/bin/sqlcmd -C -S ${backup_job_db_host}\,${backup_job_db_port} -U ${backup_job_db_user} -P ${backup_job_db_pass} -Q "BACKUP LOG [${backup_job_db_name}] TO DISK = N'${TEMP_PATH}/${target}' WITH NOFORMAT, NOINIT, NAME = '${backup_job_db_name}-log', SKIP, NOREWIND, NOUNLOAD, STATS = 10" exit_code=$? check_exit_code backup $target + file_encryption generate_checksum move_dbbackup check_exit_code move $target post_dbbackup "${backup_job_db_name}" ;; - }; + esac } backup_mysql() { @@ -518,6 +537,7 @@ backup_mysql() { run_as_user ${nice} mysqldump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_dump_opts} $db | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${target}" > /dev/null exit_code=$? check_exit_code backup $target + file_encryption generate_checksum move_dbbackup check_exit_code move $target @@ -534,6 +554,7 @@ backup_mysql() { run_as_user ${nice} mysqldump --max-allowed-packet=${backup_job_mysql_max_allowed_packet} -h ${backup_job_db_host} -P ${backup_job_db_port} -u${backup_job_db_user} ${single_transaction} ${stored_procedures} ${mysql_tls_args} ${backup_job_extra_opts} ${backup_job_extra_dump_opts} --databases $(echo ${db_names} | xargs) | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${target}" > /dev/null exit_code=$? check_exit_code backup $target + file_encryption generate_checksum move_dbbackup check_exit_code move $target @@ -575,6 +596,7 @@ backup_pgsql() { run_as_user ${nice} pg_dump -h ${backup_job_db_host} -p ${backup_job_db_port} -U ${backup_job_db_user} $db ${backup_job_extra_opts} ${backup_job_extra_dump_opts} | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${target}" > /dev/null exit_code=$? check_exit_code backup $target + file_encryption generate_checksum move_dbbackup check_exit_code move $target @@ -582,13 +604,13 @@ backup_pgsql() { done prepare_dbbackup target=pgsql_globals_${backup_job_db_host,,}_${now}.sql - ltarget=pgsql_globals_${backup_job_db_host,,} compression pre_dbbackup "globals" print_notice "Dumping PostgresSQL globals: with 'pg_dumpall -g' ${compression_string}" run_as_user ${nice} pg_dumpall -h ${backup_job_db_host} -U ${backup_job_db_user} -p ${backup_job_db_port} -g ${backup_job_extra_opts} ${backup_job_extra_dump_opts} | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${target}" > /dev/null exit_code=$? check_exit_code $target + file_encryption generate_checksum move_dbbackup check_exit_code move $target @@ -612,6 +634,7 @@ backup_pgsql() { run_as_user ${nice} pg_dumpall -h ${backup_job_db_host} -p ${backup_job_db_port} -U ${backup_job_db_user} ${pgexclude_arg} ${backup_job_extra_opts} ${backup_job_extra_dump_opts} | ${compress_cmd} | run_as_user tee "${TEMP_PATH}"/"${target}" > /dev/null exit_code=$? check_exit_code backup $target + file_encryption generate_checksum move_dbbackup check_exit_code move $target @@ -644,6 +667,7 @@ backup_redis() { pre_dbbackup all run_as_user ${compress_cmd} "${TEMP_PATH}/${target_original}" check_exit_code backup $target + file_encryption generate_checksum move_dbbackup check_exit_code move $target @@ -663,6 +687,7 @@ backup_sqlite3() { exit_code=$? check_exit_code backup $target run_as_user ${nice} cat "${TEMP_PATH}"/backup.sqlite3 | ${dir_compress_cmd} | run_as_user tee "${TEMP_PATH}/${target}" > /dev/null + file_encryption generate_checksum move_dbbackup check_exit_code move $target @@ -982,6 +1007,46 @@ db_backup_container_init() { touch /tmp/.container/db-backup-backups } +file_encryption() { + if var_true "${backup_job_encrypt}" ; then + if [ "${exit_code}" = "0" ] ; then + print_debug "Encrypting" + output_off + if [ -n "${backup_job_encrypt_passphrase}" ] && [ -n "${backup_job_encrypt_pubkey}" ]; then + print_error "Can't encrypt as both ENCRYPT_PASSPHRASE and ENCRYPT_PUBKEY exist!" + return + elif [ -n "${backup_job_encrypt_passphrase}" ] && [ -z "${backup_job_encrypt_pubkey}" ]; then + print_notice "Encrypting with GPG Passphrase" + encrypt_routines_start_time=$(date +'%s') + encrypt_tmp_dir=$(run_as_user mktemp -d) + echo "${backup_job_encrypt_passphrase}" | silent run_as_user ${nice} gpg --batch --home ${encrypt_tmp_dir} --yes --passphrase-fd 0 -c "${TEMP_PATH}"/"${target}" + rm -rf "${encrypt_tmp_dir}" + elif [ -z "${backup_job_encrypt_passphrase}" ] && [ -n "${backup_job_encrypt_pubkey}" ]; then + if [ -f "${backup_job_encrypt_pubkey}" ]; then + encrypt_routines_start_time=$(date +'%s') + print_notice "Encrypting with GPG Public Key" + encrypt_tmp_dir=$(run_as_user mktemp -d) + silent run_as_user ${nice} gpg --batch --yes --home "${encrypt_tmp_dir}" --recipient-file "${backup_job_encrypt_pubkey}" -c "${TEMP_PATH}"/"${target}" + rm -rf "${encrypt_tmp_dir}" + fi + fi + if [ -f "${TEMP_PATH}"/"${target}".gpg ]; then + rm -rf "${TEMP_PATH}"/"${target}" + target="${target}.gpg" + + encrypt_routines_finish_time=$(date +'%s') + encrypt_routines_total_time=$(echo $((encrypt_routines_finish_time-encrypt_routines_start_time))) + zabbix_encrypt_time=$(cat <