diff --git a/dev/debug_obackup.sh b/dev/debug_obackup.sh index 0aa7f9c..ac77539 100755 --- a/dev/debug_obackup.sh +++ b/dev/debug_obackup.sh @@ -1,21 +1,16 @@ #!/usr/bin/env bash -#TODO: test bad return of _GetDirectoriesSizeRemote -#TODO(critical): test RotateCopies remote -#TODO(high): check paths with spaces (again) - fails -#TODO(low): doc obackup-rerun is minimal 1 and not 0 - ###### Remote push/pull (or local) backup script for files & databases PROGRAM="obackup" AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" PROGRAM_VERSION=2.1-dev -PROGRAM_BUILD=2016081803 +PROGRAM_BUILD=2016081806 IS_STABLE=no #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016081801 +## FUNC_BUILD=2016081802 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode @@ -589,12 +584,11 @@ function WaitForTaskCompletion { local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. local caller_name="${4}" # Who called this function - local exit_on_error="${5:-false}" # Should the function exit on subprocess errors - local counting="${6:-true}" # Count time since function launch if true, script launch if false - local keep_logging="${7:-0}" # Log a standby message every X seconds. Set to zero to disable logging + local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false + local keep_logging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG - __CheckArguments 7 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 6 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once local log_ttime=0 # local time instance for comparaison @@ -681,12 +675,7 @@ function WaitForTaskCompletion { done Logger "${FUNCNAME[0]} ended for [$caller_name] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG - if [ $exit_on_error == true ] && [ $errorcount -gt 0 ]; then - Logger "Stopping execution." "CRITICAL" - exit 1337 - else - return $errorcount - fi + return $errorcount } function CleanUp { @@ -1437,7 +1426,7 @@ function CheckCurrentConfig { fi # Check all variables that should contain "yes" or "no" - declare -a yes_no_vars=(SQL_BACKUP FILE_BACKUP ENCRYPTION CREATE_DIRS KEEP_ABSOLUTE_PATHS GET_BACKUP_SIZE SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING SUDO_EXEC DATABASES_ALL PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS RSYNC_COMPRESS PARTIAL DELETE_VANISHED_FILES DELTA_COPIES ROTATE_SQL_BACKUPS ROTATE_FILE_BACKUPS STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR) + declare -a yes_no_vars=(SQL_BACKUP FILE_BACKUP ENCRYPTION CREATE_DIRS KEEP_ABSOLUTE_PATHS GET_BACKUP_SIZE SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING SUDO_EXEC DATABASES_ALL PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS RSYNC_COMPRESS PARTIAL DELETE_VANISHED_FILES DELTA_COPIES ROTATE_SQL_BACKUPS ROTATE_FILE_BACKUPS STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR) for i in "${yes_no_vars[@]}"; do test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it with the update script if using and old version.\" \"CRITICAL\"; exit 1; fi" eval "$test" @@ -1487,7 +1476,7 @@ function _ListDatabasesLocal { sql_cmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2>&1" Logger "cmd: $sql_cmd" "DEBUG" eval "$sql_cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" else @@ -1510,7 +1499,7 @@ function _ListDatabasesRemote { sql_cmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID\" 2>&1" Logger "cmd: $sql_cmd" "DEBUG" eval "$sql_cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" else @@ -1615,7 +1604,7 @@ function _ListRecursiveBackupDirectoriesLocal { cmd="$FIND_CMD -L $directory/ -mindepth 1 -maxdepth 1 -type d >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Could not enumerate directories in [$directory]." "ERROR" if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then @@ -1645,7 +1634,7 @@ function _ListRecursiveBackupDirectoriesRemote { cmd=$SSH_CMD' "'$COMMAND_SUDO' '$REMOTE_FIND_CMD' -L '$directory'/ -mindepth 1 -maxdepth 1 -type d" >> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Could not enumerate directories in [$directory]." "ERROR" if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then @@ -1742,7 +1731,7 @@ function _GetDirectoriesSizeLocal { cmd="du -cs $dir_list | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING # $cmd will return 0 even if some errors found, so we need to check if there is an error output if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then Logger "Could not get files size for some or all directories." "ERROR" @@ -1773,7 +1762,7 @@ function _GetDirectoriesSizeRemote { cmd=$SSH_CMD' '$COMMAND_SUDO' du -cs '$dir_list' | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING # $cmd will return 0 even if some errors found, so we need to check if there is an error output if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then Logger "Could not get files size for some or all directories." "ERROR" @@ -1837,7 +1826,7 @@ function _CreateDirectoryRemote { cmd=$SSH_CMD' "if ! [ -d \"'$dir_to_create'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$dir_to_create'\"; fi" > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot create remote directory [$dir_to_create]." "CRITICAL" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" @@ -1911,7 +1900,7 @@ function GetDiskSpaceRemote { cmd=$SSH_CMD' "if [ -d \"'$path_to_check'\" ]; then '$COMMAND_SUDO' df -P \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then DISK_SPACE=0 Logger "Cannot get disk space in [$path_to_check] on remote system." "ERROR" @@ -2028,7 +2017,7 @@ function _BackupDatabaseLocalToLocal { Logger "cmd: $dry_sql_cmd" "DEBUG" eval "$dry_sql_cmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -2061,7 +2050,7 @@ function _BackupDatabaseLocalToRemote { Logger "cmd: $dry_sql_cmd" "DEBUG" eval "$dry_sql_cmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -2094,7 +2083,7 @@ function _BackupDatabaseRemoteToLocal { Logger "cmd: $dry_sql_cmd" "DEBUG" eval "$dry_sql_cmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -2141,11 +2130,21 @@ function BackupDatabases { do Logger "Backing up database [$database]." "NOTICE" BackupDatabase $database & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING CheckTotalExecutionTime done } +function EncryptFiles { + local filePath="${1}" # Path of files to encrypt + local tmpPath="${2}" + + __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + #crypt_cmd source temp + # Send files to remote, rotate & copy +} + function Rsync { local backup_directory="${1}" # Which directory to backup local is_recursive="${2}" # Backup only files at toplevel of directory @@ -2189,7 +2188,7 @@ function Rsync { Logger "cmd: $rsync_cmd" "DEBUG" eval "$rsync_cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Failed to backup [$backup_directory] to [$file_storage_path]." "ERROR" Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" @@ -2230,7 +2229,7 @@ function Duplicity { Logger "cmd: $duplicity_cmd" "DEBUG" eval "$duplicity_cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Failed to backup [$backup_directory] to [$file_storage_path]." "ERROR" Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" @@ -2306,7 +2305,6 @@ function _RotateBackupsLocal { local cmd local path - #TODO(low): check if mindepth / maxdepth is compatible BSD & MSYS find "$backup_path" -mindepth 1 -maxdepth 1 ! -iname "*.$PROGRAM.*" -print0 | while IFS= read -r -d $'\0' backup; do copy=$rotate_copies while [ $copy -gt 1 ]; do @@ -2316,7 +2314,7 @@ function _RotateBackupsLocal { cmd="rm -rf \"$path\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot delete oldest copy [$path]." "ERROR" fi @@ -2328,7 +2326,7 @@ function _RotateBackupsLocal { cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR" fi @@ -2342,7 +2340,7 @@ function _RotateBackupsLocal { cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -2351,7 +2349,7 @@ function _RotateBackupsLocal { cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -2360,7 +2358,7 @@ function _RotateBackupsLocal { cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -2474,7 +2472,7 @@ function _RotateBackupsRemoteSSH { ENDSSH - WaitForTaskCompletion $! 1800 0 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 1800 0 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Could not rotate backups in [$backup_path]." "ERROR" Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" diff --git a/obackup.sh b/obackup.sh index a2b14c4..65a4d94 100755 --- a/obackup.sh +++ b/obackup.sh @@ -1,21 +1,16 @@ #!/usr/bin/env bash -#TODO: test bad return of _GetDirectoriesSizeRemote -#TODO(critical): test RotateCopies remote -#TODO(high): check paths with spaces (again) - fails -#TODO(low): doc obackup-rerun is minimal 1 and not 0 - ###### Remote push/pull (or local) backup script for files & databases PROGRAM="obackup" AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" PROGRAM_VERSION=2.1-dev -PROGRAM_BUILD=2016081803 +PROGRAM_BUILD=2016081806 IS_STABLE=no #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016081801 +## FUNC_BUILD=2016081802 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode @@ -572,9 +567,8 @@ function WaitForTaskCompletion { local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. local caller_name="${4}" # Who called this function - local exit_on_error="${5:-false}" # Should the function exit on subprocess errors - local counting="${6:-true}" # Count time since function launch if true, script launch if false - local keep_logging="${7:-0}" # Log a standby message every X seconds. Set to zero to disable logging + local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false + local keep_logging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once @@ -661,12 +655,7 @@ function WaitForTaskCompletion { sleep $SLEEP_TIME done - if [ $exit_on_error == true ] && [ $errorcount -gt 0 ]; then - Logger "Stopping execution." "CRITICAL" - exit 1337 - else - return $errorcount - fi + return $errorcount } function CleanUp { @@ -1361,7 +1350,7 @@ function CheckCurrentConfig { fi # Check all variables that should contain "yes" or "no" - declare -a yes_no_vars=(SQL_BACKUP FILE_BACKUP ENCRYPTION CREATE_DIRS KEEP_ABSOLUTE_PATHS GET_BACKUP_SIZE SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING SUDO_EXEC DATABASES_ALL PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS RSYNC_COMPRESS PARTIAL DELETE_VANISHED_FILES DELTA_COPIES ROTATE_SQL_BACKUPS ROTATE_FILE_BACKUPS STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR) + declare -a yes_no_vars=(SQL_BACKUP FILE_BACKUP ENCRYPTION CREATE_DIRS KEEP_ABSOLUTE_PATHS GET_BACKUP_SIZE SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING SUDO_EXEC DATABASES_ALL PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS RSYNC_COMPRESS PARTIAL DELETE_VANISHED_FILES DELTA_COPIES ROTATE_SQL_BACKUPS ROTATE_FILE_BACKUPS STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR) for i in "${yes_no_vars[@]}"; do test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it with the update script if using and old version.\" \"CRITICAL\"; exit 1; fi" eval "$test" @@ -1409,7 +1398,7 @@ function _ListDatabasesLocal { sql_cmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2>&1" Logger "cmd: $sql_cmd" "DEBUG" eval "$sql_cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" else @@ -1431,7 +1420,7 @@ function _ListDatabasesRemote { sql_cmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID\" 2>&1" Logger "cmd: $sql_cmd" "DEBUG" eval "$sql_cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" else @@ -1534,7 +1523,7 @@ function _ListRecursiveBackupDirectoriesLocal { cmd="$FIND_CMD -L $directory/ -mindepth 1 -maxdepth 1 -type d >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Could not enumerate directories in [$directory]." "ERROR" if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then @@ -1563,7 +1552,7 @@ function _ListRecursiveBackupDirectoriesRemote { cmd=$SSH_CMD' "'$COMMAND_SUDO' '$REMOTE_FIND_CMD' -L '$directory'/ -mindepth 1 -maxdepth 1 -type d" >> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Could not enumerate directories in [$directory]." "ERROR" if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then @@ -1658,7 +1647,7 @@ function _GetDirectoriesSizeLocal { cmd="du -cs $dir_list | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING # $cmd will return 0 even if some errors found, so we need to check if there is an error output if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then Logger "Could not get files size for some or all directories." "ERROR" @@ -1688,7 +1677,7 @@ function _GetDirectoriesSizeRemote { cmd=$SSH_CMD' '$COMMAND_SUDO' du -cs '$dir_list' | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING # $cmd will return 0 even if some errors found, so we need to check if there is an error output if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then Logger "Could not get files size for some or all directories." "ERROR" @@ -1749,7 +1738,7 @@ function _CreateDirectoryRemote { cmd=$SSH_CMD' "if ! [ -d \"'$dir_to_create'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$dir_to_create'\"; fi" > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot create remote directory [$dir_to_create]." "CRITICAL" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" @@ -1820,7 +1809,7 @@ function GetDiskSpaceRemote { cmd=$SSH_CMD' "if [ -d \"'$path_to_check'\" ]; then '$COMMAND_SUDO' df -P \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then DISK_SPACE=0 Logger "Cannot get disk space in [$path_to_check] on remote system." "ERROR" @@ -1935,7 +1924,7 @@ function _BackupDatabaseLocalToLocal { Logger "cmd: $dry_sql_cmd" "DEBUG" eval "$dry_sql_cmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -1967,7 +1956,7 @@ function _BackupDatabaseLocalToRemote { Logger "cmd: $dry_sql_cmd" "DEBUG" eval "$dry_sql_cmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -1999,7 +1988,7 @@ function _BackupDatabaseRemoteToLocal { Logger "cmd: $dry_sql_cmd" "DEBUG" eval "$dry_sql_cmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -2044,11 +2033,20 @@ function BackupDatabases { do Logger "Backing up database [$database]." "NOTICE" BackupDatabase $database & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING CheckTotalExecutionTime done } +function EncryptFiles { + local filePath="${1}" # Path of files to encrypt + local tmpPath="${2}" + + + #crypt_cmd source temp + # Send files to remote, rotate & copy +} + function Rsync { local backup_directory="${1}" # Which directory to backup local is_recursive="${2}" # Backup only files at toplevel of directory @@ -2091,7 +2089,7 @@ function Rsync { Logger "cmd: $rsync_cmd" "DEBUG" eval "$rsync_cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Failed to backup [$backup_directory] to [$file_storage_path]." "ERROR" Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" @@ -2131,7 +2129,7 @@ function Duplicity { Logger "cmd: $duplicity_cmd" "DEBUG" eval "$duplicity_cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Failed to backup [$backup_directory] to [$file_storage_path]." "ERROR" Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" @@ -2204,7 +2202,6 @@ function _RotateBackupsLocal { local cmd local path - #TODO(low): check if mindepth / maxdepth is compatible BSD & MSYS find "$backup_path" -mindepth 1 -maxdepth 1 ! -iname "*.$PROGRAM.*" -print0 | while IFS= read -r -d $'\0' backup; do copy=$rotate_copies while [ $copy -gt 1 ]; do @@ -2214,7 +2211,7 @@ function _RotateBackupsLocal { cmd="rm -rf \"$path\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot delete oldest copy [$path]." "ERROR" fi @@ -2226,7 +2223,7 @@ function _RotateBackupsLocal { cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR" fi @@ -2240,7 +2237,7 @@ function _RotateBackupsLocal { cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -2249,7 +2246,7 @@ function _RotateBackupsLocal { cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -2258,7 +2255,7 @@ function _RotateBackupsLocal { cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -2366,7 +2363,7 @@ function _RotateBackupsRemoteSSH { ENDSSH - WaitForTaskCompletion $! 1800 0 ${FUNCNAME[0]} false true $KEEP_LOGGING + WaitForTaskCompletion $! 1800 0 ${FUNCNAME[0]} true $KEEP_LOGGING if [ $? != 0 ]; then Logger "Could not rotate backups in [$backup_path]." "ERROR" Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"