From 31eaee7993b79b13d6471815a2ada391c0427568 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 19 Dec 2016 22:57:21 +0100 Subject: [PATCH] Updated multiple comments and remote sudo execs --- dev/debug_obackup.sh | 187 ++++++++++++++++++++----------------------- dev/n_obackup.sh | 58 ++++++++------ 2 files changed, 118 insertions(+), 127 deletions(-) diff --git a/dev/debug_obackup.sh b/dev/debug_obackup.sh index c9ed985..f4a57ea 100755 --- a/dev/debug_obackup.sh +++ b/dev/debug_obackup.sh @@ -9,34 +9,27 @@ PROGRAM="obackup" AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" PROGRAM_VERSION=2.1-dev -PROGRAM_BUILD=2016113001 +PROGRAM_BUILD=2016120401 IS_STABLE=no #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016112902 +_OFUNCTIONS_VERSION=2.0 +_OFUNCTIONS_BUILD=2016120401 ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: ## PROGRAM=program-name ## INSTANCE_ID=program-instance-name ## _DEBUG=yes/no -## _LOGGER_LOGGER_SILENT=true/false -## _LOGGER_LOGGER_VERBOSE=true/false +## _LOGGER_SILENT=true/false +## _LOGGER_VERBOSE=true/false ## _LOGGER_ERR_ONLY=true/false ## _LOGGER_PREFIX="date"/"time"/"" ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID -## META ISSUES -## -## Updated _LOGGER_STDERR -## Updated WaitForTaskCompletion syntax -## Updated ParallelExec syntax -## SendEmail WinNT10 & msys are two totally different beasts. Document in sync.conf and host_backup.conf - - if ! type "$BASH" > /dev/null; then echo "Please run this script only with bash shell. Tested on bash >= 3.2" exit 127 @@ -204,11 +197,11 @@ function Logger { _Logger "$prefix$value" "$prefix$value" return fi - elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG - if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG - _Logger "$prefix$value" "$prefix$value" #__WITH_PARANOIA_DEBUG - return #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG + elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG + if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG + _Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG + return #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG else _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" _Logger "Value was: $prefix$value" @@ -383,8 +376,7 @@ function SendEmail { local smtpUser="${9}" local smtpPassword="${10}" - # CheckArguments will report a warning that can be ignored if used in Windows with paranoia debug enabled - __CheckArguments 4 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 3-10 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG local mail_no_attachment= local attachment_command= @@ -423,7 +415,8 @@ function SendEmail { fi if type mutt > /dev/null 2>&1 ; then - echo "$message" | $(type -p mutt) -x -s "$subject" "$destinationMails" $attachment_command + # We need to replace spaces with comma in order for mutt to be able to process multiple destinations + echo "$message" | $(type -p mutt) -x -s "$subject" "${destinationMails// /,}" $attachment_command if [ $? != 0 ]; then Logger "Cannot send mail via $(type -p mutt) !!!" "WARN" else @@ -591,8 +584,8 @@ function joinString { } # Time control function for background processes, suitable for multiple synchronous processes -# Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 -# Warning: Don't imbricate this function into another run if you plan to use the global variable output +# Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 +# Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached # Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true false "${FUNCNAME[0]}" @@ -634,7 +627,9 @@ function WaitForTaskCompletion { IFS=';' read -a pidsArray <<< "$pids" pidCount=${#pidsArray[@]} - WAIT_FOR_TASK_COMPLETION="" + # Set global var default + eval "WAIT_FOR_TASK_COMPLETION_$callerName=\"\"" + eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=false" while [ ${#pidsArray[@]} -gt 0 ]; do newPidsArray=() @@ -681,6 +676,7 @@ function WaitForTaskCompletion { if [ $noErrorLog != true ]; then SendAlert true fi + eval "HARD_MAX_EXEC_TIME_REACHED_$callerName=true" return $errorcount fi @@ -699,10 +695,11 @@ function WaitForTaskCompletion { if [ $retval -ne 0 ]; then errorcount=$((errorcount+1)) Logger "${FUNCNAME[0]} called by [$callerName] finished monitoring [$pid] with exitcode [$retval]." "DEBUG" - if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then - WAIT_FOR_TASK_COMPLETION="$pid:$retval" + # Welcome to variable variable bash hell + if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$callerName\")" == "" ]; then + eval "WAIT_FOR_TASK_COMPLETION_$callerName=\"$pid:$retval\"" else - WAIT_FOR_TASK_COMPLETION=";$pid:$retval" + eval "WAIT_FOR_TASK_COMPLETION_$callerName=\";$pid:$retval\"" fi fi fi @@ -734,6 +731,7 @@ function WaitForTaskCompletion { # Returns the number of non zero exit codes from commands # Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets # Only 2 first arguments are mandatory +# Sets a global variable called HARD_MAX_EXEC_TIME_REACHED to true if hardMaxTime is reached function ParallelExec { local numberOfProcesses="${1}" # Number of simultaneous commands to run @@ -769,6 +767,8 @@ function ParallelExec { local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG + HARD_MAX_EXEC_TIME_REACHED=false + if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once fi @@ -828,10 +828,10 @@ function ParallelExec { done if [ $noErrorLog != true ]; then SendAlert true - else - # Return the number of commands that haven't run / finished run - return $(($commandCount - $counter + ${#pidsArray[@]})) fi + HARD_MAX_EXEC_TIME_REACHED=true + # Return the number of commands that haven't run / finished run + return $(($commandCount - $counter + ${#pidsArray[@]})) fi while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do @@ -1080,7 +1080,7 @@ function GetLocalOS { LOCAL_OS="BusyBox" ;; *) - if [ "$IGNORE_OS_TYPE" == "yes" ]; then #TODO(doc): Undocumented option + if [ "$IGNORE_OS_TYPE" == "yes" ]; then Logger "Running on unknown local OS [$localOsVar]." "WARN" return fi @@ -1370,15 +1370,13 @@ function __CheckArguments { IFS='-' read minArgs maxArgs <<< "$numberOfArguments" fi - if [ "$_PARANOIA_DEBUG" == "yes" ]; then - Logger "Entering function [$functionName]." "DEBUG" - fi + Logger "Entering function [$functionName]." "PARANOIA_DEBUG" if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then Logger "Function $functionName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments. see log file." "ERROR" Logger "Arguments passed: $argList" "ERROR" else - Logger "Arguments passed: $argList" "VERBOSE" + Logger "Arguments passed: $argList" "PARANOIA_DEBUG" fi fi } @@ -1398,13 +1396,13 @@ function RsyncPatternsAdd { while [ -n "$rest" ] do # Take the string until first occurence until $PATH_SEPARATOR_CHAR - str=${rest%%;*} #TODO: replace ; with $PATH_SEPARATOR_CHAR + str="${rest%%$PATH_SEPARATOR_CHAR*}" # Handle the last case if [ "$rest" = "${rest/$PATH_SEPARATOR_CHAR/}" ]; then rest= else # Cut everything before the first occurence of $PATH_SEPARATOR_CHAR - rest=${rest#*$PATH_SEPARATOR_CHAR} + rest="${rest#*$PATH_SEPARATOR_CHAR}" fi if [ "$RSYNC_PATTERNS" == "" ]; then RSYNC_PATTERNS="--"$patternType"=\"$str\"" @@ -1508,49 +1506,6 @@ function PreInit { if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then COMPRESSION_LEVEL=3 fi - - #TODO: Remote OS isn't defined yet - ## Busybox fix (Termux xz command doesn't support compression at all) - if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then - compressionString="" - if type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - else - compressionString=" -$COMPRESSION_LEVEL" - - if type xz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| xz -c$compressionString" - COMPRESSION_EXTENSION=.xz - elif type lzma > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| lzma -c$compressionString" - COMPRESSION_EXTENSION=.lzma - elif type pigz > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| pigz -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - elif type gzip > /dev/null 2>&1 - then - COMPRESSION_PROGRAM="| gzip -c$compressionString" - COMPRESSION_EXTENSION=.gz - # obackup specific - COMPRESSION_OPTIONS=--rsyncable - else - COMPRESSION_PROGRAM= - COMPRESSION_EXTENSION= - fi - fi - ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" } function PostInit { @@ -1573,7 +1528,7 @@ function PostInit { fi } -function InitLocalOSSettings { +function InitLocalOSDependingSettings { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG ## If running under Msys, some commands do not run the same way @@ -1609,7 +1564,7 @@ function InitLocalOSSettings { fi } -function InitRemoteOSSettings { +function InitRemoteOSDependingSettings { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG if [ "$REMOTE_OS" == "msys" ]; then @@ -1627,11 +1582,6 @@ function InitRemoteOSSettings { REMOTE_STAT_CTIME_MTIME_CMD="stat -c \\\"%n;%Z;%Y\\\"" fi -} - -function InitRsyncSettings { - __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - ## Set rsync default arguments RSYNC_ARGS="-rltD" if [ "$_DRYRUN" == true ]; then @@ -1693,6 +1643,48 @@ function InitRsyncSettings { else RSYNC_ARGS=$RSYNC_ARGS" --whole-file" fi + + ## Busybox fix (Termux xz command doesn't support compression at all) + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then + compressionString="" + if type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + else + compressionString=" -$COMPRESSION_LEVEL" + + if type xz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| xz -c$compressionString" + COMPRESSION_EXTENSION=.xz + elif type lzma > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| lzma -c$compressionString" + COMPRESSION_EXTENSION=.lzma + elif type pigz > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| pigz -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + elif type gzip > /dev/null 2>&1 + then + COMPRESSION_PROGRAM="| gzip -c$compressionString" + COMPRESSION_EXTENSION=.gz + # obackup specific + COMPRESSION_OPTIONS=--rsyncable + else + COMPRESSION_PROGRAM= + COMPRESSION_EXTENSION= + fi + fi + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" } ## IFS debug function @@ -1769,7 +1761,7 @@ function TrapQuit { exitcode=2 else RunAfterHook - Logger "$PROGRAM finshed without errors." "NOTICE" + Logger "$PROGRAM finshed." "ALWAYS" exitcode=0 fi @@ -3158,11 +3150,6 @@ function RotateBackups { fi } -function SetTraps { - trap TrapStop INT QUIT TERM HUP - trap TrapQuit EXIT -} - function Init { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG @@ -3170,6 +3157,9 @@ function Init { local hosturiandpath local hosturi + trap TrapStop INT QUIT TERM HUP + trap TrapQuit EXIT + ## Test if target dir is a ssh uri, and if yes, break it down it its values if [ "${REMOTE_SYSTEM_URI:0:6}" == "ssh://" ] && [ "$BACKUP_TYPE" != "local" ]; then REMOTE_OPERATION="yes" @@ -3372,7 +3362,6 @@ function GetCommandlineArguments { done } -SetTraps GetCommandlineArguments "$@" if [ "$_DECRYPT_MODE" == true ]; then CheckCryptEnvironnment @@ -3411,24 +3400,20 @@ fi DATE=$(date) Logger "--------------------------------------------------------------------" "NOTICE" -Logger "$DRY_WARNING$DATE - $PROGRAM v$PROGRAM_VERSION $BACKUP_TYPE script begin." "NOTICE" +Logger "$DRY_WARNING$DATE - $PROGRAM v$PROGRAM_VERSION $BACKUP_TYPE script begin." "ALWAYS" Logger "--------------------------------------------------------------------" "NOTICE" Logger "Backup instance [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE" GetLocalOS -InitLocalOSSettings +InitLocalOSDependingSettings CheckRunningInstances PreInit Init CheckEnvironment PostInit CheckCurrentConfig - -if [ "$REMOTE_OPERATION" == "yes" ]; then - GetRemoteOS - InitRemoteOSSettings -fi -InitRsyncSettings +GetRemoteOS +InitRemoteOSDependingSettings if [ $no_maxtime == true ]; then SOFT_MAX_EXEC_TIME_DB_TASK=0 diff --git a/dev/n_obackup.sh b/dev/n_obackup.sh index 34a1c80..f4b47e7 100755 --- a/dev/n_obackup.sh +++ b/dev/n_obackup.sh @@ -9,7 +9,7 @@ PROGRAM="obackup" AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" PROGRAM_VERSION=2.1-dev -PROGRAM_BUILD=2016120401 +PROGRAM_BUILD=2016121901 IS_STABLE=no source "./ofunctions.sh" @@ -228,7 +228,7 @@ function _ListDatabasesLocal { sqlCmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2>&1" Logger "cmd: $sqlCmd" "DEBUG" eval "$sqlCmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false if [ $? -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" else @@ -251,7 +251,7 @@ function _ListDatabasesRemote { sqlCmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID\" 2>&1" Logger "cmd: $sqlCmd" "DEBUG" eval "$sqlCmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false if [ $? -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" else @@ -352,7 +352,7 @@ function _ListRecursiveBackupDirectoriesLocal { cmd="$FIND_CMD -L $directory/ -mindepth 1 -maxdepth 1 -type d >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false if [ $? != 0 ]; then Logger "Could not enumerate directories in [$directory]." "ERROR" if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then @@ -377,13 +377,15 @@ function _ListRecursiveBackupDirectoriesRemote { local directory local retval + #TODO(high): refactor this using bash heredoc in order to have all those find commands in one big ssh call + #TODO(high): add command_sudo to the heredoc and remove it from find cmd IFS=$PATH_SEPARATOR_CHAR read -r -a directories <<< "$RECURSIVE_DIRECTORY_LIST" for directory in "${directories[@]}"; do #TODO(med): Uses local home directory for remote lookup... cmd=$SSH_CMD' "'$COMMAND_SUDO' '$REMOTE_FIND_CMD' -L '$directory'/ -mindepth 1 -maxdepth 1 -type d" >> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false if [ $? != 0 ]; then Logger "Could not enumerate directories in [$directory]." "ERROR" if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then @@ -480,7 +482,7 @@ function _GetDirectoriesSizeLocal { cmd="du -cs $dir_list | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false # $cmd will return 0 even if some errors found, so we need to check if there is an error output if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then Logger "Could not get files size for some or all directories." "ERROR" @@ -510,11 +512,12 @@ function _GetDirectoriesSizeRemote { local cmd + #TODO(med): check if heredoc needed for compat # Error output is different from stdout because not all files in list may fail at once cmd=$SSH_CMD' '$COMMAND_SUDO' du -cs '$dir_list' | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false # $cmd will return 0 even if some errors found, so we need to check if there is an error output if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then Logger "Could not get files size for some or all directories." "ERROR" @@ -578,10 +581,11 @@ function _CreateDirectoryRemote { CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost + #TODO: compat bash freebsd 11 checks cmd=$SSH_CMD' "if ! [ -d \"'$dir_to_create'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$dir_to_create'\"; fi" > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false if [ $? != 0 ]; then Logger "Cannot create remote directory [$dir_to_create]." "CRITICAL" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" @@ -667,10 +671,11 @@ function GetDiskSpaceRemote { local cmd + #TODO(med): if -d will fail if cannot traverse above directories. Consider using heredoc here. cmd=$SSH_CMD' "if [ -d \"'$path_to_check'\" ]; then '$COMMAND_SUDO' '$DF_CMD' \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false if [ $? != 0 ]; then DISK_SPACE=0 Logger "Cannot get disk space in [$path_to_check] on remote system." "ERROR" @@ -848,7 +853,7 @@ function _BackupDatabaseLocalToLocal { Logger "cmd: $drySqlCmd" "DEBUG" eval "$drySqlCmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -890,7 +895,7 @@ function _BackupDatabaseLocalToRemote { Logger "cmd: $drySqlCmd" "DEBUG" eval "$drySqlCmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -932,7 +937,7 @@ function _BackupDatabaseRemoteToLocal { Logger "cmd: $drySqlCmd" "DEBUG" eval "$drySqlCmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -1155,7 +1160,7 @@ function Rsync { Logger "cmd: $rsyncCmd" "DEBUG" eval "$rsyncCmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false retval=$? if [ $retval != 0 ]; then Logger "Failed to backup [$backupDirectory] to [$fileStoragePath]." "ERROR" @@ -1273,7 +1278,7 @@ function _RotateBackupsLocal { cmd="rm -rf \"$path\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false if [ $? != 0 ]; then Logger "Cannot delete oldest copy [$path]." "ERROR" fi @@ -1285,7 +1290,7 @@ function _RotateBackupsLocal { cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false if [ $? != 0 ]; then Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR" fi @@ -1299,7 +1304,7 @@ function _RotateBackupsLocal { cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false if [ $? != 0 ]; then Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -1308,7 +1313,7 @@ function _RotateBackupsLocal { cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false if [ $? != 0 ]; then Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -1317,7 +1322,7 @@ function _RotateBackupsLocal { cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false if [ $? != 0 ]; then Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -1330,8 +1335,10 @@ function _RotateBackupsRemote { local rotate_copies="${2}" __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG -$SSH_CMD PROGRAM=$PROGRAM REMOTE_OPERATION=$REMOTE_OPERATION _DEBUG=$_DEBUG COMMAND_SUDO=$COMMAND_SUDO rotate_copies=$rotate_copies backup_path="$backup_path" 'bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & +#TODO(high): add _LOGGER_* env variables here +$SSH_CMD env PROGRAM=$PROGRAM env REMOTE_OPERATION=$REMOTE_OPERATION env _DEBUG=$_DEBUG env rotate_copies=$rotate_copies env backup_path="$backup_path" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & +#TODO(high): replace this with include statements function _RemoteLogger { local value="${1}" # What to log echo -e "$value" @@ -1378,7 +1385,7 @@ function _RotateBackupsRemoteSSH { if [ $copy -eq $rotate_copies ]; then path="$backup.$PROGRAM.$copy" if [ -f "$path" ] || [ -d "$path" ]; then - cmd="$COMMAND_SUDO rm -rf \"$path\"" + cmd="rm -rf \"$path\"" RemoteLogger "cmd: $cmd" "DEBUG" eval "$cmd" if [ $? != 0 ]; then @@ -1388,7 +1395,7 @@ function _RotateBackupsRemoteSSH { fi path="$backup.$PROGRAM.$(($copy-1))" if [ -f "$path" ] || [ -d "$path" ]; then - cmd="$COMMAND_SUDO mv \"$path\" \"$backup.$PROGRAM.$copy\"" + cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\"" RemoteLogger "cmd: $cmd" "DEBUG" eval "$cmd" if [ $? != 0 ]; then @@ -1401,7 +1408,7 @@ function _RotateBackupsRemoteSSH { # Latest file backup will not be moved if script configured for remote backup so next rsync execution will only do delta copy instead of full one if [[ $backup == *.sql.* ]]; then - cmd="$COMMAND_SUDO mv \"$backup\" \"$backup.$PROGRAM.1\"" + cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" RemoteLogger "cmd: $cmd" "DEBUG" eval "$cmd" if [ $? != 0 ]; then @@ -1409,7 +1416,7 @@ function _RotateBackupsRemoteSSH { fi elif [ "$REMOTE_OPERATION" == "yes" ]; then - cmd="$COMMAND_SUDO cp -R \"$backup\" \"$backup.$PROGRAM.1\"" + cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\"" RemoteLogger "cmd: $cmd" "DEBUG" eval "$cmd" if [ $? != 0 ]; then @@ -1417,7 +1424,7 @@ function _RotateBackupsRemoteSSH { fi else - cmd="$COMMAND_SUDO mv \"$backup\" \"$backup.$PROGRAM.1\"" + cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" RemoteLogger "cmd: $cmd" "DEBUG" eval "$cmd" if [ $? != 0 ]; then @@ -1431,7 +1438,7 @@ function _RotateBackupsRemoteSSH { ENDSSH - WaitForTaskCompletion $! 1800 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} + WaitForTaskCompletion $! 1800 0 $SLEEP_TIME $KEEP_LOGGING true true false if [ $? != 0 ]; then Logger "Could not rotate backups in [$backup_path]." "ERROR" Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" @@ -1600,7 +1607,6 @@ function Usage { # Command line argument flags _DRYRUN=false -_LOGGER_SILENT=false no_maxtime=false stats=false PARTIAL=no