diff --git a/dev/debug_obackup.sh b/dev/debug_obackup.sh index c2c0a99..c9ed985 100755 --- a/dev/debug_obackup.sh +++ b/dev/debug_obackup.sh @@ -9,12 +9,12 @@ PROGRAM="obackup" AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" PROGRAM_VERSION=2.1-dev -PROGRAM_BUILD=2016111701 +PROGRAM_BUILD=2016113001 IS_STABLE=no #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016111704 +## FUNC_BUILD=2016112902 ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: @@ -29,8 +29,13 @@ IS_STABLE=no ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID -#TODO: Rewrite Logger so we can decide what to send to stdout, stderr and logfile -#TODO: Windows checks, check sendmail & mailsend +## META ISSUES +## +## Updated _LOGGER_STDERR +## Updated WaitForTaskCompletion syntax +## Updated ParallelExec syntax +## SendEmail WinNT10 & msys are two totally different beasts. Document in sync.conf and host_backup.conf + if ! type "$BASH" > /dev/null; then echo "Please run this script only with bash shell. Tested on bash >= 3.2" @@ -65,16 +70,16 @@ fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=yes if [ ! "$_DEBUG" == "yes" ]; then _DEBUG=no - SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys _LOGGER_VERBOSE=false else - if [ "$SLEEP_TIME" == "" ]; then # Set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console - SLEEP_TIME=.05 - fi trap 'TrapError ${LINENO} $?' ERR _LOGGER_VERBOSE=true fi +if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console + SLEEP_TIME=.05 +fi + SCRIPT_PID=$$ LOCAL_USER=$(whoami) @@ -250,16 +255,20 @@ function KillChilds { done fi # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing - if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then - Logger "Sending SIGTERM to process [$pid]." "DEBUG" - kill -s TERM "$pid" - if [ $? != 0 ]; then - sleep 15 - Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" - kill -9 "$pid" + if [ "$self" == true ]; then + if kill -0 "$pid" > /dev/null 2>&1; then + kill -s TERM "$pid" + Logger "Sent SIGTERM to process [$pid]." "DEBUG" if [ $? != 0 ]; then - Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" - return 1 + sleep 15 + Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" + kill -9 "$pid" + if [ $? != 0 ]; then + Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" + return 1 + fi # Simplify the return 0 logic here + else + return 0 fi else return 0 @@ -324,7 +333,6 @@ function SendAlert { if [ -e "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID" ]; then body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID)" fi - exit if [ $ERROR_ALERT == true ]; then subject="Error alert for $INSTANCE_ID" @@ -391,7 +399,7 @@ function SendEmail { mail_no_attachment=0 fi - if [ "$LOCAL_OS" == "BUSYBOX" ]; then + if [ "$LOCAL_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ]; then if type sendmail > /dev/null 2>&1; then if [ "$ENCRYPTION" == "tls" ]; then echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$SenderMail" -H "exec openssl s_client -quiet -tls1_2 -starttls smtp -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" @@ -425,13 +433,18 @@ function SendEmail { fi if type mail > /dev/null 2>&1 ; then - if [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then + # We need to detect which version of mail is installed + if ! $(type -p mail) -V > /dev/null 2>&1; then + # This may be MacOS mail program + attachment_command="" + elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then attachment_command="-A $attachment" elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then attachment_command="-a$attachment" else attachment_command="" fi + echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails" if [ $? != 0 ]; then Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN" @@ -543,30 +556,30 @@ function Spinner { return 0 fi - case $toggle + case $_OFUNCTIONS_SPINNER_TOGGLE in 1) echo -n " \ " echo -ne "\r" - toggle="2" + _OFUNCTIONS_SPINNER_TOGGLE=2 ;; 2) echo -n " | " echo -ne "\r" - toggle="3" + _OFUNCTIONS_SPINNER_TOGGLE=3 ;; 3) echo -n " / " echo -ne "\r" - toggle="4" + _OFUNCTIONS_SPINNER_TOGGLE=4 ;; *) echo -n " - " echo -ne "\r" - toggle="1" + _OFUNCTIONS_SPINNER_TOGGLE=1 ;; esac } @@ -581,18 +594,22 @@ function joinString { # Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 # Warning: Don't imbricate this function into another run if you plan to use the global variable output +# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true false "${FUNCNAME[0]}" + function WaitForTaskCompletion { local pids="${1}" # pids to wait for, separated by semi-colon - local softMaxTime="${2}" # If program with pid $pid takes longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. - local hardMaxTime="${3}" # If program with pid $pid takes longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. - local callerName="${4}" # Who called this function - local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false - local keepLogging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging + local softMaxTime="${2:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. + local hardMaxTime="${3:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. + local sleepTime="${4:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1). + local keepLogging="${5:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging. + local counting="${6:-true}" # Count time since function has been launched (true), or since script has been launched (false) + local spinner="${7:-true}" # Show spinner (true), don't show anything (false) + local noErrorLog="${8:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true) + local callerName="${9}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]} Logger "${FUNCNAME[0]} called by [$callerName]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG - __CheckArguments 6 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + __CheckArguments 9 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once local log_ttime=0 # local time instance for comparaison local seconds_begin=$SECONDS # Seconds since the beginning of the script @@ -610,6 +627,10 @@ function WaitForTaskCompletion { local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG + if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown + local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once + fi + IFS=';' read -a pidsArray <<< "$pids" pidCount=${#pidsArray[@]} @@ -618,7 +639,9 @@ function WaitForTaskCompletion { while [ ${#pidsArray[@]} -gt 0 ]; do newPidsArray=() - Spinner + if [ $spinner == true ]; then + Spinner + fi if [ $counting == true ]; then exec_time=$(($SECONDS - $seconds_begin)) else @@ -635,33 +658,36 @@ function WaitForTaskCompletion { fi if [ $exec_time -gt $softMaxTime ]; then - if [ $soft_alert == true ] && [ $softMaxTime -ne 0 ]; then + if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN" - soft_alert=true + _SOFT_ALERT=true SendAlert true + fi + fi - fi - if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then + if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then + if [ $noErrorLog != true ]; then Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" - for pid in "${pidsArray[@]}"; do - KillChilds $pid true - if [ $? == 0 ]; then - Logger "Task with pid [$pid] stopped successfully." "NOTICE" - else - Logger "Could not stop task with pid [$pid]." "ERROR" - fi - done + fi + for pid in "${pidsArray[@]}"; do + KillChilds $pid true + if [ $? == 0 ]; then + Logger "Task with pid [$pid] stopped successfully." "NOTICE" + else + Logger "Could not stop task with pid [$pid]." "ERROR" + fi + errorcount=$((errorcount+1)) + done + if [ $noErrorLog != true ]; then SendAlert true fi + return $errorcount fi for pid in "${pidsArray[@]}"; do if [ $(IsInteger $pid) -eq 1 ]; then if kill -0 $pid > /dev/null 2>&1; then # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) - #TODO(high): have this tested on *BSD, Mac, Win & busybox. - #TODO(high): propagate changes to ParallelExec - #pidState=$(ps -p$pid -o state= 2 > /dev/null) pidState="$(eval $PROCESS_STATE_CMD)" if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then newPidsArray+=($pid) @@ -690,14 +716,15 @@ function WaitForTaskCompletion { pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU - sleep $SLEEP_TIME + sleep $sleepTime done Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG # Return exit code if only one process was monitored, else return number of errors - if [ $pidCount -eq 1 ] && [ $errorcount -eq 0 ]; then - return $errorcount + # As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value + if [ $pidCount -eq 1 ]; then + return $retval else return $errorcount fi @@ -706,17 +733,27 @@ function WaitForTaskCompletion { # Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs # Returns the number of non zero exit codes from commands # Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets -function ParallelExec { - local numberOfProcesses="${1}" # Number of simultaneous commands to run - local commandsArg="${2}" # Semi-colon separated list of commands, or file containing one command per line - local readFromFile="${3:-false}" # Is commandsArg a file or a string ? - local softMaxTime="${4:-0}" - local hardMaxTime="${5:-0}" - local callerName="${6}" # Who called this function - local counting="${7:-true}" # Count time since function has been launched if true, since script has been launched if false - local keepLogging="${8:-0}" # Log a standby message every X seconds. Set to zero to disable logging +# Only 2 first arguments are mandatory - __CheckArguments 8 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG +function ParallelExec { + local numberOfProcesses="${1}" # Number of simultaneous commands to run + local commandsArg="${2}" # Semi-colon separated list of commands, or path to file containing one command per line + local readFromFile="${3:-false}" # commandsArg is a file (true), or a string (false) + local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. + local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. + local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1). + local keepLogging="${7:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging. + local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false) + local spinner="${9:-false}" # Show spinner (true), don't show spinner (false) + local noErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true) + local callerName="${11:-false}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]} + + __CheckArguments 2-11 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local log_ttime=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function local commandCount local command @@ -732,6 +769,10 @@ function ParallelExec { local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG + if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown + local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once + fi + if [ $readFromFile == true ];then if [ -f "$commandsArg" ]; then commandCount=$(wc -l < "$commandsArg") @@ -747,15 +788,60 @@ function ParallelExec { while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do + if [ $spinner == true ]; then + Spinner + fi + + if [ $counting == true ]; then + exec_time=$(($SECONDS - $seconds_begin)) + else + exec_time=$SECONDS + fi + + if [ $keepLogging -ne 0 ]; then + if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then + if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s + log_ttime=$exec_time + Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" + fi + fi + fi + + if [ $exec_time -gt $softMaxTime ]; then + if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then + Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN" + _SOFT_ALERT=true + SendAlert true + fi + fi + if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then + if [ $noErrorLog != true ]; then + Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" + fi + for pid in "${pidsArray[@]}"; do + KillChilds $pid true + if [ $? == 0 ]; then + Logger "Task with pid [$pid] stopped successfully." "NOTICE" + else + Logger "Could not stop task with pid [$pid]." "ERROR" + fi + done + if [ $noErrorLog != true ]; then + SendAlert true + else + # Return the number of commands that haven't run / finished run + return $(($commandCount - $counter + ${#pidsArray[@]})) + fi + fi + while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do if [ $readFromFile == true ]; then - #TODO: Checked on FreeBSD 10, also check on Win command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg") else command="${commandsArray[$counter]}" fi Logger "Running command [$command]." "DEBUG" - eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & + eval "$command" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID" 2>&1 & pid=$! pidsArray+=($pid) commandsArrayPid[$pid]="$command" @@ -956,16 +1042,25 @@ function GetLocalOS { if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then localOsVar="BusyBox" else - localOsVar="$(uname -spio 2>&1)" - if [ $? != 0 ]; then - localOsVar="$(uname -v 2>&1)" + # Detecting the special ubuntu userland in Windows 10 bash + if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then + localOsVar="Microsoft" + else + localOsVar="$(uname -spio 2>&1)" if [ $? != 0 ]; then - localOsVar="$(uname)" + localOsVar="$(uname -v 2>&1)" + if [ $? != 0 ]; then + localOsVar="$(uname)" + fi fi fi fi case $localOsVar in + # Android uname contains both linux and android, keep it before linux entry + *"Android"*) + LOCAL_OS="Android" + ;; *"Linux"*) LOCAL_OS="Linux" ;; @@ -975,11 +1070,14 @@ function GetLocalOS { *"MINGW32"*|*"CYGWIN"*) LOCAL_OS="msys" ;; + *"Microsoft"*) + LOCAL_OS="WinNT10" + ;; *"Darwin"*) LOCAL_OS="MacOSX" ;; *"BusyBox"*) - LOCAL_OS="BUSYBOX" + LOCAL_OS="BusyBox" ;; *) if [ "$IGNORE_OS_TYPE" == "yes" ]; then #TODO(doc): Undocumented option @@ -998,6 +1096,10 @@ function GetLocalOS { function GetRemoteOS { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + if [ "$REMOTE_OPERATION" != "yes" ]; then + return 0 + fi + local remoteOsVar $SSH_CMD bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 @@ -1009,15 +1111,19 @@ function GetOs { if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then localOsVar="BusyBox" else - localOsVar="$(uname -spio 2>&1)" - if [ $? != 0 ]; then - localOsVar="$(uname -v 2>&1)" + # Detecting the special ubuntu userland in Windows 10 bash + if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then + localOsVar="Microsoft" + else + localOsVar="$(uname -spio 2>&1)" if [ $? != 0 ]; then - localOsVar="$(uname)" + localOsVar="$(uname -v 2>&1)" + if [ $? != 0 ]; then + localOsVar="$(uname)" + fi fi fi fi - echo "$localOsVar" } @@ -1028,6 +1134,9 @@ ENDSSH if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID") case $remoteOsVar in + *"Android"*) + REMOTE_OS="Android" + ;; *"Linux"*) REMOTE_OS="Linux" ;; @@ -1037,11 +1146,14 @@ ENDSSH *"MINGW32"*|*"CYGWIN"*) REMOTE_OS="msys" ;; + *"Microsoft"*) + REMOTE_OS="WinNT10" + ;; *"Darwin"*) REMOTE_OS="MacOSX" ;; *"BusyBox"*) - REMOTE_OS="BUSYBOX" + REMOTE_OS="BusyBox" ;; *"ssh"*|*"SSH"*) Logger "Cannot connect to remote system." "CRITICAL" @@ -1074,7 +1186,8 @@ function RunLocalCommand { Logger "Running command [$command] on local host." "NOTICE" eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & - WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING + + WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ $retval -eq 0 ]; then Logger "Command succeded." "NOTICE" @@ -1109,7 +1222,7 @@ function RunRemoteCommand { cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ $retval -eq 0 ]; then Logger "Command succeded." "NOTICE" @@ -1143,7 +1256,7 @@ function RunBeforeHook { pids="$pids;$!" fi if [ "$pids" != "" ]; then - WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} fi } @@ -1162,7 +1275,7 @@ function RunAfterHook { pids="$pids;$!" fi if [ "$pids" != "" ]; then - WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} fi } @@ -1171,18 +1284,18 @@ function CheckConnectivityRemoteHost { local retval - if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug + if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug #__WITH_PARANOIA_DEBUG if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & - WaitForTaskCompletion $! 60 180 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 60 180 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ $retval != 0 ]; then Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN" return $retval fi fi - fi + fi #__WITH_PARANOIA_DEBUG } function CheckConnectivity3rdPartyHosts { @@ -1191,14 +1304,14 @@ function CheckConnectivity3rdPartyHosts { local remote3rdPartySuccess local retval - if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug + if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug #__WITH_PARANOIA_DEBUG if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then remote3rdPartySuccess=false for i in $REMOTE_3RD_PARTY_HOSTS do eval "$PING_CMD $i > /dev/null 2>&1" & - WaitForTaskCompletion $! 180 360 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 180 360 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ $retval != 0 ]; then Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" @@ -1214,7 +1327,7 @@ function CheckConnectivity3rdPartyHosts { return 0 fi fi - fi + fi #__WITH_PARANOIA_DEBUG } #__BEGIN_WITH_PARANOIA_DEBUG @@ -1229,10 +1342,6 @@ function __CheckArguments { local minArgs local maxArgs - if [ "$_PARANOIA_DEBUG" == "yes" ]; then - Logger "Entering function [$functionName]." "DEBUG" - fi - # All arguments of the function to check are passed as array in ${4} (the function call waits for $@) # If any of the arguments contains spaces, bash things there are two aguments # In order to avoid this, we need to iterate over ${4} and count @@ -1247,22 +1356,29 @@ function __CheckArguments { if [ "$argument" = "" ]; then fetchArguments=false else - argList="$arg_list [Argument $(($iterate-3)): $argument]" + argList="$argList[Argument $(($iterate-3)): $argument] " iterate=$(($iterate+1)) fi done + countedArguments=$((iterate-4)) - if [ $(IsNumeric "$numberOfArguments") -eq 1 ]; then + if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then minArgs=$numberOfArguments maxArgs=$numberOfArguments else IFS='-' read minArgs maxArgs <<< "$numberOfArguments" fi + if [ "$_PARANOIA_DEBUG" == "yes" ]; then + Logger "Entering function [$functionName]." "DEBUG" + fi + if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then Logger "Function $functionName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments. see log file." "ERROR" Logger "Arguments passed: $argList" "ERROR" + else + Logger "Arguments passed: $argList" "VERBOSE" fi fi } @@ -1388,68 +1504,14 @@ function PreInit { COMMAND_SUDO="" fi - ## Set rsync default arguments - RSYNC_ARGS="-rltD" - if [ "$_DRYRUN" == true ]; then - RSYNC_DRY_ARG="-n" - DRY_WARNING="/!\ DRY RUN " - else - RSYNC_DRY_ARG="" - fi - - RSYNC_ATTR_ARGS="" - if [ "$PRESERVE_PERMISSIONS" != "no" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" - fi - if [ "$PRESERVE_OWNER" != "no" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o" - fi - if [ "$PRESERVE_GROUP" != "no" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g" - fi - if [ "$PRESERVE_ACL" == "yes" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A" - fi - if [ "$PRESERVE_XATTR" == "yes" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X" - fi - if [ "$RSYNC_COMPRESS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -z" - fi - if [ "$COPY_SYMLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -L" - fi - if [ "$KEEP_DIRLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -K" - fi - if [ "$PRESERVE_HARDLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -H" - fi - if [ "$CHECKSUM" == "yes" ]; then - RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" - fi - if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" - fi - - if [ "$PARTIAL" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" - RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" - fi - - if [ "$DELTA_COPIES" != "no" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" - else - RSYNC_ARGS=$RSYNC_ARGS" --whole-file" - fi - - ## Set compression executable and extension + ## Set compression executable and extension if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then COMPRESSION_LEVEL=3 fi + #TODO: Remote OS isn't defined yet ## Busybox fix (Termux xz command doesn't support compression at all) - if [ "$LOCAL_OS" == "BUSYBOX" ] || [ "$REMOTE_OS" == "BUSYBOX" ]; then + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then compressionString="" if type gzip > /dev/null 2>&1 then @@ -1526,10 +1588,13 @@ function InitLocalOSSettings { PING_CMD="ping -c 2 -i .2" fi - if [ "$LOCAL_OS" == "BUSYBOX" ]; then + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "msys" ]; then PROCESS_STATE_CMD="echo none" + DF_CMD="df" else PROCESS_STATE_CMD='ps -p$pid -o state= 2 > /dev/null' + # CentOS 5 needs -P for one line output + DF_CMD="df -P" fi ## Stat command has different syntax on Linux and FreeBSD/MacOSX @@ -1538,7 +1603,7 @@ function InitLocalOSSettings { STAT_CMD="stat -f \"%Sm\"" STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" else - # Tested on GNU stat and busybox + # Tested on GNU stat, busybox and Cygwin STAT_CMD="stat -c %y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" fi @@ -1547,13 +1612,6 @@ function InitLocalOSSettings { function InitRemoteOSSettings { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) - if [ "$PRESERVE_EXECUTABILITY" != "no" ];then - if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E" - fi - fi - if [ "$REMOTE_OS" == "msys" ]; then REMOTE_FIND_CMD=$(dirname $BASH)/find else @@ -1571,6 +1629,72 @@ function InitRemoteOSSettings { } +function InitRsyncSettings { + __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + ## Set rsync default arguments + RSYNC_ARGS="-rltD" + if [ "$_DRYRUN" == true ]; then + RSYNC_DRY_ARG="-n" + DRY_WARNING="/!\ DRY RUN " + else + RSYNC_DRY_ARG="" + fi + + RSYNC_ATTR_ARGS="" + if [ "$PRESERVE_PERMISSIONS" != "no" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" + fi + if [ "$PRESERVE_OWNER" != "no" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o" + fi + if [ "$PRESERVE_GROUP" != "no" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g" + fi + if [ "$PRESERVE_EXECUTABILITY" != "no" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" --executability" + fi + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] && [ "$LOCAL_OS" != "msys" ] && [ "$REMOTE_OS" != "MacOSX" ]; then + if [ "$PRESERVE_ACL" == "yes" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A" + fi + if [ "$PRESERVE_XATTR" == "yes" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X" + fi + else + Logger "Disabling ACL and extended attributes synchronization on [$LOCAL_OS]." "NOTICE" + fi + if [ "$RSYNC_COMPRESS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -z" + fi + if [ "$COPY_SYMLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -L" + fi + if [ "$KEEP_DIRLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -K" + fi + if [ "$PRESERVE_HARDLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -H" + fi + if [ "$CHECKSUM" == "yes" ]; then + RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" + fi + if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" + fi + + if [ "$PARTIAL" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" + RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" + fi + + if [ "$DELTA_COPIES" != "no" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" + else + RSYNC_ARGS=$RSYNC_ARGS" --whole-file" + fi +} + ## IFS debug function function PrintIFS { printf "IFS is: %q" "$IFS" @@ -1621,13 +1745,20 @@ function TrapStop { function TrapQuit { local exitcode + # Get ERROR / WARN alert flags from subprocesses that call Logger + if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID" ]; then + WARN_ALERT=1 + fi + if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID" ]; then + ERROR_ALERT=1 + fi + if [ $ERROR_ALERT == true ]; then if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then RunAfterHook fi Logger "$PROGRAM finished with errors." "ERROR" SendAlert - CleanUp exitcode=1 elif [ $WARN_ALERT == true ]; then if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then @@ -1635,12 +1766,10 @@ function TrapQuit { fi Logger "$PROGRAM finished with warnings." "WARN" SendAlert - CleanUp exitcode=2 else RunAfterHook Logger "$PROGRAM finshed without errors." "NOTICE" - CleanUp exitcode=0 fi @@ -1648,6 +1777,7 @@ function TrapQuit { rm -f "$RUN_DIR/$PROGRAM.$INSTANCE_ID" fi + CleanUp KillChilds $$ > /dev/null 2>&1 exit $exitcode } @@ -1799,7 +1929,7 @@ function _ListDatabasesLocal { sqlCmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2>&1" Logger "cmd: $sqlCmd" "DEBUG" eval "$sqlCmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" else @@ -1822,7 +1952,7 @@ function _ListDatabasesRemote { sqlCmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID\" 2>&1" Logger "cmd: $sqlCmd" "DEBUG" eval "$sqlCmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" else @@ -1923,7 +2053,7 @@ function _ListRecursiveBackupDirectoriesLocal { cmd="$FIND_CMD -L $directory/ -mindepth 1 -maxdepth 1 -type d >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Could not enumerate directories in [$directory]." "ERROR" if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then @@ -1954,7 +2084,7 @@ function _ListRecursiveBackupDirectoriesRemote { cmd=$SSH_CMD' "'$COMMAND_SUDO' '$REMOTE_FIND_CMD' -L '$directory'/ -mindepth 1 -maxdepth 1 -type d" >> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Could not enumerate directories in [$directory]." "ERROR" if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then @@ -2051,7 +2181,7 @@ function _GetDirectoriesSizeLocal { cmd="du -cs $dir_list | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} # $cmd will return 0 even if some errors found, so we need to check if there is an error output if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then Logger "Could not get files size for some or all directories." "ERROR" @@ -2085,7 +2215,7 @@ function _GetDirectoriesSizeRemote { cmd=$SSH_CMD' '$COMMAND_SUDO' du -cs '$dir_list' | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} # $cmd will return 0 even if some errors found, so we need to check if there is an error output if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then Logger "Could not get files size for some or all directories." "ERROR" @@ -2152,7 +2282,7 @@ function _CreateDirectoryRemote { cmd=$SSH_CMD' "if ! [ -d \"'$dir_to_create'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$dir_to_create'\"; fi" > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Cannot create remote directory [$dir_to_create]." "CRITICAL" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" @@ -2213,7 +2343,7 @@ function GetDiskSpaceLocal { if [ -d "$path_to_check" ]; then # Not elegant solution to make df silent on errors # No sudo on local commands, assuming you should have all the necesarry rights to check backup directories sizes - df "$path_to_check" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 + $DF_CMD "$path_to_check" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 if [ $? != 0 ]; then DISK_SPACE=0 Logger "Cannot get disk space in [$path_to_check] on local system." "ERROR" @@ -2238,10 +2368,10 @@ function GetDiskSpaceRemote { local cmd - cmd=$SSH_CMD' "if [ -d \"'$path_to_check'\" ]; then '$COMMAND_SUDO' df \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if [ -d \"'$path_to_check'\" ]; then '$COMMAND_SUDO' '$DF_CMD' \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then DISK_SPACE=0 Logger "Cannot get disk space in [$path_to_check] on remote system." "ERROR" @@ -2419,7 +2549,7 @@ function _BackupDatabaseLocalToLocal { Logger "cmd: $drySqlCmd" "DEBUG" eval "$drySqlCmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -2461,7 +2591,7 @@ function _BackupDatabaseLocalToRemote { Logger "cmd: $drySqlCmd" "DEBUG" eval "$drySqlCmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -2503,7 +2633,7 @@ function _BackupDatabaseRemoteToLocal { Logger "cmd: $drySqlCmd" "DEBUG" eval "$drySqlCmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -2726,7 +2856,7 @@ function Rsync { Logger "cmd: $rsyncCmd" "DEBUG" eval "$rsyncCmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ $retval != 0 ]; then Logger "Failed to backup [$backupDirectory] to [$fileStoragePath]." "ERROR" @@ -2844,7 +2974,7 @@ function _RotateBackupsLocal { cmd="rm -rf \"$path\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Cannot delete oldest copy [$path]." "ERROR" fi @@ -2856,7 +2986,7 @@ function _RotateBackupsLocal { cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR" fi @@ -2870,7 +3000,7 @@ function _RotateBackupsLocal { cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -2879,7 +3009,7 @@ function _RotateBackupsLocal { cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -2888,7 +3018,7 @@ function _RotateBackupsLocal { cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -3002,7 +3132,7 @@ function _RotateBackupsRemoteSSH { ENDSSH - WaitForTaskCompletion $! 1800 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 1800 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Could not rotate backups in [$backup_path]." "ERROR" Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" @@ -3298,6 +3428,7 @@ if [ "$REMOTE_OPERATION" == "yes" ]; then GetRemoteOS InitRemoteOSSettings fi +InitRsyncSettings if [ $no_maxtime == true ]; then SOFT_MAX_EXEC_TIME_DB_TASK=0 diff --git a/install.sh b/install.sh index e7ae451..b067621 100755 --- a/install.sh +++ b/install.sh @@ -4,7 +4,7 @@ PROGRAM=obackup PROGRAM_VERSION=2.1-dev PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" -SCRIPT_BUILD=2016090605 +SCRIPT_BUILD=2016112401 ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8 & 10 @@ -12,6 +12,9 @@ SCRIPT_BUILD=2016090605 #TODO: silent mode and no stats mode +# Get current install.sh path from http://stackoverflow.com/a/246128/2635443 +SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + CONF_DIR=$FAKEROOT/etc/$PROGRAM BIN_DIR="$FAKEROOT/usr/local/bin" SERVICE_DIR_INIT=$FAKEROOT/etc/init.d @@ -70,25 +73,35 @@ function urlencode() { local c="${1:i:1}" case $c in [a-zA-Z0-9.~_-]) printf "$c" ;; - *) printf '%%%02X' "'$c" ;; + *) printf '%%%02X' "'$c" ;; esac done } function SetOSSettings { + local localOsVar + USER=root - local local_os_var + # There's no good way to tell if currently running in BusyBox shell. Using sluggish way. + if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then + localOsVar="BusyBox" + else + # Detecting the special ubuntu userland in Windows 10 bash + if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then + localOsVar="Microsoft" + else + localOsVar="$(uname -spio 2>&1)" + if [ $? != 0 ]; then + localOsVar="$(uname -v 2>&1)" + if [ $? != 0 ]; then + localOsVar="$(uname)" + fi + fi + fi + fi - local_os_var="$(uname -spio 2>&1)" - if [ $? != 0 ]; then - local_os_var="$(uname -v 2>&1)" - if [ $? != 0 ]; then - local_os_var="$(uname)" - fi - fi - - case $local_os_var in + case $localOsVar in *"BSD"*) GROUP=wheel ;; @@ -109,7 +122,7 @@ function SetOSSettings { exit 1 fi - OS=$(urlencode "$local_os_var") + OS=$(urlencode "$localOsVar") } function GetInit { @@ -140,34 +153,34 @@ function CreateConfDir { } function CopyExampleFiles { - if [ -f "./sync.conf.example" ]; then - cp "./sync.conf.example" "$CONF_DIR/sync.conf.example" + if [ -f "$SCRIPT_PATH/sync.conf.example" ]; then + cp "$SCRIPT_PATH/sync.conf.example" "$CONF_DIR/sync.conf.example" fi - if [ -f "./host_backup.conf.example" ]; then - cp "./host_backup.conf.example" "$CONF_DIR/host_backup.conf.example" + if [ -f "$SCRIPT_PATH/host_backup.conf.example" ]; then + cp "$SCRIPT_PATH/host_backup.conf.example" "$CONF_DIR/host_backup.conf.example" fi - if [ -f "./exlude.list.example" ]; then - cp "./exclude.list.example" "$CONF_DIR/exclude.list.example" + if [ -f "$SCRIPT_PATH/exlude.list.example" ]; then + cp "$SCRIPT_PATH/exclude.list.example" "$CONF_DIR/exclude.list.example" fi - if [ -f "./snapshot.conf.example" ]; then - cp "./snapshot.conf.example" "$CONF_DIR/snapshot.conf.example" + if [ -f "$SCRIPT_PATH/snapshot.conf.example" ]; then + cp "$SCRIPT_PATH/snapshot.conf.example" "$CONF_DIR/snapshot.conf.example" fi - if [ -f "./default.conf" ]; then + if [ -f "$SCRIPT_PATH/default.conf" ]; then if [ -f "$CONF_DIR/default.conf" ]; then - cp "./default.conf" "$CONF_DIR/default.conf.new" + cp "$SCRIPT_PATH/default.conf" "$CONF_DIR/default.conf.new" QuickLogger "Copied default.conf to [$CONF_DIR/default.conf.new]." else - cp "./default.conf" "$CONF_DIR/default.conf" + cp "$SCRIPT_PATH/default.conf" "$CONF_DIR/default.conf" fi fi } function CopyProgram { - cp "./$PROGRAM_BINARY" "$BIN_DIR" + cp "$SCRIPT_PATH/$PROGRAM_BINARY" "$BIN_DIR" if [ $? != 0 ]; then QuickLogger "Cannot copy $PROGRAM_BINARY to [$BIN_DIR]. Make sure to run install script in the directory containing all other files." QuickLogger "Also make sure you have permissions to write to [$BIN_DIR]." @@ -177,8 +190,8 @@ function CopyProgram { QuickLogger "Copied $PROGRAM_BINARY to [$BIN_DIR]." fi - if [ -f "./$PROGRAM_BATCH" ]; then - cp "./$PROGRAM_BATCH" "$BIN_DIR" + if [ -f "$SCRIPT_PATH/$PROGRAM_BATCH" ]; then + cp "$SCRIPT_PATH/$PROGRAM_BATCH" "$BIN_DIR" if [ $? != 0 ]; then QuickLogger "Cannot copy $PROGRAM_BATCH to [$BIN_DIR]." else @@ -187,8 +200,8 @@ function CopyProgram { fi fi - if [ -f "./ssh_filter.sh" ]; then - cp "./ssh_filter.sh" "$BIN_DIR" + if [ -f "$SCRIPT_PATH/ssh_filter.sh" ]; then + cp "$SCRIPT_PATH/ssh_filter.sh" "$BIN_DIR" if [ $? != 0 ]; then QuickLogger "Cannot copy ssh_filter.sh to [$BIN_DIR]." else @@ -203,8 +216,8 @@ function CopyProgram { function CopyServiceFiles { # OSYNC SPECIFIC - if ([ "$init" == "systemd" ] && [ -f "./$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then - cp "./$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM" && cp "./$OSYNC_SERVICE_FILE_SYSTEMD_USER" "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM" + if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then + cp "$SCRIPT_PATH/$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM" && cp "$SCRIPT_PATH/$OSYNC_SERVICE_FILE_SYSTEMD_USER" "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM" if [ $? != 0 ]; then QuickLogger "Cannot copy the systemd file to [$SERVICE_DIR_SYSTEMD_SYSTEM] or [$SERVICE_DIR_SYSTEMD_USER]." else @@ -213,8 +226,8 @@ function CopyServiceFiles { QuickLogger "Can be enabled on boot with [systemctl enable osync-srv@instance.conf]." QuickLogger "In userland, active with [systemctl --user start osync-srv@instance.conf]." fi - elif ([ "$init" == "initV" ] && [ -f "./$OSYNC_SERVICE_FILE_INIT" ]); then - cp "./$OSYNC_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT" + elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$OSYNC_SERVICE_FILE_INIT" ]); then + cp "$SCRIPT_PATH/$OSYNC_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT" if [ $? != 0 ]; then QuickLogger "Cannot copy osync-srv to [$SERVICE_DIR_INIT]." else @@ -226,8 +239,8 @@ function CopyServiceFiles { fi # PMOCR SPECIFIC - if ([ "$init" == "systemd" ] && [ -f "./$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then - cp "./$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM" + if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then + cp "$SCRIPT_PATH/$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM" if [ $? != 0 ]; then QuickLogger "Cannot copy the systemd file to [$SERVICE_DIR_SYSTEMD_SYSTEM] or [$SERVICE_DIR_SYSTEMD_USER]." else @@ -235,8 +248,8 @@ function CopyServiceFiles { QuickLogger "Can be activated with [systemctl start pmocr-srv@default.conf] where default.conf is the name of the config file in $CONF_DIR." QuickLogger "Can be enabled on boot with [systemctl enable pmocr-srv@default.conf]." fi - elif ([ "$init" == "initV" ] && [ -f "./$PMOCR_SERVICE_FILE_INIT" ]); then - cp "./$PMOCR_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT" + elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$PMOCR_SERVICE_FILE_INIT" ]); then + cp "$SCRIPT_PATH/$PMOCR_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT" if [ $? != 0 ]; then QuickLogger "Cannot copy pmoct-srv to [$SERVICE_DIR_INIT]." else diff --git a/obackup-batch.sh b/obackup-batch.sh index 4d52657..6a4a05b 100755 --- a/obackup-batch.sh +++ b/obackup-batch.sh @@ -3,7 +3,7 @@ SUBPROGRAM=obackup PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones AUTHOR="(L) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr - ozy@netpower.fr" -PROGRAM_BUILD=2016082901 +PROGRAM_BUILD=2016112402 ## Runs an osync /obackup instance for every conf file found ## If an instance fails, run it again if time permits @@ -13,9 +13,6 @@ if ! type "$BASH" > /dev/null; then exit 127 fi -## Configuration file path. The path where all the osync / obackup conf files are, usually /etc/osync or /etc/obackup -CONF_FILE_PATH=/etc/$SUBPROGRAM - ## If maximum execution time is not reached, failed instances will be rerun. Max exec time is in seconds. Example is set to 10 hours. MAX_EXECUTION_TIME=36000 @@ -74,6 +71,10 @@ function CheckEnvironment { else SUBPROGRAM_EXECUTABLE=$(type -p $SUBPROGRAM.sh) fi + + if [ "$CONF_FILE_PATH" == "" ]; then + Usage + fi } function Batch { @@ -111,7 +112,7 @@ function Batch { Logger "$SUBPROGRAM instances will be run for: $runList" "NOTICE" for confFile in $runList do - $SUBPROGRAM_EXECUTABLE "$confFile" $opts & + $SUBPROGRAM_EXECUTABLE "$confFile" --silent $opts & wait $! result=$? if [ $result != 0 ]; then @@ -141,16 +142,18 @@ function Usage { echo $CONTACT echo "" echo "Batch script to sequentially run osync or obackup instances and rerun failed ones." - echo "Usage: $SUBPROGRAM-batch.sh [OPTIONS]" + echo "Usage: $PROGRAM.sh [OPTIONS] [$SUBPROGRAM OPTIONS]" echo "" echo "[OPTIONS]" echo "--path=/path/to/conf Path to osync / obackup conf files, defaults to /etc/osync or /etc/obackup" echo "--max-runs=X Number of max runs per instance, (defaults to 3)" echo "--max-exec-time=X Retry failed instances only if max execution time not reached (defaults to 36000 seconds). Set to 0 to bypass execution time check" - echo "--no-maxtime Run osync / obackup without honoring conf file defined timeouts" - echo "--dry Will run osync / obackup without actually doing anything; just testing" - echo "--silent Will run osync / obackup without any output to stdout, used for cron jobs" - echo "--verbose Increases output" + echo "[$SUBPROGRAM OPTIONS]" + echo "Specify whatever options $PROGRAM accepts. Example" + echo "$PROGRAM.sh --path=/etc/$SUBPROGRAM --no-maxtime" + echo "" + echo "No output will be written to stdout/stderr." + echo "Verify log file in [$LOG_FILE]." exit 128 } @@ -158,18 +161,6 @@ opts="" for i in "$@" do case $i in - --silent) - opts=$opts" --silent" - ;; - --dry) - opts=$opts" --dry" - ;; - --verbose) - opts=$opts" --verbose" - ;; - --no-maxtime) - opts=$opts" --no-maxtime" - ;; --path=*) CONF_FILE_PATH=${i##*=} ;; @@ -183,8 +174,7 @@ do Usage ;; *) - Logger "Unknown param '$i'" "CRITICAL" - Usage + opts="$i " ;; esac done diff --git a/obackup.sh b/obackup.sh index e28382b..e6c5a3a 100755 --- a/obackup.sh +++ b/obackup.sh @@ -9,12 +9,12 @@ PROGRAM="obackup" AUTHOR="(C) 2013-2016 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" PROGRAM_VERSION=2.1-dev -PROGRAM_BUILD=2016111701 +PROGRAM_BUILD=2016113001 IS_STABLE=no #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016111704 +## FUNC_BUILD=2016112902 ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## To use in a program, define the following variables: @@ -29,8 +29,13 @@ IS_STABLE=no ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID -#TODO: Rewrite Logger so we can decide what to send to stdout, stderr and logfile -#TODO: Windows checks, check sendmail & mailsend +## META ISSUES +## +## Updated _LOGGER_STDERR +## Updated WaitForTaskCompletion syntax +## Updated ParallelExec syntax +## SendEmail WinNT10 & msys are two totally different beasts. Document in sync.conf and host_backup.conf + if ! type "$BASH" > /dev/null; then echo "Please run this script only with bash shell. Tested on bash >= 3.2" @@ -61,16 +66,16 @@ WARN_ALERT=false ## allow debugging from command line with _DEBUG=yes if [ ! "$_DEBUG" == "yes" ]; then _DEBUG=no - SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys _LOGGER_VERBOSE=false else - if [ "$SLEEP_TIME" == "" ]; then # Set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console - SLEEP_TIME=.05 - fi trap 'TrapError ${LINENO} $?' ERR _LOGGER_VERBOSE=true fi +if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console + SLEEP_TIME=.05 +fi + SCRIPT_PID=$$ LOCAL_USER=$(whoami) @@ -237,16 +242,20 @@ function KillChilds { done fi # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing - if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then - Logger "Sending SIGTERM to process [$pid]." "DEBUG" - kill -s TERM "$pid" - if [ $? != 0 ]; then - sleep 15 - Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" - kill -9 "$pid" + if [ "$self" == true ]; then + if kill -0 "$pid" > /dev/null 2>&1; then + kill -s TERM "$pid" + Logger "Sent SIGTERM to process [$pid]." "DEBUG" if [ $? != 0 ]; then - Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" - return 1 + sleep 15 + Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" + kill -9 "$pid" + if [ $? != 0 ]; then + Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" + return 1 + fi # Simplify the return 0 logic here + else + return 0 fi else return 0 @@ -309,7 +318,6 @@ function SendAlert { if [ -e "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID" ]; then body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID)" fi - exit if [ $ERROR_ALERT == true ]; then subject="Error alert for $INSTANCE_ID" @@ -375,7 +383,7 @@ function SendEmail { mail_no_attachment=0 fi - if [ "$LOCAL_OS" == "BUSYBOX" ]; then + if [ "$LOCAL_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ]; then if type sendmail > /dev/null 2>&1; then if [ "$ENCRYPTION" == "tls" ]; then echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$SenderMail" -H "exec openssl s_client -quiet -tls1_2 -starttls smtp -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" @@ -409,13 +417,18 @@ function SendEmail { fi if type mail > /dev/null 2>&1 ; then - if [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then + # We need to detect which version of mail is installed + if ! $(type -p mail) -V > /dev/null 2>&1; then + # This may be MacOS mail program + attachment_command="" + elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then attachment_command="-A $attachment" elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then attachment_command="-a$attachment" else attachment_command="" fi + echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails" if [ $? != 0 ]; then Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN" @@ -526,30 +539,30 @@ function Spinner { return 0 fi - case $toggle + case $_OFUNCTIONS_SPINNER_TOGGLE in 1) echo -n " \ " echo -ne "\r" - toggle="2" + _OFUNCTIONS_SPINNER_TOGGLE=2 ;; 2) echo -n " | " echo -ne "\r" - toggle="3" + _OFUNCTIONS_SPINNER_TOGGLE=3 ;; 3) echo -n " / " echo -ne "\r" - toggle="4" + _OFUNCTIONS_SPINNER_TOGGLE=4 ;; *) echo -n " - " echo -ne "\r" - toggle="1" + _OFUNCTIONS_SPINNER_TOGGLE=1 ;; esac } @@ -564,16 +577,20 @@ function joinString { # Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 # Warning: Don't imbricate this function into another run if you plan to use the global variable output +# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true false "${FUNCNAME[0]}" + function WaitForTaskCompletion { local pids="${1}" # pids to wait for, separated by semi-colon - local softMaxTime="${2}" # If program with pid $pid takes longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. - local hardMaxTime="${3}" # If program with pid $pid takes longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. - local callerName="${4}" # Who called this function - local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false - local keepLogging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging + local softMaxTime="${2:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. + local hardMaxTime="${3:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. + local sleepTime="${4:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1). + local keepLogging="${5:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging. + local counting="${6:-true}" # Count time since function has been launched (true), or since script has been launched (false) + local spinner="${7:-true}" # Show spinner (true), don't show anything (false) + local noErrorLog="${8:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true) + local callerName="${9}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]} - local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once local log_ttime=0 # local time instance for comparaison local seconds_begin=$SECONDS # Seconds since the beginning of the script @@ -590,6 +607,10 @@ function WaitForTaskCompletion { local newPidsArray # New array of currently running pids + if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown + local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once + fi + IFS=';' read -a pidsArray <<< "$pids" pidCount=${#pidsArray[@]} @@ -598,7 +619,9 @@ function WaitForTaskCompletion { while [ ${#pidsArray[@]} -gt 0 ]; do newPidsArray=() - Spinner + if [ $spinner == true ]; then + Spinner + fi if [ $counting == true ]; then exec_time=$(($SECONDS - $seconds_begin)) else @@ -615,33 +638,36 @@ function WaitForTaskCompletion { fi if [ $exec_time -gt $softMaxTime ]; then - if [ $soft_alert == true ] && [ $softMaxTime -ne 0 ]; then + if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN" - soft_alert=true + _SOFT_ALERT=true SendAlert true + fi + fi - fi - if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then + if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then + if [ $noErrorLog != true ]; then Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" - for pid in "${pidsArray[@]}"; do - KillChilds $pid true - if [ $? == 0 ]; then - Logger "Task with pid [$pid] stopped successfully." "NOTICE" - else - Logger "Could not stop task with pid [$pid]." "ERROR" - fi - done + fi + for pid in "${pidsArray[@]}"; do + KillChilds $pid true + if [ $? == 0 ]; then + Logger "Task with pid [$pid] stopped successfully." "NOTICE" + else + Logger "Could not stop task with pid [$pid]." "ERROR" + fi + errorcount=$((errorcount+1)) + done + if [ $noErrorLog != true ]; then SendAlert true fi + return $errorcount fi for pid in "${pidsArray[@]}"; do if [ $(IsInteger $pid) -eq 1 ]; then if kill -0 $pid > /dev/null 2>&1; then # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) - #TODO(high): have this tested on *BSD, Mac, Win & busybox. - #TODO(high): propagate changes to ParallelExec - #pidState=$(ps -p$pid -o state= 2 > /dev/null) pidState="$(eval $PROCESS_STATE_CMD)" if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then newPidsArray+=($pid) @@ -666,13 +692,14 @@ function WaitForTaskCompletion { pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU - sleep $SLEEP_TIME + sleep $sleepTime done # Return exit code if only one process was monitored, else return number of errors - if [ $pidCount -eq 1 ] && [ $errorcount -eq 0 ]; then - return $errorcount + # As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value + if [ $pidCount -eq 1 ]; then + return $retval else return $errorcount fi @@ -681,16 +708,26 @@ function WaitForTaskCompletion { # Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs # Returns the number of non zero exit codes from commands # Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets -function ParallelExec { - local numberOfProcesses="${1}" # Number of simultaneous commands to run - local commandsArg="${2}" # Semi-colon separated list of commands, or file containing one command per line - local readFromFile="${3:-false}" # Is commandsArg a file or a string ? - local softMaxTime="${4:-0}" - local hardMaxTime="${5:-0}" - local callerName="${6}" # Who called this function - local counting="${7:-true}" # Count time since function has been launched if true, since script has been launched if false - local keepLogging="${8:-0}" # Log a standby message every X seconds. Set to zero to disable logging +# Only 2 first arguments are mandatory +function ParallelExec { + local numberOfProcesses="${1}" # Number of simultaneous commands to run + local commandsArg="${2}" # Semi-colon separated list of commands, or path to file containing one command per line + local readFromFile="${3:-false}" # commandsArg is a file (true), or a string (false) + local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. + local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. + local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1). + local keepLogging="${7:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging. + local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false) + local spinner="${9:-false}" # Show spinner (true), don't show spinner (false) + local noErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true) + local callerName="${11:-false}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]} + + + local log_ttime=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function local commandCount local command @@ -705,6 +742,10 @@ function ParallelExec { local commandsArrayPid + if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown + local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once + fi + if [ $readFromFile == true ];then if [ -f "$commandsArg" ]; then commandCount=$(wc -l < "$commandsArg") @@ -720,15 +761,60 @@ function ParallelExec { while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do + if [ $spinner == true ]; then + Spinner + fi + + if [ $counting == true ]; then + exec_time=$(($SECONDS - $seconds_begin)) + else + exec_time=$SECONDS + fi + + if [ $keepLogging -ne 0 ]; then + if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then + if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s + log_ttime=$exec_time + Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE" + fi + fi + fi + + if [ $exec_time -gt $softMaxTime ]; then + if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then + Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN" + _SOFT_ALERT=true + SendAlert true + fi + fi + if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then + if [ $noErrorLog != true ]; then + Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" + fi + for pid in "${pidsArray[@]}"; do + KillChilds $pid true + if [ $? == 0 ]; then + Logger "Task with pid [$pid] stopped successfully." "NOTICE" + else + Logger "Could not stop task with pid [$pid]." "ERROR" + fi + done + if [ $noErrorLog != true ]; then + SendAlert true + else + # Return the number of commands that haven't run / finished run + return $(($commandCount - $counter + ${#pidsArray[@]})) + fi + fi + while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do if [ $readFromFile == true ]; then - #TODO: Checked on FreeBSD 10, also check on Win command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg") else command="${commandsArray[$counter]}" fi Logger "Running command [$command]." "DEBUG" - eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & + eval "$command" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID" 2>&1 & pid=$! pidsArray+=($pid) commandsArrayPid[$pid]="$command" @@ -923,16 +1009,25 @@ function GetLocalOS { if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then localOsVar="BusyBox" else - localOsVar="$(uname -spio 2>&1)" - if [ $? != 0 ]; then - localOsVar="$(uname -v 2>&1)" + # Detecting the special ubuntu userland in Windows 10 bash + if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then + localOsVar="Microsoft" + else + localOsVar="$(uname -spio 2>&1)" if [ $? != 0 ]; then - localOsVar="$(uname)" + localOsVar="$(uname -v 2>&1)" + if [ $? != 0 ]; then + localOsVar="$(uname)" + fi fi fi fi case $localOsVar in + # Android uname contains both linux and android, keep it before linux entry + *"Android"*) + LOCAL_OS="Android" + ;; *"Linux"*) LOCAL_OS="Linux" ;; @@ -942,11 +1037,14 @@ function GetLocalOS { *"MINGW32"*|*"CYGWIN"*) LOCAL_OS="msys" ;; + *"Microsoft"*) + LOCAL_OS="WinNT10" + ;; *"Darwin"*) LOCAL_OS="MacOSX" ;; *"BusyBox"*) - LOCAL_OS="BUSYBOX" + LOCAL_OS="BusyBox" ;; *) if [ "$IGNORE_OS_TYPE" == "yes" ]; then #TODO(doc): Undocumented option @@ -964,6 +1062,10 @@ function GetLocalOS { function GetRemoteOS { + if [ "$REMOTE_OPERATION" != "yes" ]; then + return 0 + fi + local remoteOsVar $SSH_CMD bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 @@ -975,15 +1077,19 @@ function GetOs { if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then localOsVar="BusyBox" else - localOsVar="$(uname -spio 2>&1)" - if [ $? != 0 ]; then - localOsVar="$(uname -v 2>&1)" + # Detecting the special ubuntu userland in Windows 10 bash + if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then + localOsVar="Microsoft" + else + localOsVar="$(uname -spio 2>&1)" if [ $? != 0 ]; then - localOsVar="$(uname)" + localOsVar="$(uname -v 2>&1)" + if [ $? != 0 ]; then + localOsVar="$(uname)" + fi fi fi fi - echo "$localOsVar" } @@ -994,6 +1100,9 @@ ENDSSH if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID") case $remoteOsVar in + *"Android"*) + REMOTE_OS="Android" + ;; *"Linux"*) REMOTE_OS="Linux" ;; @@ -1003,11 +1112,14 @@ ENDSSH *"MINGW32"*|*"CYGWIN"*) REMOTE_OS="msys" ;; + *"Microsoft"*) + REMOTE_OS="WinNT10" + ;; *"Darwin"*) REMOTE_OS="MacOSX" ;; *"BusyBox"*) - REMOTE_OS="BUSYBOX" + REMOTE_OS="BusyBox" ;; *"ssh"*|*"SSH"*) Logger "Cannot connect to remote system." "CRITICAL" @@ -1039,7 +1151,8 @@ function RunLocalCommand { Logger "Running command [$command] on local host." "NOTICE" eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & - WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING + + WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ $retval -eq 0 ]; then Logger "Command succeded." "NOTICE" @@ -1073,7 +1186,7 @@ function RunRemoteCommand { cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ $retval -eq 0 ]; then Logger "Command succeded." "NOTICE" @@ -1106,7 +1219,7 @@ function RunBeforeHook { pids="$pids;$!" fi if [ "$pids" != "" ]; then - WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} fi } @@ -1124,7 +1237,7 @@ function RunAfterHook { pids="$pids;$!" fi if [ "$pids" != "" ]; then - WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} fi } @@ -1132,18 +1245,16 @@ function CheckConnectivityRemoteHost { local retval - if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & - WaitForTaskCompletion $! 60 180 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 60 180 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ $retval != 0 ]; then Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN" return $retval fi fi - fi } function CheckConnectivity3rdPartyHosts { @@ -1151,14 +1262,13 @@ function CheckConnectivity3rdPartyHosts { local remote3rdPartySuccess local retval - if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then remote3rdPartySuccess=false for i in $REMOTE_3RD_PARTY_HOSTS do eval "$PING_CMD $i > /dev/null 2>&1" & - WaitForTaskCompletion $! 180 360 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 180 360 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ $retval != 0 ]; then Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" @@ -1174,11 +1284,8 @@ function CheckConnectivity3rdPartyHosts { return 0 fi fi - fi } -#__BEGIN_WITH_PARANOIA_DEBUG -#__END_WITH_PARANOIA_DEBUG function RsyncPatternsAdd { local patternType="${1}" # exclude or include @@ -1295,68 +1402,14 @@ function PreInit { COMMAND_SUDO="" fi - ## Set rsync default arguments - RSYNC_ARGS="-rltD" - if [ "$_DRYRUN" == true ]; then - RSYNC_DRY_ARG="-n" - DRY_WARNING="/!\ DRY RUN " - else - RSYNC_DRY_ARG="" - fi - - RSYNC_ATTR_ARGS="" - if [ "$PRESERVE_PERMISSIONS" != "no" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" - fi - if [ "$PRESERVE_OWNER" != "no" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o" - fi - if [ "$PRESERVE_GROUP" != "no" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g" - fi - if [ "$PRESERVE_ACL" == "yes" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A" - fi - if [ "$PRESERVE_XATTR" == "yes" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X" - fi - if [ "$RSYNC_COMPRESS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -z" - fi - if [ "$COPY_SYMLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -L" - fi - if [ "$KEEP_DIRLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -K" - fi - if [ "$PRESERVE_HARDLINKS" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" -H" - fi - if [ "$CHECKSUM" == "yes" ]; then - RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" - fi - if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" - fi - - if [ "$PARTIAL" == "yes" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" - RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" - fi - - if [ "$DELTA_COPIES" != "no" ]; then - RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" - else - RSYNC_ARGS=$RSYNC_ARGS" --whole-file" - fi - - ## Set compression executable and extension + ## Set compression executable and extension if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then COMPRESSION_LEVEL=3 fi + #TODO: Remote OS isn't defined yet ## Busybox fix (Termux xz command doesn't support compression at all) - if [ "$LOCAL_OS" == "BUSYBOX" ] || [ "$REMOTE_OS" == "BUSYBOX" ]; then + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then compressionString="" if type gzip > /dev/null 2>&1 then @@ -1431,10 +1484,13 @@ function InitLocalOSSettings { PING_CMD="ping -c 2 -i .2" fi - if [ "$LOCAL_OS" == "BUSYBOX" ]; then + if [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "msys" ]; then PROCESS_STATE_CMD="echo none" + DF_CMD="df" else PROCESS_STATE_CMD='ps -p$pid -o state= 2 > /dev/null' + # CentOS 5 needs -P for one line output + DF_CMD="df -P" fi ## Stat command has different syntax on Linux and FreeBSD/MacOSX @@ -1443,7 +1499,7 @@ function InitLocalOSSettings { STAT_CMD="stat -f \"%Sm\"" STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" else - # Tested on GNU stat and busybox + # Tested on GNU stat, busybox and Cygwin STAT_CMD="stat -c %y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" fi @@ -1451,13 +1507,6 @@ function InitLocalOSSettings { function InitRemoteOSSettings { - ## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability) - if [ "$PRESERVE_EXECUTABILITY" != "no" ];then - if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then - RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E" - fi - fi - if [ "$REMOTE_OS" == "msys" ]; then REMOTE_FIND_CMD=$(dirname $BASH)/find else @@ -1475,6 +1524,71 @@ function InitRemoteOSSettings { } +function InitRsyncSettings { + + ## Set rsync default arguments + RSYNC_ARGS="-rltD" + if [ "$_DRYRUN" == true ]; then + RSYNC_DRY_ARG="-n" + DRY_WARNING="/!\ DRY RUN " + else + RSYNC_DRY_ARG="" + fi + + RSYNC_ATTR_ARGS="" + if [ "$PRESERVE_PERMISSIONS" != "no" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p" + fi + if [ "$PRESERVE_OWNER" != "no" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o" + fi + if [ "$PRESERVE_GROUP" != "no" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g" + fi + if [ "$PRESERVE_EXECUTABILITY" != "no" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" --executability" + fi + if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] && [ "$LOCAL_OS" != "msys" ] && [ "$REMOTE_OS" != "MacOSX" ]; then + if [ "$PRESERVE_ACL" == "yes" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A" + fi + if [ "$PRESERVE_XATTR" == "yes" ]; then + RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X" + fi + else + Logger "Disabling ACL and extended attributes synchronization on [$LOCAL_OS]." "NOTICE" + fi + if [ "$RSYNC_COMPRESS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -z" + fi + if [ "$COPY_SYMLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -L" + fi + if [ "$KEEP_DIRLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -K" + fi + if [ "$PRESERVE_HARDLINKS" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" -H" + fi + if [ "$CHECKSUM" == "yes" ]; then + RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum" + fi + if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH" + fi + + if [ "$PARTIAL" == "yes" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\"" + RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\"" + fi + + if [ "$DELTA_COPIES" != "no" ]; then + RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file" + else + RSYNC_ARGS=$RSYNC_ARGS" --whole-file" + fi +} + ## IFS debug function function PrintIFS { printf "IFS is: %q" "$IFS" @@ -1525,13 +1639,20 @@ function TrapStop { function TrapQuit { local exitcode + # Get ERROR / WARN alert flags from subprocesses that call Logger + if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID" ]; then + WARN_ALERT=1 + fi + if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID" ]; then + ERROR_ALERT=1 + fi + if [ $ERROR_ALERT == true ]; then if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then RunAfterHook fi Logger "$PROGRAM finished with errors." "ERROR" SendAlert - CleanUp exitcode=1 elif [ $WARN_ALERT == true ]; then if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then @@ -1539,12 +1660,10 @@ function TrapQuit { fi Logger "$PROGRAM finished with warnings." "WARN" SendAlert - CleanUp exitcode=2 else RunAfterHook Logger "$PROGRAM finshed without errors." "NOTICE" - CleanUp exitcode=0 fi @@ -1552,6 +1671,7 @@ function TrapQuit { rm -f "$RUN_DIR/$PROGRAM.$INSTANCE_ID" fi + CleanUp KillChilds $$ > /dev/null 2>&1 exit $exitcode } @@ -1699,7 +1819,7 @@ function _ListDatabasesLocal { sqlCmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2>&1" Logger "cmd: $sqlCmd" "DEBUG" eval "$sqlCmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" else @@ -1721,7 +1841,7 @@ function _ListDatabasesRemote { sqlCmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID\" 2>&1" Logger "cmd: $sqlCmd" "DEBUG" eval "$sqlCmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" else @@ -1820,7 +1940,7 @@ function _ListRecursiveBackupDirectoriesLocal { cmd="$FIND_CMD -L $directory/ -mindepth 1 -maxdepth 1 -type d >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Could not enumerate directories in [$directory]." "ERROR" if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then @@ -1850,7 +1970,7 @@ function _ListRecursiveBackupDirectoriesRemote { cmd=$SSH_CMD' "'$COMMAND_SUDO' '$REMOTE_FIND_CMD' -L '$directory'/ -mindepth 1 -maxdepth 1 -type d" >> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Could not enumerate directories in [$directory]." "ERROR" if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then @@ -1945,7 +2065,7 @@ function _GetDirectoriesSizeLocal { cmd="du -cs $dir_list | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} # $cmd will return 0 even if some errors found, so we need to check if there is an error output if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then Logger "Could not get files size for some or all directories." "ERROR" @@ -1978,7 +2098,7 @@ function _GetDirectoriesSizeRemote { cmd=$SSH_CMD' '$COMMAND_SUDO' du -cs '$dir_list' | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} # $cmd will return 0 even if some errors found, so we need to check if there is an error output if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then Logger "Could not get files size for some or all directories." "ERROR" @@ -2042,7 +2162,7 @@ function _CreateDirectoryRemote { cmd=$SSH_CMD' "if ! [ -d \"'$dir_to_create'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$dir_to_create'\"; fi" > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Cannot create remote directory [$dir_to_create]." "CRITICAL" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" @@ -2101,7 +2221,7 @@ function GetDiskSpaceLocal { if [ -d "$path_to_check" ]; then # Not elegant solution to make df silent on errors # No sudo on local commands, assuming you should have all the necesarry rights to check backup directories sizes - df "$path_to_check" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 + $DF_CMD "$path_to_check" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 if [ $? != 0 ]; then DISK_SPACE=0 Logger "Cannot get disk space in [$path_to_check] on local system." "ERROR" @@ -2125,10 +2245,10 @@ function GetDiskSpaceRemote { local cmd - cmd=$SSH_CMD' "if [ -d \"'$path_to_check'\" ]; then '$COMMAND_SUDO' df \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' + cmd=$SSH_CMD' "if [ -d \"'$path_to_check'\" ]; then '$COMMAND_SUDO' '$DF_CMD' \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then DISK_SPACE=0 Logger "Cannot get disk space in [$path_to_check] on remote system." "ERROR" @@ -2304,7 +2424,7 @@ function _BackupDatabaseLocalToLocal { Logger "cmd: $drySqlCmd" "DEBUG" eval "$drySqlCmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -2345,7 +2465,7 @@ function _BackupDatabaseLocalToRemote { Logger "cmd: $drySqlCmd" "DEBUG" eval "$drySqlCmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -2386,7 +2506,7 @@ function _BackupDatabaseRemoteToLocal { Logger "cmd: $drySqlCmd" "DEBUG" eval "$drySqlCmd" & fi - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" @@ -2604,7 +2724,7 @@ function Rsync { Logger "cmd: $rsyncCmd" "DEBUG" eval "$rsyncCmd" & - WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} retval=$? if [ $retval != 0 ]; then Logger "Failed to backup [$backupDirectory] to [$fileStoragePath]." "ERROR" @@ -2719,7 +2839,7 @@ function _RotateBackupsLocal { cmd="rm -rf \"$path\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Cannot delete oldest copy [$path]." "ERROR" fi @@ -2731,7 +2851,7 @@ function _RotateBackupsLocal { cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR" fi @@ -2745,7 +2865,7 @@ function _RotateBackupsLocal { cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -2754,7 +2874,7 @@ function _RotateBackupsLocal { cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -2763,7 +2883,7 @@ function _RotateBackupsLocal { cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" Logger "cmd: $cmd" "DEBUG" eval "$cmd" & - WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" fi @@ -2871,7 +2991,7 @@ function _RotateBackupsRemoteSSH { ENDSSH - WaitForTaskCompletion $! 1800 0 ${FUNCNAME[0]} true $KEEP_LOGGING + WaitForTaskCompletion $! 1800 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]} if [ $? != 0 ]; then Logger "Could not rotate backups in [$backup_path]." "ERROR" Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" @@ -3163,6 +3283,7 @@ if [ "$REMOTE_OPERATION" == "yes" ]; then GetRemoteOS InitRemoteOSSettings fi +InitRsyncSettings if [ $no_maxtime == true ]; then SOFT_MAX_EXEC_TIME_DB_TASK=0