Linux
/
obackup
mirror of https://github.com/deajan/obackup.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Rebuilt targets

This commit is contained in:
deajan 2016-11-30 14:04:41 +01:00
parent 161599617d
commit ff791f2ccf
4 changed files with 668 additions and 413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

489
dev/debug_obackup.sh
View File

@ -9,12 +9,12 @@ PROGRAM="obackup"
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
PROGRAM_VERSION=2.1-dev PROGRAM_VERSION=2.1-dev
PROGRAM_BUILD=2016111701 PROGRAM_BUILD=2016113001
IS_STABLE=no IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016111704 ## FUNC_BUILD=2016112902
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
@ -29,8 +29,13 @@ IS_STABLE=no
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID ## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID
#TODO: Rewrite Logger so we can decide what to send to stdout, stderr and logfile ## META ISSUES
#TODO: Windows checks, check sendmail & mailsend ##
## Updated _LOGGER_STDERR
## Updated WaitForTaskCompletion syntax
## Updated ParallelExec syntax
## SendEmail WinNT10 & msys are two totally different beasts. Document in sync.conf and host_backup.conf
if ! type "$BASH" > /dev/null; then if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2" echo "Please run this script only with bash shell. Tested on bash >= 3.2"
@ -65,16 +70,16 @@ fi #__WITH_PARANOIA_DEBUG
## allow debugging from command line with _DEBUG=yes ## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no _DEBUG=no
SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys
_LOGGER_VERBOSE=false _LOGGER_VERBOSE=false
else else
if [ "$SLEEP_TIME" == "" ]; then # Set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=.05
fi
trap 'TrapError ${LINENO} $?' ERR trap 'TrapError ${LINENO} $?' ERR
_LOGGER_VERBOSE=true _LOGGER_VERBOSE=true
fi fi
if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=.05
fi
SCRIPT_PID=$$ SCRIPT_PID=$$
LOCAL_USER=$(whoami) LOCAL_USER=$(whoami)
@ -250,16 +255,20 @@ function KillChilds {
done done
fi fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then if [ "$self" == true ]; then
Logger "Sending SIGTERM to process [$pid]." "DEBUG" if kill -0 "$pid" > /dev/null 2>&1; then
kill -s TERM "$pid" kill -s TERM "$pid"
if [ $? != 0 ]; then Logger "Sent SIGTERM to process [$pid]." "DEBUG"
sleep 15
Logger "Sending SIGTERM to process [$pid] failed." "DEBUG"
kill -9 "$pid"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" sleep 15
return 1 Logger "Sending SIGTERM to process [$pid] failed." "DEBUG"
kill -9 "$pid"
if [ $? != 0 ]; then
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
return 1
fi # Simplify the return 0 logic here
else
return 0
fi fi
else else
return 0 return 0
@ -324,7 +333,6 @@ function SendAlert {
if [ -e "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID" ]; then if [ -e "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID" ]; then
body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID)" body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID)"
fi fi
exit
if [ $ERROR_ALERT == true ]; then if [ $ERROR_ALERT == true ]; then
subject="Error alert for $INSTANCE_ID" subject="Error alert for $INSTANCE_ID"
@ -391,7 +399,7 @@ function SendEmail {
mail_no_attachment=0 mail_no_attachment=0
fi fi
if [ "$LOCAL_OS" == "BUSYBOX" ]; then if [ "$LOCAL_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ]; then
if type sendmail > /dev/null 2>&1; then if type sendmail > /dev/null 2>&1; then
if [ "$ENCRYPTION" == "tls" ]; then if [ "$ENCRYPTION" == "tls" ]; then
echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$SenderMail" -H "exec openssl s_client -quiet -tls1_2 -starttls smtp -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$SenderMail" -H "exec openssl s_client -quiet -tls1_2 -starttls smtp -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails"
@ -425,13 +433,18 @@ function SendEmail {
fi fi
if type mail > /dev/null 2>&1 ; then if type mail > /dev/null 2>&1 ; then
if [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then # We need to detect which version of mail is installed
if ! $(type -p mail) -V > /dev/null 2>&1; then
# This may be MacOS mail program
attachment_command=""
elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then
attachment_command="-A $attachment" attachment_command="-A $attachment"
elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then
attachment_command="-a$attachment" attachment_command="-a$attachment"
else else
attachment_command="" attachment_command=""
fi fi
echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails" echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN" Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN"
@ -543,30 +556,30 @@ function Spinner {
return 0 return 0
fi fi
case $toggle case $_OFUNCTIONS_SPINNER_TOGGLE
in in
1) 1)
echo -n " \ " echo -n " \ "
echo -ne "\r" echo -ne "\r"
toggle="2" _OFUNCTIONS_SPINNER_TOGGLE=2
;; ;;
2) 2)
echo -n " | " echo -n " | "
echo -ne "\r" echo -ne "\r"
toggle="3" _OFUNCTIONS_SPINNER_TOGGLE=3
;; ;;
3) 3)
echo -n " / " echo -n " / "
echo -ne "\r" echo -ne "\r"
toggle="4" _OFUNCTIONS_SPINNER_TOGGLE=4
;; ;;
*) *)
echo -n " - " echo -n " - "
echo -ne "\r" echo -ne "\r"
toggle="1" _OFUNCTIONS_SPINNER_TOGGLE=1
;; ;;
esac esac
} }
@ -581,18 +594,22 @@ function joinString {
# Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 # Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2
# Warning: Don't imbricate this function into another run if you plan to use the global variable output # Warning: Don't imbricate this function into another run if you plan to use the global variable output
# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true false "${FUNCNAME[0]}"
function WaitForTaskCompletion { function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon local pids="${1}" # pids to wait for, separated by semi-colon
local softMaxTime="${2}" # If program with pid $pid takes longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. local softMaxTime="${2:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${3}" # If program with pid $pid takes longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. local hardMaxTime="${3:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local callerName="${4}" # Who called this function local sleepTime="${4:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false local keepLogging="${5:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local keepLogging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging local counting="${6:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${7:-true}" # Show spinner (true), don't show anything (false)
local noErrorLog="${8:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local callerName="${9}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]}
Logger "${FUNCNAME[0]} called by [$callerName]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} called by [$callerName]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 6 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 9 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once
local log_ttime=0 # local time instance for comparaison local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script local seconds_begin=$SECONDS # Seconds since the beginning of the script
@ -610,6 +627,10 @@ function WaitForTaskCompletion {
local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG
if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown
local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once
fi
IFS=';' read -a pidsArray <<< "$pids" IFS=';' read -a pidsArray <<< "$pids"
pidCount=${#pidsArray[@]} pidCount=${#pidsArray[@]}
@ -618,7 +639,9 @@ function WaitForTaskCompletion {
while [ ${#pidsArray[@]} -gt 0 ]; do while [ ${#pidsArray[@]} -gt 0 ]; do
newPidsArray=() newPidsArray=()
Spinner if [ $spinner == true ]; then
Spinner
fi
if [ $counting == true ]; then if [ $counting == true ]; then
exec_time=$(($SECONDS - $seconds_begin)) exec_time=$(($SECONDS - $seconds_begin))
else else
@ -635,33 +658,36 @@ function WaitForTaskCompletion {
fi fi
if [ $exec_time -gt $softMaxTime ]; then if [ $exec_time -gt $softMaxTime ]; then
if [ $soft_alert == true ] && [ $softMaxTime -ne 0 ]; then if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN" Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
soft_alert=true _SOFT_ALERT=true
SendAlert true SendAlert true
fi
fi
fi if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then if [ $noErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
for pid in "${pidsArray[@]}"; do fi
KillChilds $pid true for pid in "${pidsArray[@]}"; do
if [ $? == 0 ]; then KillChilds $pid true
Logger "Task with pid [$pid] stopped successfully." "NOTICE" if [ $? == 0 ]; then
else Logger "Task with pid [$pid] stopped successfully." "NOTICE"
Logger "Could not stop task with pid [$pid]." "ERROR" else
fi Logger "Could not stop task with pid [$pid]." "ERROR"
done fi
errorcount=$((errorcount+1))
done
if [ $noErrorLog != true ]; then
SendAlert true SendAlert true
fi fi
return $errorcount
fi fi
for pid in "${pidsArray[@]}"; do for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then if [ $(IsInteger $pid) -eq 1 ]; then
if kill -0 $pid > /dev/null 2>&1; then if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
#TODO(high): have this tested on *BSD, Mac, Win & busybox.
#TODO(high): propagate changes to ParallelExec
#pidState=$(ps -p$pid -o state= 2 > /dev/null)
pidState="$(eval $PROCESS_STATE_CMD)" pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
newPidsArray+=($pid) newPidsArray+=($pid)
@ -690,14 +716,15 @@ function WaitForTaskCompletion {
pidsArray=("${newPidsArray[@]}") pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU # Trivial wait time for bash to not eat up all CPU
sleep $SLEEP_TIME sleep $sleepTime
done done
Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# Return exit code if only one process was monitored, else return number of errors # Return exit code if only one process was monitored, else return number of errors
if [ $pidCount -eq 1 ] && [ $errorcount -eq 0 ]; then # As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
return $errorcount if [ $pidCount -eq 1 ]; then
return $retval
else else
return $errorcount return $errorcount
fi fi
@ -706,17 +733,27 @@ function WaitForTaskCompletion {
# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs # Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
# Returns the number of non zero exit codes from commands # Returns the number of non zero exit codes from commands
# Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets # Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets
function ParallelExec { # Only 2 first arguments are mandatory
local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or file containing one command per line
local readFromFile="${3:-false}" # Is commandsArg a file or a string ?
local softMaxTime="${4:-0}"
local hardMaxTime="${5:-0}"
local callerName="${6}" # Who called this function
local counting="${7:-true}" # Count time since function has been launched if true, since script has been launched if false
local keepLogging="${8:-0}" # Log a standby message every X seconds. Set to zero to disable logging
__CheckArguments 8 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG function ParallelExec {
local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or path to file containing one command per line
local readFromFile="${3:-false}" # commandsArg is a file (true), or a string (false)
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-false}" # Show spinner (true), don't show spinner (false)
local noErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local callerName="${11:-false}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]}
__CheckArguments 2-11 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local commandCount local commandCount
local command local command
@ -732,6 +769,10 @@ function ParallelExec {
local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG
if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown
local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once
fi
if [ $readFromFile == true ];then if [ $readFromFile == true ];then
if [ -f "$commandsArg" ]; then if [ -f "$commandsArg" ]; then
commandCount=$(wc -l < "$commandsArg") commandCount=$(wc -l < "$commandsArg")
@ -747,15 +788,60 @@ function ParallelExec {
while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do
if [ $spinner == true ]; then
Spinner
fi
if [ $counting == true ]; then
exec_time=$(($SECONDS - $seconds_begin))
else
exec_time=$SECONDS
fi
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
log_ttime=$exec_time
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
_SOFT_ALERT=true
SendAlert true
fi
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Task with pid [$pid] stopped successfully." "NOTICE"
else
Logger "Could not stop task with pid [$pid]." "ERROR"
fi
done
if [ $noErrorLog != true ]; then
SendAlert true
else
# Return the number of commands that haven't run / finished run
return $(($commandCount - $counter + ${#pidsArray[@]}))
fi
fi
while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then if [ $readFromFile == true ]; then
#TODO: Checked on FreeBSD 10, also check on Win
command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg") command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg")
else else
command="${commandsArray[$counter]}" command="${commandsArray[$counter]}"
fi fi
Logger "Running command [$command]." "DEBUG" Logger "Running command [$command]." "DEBUG"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & eval "$command" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID" 2>&1 &
pid=$! pid=$!
pidsArray+=($pid) pidsArray+=($pid)
commandsArrayPid[$pid]="$command" commandsArrayPid[$pid]="$command"
@ -956,16 +1042,25 @@ function GetLocalOS {
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox" localOsVar="BusyBox"
else else
localOsVar="$(uname -spio 2>&1)" # Detecting the special ubuntu userland in Windows 10 bash
if [ $? != 0 ]; then if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
localOsVar="$(uname -v 2>&1)" localOsVar="Microsoft"
else
localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then if [ $? != 0 ]; then
localOsVar="$(uname)" localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname)"
fi
fi fi
fi fi
fi fi
case $localOsVar in case $localOsVar in
# Android uname contains both linux and android, keep it before linux entry
*"Android"*)
LOCAL_OS="Android"
;;
*"Linux"*) *"Linux"*)
LOCAL_OS="Linux" LOCAL_OS="Linux"
;; ;;
@ -975,11 +1070,14 @@ function GetLocalOS {
*"MINGW32"*|*"CYGWIN"*) *"MINGW32"*|*"CYGWIN"*)
LOCAL_OS="msys" LOCAL_OS="msys"
;; ;;
*"Microsoft"*)
LOCAL_OS="WinNT10"
;;
*"Darwin"*) *"Darwin"*)
LOCAL_OS="MacOSX" LOCAL_OS="MacOSX"
;; ;;
*"BusyBox"*) *"BusyBox"*)
LOCAL_OS="BUSYBOX" LOCAL_OS="BusyBox"
;; ;;
*) *)
if [ "$IGNORE_OS_TYPE" == "yes" ]; then #TODO(doc): Undocumented option if [ "$IGNORE_OS_TYPE" == "yes" ]; then #TODO(doc): Undocumented option
@ -998,6 +1096,10 @@ function GetLocalOS {
function GetRemoteOS { function GetRemoteOS {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ "$REMOTE_OPERATION" != "yes" ]; then
return 0
fi
local remoteOsVar local remoteOsVar
$SSH_CMD bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 $SSH_CMD bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1
@ -1009,15 +1111,19 @@ function GetOs {
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox" localOsVar="BusyBox"
else else
localOsVar="$(uname -spio 2>&1)" # Detecting the special ubuntu userland in Windows 10 bash
if [ $? != 0 ]; then if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
localOsVar="$(uname -v 2>&1)" localOsVar="Microsoft"
else
localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then if [ $? != 0 ]; then
localOsVar="$(uname)" localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname)"
fi
fi fi
fi fi
fi fi
echo "$localOsVar" echo "$localOsVar"
} }
@ -1028,6 +1134,9 @@ ENDSSH
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID") remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID")
case $remoteOsVar in case $remoteOsVar in
*"Android"*)
REMOTE_OS="Android"
;;
*"Linux"*) *"Linux"*)
REMOTE_OS="Linux" REMOTE_OS="Linux"
;; ;;
@ -1037,11 +1146,14 @@ ENDSSH
*"MINGW32"*|*"CYGWIN"*) *"MINGW32"*|*"CYGWIN"*)
REMOTE_OS="msys" REMOTE_OS="msys"
;; ;;
*"Microsoft"*)
REMOTE_OS="WinNT10"
;;
*"Darwin"*) *"Darwin"*)
REMOTE_OS="MacOSX" REMOTE_OS="MacOSX"
;; ;;
*"BusyBox"*) *"BusyBox"*)
REMOTE_OS="BUSYBOX" REMOTE_OS="BusyBox"
;; ;;
*"ssh"*|*"SSH"*) *"ssh"*|*"SSH"*)
Logger "Cannot connect to remote system." "CRITICAL" Logger "Cannot connect to remote system." "CRITICAL"
@ -1074,7 +1186,8 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE" Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1109,7 +1222,7 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1143,7 +1256,7 @@ function RunBeforeHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -1162,7 +1275,7 @@ function RunAfterHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -1171,18 +1284,18 @@ function CheckConnectivityRemoteHost {
local retval local retval
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug #__WITH_PARANOIA_DEBUG
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
WaitForTaskCompletion $! 60 180 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 60 180 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN" Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
return $retval return $retval
fi fi
fi fi
fi fi #__WITH_PARANOIA_DEBUG
} }
function CheckConnectivity3rdPartyHosts { function CheckConnectivity3rdPartyHosts {
@ -1191,14 +1304,14 @@ function CheckConnectivity3rdPartyHosts {
local remote3rdPartySuccess local remote3rdPartySuccess
local retval local retval
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug #__WITH_PARANOIA_DEBUG
if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then
remote3rdPartySuccess=false remote3rdPartySuccess=false
for i in $REMOTE_3RD_PARTY_HOSTS for i in $REMOTE_3RD_PARTY_HOSTS
do do
eval "$PING_CMD $i > /dev/null 2>&1" & eval "$PING_CMD $i > /dev/null 2>&1" &
WaitForTaskCompletion $! 180 360 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 180 360 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
@ -1214,7 +1327,7 @@ function CheckConnectivity3rdPartyHosts {
return 0 return 0
fi fi
fi fi
fi fi #__WITH_PARANOIA_DEBUG
} }
#__BEGIN_WITH_PARANOIA_DEBUG #__BEGIN_WITH_PARANOIA_DEBUG
@ -1229,10 +1342,6 @@ function __CheckArguments {
local minArgs local minArgs
local maxArgs local maxArgs
if [ "$_PARANOIA_DEBUG" == "yes" ]; then
Logger "Entering function [$functionName]." "DEBUG"
fi
# All arguments of the function to check are passed as array in ${4} (the function call waits for $@) # All arguments of the function to check are passed as array in ${4} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments # If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${4} and count # In order to avoid this, we need to iterate over ${4} and count
@ -1247,22 +1356,29 @@ function __CheckArguments {
if [ "$argument" = "" ]; then if [ "$argument" = "" ]; then
fetchArguments=false fetchArguments=false
else else
argList="$arg_list [Argument $(($iterate-3)): $argument]" argList="$argList[Argument $(($iterate-3)): $argument] "
iterate=$(($iterate+1)) iterate=$(($iterate+1))
fi fi
done done
countedArguments=$((iterate-4)) countedArguments=$((iterate-4))
if [ $(IsNumeric "$numberOfArguments") -eq 1 ]; then if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
minArgs=$numberOfArguments minArgs=$numberOfArguments
maxArgs=$numberOfArguments maxArgs=$numberOfArguments
else else
IFS='-' read minArgs maxArgs <<< "$numberOfArguments" IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
fi fi
if [ "$_PARANOIA_DEBUG" == "yes" ]; then
Logger "Entering function [$functionName]." "DEBUG"
fi
if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
Logger "Function $functionName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments. see log file." "ERROR" Logger "Function $functionName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments. see log file." "ERROR"
Logger "Arguments passed: $argList" "ERROR" Logger "Arguments passed: $argList" "ERROR"
else
Logger "Arguments passed: $argList" "VERBOSE"
fi fi
fi fi
} }
@ -1388,68 +1504,14 @@ function PreInit {
COMMAND_SUDO="" COMMAND_SUDO=""
fi fi
## Set rsync default arguments ## Set compression executable and extension
RSYNC_ARGS="-rltD"
if [ "$_DRYRUN" == true ]; then
RSYNC_DRY_ARG="-n"
DRY_WARNING="/!\ DRY RUN "
else
RSYNC_DRY_ARG=""
fi
RSYNC_ATTR_ARGS=""
if [ "$PRESERVE_PERMISSIONS" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p"
fi
if [ "$PRESERVE_OWNER" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o"
fi
if [ "$PRESERVE_GROUP" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g"
fi
if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z"
fi
if [ "$COPY_SYMLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -L"
fi
if [ "$KEEP_DIRLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -K"
fi
if [ "$PRESERVE_HARDLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -H"
fi
if [ "$CHECKSUM" == "yes" ]; then
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi
if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi
if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
## Set compression executable and extension
if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then
COMPRESSION_LEVEL=3 COMPRESSION_LEVEL=3
fi fi
#TODO: Remote OS isn't defined yet
## Busybox fix (Termux xz command doesn't support compression at all) ## Busybox fix (Termux xz command doesn't support compression at all)
if [ "$LOCAL_OS" == "BUSYBOX" ] || [ "$REMOTE_OS" == "BUSYBOX" ]; then if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then
compressionString="" compressionString=""
if type gzip > /dev/null 2>&1 if type gzip > /dev/null 2>&1
then then
@ -1526,10 +1588,13 @@ function InitLocalOSSettings {
PING_CMD="ping -c 2 -i .2" PING_CMD="ping -c 2 -i .2"
fi fi
if [ "$LOCAL_OS" == "BUSYBOX" ]; then if [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "msys" ]; then
PROCESS_STATE_CMD="echo none" PROCESS_STATE_CMD="echo none"
DF_CMD="df"
else else
PROCESS_STATE_CMD='ps -p$pid -o state= 2 > /dev/null' PROCESS_STATE_CMD='ps -p$pid -o state= 2 > /dev/null'
# CentOS 5 needs -P for one line output
DF_CMD="df -P"
fi fi
## Stat command has different syntax on Linux and FreeBSD/MacOSX ## Stat command has different syntax on Linux and FreeBSD/MacOSX
@ -1538,7 +1603,7 @@ function InitLocalOSSettings {
STAT_CMD="stat -f \"%Sm\"" STAT_CMD="stat -f \"%Sm\""
STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m"
else else
# Tested on GNU stat and busybox # Tested on GNU stat, busybox and Cygwin
STAT_CMD="stat -c %y" STAT_CMD="stat -c %y"
STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y"
fi fi
@ -1547,13 +1612,6 @@ function InitLocalOSSettings {
function InitRemoteOSSettings { function InitRemoteOSSettings {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
if [ "$PRESERVE_EXECUTABILITY" != "no" ];then
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E"
fi
fi
if [ "$REMOTE_OS" == "msys" ]; then if [ "$REMOTE_OS" == "msys" ]; then
REMOTE_FIND_CMD=$(dirname $BASH)/find REMOTE_FIND_CMD=$(dirname $BASH)/find
else else
@ -1571,6 +1629,72 @@ function InitRemoteOSSettings {
} }
function InitRsyncSettings {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
## Set rsync default arguments
RSYNC_ARGS="-rltD"
if [ "$_DRYRUN" == true ]; then
RSYNC_DRY_ARG="-n"
DRY_WARNING="/!\ DRY RUN "
else
RSYNC_DRY_ARG=""
fi
RSYNC_ATTR_ARGS=""
if [ "$PRESERVE_PERMISSIONS" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p"
fi
if [ "$PRESERVE_OWNER" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o"
fi
if [ "$PRESERVE_GROUP" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g"
fi
if [ "$PRESERVE_EXECUTABILITY" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" --executability"
fi
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] && [ "$LOCAL_OS" != "msys" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi
else
Logger "Disabling ACL and extended attributes synchronization on [$LOCAL_OS]." "NOTICE"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z"
fi
if [ "$COPY_SYMLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -L"
fi
if [ "$KEEP_DIRLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -K"
fi
if [ "$PRESERVE_HARDLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -H"
fi
if [ "$CHECKSUM" == "yes" ]; then
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi
if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi
if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
}
## IFS debug function ## IFS debug function
function PrintIFS { function PrintIFS {
printf "IFS is: %q" "$IFS" printf "IFS is: %q" "$IFS"
@ -1621,13 +1745,20 @@ function TrapStop {
function TrapQuit { function TrapQuit {
local exitcode local exitcode
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID" ]; then
WARN_ALERT=1
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID" ]; then
ERROR_ALERT=1
fi
if [ $ERROR_ALERT == true ]; then if [ $ERROR_ALERT == true ]; then
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then
RunAfterHook RunAfterHook
fi fi
Logger "$PROGRAM finished with errors." "ERROR" Logger "$PROGRAM finished with errors." "ERROR"
SendAlert SendAlert
CleanUp
exitcode=1 exitcode=1
elif [ $WARN_ALERT == true ]; then elif [ $WARN_ALERT == true ]; then
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then
@ -1635,12 +1766,10 @@ function TrapQuit {
fi fi
Logger "$PROGRAM finished with warnings." "WARN" Logger "$PROGRAM finished with warnings." "WARN"
SendAlert SendAlert
CleanUp
exitcode=2 exitcode=2
else else
RunAfterHook RunAfterHook
Logger "$PROGRAM finshed without errors." "NOTICE" Logger "$PROGRAM finshed without errors." "NOTICE"
CleanUp
exitcode=0 exitcode=0
fi fi
@ -1648,6 +1777,7 @@ function TrapQuit {
rm -f "$RUN_DIR/$PROGRAM.$INSTANCE_ID" rm -f "$RUN_DIR/$PROGRAM.$INSTANCE_ID"
fi fi
CleanUp
KillChilds $$ > /dev/null 2>&1 KillChilds $$ > /dev/null 2>&1
exit $exitcode exit $exitcode
} }
@ -1799,7 +1929,7 @@ function _ListDatabasesLocal {
sqlCmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2>&1" sqlCmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2>&1"
Logger "cmd: $sqlCmd" "DEBUG" Logger "cmd: $sqlCmd" "DEBUG"
eval "$sqlCmd" & eval "$sqlCmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
Logger "Listing databases succeeded." "NOTICE" Logger "Listing databases succeeded." "NOTICE"
else else
@ -1822,7 +1952,7 @@ function _ListDatabasesRemote {
sqlCmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID\" 2>&1" sqlCmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID\" 2>&1"
Logger "cmd: $sqlCmd" "DEBUG" Logger "cmd: $sqlCmd" "DEBUG"
eval "$sqlCmd" & eval "$sqlCmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
Logger "Listing databases succeeded." "NOTICE" Logger "Listing databases succeeded." "NOTICE"
else else
@ -1923,7 +2053,7 @@ function _ListRecursiveBackupDirectoriesLocal {
cmd="$FIND_CMD -L $directory/ -mindepth 1 -maxdepth 1 -type d >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" cmd="$FIND_CMD -L $directory/ -mindepth 1 -maxdepth 1 -type d >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not enumerate directories in [$directory]." "ERROR" Logger "Could not enumerate directories in [$directory]." "ERROR"
if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then
@ -1954,7 +2084,7 @@ function _ListRecursiveBackupDirectoriesRemote {
cmd=$SSH_CMD' "'$COMMAND_SUDO' '$REMOTE_FIND_CMD' -L '$directory'/ -mindepth 1 -maxdepth 1 -type d" >> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID cmd=$SSH_CMD' "'$COMMAND_SUDO' '$REMOTE_FIND_CMD' -L '$directory'/ -mindepth 1 -maxdepth 1 -type d" >> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not enumerate directories in [$directory]." "ERROR" Logger "Could not enumerate directories in [$directory]." "ERROR"
if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then
@ -2051,7 +2181,7 @@ function _GetDirectoriesSizeLocal {
cmd="du -cs $dir_list | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" cmd="du -cs $dir_list | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
# $cmd will return 0 even if some errors found, so we need to check if there is an error output # $cmd will return 0 even if some errors found, so we need to check if there is an error output
if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then
Logger "Could not get files size for some or all directories." "ERROR" Logger "Could not get files size for some or all directories." "ERROR"
@ -2085,7 +2215,7 @@ function _GetDirectoriesSizeRemote {
cmd=$SSH_CMD' '$COMMAND_SUDO' du -cs '$dir_list' | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID cmd=$SSH_CMD' '$COMMAND_SUDO' du -cs '$dir_list' | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
# $cmd will return 0 even if some errors found, so we need to check if there is an error output # $cmd will return 0 even if some errors found, so we need to check if there is an error output
if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then
Logger "Could not get files size for some or all directories." "ERROR" Logger "Could not get files size for some or all directories." "ERROR"
@ -2152,7 +2282,7 @@ function _CreateDirectoryRemote {
cmd=$SSH_CMD' "if ! [ -d \"'$dir_to_create'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$dir_to_create'\"; fi" > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2>&1' cmd=$SSH_CMD' "if ! [ -d \"'$dir_to_create'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$dir_to_create'\"; fi" > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot create remote directory [$dir_to_create]." "CRITICAL" Logger "Cannot create remote directory [$dir_to_create]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
@ -2213,7 +2343,7 @@ function GetDiskSpaceLocal {
if [ -d "$path_to_check" ]; then if [ -d "$path_to_check" ]; then
# Not elegant solution to make df silent on errors # Not elegant solution to make df silent on errors
# No sudo on local commands, assuming you should have all the necesarry rights to check backup directories sizes # No sudo on local commands, assuming you should have all the necesarry rights to check backup directories sizes
df "$path_to_check" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 $DF_CMD "$path_to_check" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1
if [ $? != 0 ]; then if [ $? != 0 ]; then
DISK_SPACE=0 DISK_SPACE=0
Logger "Cannot get disk space in [$path_to_check] on local system." "ERROR" Logger "Cannot get disk space in [$path_to_check] on local system." "ERROR"
@ -2238,10 +2368,10 @@ function GetDiskSpaceRemote {
local cmd local cmd
cmd=$SSH_CMD' "if [ -d \"'$path_to_check'\" ]; then '$COMMAND_SUDO' df \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if [ -d \"'$path_to_check'\" ]; then '$COMMAND_SUDO' '$DF_CMD' \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
DISK_SPACE=0 DISK_SPACE=0
Logger "Cannot get disk space in [$path_to_check] on remote system." "ERROR" Logger "Cannot get disk space in [$path_to_check] on remote system." "ERROR"
@ -2419,7 +2549,7 @@ function _BackupDatabaseLocalToLocal {
Logger "cmd: $drySqlCmd" "DEBUG" Logger "cmd: $drySqlCmd" "DEBUG"
eval "$drySqlCmd" & eval "$drySqlCmd" &
fi fi
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then
Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR"
@ -2461,7 +2591,7 @@ function _BackupDatabaseLocalToRemote {
Logger "cmd: $drySqlCmd" "DEBUG" Logger "cmd: $drySqlCmd" "DEBUG"
eval "$drySqlCmd" & eval "$drySqlCmd" &
fi fi
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then
Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR"
@ -2503,7 +2633,7 @@ function _BackupDatabaseRemoteToLocal {
Logger "cmd: $drySqlCmd" "DEBUG" Logger "cmd: $drySqlCmd" "DEBUG"
eval "$drySqlCmd" & eval "$drySqlCmd" &
fi fi
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then
Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR"
@ -2726,7 +2856,7 @@ function Rsync {
Logger "cmd: $rsyncCmd" "DEBUG" Logger "cmd: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" & eval "$rsyncCmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Failed to backup [$backupDirectory] to [$fileStoragePath]." "ERROR" Logger "Failed to backup [$backupDirectory] to [$fileStoragePath]." "ERROR"
@ -2844,7 +2974,7 @@ function _RotateBackupsLocal {
cmd="rm -rf \"$path\"" cmd="rm -rf \"$path\""
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot delete oldest copy [$path]." "ERROR" Logger "Cannot delete oldest copy [$path]." "ERROR"
fi fi
@ -2856,7 +2986,7 @@ function _RotateBackupsLocal {
cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\"" cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\""
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR" Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR"
fi fi
@ -2870,7 +3000,7 @@ function _RotateBackupsLocal {
cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" cmd="mv \"$backup\" \"$backup.$PROGRAM.1\""
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi fi
@ -2879,7 +3009,7 @@ function _RotateBackupsLocal {
cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\"" cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\""
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR" Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi fi
@ -2888,7 +3018,7 @@ function _RotateBackupsLocal {
cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" cmd="mv \"$backup\" \"$backup.$PROGRAM.1\""
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi fi
@ -3002,7 +3132,7 @@ function _RotateBackupsRemoteSSH {
ENDSSH ENDSSH
WaitForTaskCompletion $! 1800 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 1800 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not rotate backups in [$backup_path]." "ERROR" Logger "Could not rotate backups in [$backup_path]." "ERROR"
Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
@ -3298,6 +3428,7 @@ if [ "$REMOTE_OPERATION" == "yes" ]; then
GetRemoteOS GetRemoteOS
InitRemoteOSSettings InitRemoteOSSettings
fi fi
InitRsyncSettings
if [ $no_maxtime == true ]; then if [ $no_maxtime == true ]; then
SOFT_MAX_EXEC_TIME_DB_TASK=0 SOFT_MAX_EXEC_TIME_DB_TASK=0

87
install.sh
View File

@ -4,7 +4,7 @@ PROGRAM=obackup
PROGRAM_VERSION=2.1-dev PROGRAM_VERSION=2.1-dev
PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh" PROGRAM_BATCH=$PROGRAM"-batch.sh"
SCRIPT_BUILD=2016090605 SCRIPT_BUILD=2016112401
## osync / obackup / pmocr / zsnap install script ## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8 & 10 ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8 & 10
@ -12,6 +12,9 @@ SCRIPT_BUILD=2016090605
#TODO: silent mode and no stats mode #TODO: silent mode and no stats mode
# Get current install.sh path from http://stackoverflow.com/a/246128/2635443
SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
CONF_DIR=$FAKEROOT/etc/$PROGRAM CONF_DIR=$FAKEROOT/etc/$PROGRAM
BIN_DIR="$FAKEROOT/usr/local/bin" BIN_DIR="$FAKEROOT/usr/local/bin"
SERVICE_DIR_INIT=$FAKEROOT/etc/init.d SERVICE_DIR_INIT=$FAKEROOT/etc/init.d
@ -70,25 +73,35 @@ function urlencode() {
local c="${1:i:1}" local c="${1:i:1}"
case $c in case $c in
[a-zA-Z0-9.~_-]) printf "$c" ;; [a-zA-Z0-9.~_-]) printf "$c" ;;
*) printf '%%%02X' "'$c" ;; *) printf '%%%02X' "'$c" ;;
esac esac
done done
} }
function SetOSSettings { function SetOSSettings {
local localOsVar
USER=root USER=root
local local_os_var # There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
else
# Detecting the special ubuntu userland in Windows 10 bash
if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
localOsVar="Microsoft"
else
localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname)"
fi
fi
fi
fi
local_os_var="$(uname -spio 2>&1)" case $localOsVar in
if [ $? != 0 ]; then
local_os_var="$(uname -v 2>&1)"
if [ $? != 0 ]; then
local_os_var="$(uname)"
fi
fi
case $local_os_var in
*"BSD"*) *"BSD"*)
GROUP=wheel GROUP=wheel
;; ;;
@ -109,7 +122,7 @@ function SetOSSettings {
exit 1 exit 1
fi fi
OS=$(urlencode "$local_os_var") OS=$(urlencode "$localOsVar")
} }
function GetInit { function GetInit {
@ -140,34 +153,34 @@ function CreateConfDir {
} }
function CopyExampleFiles { function CopyExampleFiles {
if [ -f "./sync.conf.example" ]; then if [ -f "$SCRIPT_PATH/sync.conf.example" ]; then
cp "./sync.conf.example" "$CONF_DIR/sync.conf.example" cp "$SCRIPT_PATH/sync.conf.example" "$CONF_DIR/sync.conf.example"
fi fi
if [ -f "./host_backup.conf.example" ]; then if [ -f "$SCRIPT_PATH/host_backup.conf.example" ]; then
cp "./host_backup.conf.example" "$CONF_DIR/host_backup.conf.example" cp "$SCRIPT_PATH/host_backup.conf.example" "$CONF_DIR/host_backup.conf.example"
fi fi
if [ -f "./exlude.list.example" ]; then if [ -f "$SCRIPT_PATH/exlude.list.example" ]; then
cp "./exclude.list.example" "$CONF_DIR/exclude.list.example" cp "$SCRIPT_PATH/exclude.list.example" "$CONF_DIR/exclude.list.example"
fi fi
if [ -f "./snapshot.conf.example" ]; then if [ -f "$SCRIPT_PATH/snapshot.conf.example" ]; then
cp "./snapshot.conf.example" "$CONF_DIR/snapshot.conf.example" cp "$SCRIPT_PATH/snapshot.conf.example" "$CONF_DIR/snapshot.conf.example"
fi fi
if [ -f "./default.conf" ]; then if [ -f "$SCRIPT_PATH/default.conf" ]; then
if [ -f "$CONF_DIR/default.conf" ]; then if [ -f "$CONF_DIR/default.conf" ]; then
cp "./default.conf" "$CONF_DIR/default.conf.new" cp "$SCRIPT_PATH/default.conf" "$CONF_DIR/default.conf.new"
QuickLogger "Copied default.conf to [$CONF_DIR/default.conf.new]." QuickLogger "Copied default.conf to [$CONF_DIR/default.conf.new]."
else else
cp "./default.conf" "$CONF_DIR/default.conf" cp "$SCRIPT_PATH/default.conf" "$CONF_DIR/default.conf"
fi fi
fi fi
} }
function CopyProgram { function CopyProgram {
cp "./$PROGRAM_BINARY" "$BIN_DIR" cp "$SCRIPT_PATH/$PROGRAM_BINARY" "$BIN_DIR"
if [ $? != 0 ]; then if [ $? != 0 ]; then
QuickLogger "Cannot copy $PROGRAM_BINARY to [$BIN_DIR]. Make sure to run install script in the directory containing all other files." QuickLogger "Cannot copy $PROGRAM_BINARY to [$BIN_DIR]. Make sure to run install script in the directory containing all other files."
QuickLogger "Also make sure you have permissions to write to [$BIN_DIR]." QuickLogger "Also make sure you have permissions to write to [$BIN_DIR]."
@ -177,8 +190,8 @@ function CopyProgram {
QuickLogger "Copied $PROGRAM_BINARY to [$BIN_DIR]." QuickLogger "Copied $PROGRAM_BINARY to [$BIN_DIR]."
fi fi
if [ -f "./$PROGRAM_BATCH" ]; then if [ -f "$SCRIPT_PATH/$PROGRAM_BATCH" ]; then
cp "./$PROGRAM_BATCH" "$BIN_DIR" cp "$SCRIPT_PATH/$PROGRAM_BATCH" "$BIN_DIR"
if [ $? != 0 ]; then if [ $? != 0 ]; then
QuickLogger "Cannot copy $PROGRAM_BATCH to [$BIN_DIR]." QuickLogger "Cannot copy $PROGRAM_BATCH to [$BIN_DIR]."
else else
@ -187,8 +200,8 @@ function CopyProgram {
fi fi
fi fi
if [ -f "./ssh_filter.sh" ]; then if [ -f "$SCRIPT_PATH/ssh_filter.sh" ]; then
cp "./ssh_filter.sh" "$BIN_DIR" cp "$SCRIPT_PATH/ssh_filter.sh" "$BIN_DIR"
if [ $? != 0 ]; then if [ $? != 0 ]; then
QuickLogger "Cannot copy ssh_filter.sh to [$BIN_DIR]." QuickLogger "Cannot copy ssh_filter.sh to [$BIN_DIR]."
else else
@ -203,8 +216,8 @@ function CopyProgram {
function CopyServiceFiles { function CopyServiceFiles {
# OSYNC SPECIFIC # OSYNC SPECIFIC
if ([ "$init" == "systemd" ] && [ -f "./$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
cp "./$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM" && cp "./$OSYNC_SERVICE_FILE_SYSTEMD_USER" "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM" cp "$SCRIPT_PATH/$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM" && cp "$SCRIPT_PATH/$OSYNC_SERVICE_FILE_SYSTEMD_USER" "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM"
if [ $? != 0 ]; then if [ $? != 0 ]; then
QuickLogger "Cannot copy the systemd file to [$SERVICE_DIR_SYSTEMD_SYSTEM] or [$SERVICE_DIR_SYSTEMD_USER]." QuickLogger "Cannot copy the systemd file to [$SERVICE_DIR_SYSTEMD_SYSTEM] or [$SERVICE_DIR_SYSTEMD_USER]."
else else
@ -213,8 +226,8 @@ function CopyServiceFiles {
QuickLogger "Can be enabled on boot with [systemctl enable osync-srv@instance.conf]." QuickLogger "Can be enabled on boot with [systemctl enable osync-srv@instance.conf]."
QuickLogger "In userland, active with [systemctl --user start osync-srv@instance.conf]." QuickLogger "In userland, active with [systemctl --user start osync-srv@instance.conf]."
fi fi
elif ([ "$init" == "initV" ] && [ -f "./$OSYNC_SERVICE_FILE_INIT" ]); then elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$OSYNC_SERVICE_FILE_INIT" ]); then
cp "./$OSYNC_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT" cp "$SCRIPT_PATH/$OSYNC_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT"
if [ $? != 0 ]; then if [ $? != 0 ]; then
QuickLogger "Cannot copy osync-srv to [$SERVICE_DIR_INIT]." QuickLogger "Cannot copy osync-srv to [$SERVICE_DIR_INIT]."
else else
@ -226,8 +239,8 @@ function CopyServiceFiles {
fi fi
# PMOCR SPECIFIC # PMOCR SPECIFIC
if ([ "$init" == "systemd" ] && [ -f "./$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
cp "./$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM" cp "$SCRIPT_PATH/$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM"
if [ $? != 0 ]; then if [ $? != 0 ]; then
QuickLogger "Cannot copy the systemd file to [$SERVICE_DIR_SYSTEMD_SYSTEM] or [$SERVICE_DIR_SYSTEMD_USER]." QuickLogger "Cannot copy the systemd file to [$SERVICE_DIR_SYSTEMD_SYSTEM] or [$SERVICE_DIR_SYSTEMD_USER]."
else else
@ -235,8 +248,8 @@ function CopyServiceFiles {
QuickLogger "Can be activated with [systemctl start pmocr-srv@default.conf] where default.conf is the name of the config file in $CONF_DIR." QuickLogger "Can be activated with [systemctl start pmocr-srv@default.conf] where default.conf is the name of the config file in $CONF_DIR."
QuickLogger "Can be enabled on boot with [systemctl enable pmocr-srv@default.conf]." QuickLogger "Can be enabled on boot with [systemctl enable pmocr-srv@default.conf]."
fi fi
elif ([ "$init" == "initV" ] && [ -f "./$PMOCR_SERVICE_FILE_INIT" ]); then elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$PMOCR_SERVICE_FILE_INIT" ]); then
cp "./$PMOCR_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT" cp "$SCRIPT_PATH/$PMOCR_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT"
if [ $? != 0 ]; then if [ $? != 0 ]; then
QuickLogger "Cannot copy pmoct-srv to [$SERVICE_DIR_INIT]." QuickLogger "Cannot copy pmoct-srv to [$SERVICE_DIR_INIT]."
else else

38
obackup-batch.sh
View File

@ -3,7 +3,7 @@ SUBPROGRAM=obackup
PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
AUTHOR="(L) 2013-2016 by Orsiris de Jong" AUTHOR="(L) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr - ozy@netpower.fr" CONTACT="http://www.netpower.fr - ozy@netpower.fr"
PROGRAM_BUILD=2016082901 PROGRAM_BUILD=2016112402
## Runs an osync /obackup instance for every conf file found ## Runs an osync /obackup instance for every conf file found
## If an instance fails, run it again if time permits ## If an instance fails, run it again if time permits
@ -13,9 +13,6 @@ if ! type "$BASH" > /dev/null; then
exit 127 exit 127
fi fi
## Configuration file path. The path where all the osync / obackup conf files are, usually /etc/osync or /etc/obackup
CONF_FILE_PATH=/etc/$SUBPROGRAM
## If maximum execution time is not reached, failed instances will be rerun. Max exec time is in seconds. Example is set to 10 hours. ## If maximum execution time is not reached, failed instances will be rerun. Max exec time is in seconds. Example is set to 10 hours.
MAX_EXECUTION_TIME=36000 MAX_EXECUTION_TIME=36000
@ -74,6 +71,10 @@ function CheckEnvironment {
else else
SUBPROGRAM_EXECUTABLE=$(type -p $SUBPROGRAM.sh) SUBPROGRAM_EXECUTABLE=$(type -p $SUBPROGRAM.sh)
fi fi
if [ "$CONF_FILE_PATH" == "" ]; then
Usage
fi
} }
function Batch { function Batch {
@ -111,7 +112,7 @@ function Batch {
Logger "$SUBPROGRAM instances will be run for: $runList" "NOTICE" Logger "$SUBPROGRAM instances will be run for: $runList" "NOTICE"
for confFile in $runList for confFile in $runList
do do
$SUBPROGRAM_EXECUTABLE "$confFile" $opts & $SUBPROGRAM_EXECUTABLE "$confFile" --silent $opts &
wait $! wait $!
result=$? result=$?
if [ $result != 0 ]; then if [ $result != 0 ]; then
@ -141,16 +142,18 @@ function Usage {
echo $CONTACT echo $CONTACT
echo "" echo ""
echo "Batch script to sequentially run osync or obackup instances and rerun failed ones." echo "Batch script to sequentially run osync or obackup instances and rerun failed ones."
echo "Usage: $SUBPROGRAM-batch.sh [OPTIONS]" echo "Usage: $PROGRAM.sh [OPTIONS] [$SUBPROGRAM OPTIONS]"
echo "" echo ""
echo "[OPTIONS]" echo "[OPTIONS]"
echo "--path=/path/to/conf Path to osync / obackup conf files, defaults to /etc/osync or /etc/obackup" echo "--path=/path/to/conf Path to osync / obackup conf files, defaults to /etc/osync or /etc/obackup"
echo "--max-runs=X Number of max runs per instance, (defaults to 3)" echo "--max-runs=X Number of max runs per instance, (defaults to 3)"
echo "--max-exec-time=X Retry failed instances only if max execution time not reached (defaults to 36000 seconds). Set to 0 to bypass execution time check" echo "--max-exec-time=X Retry failed instances only if max execution time not reached (defaults to 36000 seconds). Set to 0 to bypass execution time check"
echo "--no-maxtime Run osync / obackup without honoring conf file defined timeouts" echo "[$SUBPROGRAM OPTIONS]"
echo "--dry Will run osync / obackup without actually doing anything; just testing" echo "Specify whatever options $PROGRAM accepts. Example"
echo "--silent Will run osync / obackup without any output to stdout, used for cron jobs" echo "$PROGRAM.sh --path=/etc/$SUBPROGRAM --no-maxtime"
echo "--verbose Increases output" echo ""
echo "No output will be written to stdout/stderr."
echo "Verify log file in [$LOG_FILE]."
exit 128 exit 128
} }
@ -158,18 +161,6 @@ opts=""
for i in "$@" for i in "$@"
do do
case $i in case $i in
--silent)
opts=$opts" --silent"
;;
--dry)
opts=$opts" --dry"
;;
--verbose)
opts=$opts" --verbose"
;;
--no-maxtime)
opts=$opts" --no-maxtime"
;;
--path=*) --path=*)
CONF_FILE_PATH=${i##*=} CONF_FILE_PATH=${i##*=}
;; ;;
@ -183,8 +174,7 @@ do
Usage Usage
;; ;;
*) *)
Logger "Unknown param '$i'" "CRITICAL" opts="$i "
Usage
;; ;;
esac esac
done done

467
obackup.sh
View File

@ -9,12 +9,12 @@ PROGRAM="obackup"
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
PROGRAM_VERSION=2.1-dev PROGRAM_VERSION=2.1-dev
PROGRAM_BUILD=2016111701 PROGRAM_BUILD=2016113001
IS_STABLE=no IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016111704 ## FUNC_BUILD=2016112902
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
@ -29,8 +29,13 @@ IS_STABLE=no
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID ## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID
#TODO: Rewrite Logger so we can decide what to send to stdout, stderr and logfile ## META ISSUES
#TODO: Windows checks, check sendmail & mailsend ##
## Updated _LOGGER_STDERR
## Updated WaitForTaskCompletion syntax
## Updated ParallelExec syntax
## SendEmail WinNT10 & msys are two totally different beasts. Document in sync.conf and host_backup.conf
if ! type "$BASH" > /dev/null; then if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2" echo "Please run this script only with bash shell. Tested on bash >= 3.2"
@ -61,16 +66,16 @@ WARN_ALERT=false
## allow debugging from command line with _DEBUG=yes ## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no _DEBUG=no
SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys
_LOGGER_VERBOSE=false _LOGGER_VERBOSE=false
else else
if [ "$SLEEP_TIME" == "" ]; then # Set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=.05
fi
trap 'TrapError ${LINENO} $?' ERR trap 'TrapError ${LINENO} $?' ERR
_LOGGER_VERBOSE=true _LOGGER_VERBOSE=true
fi fi
if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=.05
fi
SCRIPT_PID=$$ SCRIPT_PID=$$
LOCAL_USER=$(whoami) LOCAL_USER=$(whoami)
@ -237,16 +242,20 @@ function KillChilds {
done done
fi fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then if [ "$self" == true ]; then
Logger "Sending SIGTERM to process [$pid]." "DEBUG" if kill -0 "$pid" > /dev/null 2>&1; then
kill -s TERM "$pid" kill -s TERM "$pid"
if [ $? != 0 ]; then Logger "Sent SIGTERM to process [$pid]." "DEBUG"
sleep 15
Logger "Sending SIGTERM to process [$pid] failed." "DEBUG"
kill -9 "$pid"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" sleep 15
return 1 Logger "Sending SIGTERM to process [$pid] failed." "DEBUG"
kill -9 "$pid"
if [ $? != 0 ]; then
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
return 1
fi # Simplify the return 0 logic here
else
return 0
fi fi
else else
return 0 return 0
@ -309,7 +318,6 @@ function SendAlert {
if [ -e "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID" ]; then if [ -e "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID" ]; then
body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID)" body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID)"
fi fi
exit
if [ $ERROR_ALERT == true ]; then if [ $ERROR_ALERT == true ]; then
subject="Error alert for $INSTANCE_ID" subject="Error alert for $INSTANCE_ID"
@ -375,7 +383,7 @@ function SendEmail {
mail_no_attachment=0 mail_no_attachment=0
fi fi
if [ "$LOCAL_OS" == "BUSYBOX" ]; then if [ "$LOCAL_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ]; then
if type sendmail > /dev/null 2>&1; then if type sendmail > /dev/null 2>&1; then
if [ "$ENCRYPTION" == "tls" ]; then if [ "$ENCRYPTION" == "tls" ]; then
echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$SenderMail" -H "exec openssl s_client -quiet -tls1_2 -starttls smtp -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails" echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$SenderMail" -H "exec openssl s_client -quiet -tls1_2 -starttls smtp -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails"
@ -409,13 +417,18 @@ function SendEmail {
fi fi
if type mail > /dev/null 2>&1 ; then if type mail > /dev/null 2>&1 ; then
if [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then # We need to detect which version of mail is installed
if ! $(type -p mail) -V > /dev/null 2>&1; then
# This may be MacOS mail program
attachment_command=""
elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then
attachment_command="-A $attachment" attachment_command="-A $attachment"
elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then
attachment_command="-a$attachment" attachment_command="-a$attachment"
else else
attachment_command="" attachment_command=""
fi fi
echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails" echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN" Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN"
@ -526,30 +539,30 @@ function Spinner {
return 0 return 0
fi fi
case $toggle case $_OFUNCTIONS_SPINNER_TOGGLE
in in
1) 1)
echo -n " \ " echo -n " \ "
echo -ne "\r" echo -ne "\r"
toggle="2" _OFUNCTIONS_SPINNER_TOGGLE=2
;; ;;
2) 2)
echo -n " | " echo -n " | "
echo -ne "\r" echo -ne "\r"
toggle="3" _OFUNCTIONS_SPINNER_TOGGLE=3
;; ;;
3) 3)
echo -n " / " echo -n " / "
echo -ne "\r" echo -ne "\r"
toggle="4" _OFUNCTIONS_SPINNER_TOGGLE=4
;; ;;
*) *)
echo -n " - " echo -n " - "
echo -ne "\r" echo -ne "\r"
toggle="1" _OFUNCTIONS_SPINNER_TOGGLE=1
;; ;;
esac esac
} }
@ -564,16 +577,20 @@ function joinString {
# Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 # Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2
# Warning: Don't imbricate this function into another run if you plan to use the global variable output # Warning: Don't imbricate this function into another run if you plan to use the global variable output
# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true false "${FUNCNAME[0]}"
function WaitForTaskCompletion { function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon local pids="${1}" # pids to wait for, separated by semi-colon
local softMaxTime="${2}" # If program with pid $pid takes longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. local softMaxTime="${2:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${3}" # If program with pid $pid takes longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. local hardMaxTime="${3:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local callerName="${4}" # Who called this function local sleepTime="${4:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false local keepLogging="${5:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local keepLogging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging local counting="${6:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${7:-true}" # Show spinner (true), don't show anything (false)
local noErrorLog="${8:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local callerName="${9}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]}
local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once
local log_ttime=0 # local time instance for comparaison local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script local seconds_begin=$SECONDS # Seconds since the beginning of the script
@ -590,6 +607,10 @@ function WaitForTaskCompletion {
local newPidsArray # New array of currently running pids local newPidsArray # New array of currently running pids
if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown
local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once
fi
IFS=';' read -a pidsArray <<< "$pids" IFS=';' read -a pidsArray <<< "$pids"
pidCount=${#pidsArray[@]} pidCount=${#pidsArray[@]}
@ -598,7 +619,9 @@ function WaitForTaskCompletion {
while [ ${#pidsArray[@]} -gt 0 ]; do while [ ${#pidsArray[@]} -gt 0 ]; do
newPidsArray=() newPidsArray=()
Spinner if [ $spinner == true ]; then
Spinner
fi
if [ $counting == true ]; then if [ $counting == true ]; then
exec_time=$(($SECONDS - $seconds_begin)) exec_time=$(($SECONDS - $seconds_begin))
else else
@ -615,33 +638,36 @@ function WaitForTaskCompletion {
fi fi
if [ $exec_time -gt $softMaxTime ]; then if [ $exec_time -gt $softMaxTime ]; then
if [ $soft_alert == true ] && [ $softMaxTime -ne 0 ]; then if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN" Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
soft_alert=true _SOFT_ALERT=true
SendAlert true SendAlert true
fi
fi
fi if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then if [ $noErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
for pid in "${pidsArray[@]}"; do fi
KillChilds $pid true for pid in "${pidsArray[@]}"; do
if [ $? == 0 ]; then KillChilds $pid true
Logger "Task with pid [$pid] stopped successfully." "NOTICE" if [ $? == 0 ]; then
else Logger "Task with pid [$pid] stopped successfully." "NOTICE"
Logger "Could not stop task with pid [$pid]." "ERROR" else
fi Logger "Could not stop task with pid [$pid]." "ERROR"
done fi
errorcount=$((errorcount+1))
done
if [ $noErrorLog != true ]; then
SendAlert true SendAlert true
fi fi
return $errorcount
fi fi
for pid in "${pidsArray[@]}"; do for pid in "${pidsArray[@]}"; do
if [ $(IsInteger $pid) -eq 1 ]; then if [ $(IsInteger $pid) -eq 1 ]; then
if kill -0 $pid > /dev/null 2>&1; then if kill -0 $pid > /dev/null 2>&1; then
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
#TODO(high): have this tested on *BSD, Mac, Win & busybox.
#TODO(high): propagate changes to ParallelExec
#pidState=$(ps -p$pid -o state= 2 > /dev/null)
pidState="$(eval $PROCESS_STATE_CMD)" pidState="$(eval $PROCESS_STATE_CMD)"
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
newPidsArray+=($pid) newPidsArray+=($pid)
@ -666,13 +692,14 @@ function WaitForTaskCompletion {
pidsArray=("${newPidsArray[@]}") pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU # Trivial wait time for bash to not eat up all CPU
sleep $SLEEP_TIME sleep $sleepTime
done done
# Return exit code if only one process was monitored, else return number of errors # Return exit code if only one process was monitored, else return number of errors
if [ $pidCount -eq 1 ] && [ $errorcount -eq 0 ]; then # As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
return $errorcount if [ $pidCount -eq 1 ]; then
return $retval
else else
return $errorcount return $errorcount
fi fi
@ -681,16 +708,26 @@ function WaitForTaskCompletion {
# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs # Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
# Returns the number of non zero exit codes from commands # Returns the number of non zero exit codes from commands
# Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets # Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets
function ParallelExec { # Only 2 first arguments are mandatory
local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or file containing one command per line
local readFromFile="${3:-false}" # Is commandsArg a file or a string ?
local softMaxTime="${4:-0}"
local hardMaxTime="${5:-0}"
local callerName="${6}" # Who called this function
local counting="${7:-true}" # Count time since function has been launched if true, since script has been launched if false
local keepLogging="${8:-0}" # Log a standby message every X seconds. Set to zero to disable logging
function ParallelExec {
local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or path to file containing one command per line
local readFromFile="${3:-false}" # commandsArg is a file (true), or a string (false)
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-false}" # Show spinner (true), don't show spinner (false)
local noErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local callerName="${11:-false}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]}
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local commandCount local commandCount
local command local command
@ -705,6 +742,10 @@ function ParallelExec {
local commandsArrayPid local commandsArrayPid
if [ $counting == true ]; then # If counting == false _SOFT_ALERT should be a global value so no more than one soft alert is shown
local _SOFT_ALERT=false # Does a soft alert need to be triggered, if yes, send an alert once
fi
if [ $readFromFile == true ];then if [ $readFromFile == true ];then
if [ -f "$commandsArg" ]; then if [ -f "$commandsArg" ]; then
commandCount=$(wc -l < "$commandsArg") commandCount=$(wc -l < "$commandsArg")
@ -720,15 +761,60 @@ function ParallelExec {
while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do
if [ $spinner == true ]; then
Spinner
fi
if [ $counting == true ]; then
exec_time=$(($SECONDS - $seconds_begin))
else
exec_time=$SECONDS
fi
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
log_ttime=$exec_time
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ "$_SOFT_ALERT" != true ] && [ $softMaxTime -ne 0 ] && [ $noErrorLog != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
_SOFT_ALERT=true
SendAlert true
fi
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noErrorLog != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Task with pid [$pid] stopped successfully." "NOTICE"
else
Logger "Could not stop task with pid [$pid]." "ERROR"
fi
done
if [ $noErrorLog != true ]; then
SendAlert true
else
# Return the number of commands that haven't run / finished run
return $(($commandCount - $counter + ${#pidsArray[@]}))
fi
fi
while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then if [ $readFromFile == true ]; then
#TODO: Checked on FreeBSD 10, also check on Win
command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg") command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg")
else else
command="${commandsArray[$counter]}" command="${commandsArray[$counter]}"
fi fi
Logger "Running command [$command]." "DEBUG" Logger "Running command [$command]." "DEBUG"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & eval "$command" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID" 2>&1 &
pid=$! pid=$!
pidsArray+=($pid) pidsArray+=($pid)
commandsArrayPid[$pid]="$command" commandsArrayPid[$pid]="$command"
@ -923,16 +1009,25 @@ function GetLocalOS {
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox" localOsVar="BusyBox"
else else
localOsVar="$(uname -spio 2>&1)" # Detecting the special ubuntu userland in Windows 10 bash
if [ $? != 0 ]; then if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
localOsVar="$(uname -v 2>&1)" localOsVar="Microsoft"
else
localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then if [ $? != 0 ]; then
localOsVar="$(uname)" localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname)"
fi
fi fi
fi fi
fi fi
case $localOsVar in case $localOsVar in
# Android uname contains both linux and android, keep it before linux entry
*"Android"*)
LOCAL_OS="Android"
;;
*"Linux"*) *"Linux"*)
LOCAL_OS="Linux" LOCAL_OS="Linux"
;; ;;
@ -942,11 +1037,14 @@ function GetLocalOS {
*"MINGW32"*|*"CYGWIN"*) *"MINGW32"*|*"CYGWIN"*)
LOCAL_OS="msys" LOCAL_OS="msys"
;; ;;
*"Microsoft"*)
LOCAL_OS="WinNT10"
;;
*"Darwin"*) *"Darwin"*)
LOCAL_OS="MacOSX" LOCAL_OS="MacOSX"
;; ;;
*"BusyBox"*) *"BusyBox"*)
LOCAL_OS="BUSYBOX" LOCAL_OS="BusyBox"
;; ;;
*) *)
if [ "$IGNORE_OS_TYPE" == "yes" ]; then #TODO(doc): Undocumented option if [ "$IGNORE_OS_TYPE" == "yes" ]; then #TODO(doc): Undocumented option
@ -964,6 +1062,10 @@ function GetLocalOS {
function GetRemoteOS { function GetRemoteOS {
if [ "$REMOTE_OPERATION" != "yes" ]; then
return 0
fi
local remoteOsVar local remoteOsVar
$SSH_CMD bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 $SSH_CMD bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1
@ -975,15 +1077,19 @@ function GetOs {
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox" localOsVar="BusyBox"
else else
localOsVar="$(uname -spio 2>&1)" # Detecting the special ubuntu userland in Windows 10 bash
if [ $? != 0 ]; then if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
localOsVar="$(uname -v 2>&1)" localOsVar="Microsoft"
else
localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then if [ $? != 0 ]; then
localOsVar="$(uname)" localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname)"
fi
fi fi
fi fi
fi fi
echo "$localOsVar" echo "$localOsVar"
} }
@ -994,6 +1100,9 @@ ENDSSH
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID") remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID")
case $remoteOsVar in case $remoteOsVar in
*"Android"*)
REMOTE_OS="Android"
;;
*"Linux"*) *"Linux"*)
REMOTE_OS="Linux" REMOTE_OS="Linux"
;; ;;
@ -1003,11 +1112,14 @@ ENDSSH
*"MINGW32"*|*"CYGWIN"*) *"MINGW32"*|*"CYGWIN"*)
REMOTE_OS="msys" REMOTE_OS="msys"
;; ;;
*"Microsoft"*)
REMOTE_OS="WinNT10"
;;
*"Darwin"*) *"Darwin"*)
REMOTE_OS="MacOSX" REMOTE_OS="MacOSX"
;; ;;
*"BusyBox"*) *"BusyBox"*)
REMOTE_OS="BUSYBOX" REMOTE_OS="BusyBox"
;; ;;
*"ssh"*|*"SSH"*) *"ssh"*|*"SSH"*)
Logger "Cannot connect to remote system." "CRITICAL" Logger "Cannot connect to remote system." "CRITICAL"
@ -1039,7 +1151,8 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE" Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1073,7 +1186,7 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1106,7 +1219,7 @@ function RunBeforeHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -1124,7 +1237,7 @@ function RunAfterHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -1132,18 +1245,16 @@ function CheckConnectivityRemoteHost {
local retval local retval
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
WaitForTaskCompletion $! 60 180 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 60 180 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN" Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
return $retval return $retval
fi fi
fi fi
fi
} }
function CheckConnectivity3rdPartyHosts { function CheckConnectivity3rdPartyHosts {
@ -1151,14 +1262,13 @@ function CheckConnectivity3rdPartyHosts {
local remote3rdPartySuccess local remote3rdPartySuccess
local retval local retval
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug
if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then
remote3rdPartySuccess=false remote3rdPartySuccess=false
for i in $REMOTE_3RD_PARTY_HOSTS for i in $REMOTE_3RD_PARTY_HOSTS
do do
eval "$PING_CMD $i > /dev/null 2>&1" & eval "$PING_CMD $i > /dev/null 2>&1" &
WaitForTaskCompletion $! 180 360 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 180 360 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
@ -1174,11 +1284,8 @@ function CheckConnectivity3rdPartyHosts {
return 0 return 0
fi fi
fi fi
fi
} }
#__BEGIN_WITH_PARANOIA_DEBUG
#__END_WITH_PARANOIA_DEBUG
function RsyncPatternsAdd { function RsyncPatternsAdd {
local patternType="${1}" # exclude or include local patternType="${1}" # exclude or include
@ -1295,68 +1402,14 @@ function PreInit {
COMMAND_SUDO="" COMMAND_SUDO=""
fi fi
## Set rsync default arguments ## Set compression executable and extension
RSYNC_ARGS="-rltD"
if [ "$_DRYRUN" == true ]; then
RSYNC_DRY_ARG="-n"
DRY_WARNING="/!\ DRY RUN "
else
RSYNC_DRY_ARG=""
fi
RSYNC_ATTR_ARGS=""
if [ "$PRESERVE_PERMISSIONS" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p"
fi
if [ "$PRESERVE_OWNER" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o"
fi
if [ "$PRESERVE_GROUP" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g"
fi
if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z"
fi
if [ "$COPY_SYMLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -L"
fi
if [ "$KEEP_DIRLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -K"
fi
if [ "$PRESERVE_HARDLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -H"
fi
if [ "$CHECKSUM" == "yes" ]; then
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi
if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi
if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
## Set compression executable and extension
if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then
COMPRESSION_LEVEL=3 COMPRESSION_LEVEL=3
fi fi
#TODO: Remote OS isn't defined yet
## Busybox fix (Termux xz command doesn't support compression at all) ## Busybox fix (Termux xz command doesn't support compression at all)
if [ "$LOCAL_OS" == "BUSYBOX" ] || [ "$REMOTE_OS" == "BUSYBOX" ]; then if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then
compressionString="" compressionString=""
if type gzip > /dev/null 2>&1 if type gzip > /dev/null 2>&1
then then
@ -1431,10 +1484,13 @@ function InitLocalOSSettings {
PING_CMD="ping -c 2 -i .2" PING_CMD="ping -c 2 -i .2"
fi fi
if [ "$LOCAL_OS" == "BUSYBOX" ]; then if [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "msys" ]; then
PROCESS_STATE_CMD="echo none" PROCESS_STATE_CMD="echo none"
DF_CMD="df"
else else
PROCESS_STATE_CMD='ps -p$pid -o state= 2 > /dev/null' PROCESS_STATE_CMD='ps -p$pid -o state= 2 > /dev/null'
# CentOS 5 needs -P for one line output
DF_CMD="df -P"
fi fi
## Stat command has different syntax on Linux and FreeBSD/MacOSX ## Stat command has different syntax on Linux and FreeBSD/MacOSX
@ -1443,7 +1499,7 @@ function InitLocalOSSettings {
STAT_CMD="stat -f \"%Sm\"" STAT_CMD="stat -f \"%Sm\""
STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m"
else else
# Tested on GNU stat and busybox # Tested on GNU stat, busybox and Cygwin
STAT_CMD="stat -c %y" STAT_CMD="stat -c %y"
STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y"
fi fi
@ -1451,13 +1507,6 @@ function InitLocalOSSettings {
function InitRemoteOSSettings { function InitRemoteOSSettings {
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
if [ "$PRESERVE_EXECUTABILITY" != "no" ];then
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E"
fi
fi
if [ "$REMOTE_OS" == "msys" ]; then if [ "$REMOTE_OS" == "msys" ]; then
REMOTE_FIND_CMD=$(dirname $BASH)/find REMOTE_FIND_CMD=$(dirname $BASH)/find
else else
@ -1475,6 +1524,71 @@ function InitRemoteOSSettings {
} }
function InitRsyncSettings {
## Set rsync default arguments
RSYNC_ARGS="-rltD"
if [ "$_DRYRUN" == true ]; then
RSYNC_DRY_ARG="-n"
DRY_WARNING="/!\ DRY RUN "
else
RSYNC_DRY_ARG=""
fi
RSYNC_ATTR_ARGS=""
if [ "$PRESERVE_PERMISSIONS" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p"
fi
if [ "$PRESERVE_OWNER" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o"
fi
if [ "$PRESERVE_GROUP" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g"
fi
if [ "$PRESERVE_EXECUTABILITY" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" --executability"
fi
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] && [ "$LOCAL_OS" != "msys" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi
else
Logger "Disabling ACL and extended attributes synchronization on [$LOCAL_OS]." "NOTICE"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z"
fi
if [ "$COPY_SYMLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -L"
fi
if [ "$KEEP_DIRLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -K"
fi
if [ "$PRESERVE_HARDLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -H"
fi
if [ "$CHECKSUM" == "yes" ]; then
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi
if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi
if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
}
## IFS debug function ## IFS debug function
function PrintIFS { function PrintIFS {
printf "IFS is: %q" "$IFS" printf "IFS is: %q" "$IFS"
@ -1525,13 +1639,20 @@ function TrapStop {
function TrapQuit { function TrapQuit {
local exitcode local exitcode
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID" ]; then
WARN_ALERT=1
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID" ]; then
ERROR_ALERT=1
fi
if [ $ERROR_ALERT == true ]; then if [ $ERROR_ALERT == true ]; then
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then
RunAfterHook RunAfterHook
fi fi
Logger "$PROGRAM finished with errors." "ERROR" Logger "$PROGRAM finished with errors." "ERROR"
SendAlert SendAlert
CleanUp
exitcode=1 exitcode=1
elif [ $WARN_ALERT == true ]; then elif [ $WARN_ALERT == true ]; then
if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then if [ "$RUN_AFTER_CMD_ON_ERROR" == "yes" ]; then
@ -1539,12 +1660,10 @@ function TrapQuit {
fi fi
Logger "$PROGRAM finished with warnings." "WARN" Logger "$PROGRAM finished with warnings." "WARN"
SendAlert SendAlert
CleanUp
exitcode=2 exitcode=2
else else
RunAfterHook RunAfterHook
Logger "$PROGRAM finshed without errors." "NOTICE" Logger "$PROGRAM finshed without errors." "NOTICE"
CleanUp
exitcode=0 exitcode=0
fi fi
@ -1552,6 +1671,7 @@ function TrapQuit {
rm -f "$RUN_DIR/$PROGRAM.$INSTANCE_ID" rm -f "$RUN_DIR/$PROGRAM.$INSTANCE_ID"
fi fi
CleanUp
KillChilds $$ > /dev/null 2>&1 KillChilds $$ > /dev/null 2>&1
exit $exitcode exit $exitcode
} }
@ -1699,7 +1819,7 @@ function _ListDatabasesLocal {
sqlCmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2>&1" sqlCmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2>&1"
Logger "cmd: $sqlCmd" "DEBUG" Logger "cmd: $sqlCmd" "DEBUG"
eval "$sqlCmd" & eval "$sqlCmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
Logger "Listing databases succeeded." "NOTICE" Logger "Listing databases succeeded." "NOTICE"
else else
@ -1721,7 +1841,7 @@ function _ListDatabasesRemote {
sqlCmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID\" 2>&1" sqlCmd="$SSH_CMD \"mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID\" 2>&1"
Logger "cmd: $sqlCmd" "DEBUG" Logger "cmd: $sqlCmd" "DEBUG"
eval "$sqlCmd" & eval "$sqlCmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
Logger "Listing databases succeeded." "NOTICE" Logger "Listing databases succeeded." "NOTICE"
else else
@ -1820,7 +1940,7 @@ function _ListRecursiveBackupDirectoriesLocal {
cmd="$FIND_CMD -L $directory/ -mindepth 1 -maxdepth 1 -type d >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" cmd="$FIND_CMD -L $directory/ -mindepth 1 -maxdepth 1 -type d >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not enumerate directories in [$directory]." "ERROR" Logger "Could not enumerate directories in [$directory]." "ERROR"
if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then
@ -1850,7 +1970,7 @@ function _ListRecursiveBackupDirectoriesRemote {
cmd=$SSH_CMD' "'$COMMAND_SUDO' '$REMOTE_FIND_CMD' -L '$directory'/ -mindepth 1 -maxdepth 1 -type d" >> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID cmd=$SSH_CMD' "'$COMMAND_SUDO' '$REMOTE_FIND_CMD' -L '$directory'/ -mindepth 1 -maxdepth 1 -type d" >> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not enumerate directories in [$directory]." "ERROR" Logger "Could not enumerate directories in [$directory]." "ERROR"
if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then if [ -f $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID ]; then
@ -1945,7 +2065,7 @@ function _GetDirectoriesSizeLocal {
cmd="du -cs $dir_list | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" cmd="du -cs $dir_list | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID"
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
# $cmd will return 0 even if some errors found, so we need to check if there is an error output # $cmd will return 0 even if some errors found, so we need to check if there is an error output
if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then
Logger "Could not get files size for some or all directories." "ERROR" Logger "Could not get files size for some or all directories." "ERROR"
@ -1978,7 +2098,7 @@ function _GetDirectoriesSizeRemote {
cmd=$SSH_CMD' '$COMMAND_SUDO' du -cs '$dir_list' | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID cmd=$SSH_CMD' '$COMMAND_SUDO' du -cs '$dir_list' | tail -n1 | cut -f1 > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2> '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
# $cmd will return 0 even if some errors found, so we need to check if there is an error output # $cmd will return 0 even if some errors found, so we need to check if there is an error output
if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then if [ $? != 0 ] || [ -s $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID ]; then
Logger "Could not get files size for some or all directories." "ERROR" Logger "Could not get files size for some or all directories." "ERROR"
@ -2042,7 +2162,7 @@ function _CreateDirectoryRemote {
cmd=$SSH_CMD' "if ! [ -d \"'$dir_to_create'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$dir_to_create'\"; fi" > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2>&1' cmd=$SSH_CMD' "if ! [ -d \"'$dir_to_create'\" ]; then '$COMMAND_SUDO' mkdir -p \"'$dir_to_create'\"; fi" > '$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID' 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot create remote directory [$dir_to_create]." "CRITICAL" Logger "Cannot create remote directory [$dir_to_create]." "CRITICAL"
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
@ -2101,7 +2221,7 @@ function GetDiskSpaceLocal {
if [ -d "$path_to_check" ]; then if [ -d "$path_to_check" ]; then
# Not elegant solution to make df silent on errors # Not elegant solution to make df silent on errors
# No sudo on local commands, assuming you should have all the necesarry rights to check backup directories sizes # No sudo on local commands, assuming you should have all the necesarry rights to check backup directories sizes
df "$path_to_check" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 $DF_CMD "$path_to_check" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1
if [ $? != 0 ]; then if [ $? != 0 ]; then
DISK_SPACE=0 DISK_SPACE=0
Logger "Cannot get disk space in [$path_to_check] on local system." "ERROR" Logger "Cannot get disk space in [$path_to_check] on local system." "ERROR"
@ -2125,10 +2245,10 @@ function GetDiskSpaceRemote {
local cmd local cmd
cmd=$SSH_CMD' "if [ -d \"'$path_to_check'\" ]; then '$COMMAND_SUDO' df \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "if [ -d \"'$path_to_check'\" ]; then '$COMMAND_SUDO' '$DF_CMD' \"'$path_to_check'\"; else exit 1; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
DISK_SPACE=0 DISK_SPACE=0
Logger "Cannot get disk space in [$path_to_check] on remote system." "ERROR" Logger "Cannot get disk space in [$path_to_check] on remote system." "ERROR"
@ -2304,7 +2424,7 @@ function _BackupDatabaseLocalToLocal {
Logger "cmd: $drySqlCmd" "DEBUG" Logger "cmd: $drySqlCmd" "DEBUG"
eval "$drySqlCmd" & eval "$drySqlCmd" &
fi fi
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then
Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR"
@ -2345,7 +2465,7 @@ function _BackupDatabaseLocalToRemote {
Logger "cmd: $drySqlCmd" "DEBUG" Logger "cmd: $drySqlCmd" "DEBUG"
eval "$drySqlCmd" & eval "$drySqlCmd" &
fi fi
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then
Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR"
@ -2386,7 +2506,7 @@ function _BackupDatabaseRemoteToLocal {
Logger "cmd: $drySqlCmd" "DEBUG" Logger "cmd: $drySqlCmd" "DEBUG"
eval "$drySqlCmd" & eval "$drySqlCmd" &
fi fi
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID" ]; then
Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR" Logger "Error output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID)" "ERROR"
@ -2604,7 +2724,7 @@ function Rsync {
Logger "cmd: $rsyncCmd" "DEBUG" Logger "cmd: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" & eval "$rsyncCmd" &
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Failed to backup [$backupDirectory] to [$fileStoragePath]." "ERROR" Logger "Failed to backup [$backupDirectory] to [$fileStoragePath]." "ERROR"
@ -2719,7 +2839,7 @@ function _RotateBackupsLocal {
cmd="rm -rf \"$path\"" cmd="rm -rf \"$path\""
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot delete oldest copy [$path]." "ERROR" Logger "Cannot delete oldest copy [$path]." "ERROR"
fi fi
@ -2731,7 +2851,7 @@ function _RotateBackupsLocal {
cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\"" cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\""
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR" Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR"
fi fi
@ -2745,7 +2865,7 @@ function _RotateBackupsLocal {
cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" cmd="mv \"$backup\" \"$backup.$PROGRAM.1\""
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi fi
@ -2754,7 +2874,7 @@ function _RotateBackupsLocal {
cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\"" cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\""
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR" Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi fi
@ -2763,7 +2883,7 @@ function _RotateBackupsLocal {
cmd="mv \"$backup\" \"$backup.$PROGRAM.1\"" cmd="mv \"$backup\" \"$backup.$PROGRAM.1\""
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 3600 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR" Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
fi fi
@ -2871,7 +2991,7 @@ function _RotateBackupsRemoteSSH {
ENDSSH ENDSSH
WaitForTaskCompletion $! 1800 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 1800 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Could not rotate backups in [$backup_path]." "ERROR" Logger "Could not rotate backups in [$backup_path]." "ERROR"
Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR" Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "ERROR"
@ -3163,6 +3283,7 @@ if [ "$REMOTE_OPERATION" == "yes" ]; then
GetRemoteOS GetRemoteOS
InitRemoteOSSettings InitRemoteOSSettings
fi fi
InitRsyncSettings
if [ $no_maxtime == true ]; then if [ $no_maxtime == true ]; then
SOFT_MAX_EXEC_TIME_DB_TASK=0 SOFT_MAX_EXEC_TIME_DB_TASK=0