mirror of
https://github.com/deajan/obackup.git
synced 2024-11-15 04:03:41 +01:00
Rebuilt targets
This commit is contained in:
deajan
parent
71d7445c76
commit
2e2b757fd7
File diff suppressed because it is too large
Load Diff
618
obackup.sh
618
obackup.sh
@ -6,14 +6,21 @@
|
|||||||
PROGRAM="obackup"
|
PROGRAM="obackup"
|
||||||
AUTHOR="(C) 2013-2017 by Orsiris de Jong"
|
AUTHOR="(C) 2013-2017 by Orsiris de Jong"
|
||||||
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
|
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
|
||||||
PROGRAM_VERSION=2.1-beta4
|
PROGRAM_VERSION=2.1-beta5
|
||||||
PROGRAM_BUILD=2018011901
|
PROGRAM_BUILD=2018022401
|
||||||
IS_STABLE=no
|
IS_STABLE=no
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
_OFUNCTIONS_VERSION=2.2.0-dev
|
#TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling"
|
||||||
_OFUNCTIONS_BUILD=2018010303
|
#done: add checkRFC function (and use it for --destination-mails)
|
||||||
|
#done: ExecTasks still needs some better call argument list
|
||||||
|
#done: ExecTasks sub function relocate
|
||||||
|
#done: SendMail and SendEmail convert functions inverted, check on osync and obackup
|
||||||
|
#command line arguments don't take -AaqV for example
|
||||||
|
|
||||||
|
_OFUNCTIONS_VERSION=2.3.0-dev
|
||||||
|
_OFUNCTIONS_BUILD=2018022402
|
||||||
_OFUNCTIONS_BOOTSTRAP=true
|
_OFUNCTIONS_BOOTSTRAP=true
|
||||||
|
|
||||||
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
|
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
|
||||||
@ -394,14 +401,8 @@ function SendAlert {
|
|||||||
else
|
else
|
||||||
attachment=true
|
attachment=true
|
||||||
fi
|
fi
|
||||||
if [ -e "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" ]; then
|
|
||||||
if [ "$MAIL_BODY_CHARSET" != "" ] && type iconv > /dev/null 2>&1; then
|
body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of log"$'\n\n'"$(tail -n 1000 $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP)"
|
||||||
iconv -f UTF-8 -t $MAIL_BODY_CHARSET "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" > "$RUN_DIR/$PROGRAM._Logger.iconv.$SCRIPT_PID.$TSTAMP"
|
|
||||||
body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.iconv.$SCRIPT_PID.$TSTAMP)"
|
|
||||||
else
|
|
||||||
body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP)"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ $ERROR_ALERT == true ]; then
|
if [ $ERROR_ALERT == true ]; then
|
||||||
subject="Error alert for $INSTANCE_ID"
|
subject="Error alert for $INSTANCE_ID"
|
||||||
@ -459,6 +460,24 @@ function SendEmail {
|
|||||||
local encryption_string=
|
local encryption_string=
|
||||||
local auth_string=
|
local auth_string=
|
||||||
|
|
||||||
|
local i
|
||||||
|
|
||||||
|
for i in "${destinationMails}"; do
|
||||||
|
if [ $(CheckRFC822 "$i") -ne 1 ]; then
|
||||||
|
Logger "Given email [$i] does not seem to be valid." "WARN"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Prior to sending an email, convert its body if needed
|
||||||
|
if [ "$MAIL_BODY_CHARSET" != "" ]; then
|
||||||
|
if type iconv > /dev/null 2>&1; then
|
||||||
|
echo "$message" | iconv -f UTF-8 -t $MAIL_BODY_CHARSET -o "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP"
|
||||||
|
message="$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP)"
|
||||||
|
else
|
||||||
|
Logger "iconv utility not installed. Will not convert email charset." "NOTICE"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
if [ ! -f "$attachment" ]; then
|
if [ ! -f "$attachment" ]; then
|
||||||
attachment_command="-a $attachment"
|
attachment_command="-a $attachment"
|
||||||
mail_no_attachment=1
|
mail_no_attachment=1
|
||||||
@ -630,7 +649,6 @@ function Spinner {
|
|||||||
return 0
|
return 0
|
||||||
else
|
else
|
||||||
printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER"
|
printf " [%c] \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER"
|
||||||
#printf "\b\b\b\b\b\b"
|
|
||||||
_OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???}
|
_OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???}
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
@ -648,7 +666,7 @@ function WaitForTaskCompletion {
|
|||||||
local noErrorLog="${8:-false}"
|
local noErrorLog="${8:-false}"
|
||||||
local id="${9-base}"
|
local id="${9-base}"
|
||||||
|
|
||||||
ExecTasks "$id" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorlog" false 1 "$pids" "" ""
|
ExecTasks "$pids" "$id" false 0 0 "$softMaxTime" "$hardMaxTime" "$counting" "$sleepTime" "$keepLogging" "$spinner" "$noErrorlog"
|
||||||
}
|
}
|
||||||
|
|
||||||
# ParallelExec function emulation, now uses ExecTasks
|
# ParallelExec function emulation, now uses ExecTasks
|
||||||
@ -665,97 +683,104 @@ function ParallelExec {
|
|||||||
local noErrorLog="${10:-false}"
|
local noErrorLog="${10:-false}"
|
||||||
|
|
||||||
if [ $readFromFile == true ]; then
|
if [ $readFromFile == true ]; then
|
||||||
ExecTasks "base" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorLog" false 6 "$commandsArg" "" "$numberOfProcesses"
|
ExecTasks "$commandsArg" "base" $readFromFile 0 0 "$softMaxTime" "$hardMaxTime" "$counting" "$sleepTime" "$keepLogging" "$spinner" "$noErrorLog" false "$numberOfProcesses"
|
||||||
else
|
else
|
||||||
ExecTasks "base" 0 0 "$softMaxTime" "$hardMaxTime" "$sleepTime" "$keepLogging" "$counting" "$spinner" "$noErrorLog" false 3 "$commandsArg" "" "$numberOfProcesses"
|
ExecTasks "$commandsArg" "base" $readFromFile 0 0 "$softMaxTime" "$hardMaxTime" "$counting" "$sleepTime" "$keepLogging" "$spinner" "$noErrorLog" false "$numberOfProcesses"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
## Main asynchronous execution function
|
## Main asynchronous execution function
|
||||||
## This function can monitor given pids as background processes, and stop them if max execution time is reached. Suitable for multiple synchronous processes.
|
## Function can work in:
|
||||||
## It can also take a list of commands to execute in parallel, and stop them if max execution time is reached.
|
## WaitForTaskCompletion mode: monitors given pid in background, and stops them if max execution time is reached. Suitable for multiple synchronous pids to monitor and wait for
|
||||||
|
## ParallExec mode: takes list of commands to execute in parallel per batch, and stops them if max execution time is reahed.
|
||||||
|
|
||||||
## Function has 8 execution modes
|
## Example of improved wait $!
|
||||||
|
## ExecTasks $! "some_identifier" false 0 0 0 0 true 1 1800 false
|
||||||
|
## Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds
|
||||||
|
## sleep 15 &
|
||||||
|
## pid=$!
|
||||||
|
## sleep 20 &
|
||||||
|
## pid2=$!
|
||||||
|
## ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2"
|
||||||
|
|
||||||
# WaitForTaskCompletion mode: Monitor given pids as background processes, stop them if max execution time is reached. Suitaable for multiple synhronous processes.
|
## Example of parallel execution of four commands, only if directories exist. Warn if execution takes more than 300 seconds. Stop if takes longer than 900 seconds. Exeute max 3 commands in parallel.
|
||||||
# 1 : WaitForTaskCompletion, semi-colon separated list of pids to monitor
|
## commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr"
|
||||||
# 2 : WaitForTaskCompletion, list of pids to monior, from file, one per line
|
## conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]"
|
||||||
|
## ExecTasks "$commands" "some_identifier" false 0 0 300 900 true 1 1800 true false false 3 "$conditions"
|
||||||
|
|
||||||
# Example of improved wait $! emulation
|
## Bear in mind that given commands and conditions need to be quoted
|
||||||
# ExecTasks "some_identifier" 0 0 0 0 1 1800 true false true true 1 $!
|
|
||||||
# Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds
|
|
||||||
# sleep 15 &
|
|
||||||
# pid=$!
|
|
||||||
# sleep 20 &
|
|
||||||
# pid2=$!
|
|
||||||
# ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2"
|
|
||||||
|
|
||||||
# ParallelExecMode: Take list of commands to execute in parallel, stop them if max execution time is reached.
|
## ExecTasks has the following ofunctions subfunction requirements:
|
||||||
# Also take optional conditional arguments to verifiy before execution main commands. Conditional command exit code 0 means ready to execute. Other exit codes will ignore/postpone main command.
|
|
||||||
# 3 : ParallelExec, semi-colon separated list of commands to execute in parallel, no conditions
|
|
||||||
# 4 : ParallelExec, semi-colon separated list of commands to execute in parallel , semi-colon separated list of conditional commands, ignoring main commands on condition failures
|
|
||||||
# 5 : ParallelExec, semi-colon separated list of commands, semi-colon separated list of conditional commands, postponing main commands on condition failures
|
|
||||||
# 6 : ParallelExec, list of commands from file, one per line, no conditions
|
|
||||||
# 7 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, ignoring main commands on condition failures
|
|
||||||
# 8 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, postponing main commands on condition failures
|
|
||||||
|
|
||||||
# Exmaple: execute four du commands, only if directories exist, warn if execution takes more than 300 seconds, stop if takes longer than 900 seconds. Execute max 3 commands in parallel.
|
|
||||||
# commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr"
|
|
||||||
# conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]"
|
|
||||||
# ExecTasks "some_identifier" 0 0 300 900 1 1800 true true false false 4 "$commands" "$conditions" 3
|
|
||||||
|
|
||||||
# ParallelExecMode also creates output for commands in "$RUN_DIR/$PROGRAM.ExecTasks.$id.$SCRIPT_PID.$TSTAMP"
|
|
||||||
|
|
||||||
## ofunctions.sh subfunction requirements:
|
|
||||||
## Spinner
|
## Spinner
|
||||||
## Logger
|
## Logger
|
||||||
## JoinString
|
## JoinString
|
||||||
## KillChilds
|
## KillChilds
|
||||||
|
|
||||||
|
## Full call
|
||||||
|
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes
|
||||||
|
|
||||||
function ExecTasks {
|
function ExecTasks {
|
||||||
local id="${1:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
|
# Mandatory arguments
|
||||||
#TODO: not implemented yet
|
local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands
|
||||||
local softPerProcessTime="${2:-0}"
|
|
||||||
local hardPerProcessTime="${3:-0}"
|
# Optional arguments
|
||||||
local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
|
local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
|
||||||
local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
|
local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false)
|
||||||
local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
|
local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0
|
||||||
local keepLogging="${7:-0}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging.
|
local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0
|
||||||
local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
|
local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0
|
||||||
local spinner="${9:-true}" # Show spinner (true), do not show anything (false)
|
local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0
|
||||||
local noTimeErrorLog="${10:-false}" # Log errors when reaching soft / hard max time (false), do not log errors on those triggers (true)
|
local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false)
|
||||||
#TODO not implemented
|
local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1)
|
||||||
local noErrorLogAtAll="${11:-false}" # Do not log errros at all (false)
|
local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging
|
||||||
local execTasksMode="${12:-1}" # In which mode the function should work, see above
|
local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running
|
||||||
local mainInput="${13}" # Contains list of pids / commands or filepath to list of pids / commands
|
local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true)
|
||||||
local auxInput="${14}" # Contains list of conditional commands or filepath to list of conditional commands
|
local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks)
|
||||||
local numberOfProcesses="${15:-2}" # Number of simultaneous commands to run in ParallExec mode
|
|
||||||
|
# Parallelism specific arguments
|
||||||
|
local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode.
|
||||||
|
local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not
|
||||||
|
local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing
|
||||||
|
local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries
|
||||||
|
local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors
|
||||||
|
|
||||||
local i
|
local i
|
||||||
|
|
||||||
|
|
||||||
# Since ExecTasks takes up to 15 arguments, do a quick preflight check in DEBUG mode
|
# Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode
|
||||||
if [ "$_DEBUG" == "yes" ]; then
|
if [ "$_DEBUG" == "yes" ]; then
|
||||||
declare -a booleans=(counting spinner noTimeErrorLog noErrorLogAtAll)
|
declare -a booleans=(readFromFile counting spinner noTimeErrorLog noErrorLogsAtAll)
|
||||||
for i in "${num_vars[@]}"; do
|
for i in "${booleans[@]}"; do
|
||||||
test="if [ $i != false ] && [ $i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
|
test="if [ \$$i != false ] && [ \$$i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
|
||||||
eval "$test"
|
eval "$test"
|
||||||
done
|
done
|
||||||
declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging execTasksMode numberOfProcesses)
|
declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging numberOfProcesses maxPostponeRetries minTimeBetweenRetries)
|
||||||
for i in "${integers[@]}"; do
|
for i in "${integers[@]}"; do
|
||||||
test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
|
test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
|
||||||
eval "$test"
|
eval "$test"
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Change '-' to '_' in task id
|
||||||
|
id="${id/-/_}"
|
||||||
|
|
||||||
|
# Expand validExitCodes into array
|
||||||
|
IFS=';' read -r -a validExitCodes <<< "$validExitCodes"
|
||||||
|
|
||||||
# ParallelExec specific variables
|
# ParallelExec specific variables
|
||||||
local auxItemCount=0 # Number of conditional commands
|
local auxItemCount=0 # Number of conditional commands
|
||||||
local commandsArray=() # Array containing commands
|
local commandsArray=() # Array containing commands
|
||||||
local commandsConditionArray=() # Array containing conditional commands
|
local commandsConditionArray=() # Array containing conditional commands
|
||||||
local currentCommand # Variable containing currently processed command
|
local currentCommand # Variable containing currently processed command
|
||||||
local currentCommandCondition # Variable containing currently processed conditional command
|
local currentCommandCondition # Variable containing currently processed conditional command
|
||||||
local commandsArrayPid # Array containing pids of commands currently run
|
local commandsArrayPid=() # Array containing pids of commands currently run
|
||||||
local postPoneIfConditionFails # Boolean to check if command needs to be postponed if condition command fails
|
local postponedRetryCount=0 # Number of current postponed commands retries
|
||||||
|
local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once)
|
||||||
|
local postponedCounter=0
|
||||||
|
local isPostponedCommand=false # Is the current command from a postponed file ?
|
||||||
|
local postponedExecTime=0 # How much time has passed since last postponed condition was checked
|
||||||
|
local needsPostponing # Does currentCommand need to be postponed
|
||||||
|
local temp
|
||||||
|
|
||||||
# Common variables
|
# Common variables
|
||||||
local pid # Current pid working on
|
local pid # Current pid working on
|
||||||
@ -769,6 +794,7 @@ function ExecTasks {
|
|||||||
local exec_time=0 # Seconds since the beginning of this function
|
local exec_time=0 # Seconds since the beginning of this function
|
||||||
|
|
||||||
local retval=0 # return value of monitored pid process
|
local retval=0 # return value of monitored pid process
|
||||||
|
local subRetval=0 # return value of condition commands
|
||||||
local errorcount=0 # Number of pids that finished with errors
|
local errorcount=0 # Number of pids that finished with errors
|
||||||
local pidsArray # Array of currently running pids
|
local pidsArray # Array of currently running pids
|
||||||
local newPidsArray # New array of currently running pids for next iteration
|
local newPidsArray # New array of currently running pids for next iteration
|
||||||
@ -788,100 +814,48 @@ function ExecTasks {
|
|||||||
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
|
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
|
||||||
eval "HARD_MAX_EXEC_TIME_REACHED_$id=false"
|
eval "HARD_MAX_EXEC_TIME_REACHED_$id=false"
|
||||||
|
|
||||||
case $execTasksMode in
|
# Init function variables depending on mode
|
||||||
1)
|
|
||||||
|
if [ $numberOfProcesses -gt 0 ]; then
|
||||||
|
functionMode=ParallelExec
|
||||||
|
else
|
||||||
|
functionMode=WaitForTaskCompletion
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $readFromFile == false ]; then
|
||||||
|
if [ $functionMode == "WaitForTaskCompletion" ]; then
|
||||||
IFS=';' read -r -a pidsArray <<< "$mainInput"
|
IFS=';' read -r -a pidsArray <<< "$mainInput"
|
||||||
mainItemCount=${#pidsArray[@]}
|
mainItemCount="${#pidsArray[@]}"
|
||||||
readFromFile=false
|
|
||||||
functionMode=WaitForTaskCompletion
|
|
||||||
# Force while condition to be true
|
|
||||||
counter=$mainItemCount
|
|
||||||
;;
|
|
||||||
2)
|
|
||||||
if [ -f "$mainInput" ]; then
|
|
||||||
mainItemCount=$(wc -l < "$mainInput")
|
|
||||||
readFromFile=true
|
|
||||||
else
|
else
|
||||||
Logger "Cannot read file [$mainInput]." "WARN"
|
|
||||||
fi
|
|
||||||
functionMode=WaitForTaskCompletion
|
|
||||||
# Force while condition to be true
|
|
||||||
counter=$mainItemCount
|
|
||||||
;;
|
|
||||||
3)
|
|
||||||
IFS=';' read -r -a commandsArray <<< "$mainInput"
|
IFS=';' read -r -a commandsArray <<< "$mainInput"
|
||||||
mainItemCount=${#commandsArray[@]}
|
mainItemCount="${#commandsArray[@]}"
|
||||||
readFromFile=false
|
|
||||||
functionMode=ParallelExec
|
|
||||||
;;
|
|
||||||
4)
|
|
||||||
IFS=';' read -r -a commandsArray <<< "$mainInput"
|
|
||||||
mainItemCount=${#commandsArray[@]}
|
|
||||||
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
|
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
|
||||||
auxItemCount=${#commandsConditionArray[@]}
|
auxItemCount="${#commandsConditionArray[@]}"
|
||||||
readFromFile=false
|
fi
|
||||||
postPoneIfConditionFails=false
|
else
|
||||||
functionMode=ParallelExec
|
|
||||||
;;
|
|
||||||
5)
|
|
||||||
IFS=';' read -r -a commandsArray <<< "$mainInput"
|
|
||||||
mainItemCount=${#commandsArray[@]}
|
|
||||||
IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
|
|
||||||
auxItemCount=${#commandsConditionArray[@]}
|
|
||||||
readFromFile=false
|
|
||||||
postPoneIfConditionFails=true
|
|
||||||
functionMode=ParallelExec
|
|
||||||
;;
|
|
||||||
6)
|
|
||||||
if [ -f "$mainInput" ]; then
|
if [ -f "$mainInput" ]; then
|
||||||
mainItemCount=$(wc -l < "$mainInput")
|
mainItemCount=$(wc -l < "$mainInput")
|
||||||
readFromFile=true
|
readFromFile=true
|
||||||
else
|
else
|
||||||
Logger "Cannot read file [$mainInput]." "WARN"
|
Logger "Cannot read main file [$mainInput]." "WARN"
|
||||||
fi
|
|
||||||
functionMode=ParallelExec
|
|
||||||
;;
|
|
||||||
7)
|
|
||||||
if [ -f "$mainInput" ]; then
|
|
||||||
mainItemCount=$(wc -l < "$mainInput")
|
|
||||||
readFromFile=true
|
|
||||||
else
|
|
||||||
Logger "Cannot read file [$mainInput]." "WARN"
|
|
||||||
fi
|
fi
|
||||||
|
if [ "$auxInput" != "" ]; then
|
||||||
if [ -f "$auxInput" ]; then
|
if [ -f "$auxInput" ]; then
|
||||||
auxItemCount=$(wc -l < "$auxInput")
|
auxItemCount=$(wc -l < "$auxInput")
|
||||||
else
|
else
|
||||||
Logger "Cannot read file [$auxInput]." "WARN"
|
Logger "Cannot read aux file [$auxInput]." "WARN"
|
||||||
fi
|
fi
|
||||||
postPoneIfConditionFails=false
|
|
||||||
functionMode=ParallelExec
|
|
||||||
;;
|
|
||||||
8)
|
|
||||||
if [ -f "$mainInput" ]; then
|
|
||||||
mainItemCount=$(wc -l < "$mainInput")
|
|
||||||
readFromFile=true
|
|
||||||
else
|
|
||||||
Logger "Cannot read file [$mainInput]." "WARN"
|
|
||||||
fi
|
fi
|
||||||
if [ -f "$auxInput" ]; then
|
|
||||||
auxItemCount=$(wc -l < "$auxInput")
|
|
||||||
else
|
|
||||||
Logger "Cannot read file [$auxInput]." "WARN"
|
|
||||||
fi
|
fi
|
||||||
postPoneIfConditionFails=true
|
|
||||||
functionMode=ParallelExec
|
if [ $functionMode == "WaitForTaskCompletion" ]; then
|
||||||
;;
|
# Force first while loop condition to be true because we don't deal with counters but pids in WaitForTaskCompletion mode
|
||||||
*)
|
counter=$mainItemCount
|
||||||
Logger "Unknown exec mode for ${FUNCNAME}." "CRITICAL"
|
fi
|
||||||
exit 1
|
|
||||||
esac
|
|
||||||
|
|
||||||
|
|
||||||
#while [ ${#pidsArray[@]} -gt 0 ]; do
|
# soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop
|
||||||
# The counter -lt mainItemCount has to be false for WaitForTaskCompletion modes
|
function _ExecTasksTimeCheck {
|
||||||
while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ]; do
|
|
||||||
newPidsArray=()
|
|
||||||
|
|
||||||
if [ $spinner == true ]; then
|
if [ $spinner == true ]; then
|
||||||
Spinner
|
Spinner
|
||||||
fi
|
fi
|
||||||
@ -892,13 +866,13 @@ function ExecTasks {
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
if [ $keepLogging -ne 0 ]; then
|
if [ $keepLogging -ne 0 ]; then
|
||||||
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
|
if [ $(((exec_time + 1) % keepLogging)) -eq 0 ]; then
|
||||||
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second
|
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second
|
||||||
log_ttime=$exec_time
|
log_ttime=$exec_time
|
||||||
if [ $functionMode == "Wait" ]; then
|
if [ $functionMode == "Wait" ]; then
|
||||||
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
|
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
|
||||||
elif [ $functionMode == "ParallelExec" ]; then
|
elif [ $functionMode == "ParallelExec" ]; then
|
||||||
Logger "There are $((mainItemCount-counter)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
|
Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
@ -906,7 +880,7 @@ function ExecTasks {
|
|||||||
|
|
||||||
if [ $exec_time -gt $softMaxTime ]; then
|
if [ $exec_time -gt $softMaxTime ]; then
|
||||||
if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
|
if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
|
||||||
Logger "Max soft execution time exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
|
Logger "Max soft execution time [$softMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
|
||||||
softAlert=true
|
softAlert=true
|
||||||
SendAlert true
|
SendAlert true
|
||||||
fi
|
fi
|
||||||
@ -914,15 +888,17 @@ function ExecTasks {
|
|||||||
|
|
||||||
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
|
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
|
||||||
if [ $noTimeErrorLog != true ]; then
|
if [ $noTimeErrorLog != true ]; then
|
||||||
Logger "Max hard execution time exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
|
Logger "Max hard execution time [$hardMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
|
||||||
fi
|
fi
|
||||||
for pid in "${pidsArray[@]}"; do
|
for pid in "${pidsArray[@]}"; do
|
||||||
KillChilds $pid true
|
KillChilds $pid true
|
||||||
if [ $? == 0 ]; then
|
if [ $? == 0 ]; then
|
||||||
Logger "Task with pid [$pid] stopped successfully." "NOTICE"
|
Logger "Task with pid [$pid] stopped successfully." "NOTICE"
|
||||||
else
|
else
|
||||||
|
if [ $noErrorLogsAtAll != true ]; then
|
||||||
Logger "Could not stop task with pid [$pid]." "ERROR"
|
Logger "Could not stop task with pid [$pid]." "ERROR"
|
||||||
fi
|
fi
|
||||||
|
fi
|
||||||
errorcount=$((errorcount+1))
|
errorcount=$((errorcount+1))
|
||||||
done
|
done
|
||||||
if [ $noTimeErrorLog != true ]; then
|
if [ $noTimeErrorLog != true ]; then
|
||||||
@ -931,70 +907,14 @@ function ExecTasks {
|
|||||||
eval "HARD_MAX_EXEC_TIME_REACHED_$id=true"
|
eval "HARD_MAX_EXEC_TIME_REACHED_$id=true"
|
||||||
if [ $functionMode == "WaitForTaskCompletion" ]; then
|
if [ $functionMode == "WaitForTaskCompletion" ]; then
|
||||||
return $errorcount
|
return $errorcount
|
||||||
elif [ $functionMode == "ParallelExec" ]; then
|
else
|
||||||
return $((mainItemCount - counter + ${#pidsArray[@]}))
|
return 129
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them
|
function _ExecTasksPidsCheck {
|
||||||
if [ $functionMode == "ParallelExec" ]; then
|
newPidsArray=()
|
||||||
while [ $counter -lt "$mainItemCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
|
|
||||||
if [ $readFromFile == true ]; then
|
|
||||||
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput")
|
|
||||||
if [ $auxItemCount -ne 0 ]; then
|
|
||||||
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput")
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
currentCommand="${commandsArray[$counter]}"
|
|
||||||
if [ $auxItemCount -ne 0 ]; then
|
|
||||||
currentCommandCondition="${commandsConditionArray[$counter]}"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
executeCommand=false
|
|
||||||
if [ $auxItemCount -ne 0 ]; then
|
|
||||||
Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG"
|
|
||||||
eval "$currentCommandCondition" &
|
|
||||||
ExecTasks "subConditionCheck" 0 0 1800 3600 1 $KEEP_LOGGING true true true true 1 $!
|
|
||||||
retval=$?
|
|
||||||
if [ $retval -ne 0 ]; then
|
|
||||||
if [ $postPoneIfConditionFails == true ]; then
|
|
||||||
Logger "Condition not met for command [$currentCommand]. Postponing command." "NOTICE"
|
|
||||||
if [ $readFromFile == true ]; then
|
|
||||||
# TODO: we should not write to the original file, but create a copy instead we can write postponed commands to
|
|
||||||
echo "$currentCommand" >> "$mainInput"
|
|
||||||
echo "$currentCommandCondition" >> "$auxInput"
|
|
||||||
else
|
|
||||||
commandsArray+=($currentCommand)
|
|
||||||
commandsConditionArray+=($currentCommandCondition)
|
|
||||||
fi
|
|
||||||
mainItemCount=$((mainItemCount+1))
|
|
||||||
|
|
||||||
# Trivial sleeptime so postponed commands will not stack too fast
|
|
||||||
sleep $sleepTime
|
|
||||||
else
|
|
||||||
Logger "Condition not met for command [$currentCommand]. Ignoring command." "NOTICE"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
executeCommand=true
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
executeCommand=true
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ $executeCommand == true ]; then
|
|
||||||
Logger "Running command [$currentCommand]." "DEBUG"
|
|
||||||
eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$SCRIPT_PID.$TSTAMP" 2>&1 &
|
|
||||||
pid=$!
|
|
||||||
pidsArray+=($pid)
|
|
||||||
commandsArrayPid[$pid]="$currentCommand"
|
|
||||||
#TODO not implemented
|
|
||||||
pidsTimeArray[$pid]=$((SECONDS - seconds_begin))
|
|
||||||
else
|
|
||||||
Logger "Skipping command [$currentCommand]." "DEBUG"
|
|
||||||
fi
|
|
||||||
counter=$((counter+1))
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
for pid in "${pidsArray[@]}"; do
|
for pid in "${pidsArray[@]}"; do
|
||||||
if [ $(IsInteger $pid) -eq 1 ]; then
|
if [ $(IsInteger $pid) -eq 1 ]; then
|
||||||
@ -1002,15 +922,57 @@ function ExecTasks {
|
|||||||
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
|
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
|
||||||
pidState="$(eval $PROCESS_STATE_CMD)"
|
pidState="$(eval $PROCESS_STATE_CMD)"
|
||||||
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
|
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
|
||||||
#TODO: implement pidsTimeArray[$pid] check here
|
|
||||||
|
# Check if pid hasn't run more than soft/hard perProcessTime
|
||||||
|
pidsTimeArray[$pid]=$((SECONDS - seconds_begin))
|
||||||
|
if [ ${pidsTimeArray[$pid]} -gt $softPerProcessTime ]; then
|
||||||
|
if [ "$softAlert" != true ] && [ $softPerProcessTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
|
||||||
|
Logger "Max soft execution time [$softPerProcessTime] exceeded for pid [$pid]." "WARN"
|
||||||
|
if [ "${commandsArrayPid[$pid]}]" != "" ]; then
|
||||||
|
Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN"
|
||||||
|
fi
|
||||||
|
softAlert=true
|
||||||
|
SendAlert true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
if [ ${pidsTimeArray[$pid]} -gt $hardPerProcessTime ] && [ $hardPerProcessTime -ne 0 ]; then
|
||||||
|
if [ $noTimeErrorLog != true ] && [ $noErrorLogsAtAll != true ]; then
|
||||||
|
Logger "Max hard execution time [$hardPerProcessTime] exceeded for pid [$pid]. Stopping command execution." "ERROR"
|
||||||
|
if [ "${commandsArrayPid[$pid]}]" != "" ]; then
|
||||||
|
Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
KillChilds $pid true
|
||||||
|
if [ $? == 0 ]; then
|
||||||
|
Logger "Command with pid [$pid] stopped successfully." "NOTICE"
|
||||||
|
else
|
||||||
|
if [ $noErrorLogsAtAll != true ]; then
|
||||||
|
Logger "Could not stop command with pid [$pid]." "ERROR"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
errorcount=$((errorcount+1))
|
||||||
|
|
||||||
|
if [ $noTimeErrorLog != true ]; then
|
||||||
|
SendAlert true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
newPidsArray+=($pid)
|
newPidsArray+=($pid)
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
# pid is dead, get its exit code from wait command
|
# pid is dead, get its exit code from wait command
|
||||||
wait $pid
|
wait $pid
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -ne 0 ]; then
|
# Check for valid exit codes
|
||||||
Logger "${FUNCNAME[0]} called by [$id] finished monitoring [$pid] [$currentCommad] with exitcode [$retval]." "DEBUG"
|
if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then
|
||||||
|
if [ $noErrorLogsAtAll != true ]; then
|
||||||
|
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR"
|
||||||
|
if [ "$functionMode" == "ParallelExec" ]; then
|
||||||
|
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
errorcount=$((errorcount+1))
|
errorcount=$((errorcount+1))
|
||||||
# Welcome to variable variable bash hell
|
# Welcome to variable variable bash hell
|
||||||
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
|
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
|
||||||
@ -1019,26 +981,156 @@ function ExecTasks {
|
|||||||
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
|
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
Logger "${FUNCNAME[0]} called by [$id] finished monitoring [$pid] [$currentCommand] with exitcode [$retval]." "DEBUG"
|
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# hasPids can be false on last iteration in ParallelExec mode
|
||||||
pidsArray=("${newPidsArray[@]}")
|
pidsArray=("${newPidsArray[@]}")
|
||||||
|
|
||||||
# Trivial wait time for bash to not eat up all CPU
|
# Trivial wait time for bash to not eat up all CPU
|
||||||
sleep $sleepTime
|
sleep $sleepTime
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]; do
|
||||||
|
_ExecTasksTimeCheck
|
||||||
|
retval=$?
|
||||||
|
if [ $retval -ne 0 ]; then
|
||||||
|
return $retval;
|
||||||
|
fi
|
||||||
|
|
||||||
|
# The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them
|
||||||
|
if [ $functionMode == "ParallelExec" ]; then
|
||||||
|
while [ ${#pidsArray[@]} -lt $numberOfProcesses ] && ([ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]); do
|
||||||
|
_ExecTasksTimeCheck
|
||||||
|
retval=$?
|
||||||
|
if [ $retval -ne 0 ]; then
|
||||||
|
return $retval;
|
||||||
|
fi
|
||||||
|
|
||||||
|
executeCommand=false
|
||||||
|
isPostponedCommand=false
|
||||||
|
currentCommand=""
|
||||||
|
currentCommandCondition=""
|
||||||
|
needsPostponing=false
|
||||||
|
|
||||||
|
if [ $readFromFile == true ]; then
|
||||||
|
# awk identifies first line as 1 instead of 0 so we need to increase counter
|
||||||
|
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput")
|
||||||
|
if [ $auxItemCount -ne 0 ]; then
|
||||||
|
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if we need to fetch postponed commands
|
||||||
|
if [ "$currentCommand" == "" ]; then
|
||||||
|
currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP")
|
||||||
|
currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP")
|
||||||
|
isPostponedCommand=true
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
currentCommand="${commandsArray[$counter]}"
|
||||||
|
if [ $auxItemCount -ne 0 ]; then
|
||||||
|
currentCommandCondition="${commandsConditionArray[$counter]}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$currentCommand" == "" ]; then
|
||||||
|
currentCommand="${postponedCommandsArray[$postponedCounter]}"
|
||||||
|
currentCommandCondition="${postponedCommandsConditionArray[$postponedCounter]}"
|
||||||
|
isPostponedCommand=true
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if we execute postponed commands, or if we delay them
|
||||||
|
if [ $isPostponedCommand == true ]; then
|
||||||
|
# Get first value before '@'
|
||||||
|
postponedExecTime="${currentCommand%%@*}"
|
||||||
|
postponedExecTime=$((SECONDS-postponedExecTime))
|
||||||
|
# Get everything after first '@'
|
||||||
|
temp="${currentCommand#*@}"
|
||||||
|
# Get first value before '@'
|
||||||
|
postponedRetryCount="${temp%%@*}"
|
||||||
|
# Replace currentCommand with actual filtered currentCommand
|
||||||
|
currentCommand="${temp#*@}"
|
||||||
|
|
||||||
|
# Since we read a postponed command, we may decrase postponedItemCounter
|
||||||
|
postponedItemCount=$((postponedItemCount-1))
|
||||||
|
#Since we read one line, we need to increase the counter
|
||||||
|
postponedCounter=$((postponedCounter+1))
|
||||||
|
|
||||||
|
else
|
||||||
|
postponedRetryCount=0
|
||||||
|
postponedExecTime=0
|
||||||
|
fi
|
||||||
|
if ([ $postponedRetryCount -lt $maxPostponeRetries ] && [ $postponedExecTime -ge $((minTimeBetweenRetries)) ]) || [ $isPostponedCommand == false ]; then
|
||||||
|
if [ "$currentCommandCondition" != "" ]; then
|
||||||
|
Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG"
|
||||||
|
eval "$currentCommandCondition" &
|
||||||
|
ExecTasks $! "subConditionCheck" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING true true true
|
||||||
|
subRetval=$?
|
||||||
|
if [ $subRetval -ne 0 ]; then
|
||||||
|
# is postponing enabled ?
|
||||||
|
if [ $maxPostponeRetries -gt 0 ]; then
|
||||||
|
Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Postponing command." "NOTICE"
|
||||||
|
postponedRetryCount=$((postponedRetryCount+1))
|
||||||
|
if [ $postponedRetryCount -ge $maxPostponeRetries ]; then
|
||||||
|
Logger "Max retries reached for postponed command [$currentCommand]. Skipping command." "NOTICE"
|
||||||
|
else
|
||||||
|
needsPostponing=true
|
||||||
|
fi
|
||||||
|
postponedExecTime=0
|
||||||
|
else
|
||||||
|
Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Ignoring command." "NOTICE"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
executeCommand=true
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
executeCommand=true
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
needsPostponing=true
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $needsPostponing == true ]; then
|
||||||
|
postponedItemCount=$((postponedItemCount+1))
|
||||||
|
if [ $readFromFile == true ]; then
|
||||||
|
echo "$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP"
|
||||||
|
echo "$currentCommandCondition" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP"
|
||||||
|
else
|
||||||
|
postponedCommandsArray+=("$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand")
|
||||||
|
postponedCommandsConditionArray+=("$currentCommandCondition")
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $executeCommand == true ]; then
|
||||||
|
Logger "Running command [$currentCommand]." "DEBUG"
|
||||||
|
eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$SCRIPT_PID.$TSTAMP" 2>&1 &
|
||||||
|
pid=$!
|
||||||
|
pidsArray+=($pid)
|
||||||
|
commandsArrayPid[$pid]="$currentCommand"
|
||||||
|
# Initialize pid execution time array
|
||||||
|
pidsTimeArray[$pid]=0
|
||||||
|
else
|
||||||
|
Logger "Skipping command [$currentCommand]." "DEBUG"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $isPostponedCommand == false ]; then
|
||||||
|
counter=$((counter+1))
|
||||||
|
fi
|
||||||
|
_ExecTasksPidsCheck
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
_ExecTasksPidsCheck
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|
||||||
# Return exit code if only one process was monitored, else return number of errors
|
# Return exit code if only one process was monitored, else return number of errors
|
||||||
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
|
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
|
||||||
#WIP: return code has nothing to do with logging
|
|
||||||
#if [ $noErrorLogAtAll == true ]; then
|
|
||||||
# return 0
|
|
||||||
#fi
|
|
||||||
|
|
||||||
if [ $mainItemCount -eq 1 ]; then
|
if [ $mainItemCount -eq 1 ]; then
|
||||||
return $retval
|
return $retval
|
||||||
@ -1108,6 +1200,20 @@ function IsNumeric {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#### CheckRFC822 SUBSET ####
|
||||||
|
# Checks email address validity
|
||||||
|
function checkRFC822 {
|
||||||
|
local mail="${1}"
|
||||||
|
local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$"
|
||||||
|
|
||||||
|
if [[ $mail =~ $rfc822 ]]; then
|
||||||
|
echo 1
|
||||||
|
else
|
||||||
|
echo 0
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
#### CheckRFC822 SUBSET END ####
|
||||||
|
|
||||||
function IsInteger {
|
function IsInteger {
|
||||||
local value="${1}"
|
local value="${1}"
|
||||||
|
|
||||||
@ -1318,6 +1424,11 @@ function GetOs {
|
|||||||
GetOs
|
GetOs
|
||||||
|
|
||||||
ENDSSH
|
ENDSSH
|
||||||
|
if [ $? != 0 ]; then
|
||||||
|
Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT]." "CRITICAL"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
|
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
|
||||||
remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")
|
remoteOsVar=$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")
|
||||||
@ -1377,7 +1488,8 @@ function RunLocalCommand {
|
|||||||
Logger "Running command [$command] on local host." "NOTICE"
|
Logger "Running command [$command] on local host." "NOTICE"
|
||||||
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
|
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
|
||||||
|
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME true $SLEEP_TIME $KEEP_LOGGING
|
||||||
|
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -eq 0 ]; then
|
if [ $retval -eq 0 ]; then
|
||||||
Logger "Command succeded." "NOTICE"
|
Logger "Command succeded." "NOTICE"
|
||||||
@ -1417,7 +1529,8 @@ function RunRemoteCommand {
|
|||||||
cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
|
cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
|
||||||
Logger "cmd: $cmd" "DEBUG"
|
Logger "cmd: $cmd" "DEBUG"
|
||||||
eval "$cmd" &
|
eval "$cmd" &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME true $SLEEP_TIME $KEEP_LOGGING
|
||||||
|
#ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -eq 0 ]; then
|
if [ $retval -eq 0 ]; then
|
||||||
Logger "Command succeded." "NOTICE"
|
Logger "Command succeded." "NOTICE"
|
||||||
@ -1450,7 +1563,8 @@ function RunBeforeHook {
|
|||||||
pids="$pids;$!"
|
pids="$pids;$!"
|
||||||
fi
|
fi
|
||||||
if [ "$pids" != "" ]; then
|
if [ "$pids" != "" ]; then
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 0 0 true $SLEEP_TIME $KEEP_LOGGING
|
||||||
|
#ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1468,7 +1582,8 @@ function RunAfterHook {
|
|||||||
pids="$pids;$!"
|
pids="$pids;$!"
|
||||||
fi
|
fi
|
||||||
if [ "$pids" != "" ]; then
|
if [ "$pids" != "" ]; then
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 0 0 true $SLEEP_TIME $KEEP_LOGGING
|
||||||
|
#ExecTasks "${FUNCNAME[0]}" 0 0 0 0 true true false false 1 $pids
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1479,7 +1594,8 @@ function CheckConnectivityRemoteHost {
|
|||||||
|
|
||||||
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
|
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
|
||||||
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
|
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 60 180 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 60 180 true $SLEEP_TIME $KEEP_LOGGING
|
||||||
|
#ExecTasks "${FUNCNAME[0]}" 0 0 60 180 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval != 0 ]; then
|
if [ $retval != 0 ]; then
|
||||||
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
|
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
|
||||||
@ -1500,7 +1616,8 @@ function CheckConnectivity3rdPartyHosts {
|
|||||||
for i in $REMOTE_3RD_PARTY_HOSTS
|
for i in $REMOTE_3RD_PARTY_HOSTS
|
||||||
do
|
do
|
||||||
eval "$PING_CMD $i > /dev/null 2>&1" &
|
eval "$PING_CMD $i > /dev/null 2>&1" &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 180 360 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 60 180 true $SLEEP_TIME $KEEP_LOGGING
|
||||||
|
#ExecTasks "${FUNCNAME[0]}" 0 0 180 360 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval != 0 ]; then
|
if [ $retval != 0 ]; then
|
||||||
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
|
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
|
||||||
@ -1738,10 +1855,12 @@ function InitLocalOSDependingSettings {
|
|||||||
# Tested on BSD and Mac
|
# Tested on BSD and Mac
|
||||||
STAT_CMD="stat -f \"%Sm\""
|
STAT_CMD="stat -f \"%Sm\""
|
||||||
STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m"
|
STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m"
|
||||||
|
SED_REGEX_ARG="-E"
|
||||||
else
|
else
|
||||||
# Tested on GNU stat, busybox and Cygwin
|
# Tested on GNU stat, busybox and Cygwin
|
||||||
STAT_CMD="stat -c %y"
|
STAT_CMD="stat -c %y"
|
||||||
STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y"
|
STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y"
|
||||||
|
SED_REGEX_ARG="-r"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Set compression first time when we know what local os we have
|
# Set compression first time when we know what local os we have
|
||||||
@ -2186,7 +2305,7 @@ function _ListDatabasesLocal {
|
|||||||
sqlCmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1"
|
sqlCmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1"
|
||||||
Logger "Launching command [$sqlCmd]." "DEBUG"
|
Logger "Launching command [$sqlCmd]." "DEBUG"
|
||||||
eval "$sqlCmd" &
|
eval "$sqlCmd" &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK true $SLEEP_TIME $KEEP_LOGGING
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -eq 0 ]; then
|
if [ $retval -eq 0 ]; then
|
||||||
Logger "Listing databases succeeded." "NOTICE"
|
Logger "Listing databases succeeded." "NOTICE"
|
||||||
@ -2211,7 +2330,7 @@ function _ListDatabasesRemote {
|
|||||||
sqlCmd="$SSH_CMD \"env _REMOTE_TOKEN=$_REMOTE_TOKEN mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP\" 2>&1"
|
sqlCmd="$SSH_CMD \"env _REMOTE_TOKEN=$_REMOTE_TOKEN mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP\" 2>&1"
|
||||||
Logger "Command output: $sqlCmd" "DEBUG"
|
Logger "Command output: $sqlCmd" "DEBUG"
|
||||||
eval "$sqlCmd" &
|
eval "$sqlCmd" &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK true $SLEEP_TIME $KEEP_LOGGING
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -eq 0 ]; then
|
if [ $retval -eq 0 ]; then
|
||||||
Logger "Listing databases succeeded." "NOTICE"
|
Logger "Listing databases succeeded." "NOTICE"
|
||||||
@ -2526,7 +2645,7 @@ function ListRecursiveBackupDirectories {
|
|||||||
Logger "Listing directories to backup." "NOTICE"
|
Logger "Listing directories to backup." "NOTICE"
|
||||||
if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then
|
if [ "$BACKUP_TYPE" == "local" ] || [ "$BACKUP_TYPE" == "push" ]; then
|
||||||
_ListRecursiveBackupDirectoriesLocal &
|
_ListRecursiveBackupDirectoriesLocal &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING
|
||||||
if [ $? -eq 1 ]; then
|
if [ $? -eq 1 ]; then
|
||||||
output_file=""
|
output_file=""
|
||||||
else
|
else
|
||||||
@ -2534,7 +2653,7 @@ function ListRecursiveBackupDirectories {
|
|||||||
fi
|
fi
|
||||||
elif [ "$BACKUP_TYPE" == "pull" ]; then
|
elif [ "$BACKUP_TYPE" == "pull" ]; then
|
||||||
_ListRecursiveBackupDirectoriesRemote &
|
_ListRecursiveBackupDirectoriesRemote &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING
|
||||||
if [ $? -eq 1 ]; then
|
if [ $? -eq 1 ]; then
|
||||||
output_file=""
|
output_file=""
|
||||||
else
|
else
|
||||||
@ -2598,7 +2717,7 @@ function _GetDirectoriesSizeLocal {
|
|||||||
cmd="du -cs $dirList | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
|
cmd="du -cs $dirList | tail -n1 | cut -f1 > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
|
||||||
Logger "Launching command [$cmd]." "DEBUG"
|
Logger "Launching command [$cmd]." "DEBUG"
|
||||||
eval "$cmd" &
|
eval "$cmd" &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING
|
||||||
# $cmd will return 0 even if some errors found, so we need to check if there is an error output
|
# $cmd will return 0 even if some errors found, so we need to check if there is an error output
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -ne 0 ] || [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then
|
if [ $retval -ne 0 ] || [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then
|
||||||
@ -2752,7 +2871,7 @@ function RemoteLogger {
|
|||||||
exit $retval
|
exit $retval
|
||||||
ENDSSH
|
ENDSSH
|
||||||
# $cmd will return 0 even if some errors found, so we need to check if there is an error output
|
# $cmd will return 0 even if some errors found, so we need to check if there is an error output
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -ne 0 ] || [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then
|
if [ $retval -ne 0 ] || [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then
|
||||||
Logger "Could not get files size for some or all remote directories." "ERROR"
|
Logger "Could not get files size for some or all remote directories." "ERROR"
|
||||||
@ -2798,7 +2917,7 @@ function _CreateDirectoryLocal {
|
|||||||
if [ ! -d "$dirToCreate" ]; then
|
if [ ! -d "$dirToCreate" ]; then
|
||||||
# No sudo, you should have all necessary rights
|
# No sudo, you should have all necessary rights
|
||||||
mkdir -p "$dirToCreate" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &
|
mkdir -p "$dirToCreate" > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -ne 0 ]; then
|
if [ $retval -ne 0 ]; then
|
||||||
Logger "Cannot create directory [$dirToCreate]" "CRITICAL"
|
Logger "Cannot create directory [$dirToCreate]" "CRITICAL"
|
||||||
@ -2942,7 +3061,7 @@ function RemoteLogger {
|
|||||||
fi
|
fi
|
||||||
exit 0
|
exit 0
|
||||||
ENDSSH
|
ENDSSH
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 720 1800 true $SLEEP_TIME $KEEP_LOGGING
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -ne 0 ]; then
|
if [ $retval -ne 0 ]; then
|
||||||
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "ERROR"
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "ERROR"
|
||||||
@ -3005,7 +3124,7 @@ function GetDiskSpaceLocal {
|
|||||||
# Not elegant solution to make df silent on errors
|
# Not elegant solution to make df silent on errors
|
||||||
# No sudo on local commands, assuming you should have all the necesarry rights to check backup directories sizes
|
# No sudo on local commands, assuming you should have all the necesarry rights to check backup directories sizes
|
||||||
$DF_CMD "$pathToCheck" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
|
$DF_CMD "$pathToCheck" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -ne 0 ]; then
|
if [ $retval -ne 0 ]; then
|
||||||
DISK_SPACE=0
|
DISK_SPACE=0
|
||||||
@ -3167,7 +3286,7 @@ function _GetDiskSpaceRemoteSub {
|
|||||||
_GetDiskSpaceRemoteSub
|
_GetDiskSpaceRemoteSub
|
||||||
exit $?
|
exit $?
|
||||||
ENDSSH
|
ENDSSH
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -ne 0 ]; then
|
if [ $retval -ne 0 ]; then
|
||||||
DISK_SPACE=0
|
DISK_SPACE=0
|
||||||
@ -3345,7 +3464,7 @@ function _BackupDatabaseLocalToLocal {
|
|||||||
Logger "Launching command [$drySqlCmd]." "DEBUG"
|
Logger "Launching command [$drySqlCmd]." "DEBUG"
|
||||||
eval "$drySqlCmd" &
|
eval "$drySqlCmd" &
|
||||||
fi
|
fi
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then
|
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then
|
||||||
if [ $_DRYRUN == false ]; then
|
if [ $_DRYRUN == false ]; then
|
||||||
@ -3393,7 +3512,7 @@ function _BackupDatabaseLocalToRemote {
|
|||||||
Logger "Launching command [$drySqlCmd]." "DEBUG"
|
Logger "Launching command [$drySqlCmd]." "DEBUG"
|
||||||
eval "$drySqlCmd" &
|
eval "$drySqlCmd" &
|
||||||
fi
|
fi
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then
|
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then
|
||||||
if [ $_DRYRUN == false ]; then
|
if [ $_DRYRUN == false ]; then
|
||||||
@ -3441,7 +3560,7 @@ function _BackupDatabaseRemoteToLocal {
|
|||||||
Logger "Launching command [$drySqlCmd]." "DEBUG"
|
Logger "Launching command [$drySqlCmd]." "DEBUG"
|
||||||
eval "$drySqlCmd" &
|
eval "$drySqlCmd" &
|
||||||
fi
|
fi
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then
|
if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then
|
||||||
if [ $_DRYRUN == false ]; then
|
if [ $_DRYRUN == false ]; then
|
||||||
@ -3581,8 +3700,7 @@ function EncryptFiles {
|
|||||||
else
|
else
|
||||||
hardMaxExecTime=0
|
hardMaxExecTime=0
|
||||||
fi
|
fi
|
||||||
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING true false false $PARALLEL_ENCRYPTION_PROCESSES "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.parallel.$SCRIPT_PID.$TSTAMP"
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $softMaxExecTime $hardMaxExecTime $SLEEP_TIME $KEEP_LOGGING true true false false 6 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.parallel.$SCRIPT_PID.$TSTAMP" "" PARALLEL_ENCRYPTION_PROCESSES
|
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -ne 0 ]; then
|
if [ $retval -ne 0 ]; then
|
||||||
Logger "Encryption error." "ERROR"
|
Logger "Encryption error." "ERROR"
|
||||||
@ -3687,7 +3805,7 @@ function DecryptFiles {
|
|||||||
hardMaxExecTime=0
|
hardMaxExecTime=0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
ParallelExec $PARALLEL_ENCRYPTION_PROCESSES "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.parallel.$SCRIPT_PID.$TSTAMP" true $softMaxExecTime $hardMaxExecTime $SLEEP_TIME $KEEP_LOGGING true true false
|
ExecTasks "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.parallel.$SCRIPT_PID.$TSTAMP" "${FUNCNAME[0]}" true 0 0 $softMaxExecTime $hardMaxExecTime true $SLEEP_TIME $KEEP_LOGGING true false $PARALLEL_ENCRYPTION_PROCESSES
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -ne 0 ]; then
|
if [ $retval -ne 0 ]; then
|
||||||
Logger "Decrypting error.." "ERROR"
|
Logger "Decrypting error.." "ERROR"
|
||||||
@ -3750,7 +3868,7 @@ function Rsync {
|
|||||||
|
|
||||||
Logger "Launching command [$rsyncCmd]." "DEBUG"
|
Logger "Launching command [$rsyncCmd]." "DEBUG"
|
||||||
eval "$rsyncCmd" &
|
eval "$rsyncCmd" &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_FILE_TASK $HARD_MAX_EXEC_TIME_FILE_TASK true $SLEEP_TIME $KEEP_LOGGING
|
||||||
retval=$?
|
retval=$?
|
||||||
if [ $retval -ne 0 ]; then
|
if [ $retval -ne 0 ]; then
|
||||||
Logger "Failed to backup [$sourceDir] to [$destinationDir]." "ERROR"
|
Logger "Failed to backup [$sourceDir] to [$destinationDir]." "ERROR"
|
||||||
@ -3897,7 +4015,7 @@ function _RotateBackupsLocal {
|
|||||||
cmd="rm -rf \"$path\""
|
cmd="rm -rf \"$path\""
|
||||||
Logger "Launching command [$cmd]." "DEBUG"
|
Logger "Launching command [$cmd]." "DEBUG"
|
||||||
eval "$cmd" &
|
eval "$cmd" &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 3600 0 true $SLEEP_TIME $KEEP_LOGGING
|
||||||
if [ $? -ne 0 ]; then
|
if [ $? -ne 0 ]; then
|
||||||
Logger "Cannot delete oldest copy [$path]." "ERROR"
|
Logger "Cannot delete oldest copy [$path]." "ERROR"
|
||||||
_LOGGER_SILENT=true Logger "Command was [$cmd]." "WARN"
|
_LOGGER_SILENT=true Logger "Command was [$cmd]." "WARN"
|
||||||
@ -3910,7 +4028,7 @@ function _RotateBackupsLocal {
|
|||||||
cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\""
|
cmd="mv \"$path\" \"$backup.$PROGRAM.$copy\""
|
||||||
Logger "Launching command [$cmd]." "DEBUG"
|
Logger "Launching command [$cmd]." "DEBUG"
|
||||||
eval "$cmd" &
|
eval "$cmd" &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 3600 0 true $SLEEP_TIME $KEEP_LOGGING
|
||||||
if [ $? -ne 0 ]; then
|
if [ $? -ne 0 ]; then
|
||||||
Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR"
|
Logger "Cannot move [$path] to [$backup.$PROGRAM.$copy]." "ERROR"
|
||||||
_LOGGER_SILENT=true Logger "Command was [$cmd]." "WARN"
|
_LOGGER_SILENT=true Logger "Command was [$cmd]." "WARN"
|
||||||
@ -3925,7 +4043,7 @@ function _RotateBackupsLocal {
|
|||||||
cmd="mv \"$backup\" \"$backup.$PROGRAM.1\""
|
cmd="mv \"$backup\" \"$backup.$PROGRAM.1\""
|
||||||
Logger "Launching command [$cmd]." "DEBUG"
|
Logger "Launching command [$cmd]." "DEBUG"
|
||||||
eval "$cmd" &
|
eval "$cmd" &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 3600 0 true $SLEEP_TIME $KEEP_LOGGING
|
||||||
if [ $? -ne 0 ]; then
|
if [ $? -ne 0 ]; then
|
||||||
Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
|
Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
|
||||||
_LOGGER_SILENT=true Logger "Command was [$cmd]." "WARN"
|
_LOGGER_SILENT=true Logger "Command was [$cmd]." "WARN"
|
||||||
@ -3935,7 +4053,7 @@ function _RotateBackupsLocal {
|
|||||||
cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\""
|
cmd="cp -R \"$backup\" \"$backup.$PROGRAM.1\""
|
||||||
Logger "Launching command [$cmd]." "DEBUG"
|
Logger "Launching command [$cmd]." "DEBUG"
|
||||||
eval "$cmd" &
|
eval "$cmd" &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 3600 0 true $SLEEP_TIME $KEEP_LOGGING
|
||||||
if [ $? -ne 0 ]; then
|
if [ $? -ne 0 ]; then
|
||||||
Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR"
|
Logger "Cannot copy [$backup] to [$backup.$PROGRAM.1]." "ERROR"
|
||||||
_LOGGER_SILENT=true Logger "Command was [$cmd]." "WARN"
|
_LOGGER_SILENT=true Logger "Command was [$cmd]." "WARN"
|
||||||
@ -3945,7 +4063,7 @@ function _RotateBackupsLocal {
|
|||||||
cmd="mv \"$backup\" \"$backup.$PROGRAM.1\""
|
cmd="mv \"$backup\" \"$backup.$PROGRAM.1\""
|
||||||
Logger "Launching command [$cmd]." "DEBUG"
|
Logger "Launching command [$cmd]." "DEBUG"
|
||||||
eval "$cmd" &
|
eval "$cmd" &
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 3600 0 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 3600 0 true $SLEEP_TIME $KEEP_LOGGING
|
||||||
if [ $? -ne 0 ]; then
|
if [ $? -ne 0 ]; then
|
||||||
Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
|
Logger "Cannot move [$backup] to [$backup.$PROGRAM.1]." "ERROR"
|
||||||
_LOGGER_SILENT=true Logger "Command was [$cmd]." "WARN"
|
_LOGGER_SILENT=true Logger "Command was [$cmd]." "WARN"
|
||||||
@ -4136,7 +4254,7 @@ function _RotateBackupsRemoteSSH {
|
|||||||
|
|
||||||
ENDSSH
|
ENDSSH
|
||||||
|
|
||||||
ExecTasks "${FUNCNAME[0]}" 0 0 1800 0 $SLEEP_TIME $KEEP_LOGGING true true false false 1 $!
|
ExecTasks $! "${FUNCNAME[0]}" false 0 0 1800 0 true $SLEEP_TIME $KEEP_LOGGING
|
||||||
if [ $? -ne 0 ]; then
|
if [ $? -ne 0 ]; then
|
||||||
Logger "Could not rotate backups in [$backupPath]." "ERROR"
|
Logger "Could not rotate backups in [$backupPath]." "ERROR"
|
||||||
Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "ERROR"
|
Logger "Command output:\n $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "ERROR"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user