diff --git a/dev/debug_obackup.sh b/dev/debug_obackup.sh index d814f1b..e56ca48 100755 --- a/dev/debug_obackup.sh +++ b/dev/debug_obackup.sh @@ -8,17 +8,20 @@ PROGRAM_VERSION=2.1-dev PROGRAM_BUILD=2016080602 IS_STABLE=yes -## FUNC_BUILD=2016072703 +#### MINIMAL-FUNCTION-SET BEGIN #### + +## FUNC_BUILD=2016080601 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr +#TODO: set _LOGGER_PREFIX in other apps, specially for osync daemon mode +#TODO: set _LOGGER_STDERR in other apps + ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode if ! type "$BASH" > /dev/null; then echo "Please run this script only with bash shell. Tested on bash >= 3.2" exit 127 fi -#### obackup & osync specific code BEGIN #### - ## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log will not be unnecessary big. KEEP_LOGGING=1801 @@ -28,13 +31,12 @@ export LC_ALL=C # Standard alert mail body MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." -#### obackup & osync specific code END #### - -#### MINIMAL-FUNCTION-SET BEGIN #### - -# Environment variables +# Environment variables that can be overriden by programs _DRYRUN=0 _SILENT=0 +_LOGGER_PREFIX="date" +_LOGGER_STDERR=0 + # Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags ERROR_ALERT=0 @@ -48,7 +50,7 @@ fi #__WITH_PARANOIA_DEBUG ## allow debugging from command line with _DEBUG=yes if [ ! "$_DEBUG" == "yes" ]; then _DEBUG=no - SLEEP_TIME=.001 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys + SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys _VERBOSE=0 else SLEEP_TIME=1 @@ -90,7 +92,8 @@ ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log" function Dummy { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - sleep .1 + + sleep $SLEEP_TIME } # Sub function of Logger @@ -98,11 +101,11 @@ function _Logger { local svalue="${1}" # What to log to stdout local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value local evalue="${3}" # What to log to stderr + echo -e "$lvalue" >> "$LOG_FILE" - # Special case in daemon mode where systemctl doesn't need double timestamps - if [ "$sync_on_changes" == "1" ]; then - cat <<< "$evalue" 1>&2 # Log to stderr in daemon mode + if [ "$_LOGGER_STDERR" -eq 1 ]; then + cat <<< "$evalue" 1>&2 elif [ "$_SILENT" -eq 0 ]; then echo -e "$svalue" fi @@ -113,13 +116,13 @@ function Logger { local value="${1}" # Sentence to log (in double quotes) local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL - # Special case in daemon mode we should timestamp instead of counting seconds - if [ "$sync_on_changes" == "1" ]; then + if [ "$_LOGGER_PREFIX" == "time" ]; then + prefix="TIME: $SECONDS - " + elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="$(date) - " else - prefix="TIME: $SECONDS - " + prefix="" fi - # if [ "$level" == "CRITICAL" ]; then _Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value" @@ -157,6 +160,8 @@ function _QuickLogger { local value="${1}" local destination="${2}" # Destination: stdout, log, both + __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then echo -e "$(date) - $value" >> "$LOG_FILE" elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then @@ -168,6 +173,8 @@ function _QuickLogger { function QuickLogger { local value="${1}" + __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + if [ "$_SILENT" -eq 1 ]; then _QuickLogger "$value" "log" else @@ -177,18 +184,18 @@ function QuickLogger { # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X function KillChilds { - local pid="${1}" + local pid="${1}" # Parent pid to kill local self="${2:-false}" + if children="$(pgrep -P "$pid")"; then for child in $children; do Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG KillChilds "$child" true done fi - - # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing - if ( [ "$self" == true ] && eval $PROCESS_TEST_CMD > /dev/null 2>&1); then + # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing + if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then Logger "Sending SIGTERM to process [$pid]." "DEBUG" kill -s SIGTERM "$pid" if [ $? != 0 ]; then @@ -199,13 +206,31 @@ function KillChilds { Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" return 1 fi + else + return 0 fi - return 0 else return 0 fi } +function KillAllChilds { + local pids="${1}" # List of parent pids to kill separated by semi-colon + + __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local errorcount=0 + + IFS=';' read -a pidsArray <<< "$pids" + for pid in "${pidsArray[@]}"; do + KillChilds $pid + if [ $? != 0 ]; then + errorcount=$((errorcount+1)) + fi + done + return $errorcount +} + # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending function SendAlert { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG @@ -347,7 +372,7 @@ function SendAlert { fi fi - # If function has not returned 0 yet, assume it's critical that no alert can be sent + # If function has not returned 0 yet, assume it is critical that no alert can be sent Logger "Cannot send alert (neither mutt, mail, sendmail, mailsend, sendemail or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue # Delete tmp log file @@ -482,7 +507,7 @@ function SendEmail { fi fi - # If function has not returned 0 yet, assume it's critical that no alert can be sent + # If function has not returned 0 yet, assume it is critical that no alert can be sent Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue } @@ -515,8 +540,6 @@ function LoadConfigFile { CONFIG_FILE="$config_file" } -#### MINIMAL-FUNCTION-SET END #### - function Spinner { if [ $_SILENT -eq 1 ]; then return 0 @@ -550,6 +573,139 @@ function Spinner { esac } +function WaitForTaskCompletion { + local pids="${1}" # pids to wait for, separated by semi-colon + local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. + local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. + local caller_name="${4}" # Who called this function + local exit_on_error="${5:-false}" # Should the function exit on subprocess errors + + Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + __CheckArguments 5 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once + local log_ttime=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function + + local retval=0 # return value of monitored pid process + local errorcount=0 # Number of pids that finished with errors + + local pidCount # number of given pids + + IFS=';' read -a pidsArray <<< "$pids" + pidCount=${#pidsArray[@]} + + while [ ${#pidsArray[@]} -gt 0 ]; do + newPidsArray=() + for pid in "${pidsArray[@]}"; do + if kill -0 $pid > /dev/null 2>&1; then + newPidsArray+=($pid) + else + wait $pid + result=$? + if [ $result -ne 0 ]; then + errorcount=$((errorcount+1)) + Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG" + fi + fi + done + + Spinner + exec_time=$(($SECONDS - $seconds_begin)) + if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then + if [ $log_ttime -ne $exec_time ]; then + log_ttime=$exec_time + Logger "Current tasks still running with pids [${pidsArray[@]}]." "NOTICE" + fi + fi + + if [ $exec_time -gt $soft_max_time ]; then + if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then + Logger "Max soft execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]." "WARN" + soft_alert=1 + SendAlert + + fi + if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then + Logger "Max hard execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]. Stopping task execution." "ERROR" + KillChilds $pid + if [ $? == 0 ]; then + Logger "Task stopped successfully" "NOTICE" + #return 0 + else + errrorcount=$((errorcount+1)) + #return 1 + fi + fi + fi + + pidsArray=("${newPidsArray[@]}") + sleep $SLEEP_TIME + done + + Logger "${FUNCNAME[0]} ended for [$caller_name] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + if [ $exit_on_error == true ] && [ $errorcount -gt 0 ]; then + Logger "Stopping execution." "CRITICAL" + exit 1337 + else + return $errorcount + fi +} + +function WaitForCompletion { + local pid="${1}" # pid to wait for + local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. + local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. + local caller_name="${4}" # Who called this function + Logger "${FUNCNAME[0]} called by [$caller_name]" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + __CheckArguments 4 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + + local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once + local log_time=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function + + local retval=0 # return value of monitored pid process + + while eval "$PROCESS_TEST_CMD" > /dev/null + do + Spinner + if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then + if [ $log_time -ne $SECONDS ]; then + log_time=$SECONDS + Logger "Current task still running." "NOTICE" + fi + fi + if [ $SECONDS -gt $soft_max_time ]; then + if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then + Logger "Max soft execution time exceeded for script." "WARN" + soft_alert=1 + SendAlert + fi + if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then + Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR" + KillChilds $pid + if [ $? == 0 ]; then + Logger "Task stopped successfully" "NOTICE" + return 0 + else + return 1 + fi + fi + fi + sleep $SLEEP_TIME + done + wait $pid + retval=$? + Logger "${FUNCNAME[0]} ended for [$caller_name] with status $retval." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + return $retval +} + +#### MINIMAL-FUNCTION-SET END #### + # obsolete, use StripQuotes function SedStripQuotes { echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") @@ -730,186 +886,6 @@ function GetRemoteOS { fi } -function WaitForPids { - # Takes a list of pids separated by space as argument, and waits until all pids are finished - local errors=0 - - while [ "$#" -gt 0 ]; do - for pid in "$@"; do - shift - if kill -0 "$pid" > /dev/null 2>&1; then - Logger "[$pid] is alive." "DEBUG" - set -- "$@" "$pid" - else - wait "$pid" - result=$? - if [ $result -eq 0 ]; then - Logger "[$pid] exited okay with [$result]" "DEBUG" - else - errors=$((errors+1)) - Logger "[$pid] exited with bad status [$result]." "WARN" - fi - fi - done - sleep $SLEEP_TIME - done - return $errors -} - - -function WaitForTaskCompletion { - local pids="${1}" # list of pids to wait for, separated by a semicolon - local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. - local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. - local caller_name="${4}" # Who called this function - local should_exit="${5}" # If true, the function exits on failure - Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG - #__CheckArguments 4 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once - local log_ttime=0 # local time instance for comparaison - - local seconds_begin=$SECONDS # Seconds since the beginning of the script - local exec_time=0 # Seconds since the beginning of this function - - local retval=0 # return value of monitored pid process - - local pid - local new_pids - local result - - IFS=';' read -r -a pidarray <<< "$pids" - - while [ ${#pidarray[@]} -gt 0 ]; do - newarray="" - for index in ${!pidarray[@]}; do - pid="${pidarray[index]}" - echo "run for $pid" - if kill -0 $pid > /dev/null 2>&1; then - echo "pid [$pid] is running." - newarray+=$pid - else - wait "$pid" - result=$? - echo $result - if [ $result -eq 0 ]; then - echo "pid [$pid] is finished with exit code 0." - else - echo "pid [$pid] is finished with exit code $result." - fi - fi - done - pidarray=$newarray - sleep .05 - done -} - -sleep 1 & -pids=$! -sleep 5 & -pids="$pids;$!" -echo "Waiting for pid $pids" -#sleep 5 & -#pids="$pids;$!" - -WaitForTaskCompletion $pids 0 0 "caller" 1 - -echo "done" -exit - -function old { - - - - - #TODO: test on FreeBSD, MacOS X and msys / cygwin - while kill -0 "$pid" > /dev/null 2>&1 - do - Spinner - exec_time=$(($SECONDS - $seconds_begin)) - if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then - if [ $log_ttime -ne $exec_time ]; then - log_ttime=$exec_time - Logger "Current task still running with pid [$pid]." "NOTICE" - fi - fi - if [ $exec_time -gt $soft_max_time ]; then - if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then - Logger "Max soft execution time exceeded for task [$caller_name] with pid [$pid]." "WARN" - soft_alert=1 - SendAlert - - fi - if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then - Logger "Max hard execution time exceeded for task [$caller_name] with pid [$pid]. Stopping task execution." "ERROR" - KillChilds $pid - if [ $? == 0 ]; then - Logger "Task stopped successfully" "NOTICE" - return 0 - else - return 1 - fi - fi - fi - sleep $SLEEP_TIME - done - wait $pid - retval=$? - Logger "${FUNCNAME[0]} ended for [$caller_name] with status $retval." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG - return $retval -} - -function WaitForCompletion { - local pid="${1}" # pid to wait for - local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. - local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. - local caller_name="${4}" # Who called this function - Logger "${FUNCNAME[0]} called by [$caller_name]" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG - __CheckArguments 4 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - - local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once - local log_time=0 # local time instance for comparaison - - local seconds_begin=$SECONDS # Seconds since the beginning of the script - local exec_time=0 # Seconds since the beginning of this function - - local retval=0 # return value of monitored pid process - - #TODO: test on FreeBSD, MacOS X and msys / cygwin - while kill -0 "$pid" > /dev/null 2>&1 - do - Spinner - if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then - if [ $log_time -ne $SECONDS ]; then - log_time=$SECONDS - Logger "Current task still running." "NOTICE" - fi - fi - if [ $SECONDS -gt $soft_max_time ]; then - if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then - Logger "Max soft execution time exceeded for script." "WARN" - soft_alert=1 - SendAlert - fi - if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then - Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR" - KillChilds $pid - if [ $? == 0 ]; then - Logger "Task stopped successfully" "NOTICE" - return 0 - else - return 1 - fi - fi - fi - sleep $SLEEP_TIME - done - wait $pid - retval=$? - Logger "${FUNCNAME[0]} ended for [$caller_name] with status $retval." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG - return $retval -} - function RunLocalCommand { local command="${1}" # Command to run local hard_max_time="${2}" # Max time to wait for command to compleet @@ -979,24 +955,38 @@ function RunRemoteCommand { function RunBeforeHook { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + local pids= + if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then - RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE + RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & + pids="$!" fi if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then - RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE + RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & + pids="$pids;$!" + fi + if [ "$pids" != "" ]; then + WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false fi } function RunAfterHook { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG + local pids + if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then - RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER + RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER & + pids="$!" fi if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then - RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER + RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER & + pids="$pids;$!" + fi + if [ "$pids" != "" ]; then + WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false fi } @@ -1120,8 +1110,6 @@ function RsyncPatternsFromAdd { local pattern_from="${2}" __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - local pattern_from= - ## Check if the exclude list has a full path, and if not, add the config file path if there is one if [ "$(basename $pattern_from)" == "$pattern_from" ]; then pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" @@ -1290,7 +1278,6 @@ function InitLocalOSSettings { FIND_CMD=$(dirname $BASH)/find # PROCESS_TEST_CMD assumes there is a variable $pid # Tested on MSYS and cygwin - #TODO: remove PROCESS_TEST_CMD if kill -0 works PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid' PING_CMD='$SYSTEMROOT\system32\ping -n 2' else diff --git a/obackup.sh b/obackup.sh index 518b815..e07ae59 100755 --- a/obackup.sh +++ b/obackup.sh @@ -8,17 +8,20 @@ PROGRAM_VERSION=2.1-dev PROGRAM_BUILD=2016080602 IS_STABLE=yes -## FUNC_BUILD=2016072703 +#### MINIMAL-FUNCTION-SET BEGIN #### + +## FUNC_BUILD=2016080601 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr +#TODO: set _LOGGER_PREFIX in other apps, specially for osync daemon mode +#TODO: set _LOGGER_STDERR in other apps + ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode if ! type "$BASH" > /dev/null; then echo "Please run this script only with bash shell. Tested on bash >= 3.2" exit 127 fi -#### obackup & osync specific code BEGIN #### - ## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log will not be unnecessary big. KEEP_LOGGING=1801 @@ -28,13 +31,12 @@ export LC_ALL=C # Standard alert mail body MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." -#### obackup & osync specific code END #### - -#### MINIMAL-FUNCTION-SET BEGIN #### - -# Environment variables +# Environment variables that can be overriden by programs _DRYRUN=0 _SILENT=0 +_LOGGER_PREFIX="date" +_LOGGER_STDERR=0 + # Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags ERROR_ALERT=0 @@ -44,7 +46,7 @@ WARN_ALERT=0 ## allow debugging from command line with _DEBUG=yes if [ ! "$_DEBUG" == "yes" ]; then _DEBUG=no - SLEEP_TIME=.001 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys + SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys _VERBOSE=0 else SLEEP_TIME=1 @@ -85,7 +87,8 @@ ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log" function Dummy { - sleep .1 + + sleep $SLEEP_TIME } # Sub function of Logger @@ -93,11 +96,11 @@ function _Logger { local svalue="${1}" # What to log to stdout local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value local evalue="${3}" # What to log to stderr + echo -e "$lvalue" >> "$LOG_FILE" - # Special case in daemon mode where systemctl doesn't need double timestamps - if [ "$sync_on_changes" == "1" ]; then - cat <<< "$evalue" 1>&2 # Log to stderr in daemon mode + if [ "$_LOGGER_STDERR" -eq 1 ]; then + cat <<< "$evalue" 1>&2 elif [ "$_SILENT" -eq 0 ]; then echo -e "$svalue" fi @@ -108,13 +111,13 @@ function Logger { local value="${1}" # Sentence to log (in double quotes) local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL - # Special case in daemon mode we should timestamp instead of counting seconds - if [ "$sync_on_changes" == "1" ]; then + if [ "$_LOGGER_PREFIX" == "time" ]; then + prefix="TIME: $SECONDS - " + elif [ "$_LOGGER_PREFIX" == "date" ]; then prefix="$(date) - " else - prefix="TIME: $SECONDS - " + prefix="" fi - # if [ "$level" == "CRITICAL" ]; then _Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value" @@ -147,6 +150,7 @@ function _QuickLogger { local value="${1}" local destination="${2}" # Destination: stdout, log, both + if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then echo -e "$(date) - $value" >> "$LOG_FILE" elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then @@ -158,6 +162,7 @@ function _QuickLogger { function QuickLogger { local value="${1}" + if [ "$_SILENT" -eq 1 ]; then _QuickLogger "$value" "log" else @@ -167,17 +172,17 @@ function QuickLogger { # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X function KillChilds { - local pid="${1}" + local pid="${1}" # Parent pid to kill local self="${2:-false}" + if children="$(pgrep -P "$pid")"; then for child in $children; do KillChilds "$child" true done fi - - # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing - if ( [ "$self" == true ] && eval $PROCESS_TEST_CMD > /dev/null 2>&1); then + # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing + if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then Logger "Sending SIGTERM to process [$pid]." "DEBUG" kill -s SIGTERM "$pid" if [ $? != 0 ]; then @@ -188,13 +193,30 @@ function KillChilds { Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" return 1 fi + else + return 0 fi - return 0 else return 0 fi } +function KillAllChilds { + local pids="${1}" # List of parent pids to kill separated by semi-colon + + + local errorcount=0 + + IFS=';' read -a pidsArray <<< "$pids" + for pid in "${pidsArray[@]}"; do + KillChilds $pid + if [ $? != 0 ]; then + errorcount=$((errorcount+1)) + fi + done + return $errorcount +} + # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending function SendAlert { @@ -335,7 +357,7 @@ function SendAlert { fi fi - # If function has not returned 0 yet, assume it's critical that no alert can be sent + # If function has not returned 0 yet, assume it is critical that no alert can be sent Logger "Cannot send alert (neither mutt, mail, sendmail, mailsend, sendemail or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue # Delete tmp log file @@ -469,7 +491,7 @@ function SendEmail { fi fi - # If function has not returned 0 yet, assume it's critical that no alert can be sent + # If function has not returned 0 yet, assume it is critical that no alert can be sent Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue } @@ -501,8 +523,6 @@ function LoadConfigFile { CONFIG_FILE="$config_file" } -#### MINIMAL-FUNCTION-SET END #### - function Spinner { if [ $_SILENT -eq 1 ]; then return 0 @@ -536,6 +556,133 @@ function Spinner { esac } +function WaitForTaskCompletion { + local pids="${1}" # pids to wait for, separated by semi-colon + local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. + local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. + local caller_name="${4}" # Who called this function + local exit_on_error="${5:-false}" # Should the function exit on subprocess errors + + + local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once + local log_ttime=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function + + local retval=0 # return value of monitored pid process + local errorcount=0 # Number of pids that finished with errors + + local pidCount # number of given pids + + IFS=';' read -a pidsArray <<< "$pids" + pidCount=${#pidsArray[@]} + + while [ ${#pidsArray[@]} -gt 0 ]; do + newPidsArray=() + for pid in "${pidsArray[@]}"; do + if kill -0 $pid > /dev/null 2>&1; then + newPidsArray+=($pid) + else + wait $pid + result=$? + if [ $result -ne 0 ]; then + errorcount=$((errorcount+1)) + Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG" + fi + fi + done + + Spinner + exec_time=$(($SECONDS - $seconds_begin)) + if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then + if [ $log_ttime -ne $exec_time ]; then + log_ttime=$exec_time + Logger "Current tasks still running with pids [${pidsArray[@]}]." "NOTICE" + fi + fi + + if [ $exec_time -gt $soft_max_time ]; then + if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then + Logger "Max soft execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]." "WARN" + soft_alert=1 + SendAlert + + fi + if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then + Logger "Max hard execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]. Stopping task execution." "ERROR" + KillChilds $pid + if [ $? == 0 ]; then + Logger "Task stopped successfully" "NOTICE" + #return 0 + else + errrorcount=$((errorcount+1)) + #return 1 + fi + fi + fi + + pidsArray=("${newPidsArray[@]}") + sleep $SLEEP_TIME + done + + if [ $exit_on_error == true ] && [ $errorcount -gt 0 ]; then + Logger "Stopping execution." "CRITICAL" + exit 1337 + else + return $errorcount + fi +} + +function WaitForCompletion { + local pid="${1}" # pid to wait for + local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. + local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. + local caller_name="${4}" # Who called this function + + local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once + local log_time=0 # local time instance for comparaison + + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function + + local retval=0 # return value of monitored pid process + + while eval "$PROCESS_TEST_CMD" > /dev/null + do + Spinner + if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then + if [ $log_time -ne $SECONDS ]; then + log_time=$SECONDS + Logger "Current task still running." "NOTICE" + fi + fi + if [ $SECONDS -gt $soft_max_time ]; then + if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then + Logger "Max soft execution time exceeded for script." "WARN" + soft_alert=1 + SendAlert + fi + if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then + Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR" + KillChilds $pid + if [ $? == 0 ]; then + Logger "Task stopped successfully" "NOTICE" + return 0 + else + return 1 + fi + fi + fi + sleep $SLEEP_TIME + done + wait $pid + retval=$? + return $retval +} + +#### MINIMAL-FUNCTION-SET END #### + # obsolete, use StripQuotes function SedStripQuotes { echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") @@ -713,180 +860,6 @@ function GetRemoteOS { fi } -function WaitForPids { - # Takes a list of pids separated by space as argument, and waits until all pids are finished - local errors=0 - - while [ "$#" -gt 0 ]; do - for pid in "$@"; do - shift - if kill -0 "$pid" > /dev/null 2>&1; then - Logger "[$pid] is alive." "DEBUG" - set -- "$@" "$pid" - else - wait "$pid" - result=$? - if [ $result -eq 0 ]; then - Logger "[$pid] exited okay with [$result]" "DEBUG" - else - errors=$((errors+1)) - Logger "[$pid] exited with bad status [$result]." "WARN" - fi - fi - done - sleep $SLEEP_TIME - done - return $errors -} - - -function WaitForTaskCompletion { - local pids="${1}" # list of pids to wait for, separated by a semicolon - local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. - local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. - local caller_name="${4}" # Who called this function - local should_exit="${5}" # If true, the function exits on failure - - local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once - local log_ttime=0 # local time instance for comparaison - - local seconds_begin=$SECONDS # Seconds since the beginning of the script - local exec_time=0 # Seconds since the beginning of this function - - local retval=0 # return value of monitored pid process - - local pid - local new_pids - local result - - IFS=';' read -r -a pidarray <<< "$pids" - - while [ ${#pidarray[@]} -gt 0 ]; do - newarray="" - for index in ${!pidarray[@]}; do - pid="${pidarray[index]}" - echo "run for $pid" - if kill -0 $pid > /dev/null 2>&1; then - echo "pid [$pid] is running." - newarray+=$pid - else - wait "$pid" - result=$? - echo $result - if [ $result -eq 0 ]; then - echo "pid [$pid] is finished with exit code 0." - else - echo "pid [$pid] is finished with exit code $result." - fi - fi - done - pidarray=$newarray - sleep .05 - done -} - -sleep 1 & -pids=$! -sleep 5 & -pids="$pids;$!" -echo "Waiting for pid $pids" -#sleep 5 & -#pids="$pids;$!" - -WaitForTaskCompletion $pids 0 0 "caller" 1 - -echo "done" -exit - -function old { - - - - - #TODO: test on FreeBSD, MacOS X and msys / cygwin - while kill -0 "$pid" > /dev/null 2>&1 - do - Spinner - exec_time=$(($SECONDS - $seconds_begin)) - if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then - if [ $log_ttime -ne $exec_time ]; then - log_ttime=$exec_time - Logger "Current task still running with pid [$pid]." "NOTICE" - fi - fi - if [ $exec_time -gt $soft_max_time ]; then - if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then - Logger "Max soft execution time exceeded for task [$caller_name] with pid [$pid]." "WARN" - soft_alert=1 - SendAlert - - fi - if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then - Logger "Max hard execution time exceeded for task [$caller_name] with pid [$pid]. Stopping task execution." "ERROR" - KillChilds $pid - if [ $? == 0 ]; then - Logger "Task stopped successfully" "NOTICE" - return 0 - else - return 1 - fi - fi - fi - sleep $SLEEP_TIME - done - wait $pid - retval=$? - return $retval -} - -function WaitForCompletion { - local pid="${1}" # pid to wait for - local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. - local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0. - local caller_name="${4}" # Who called this function - - local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once - local log_time=0 # local time instance for comparaison - - local seconds_begin=$SECONDS # Seconds since the beginning of the script - local exec_time=0 # Seconds since the beginning of this function - - local retval=0 # return value of monitored pid process - - #TODO: test on FreeBSD, MacOS X and msys / cygwin - while kill -0 "$pid" > /dev/null 2>&1 - do - Spinner - if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then - if [ $log_time -ne $SECONDS ]; then - log_time=$SECONDS - Logger "Current task still running." "NOTICE" - fi - fi - if [ $SECONDS -gt $soft_max_time ]; then - if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then - Logger "Max soft execution time exceeded for script." "WARN" - soft_alert=1 - SendAlert - fi - if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then - Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR" - KillChilds $pid - if [ $? == 0 ]; then - Logger "Task stopped successfully" "NOTICE" - return 0 - else - return 1 - fi - fi - fi - sleep $SLEEP_TIME - done - wait $pid - retval=$? - return $retval -} - function RunLocalCommand { local command="${1}" # Command to run local hard_max_time="${2}" # Max time to wait for command to compleet @@ -953,23 +926,37 @@ function RunRemoteCommand { function RunBeforeHook { + local pids= + if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then - RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE + RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & + pids="$!" fi if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then - RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE + RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE & + pids="$pids;$!" + fi + if [ "$pids" != "" ]; then + WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false fi } function RunAfterHook { + local pids + if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then - RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER + RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER & + pids="$!" fi if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then - RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER + RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER & + pids="$pids;$!" + fi + if [ "$pids" != "" ]; then + WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false fi } @@ -1051,8 +1038,6 @@ function RsyncPatternsFromAdd { local pattern_type="${1}" local pattern_from="${2}" - local pattern_from= - ## Check if the exclude list has a full path, and if not, add the config file path if there is one if [ "$(basename $pattern_from)" == "$pattern_from" ]; then pattern_from="$(dirname $CONFIG_FILE)/$pattern_from" @@ -1217,7 +1202,6 @@ function InitLocalOSSettings { FIND_CMD=$(dirname $BASH)/find # PROCESS_TEST_CMD assumes there is a variable $pid # Tested on MSYS and cygwin - #TODO: remove PROCESS_TEST_CMD if kill -0 works PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid' PING_CMD='$SYSTEMROOT\system32\ping -n 2' else