From 1a98f1c855fe183cd0f0bea4ce68757dd0cb7eed Mon Sep 17 00:00:00 2001
From: deajan <ozy@netpower.fr>
Date: Sat, 6 Aug 2016 16:16:19 +0200
Subject: [PATCH] Rebuilt targets

---
 dev/debug_obackup.sh | 413 +++++++++++++++++++++----------------------
 obackup.sh           | 398 ++++++++++++++++++++---------------------
 2 files changed, 391 insertions(+), 420 deletions(-)

diff --git a/dev/debug_obackup.sh b/dev/debug_obackup.sh
index d814f1b..e56ca48 100755
--- a/dev/debug_obackup.sh
+++ b/dev/debug_obackup.sh
@@ -8,17 +8,20 @@ PROGRAM_VERSION=2.1-dev
 PROGRAM_BUILD=2016080602
 IS_STABLE=yes
 
-## FUNC_BUILD=2016072703
+#### MINIMAL-FUNCTION-SET BEGIN ####
+
+## FUNC_BUILD=2016080601
 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
 
+#TODO: set _LOGGER_PREFIX in other apps, specially for osync daemon mode
+#TODO: set _LOGGER_STDERR in other apps
+
 ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
 if ! type "$BASH" > /dev/null; then
 	echo "Please run this script only with bash shell. Tested on bash >= 3.2"
 	exit 127
 fi
 
-#### obackup & osync specific code BEGIN ####
-
 ## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log will not be unnecessary big.
 KEEP_LOGGING=1801
 
@@ -28,13 +31,12 @@ export LC_ALL=C
 # Standard alert mail body
 MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
 
-#### obackup & osync specific code END ####
-
-#### MINIMAL-FUNCTION-SET BEGIN ####
-
-# Environment variables
+# Environment variables that can be overriden by programs
 _DRYRUN=0
 _SILENT=0
+_LOGGER_PREFIX="date"
+_LOGGER_STDERR=0
+
 
 # Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags
 ERROR_ALERT=0
@@ -48,7 +50,7 @@ fi						#__WITH_PARANOIA_DEBUG
 ## allow debugging from command line with _DEBUG=yes
 if [ ! "$_DEBUG" == "yes" ]; then
 	_DEBUG=no
-	SLEEP_TIME=.001 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys
+	SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys
 	_VERBOSE=0
 else
 	SLEEP_TIME=1
@@ -90,7 +92,8 @@ ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log"
 
 function Dummy {
 	__CheckArguments 0 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
-	sleep .1
+
+	sleep $SLEEP_TIME
 }
 
 # Sub function of Logger
@@ -98,11 +101,11 @@ function _Logger {
 	local svalue="${1}" # What to log to stdout
 	local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value
 	local evalue="${3}" # What to log to stderr
+
 	echo -e "$lvalue" >> "$LOG_FILE"
 
-	# <OSYNC SPECIFIC> Special case in daemon mode where systemctl doesn't need double timestamps
-	if [ "$sync_on_changes" == "1" ]; then
-		cat <<< "$evalue" 1>&2	# Log to stderr in daemon mode
+	if [ "$_LOGGER_STDERR" -eq 1 ]; then
+		cat <<< "$evalue" 1>&2
 	elif [ "$_SILENT" -eq 0 ]; then
 		echo -e "$svalue"
 	fi
@@ -113,13 +116,13 @@ function Logger {
 	local value="${1}" # Sentence to log (in double quotes)
 	local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL
 
-	# <OSYNC SPECIFIC> Special case in daemon mode we should timestamp instead of counting seconds
-	if [ "$sync_on_changes" == "1" ]; then
+	if [ "$_LOGGER_PREFIX" == "time" ]; then
+		prefix="TIME: $SECONDS - "
+	elif [ "$_LOGGER_PREFIX" == "date" ]; then
 		prefix="$(date) - "
 	else
-		prefix="TIME: $SECONDS - "
+		prefix=""
 	fi
-	# </OSYNC SPECIFIC>
 
 	if [ "$level" == "CRITICAL" ]; then
 		_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value"
@@ -157,6 +160,8 @@ function _QuickLogger {
 	local value="${1}"
 	local destination="${2}" # Destination: stdout, log, both
 
+	__CheckArguments 2 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
+
 	if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then
 		echo -e "$(date) - $value" >> "$LOG_FILE"
 	elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then
@@ -168,6 +173,8 @@ function _QuickLogger {
 function QuickLogger {
 	local value="${1}"
 
+	__CheckArguments 1 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
+
 	if [ "$_SILENT" -eq 1 ]; then
 		_QuickLogger "$value" "log"
 	else
@@ -177,18 +184,18 @@ function QuickLogger {
 
 # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
 function KillChilds {
-	local pid="${1}"
+	local pid="${1}" # Parent pid to kill
 	local self="${2:-false}"
 
+
 	if children="$(pgrep -P "$pid")"; then
 		for child in $children; do
 			Logger "Launching KillChilds \"$child\" true" "DEBUG"	#__WITH_PARANOIA_DEBUG
 			KillChilds "$child" true
 		done
 	fi
-
-	# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
-	if ( [ "$self" == true ] && eval $PROCESS_TEST_CMD > /dev/null 2>&1); then
+		# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
+	if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then
 		Logger "Sending SIGTERM to process [$pid]." "DEBUG"
 		kill -s SIGTERM "$pid"
 		if [ $? != 0 ]; then
@@ -199,13 +206,31 @@ function KillChilds {
 				Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
 				return 1
 			fi
+		else
+			return 0
 		fi
-		return 0
 	else
 		return 0
 	fi
 }
 
+function KillAllChilds {
+	local pids="${1}" # List of parent pids to kill separated by semi-colon
+
+	__CheckArguments 1 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
+
+	local errorcount=0
+
+	IFS=';' read -a pidsArray <<< "$pids"
+	for pid in "${pidsArray[@]}"; do
+		KillChilds $pid
+		if [ $? != 0 ]; then
+			errorcount=$((errorcount+1))
+			fi
+	done
+	return $errorcount
+}
+
 # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
 function SendAlert {
 	__CheckArguments 0 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
@@ -347,7 +372,7 @@ function SendAlert {
 		fi
 	fi
 
-	# If function has not returned 0 yet, assume it's critical that no alert can be sent
+	# If function has not returned 0 yet, assume it is critical that no alert can be sent
 	Logger "Cannot send alert (neither mutt, mail, sendmail, mailsend, sendemail or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue
 
 	# Delete tmp log file
@@ -482,7 +507,7 @@ function SendEmail {
 		fi
 	fi
 
-	# If function has not returned 0 yet, assume it's critical that no alert can be sent
+	# If function has not returned 0 yet, assume it is critical that no alert can be sent
 	Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue
 }
 
@@ -515,8 +540,6 @@ function LoadConfigFile {
 	CONFIG_FILE="$config_file"
 }
 
-#### MINIMAL-FUNCTION-SET END ####
-
 function Spinner {
 	if [ $_SILENT -eq 1 ]; then
 		return 0
@@ -550,6 +573,139 @@ function Spinner {
 	esac
 }
 
+function WaitForTaskCompletion {
+	local pids="${1}" # pids to wait for, separated by semi-colon
+	local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
+	local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
+	local caller_name="${4}" # Who called this function
+	local exit_on_error="${5:-false}" # Should the function exit on subprocess errors
+
+	Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG"	#__WITH_PARANOIA_DEBUG
+	__CheckArguments 5 $# ${FUNCNAME[0]} "$@"				#__WITH_PARANOIA_DEBUG
+
+	local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
+	local log_ttime=0 # local time instance for comparaison
+
+	local seconds_begin=$SECONDS # Seconds since the beginning of the script
+	local exec_time=0 # Seconds since the beginning of this function
+
+	local retval=0 # return value of monitored pid process
+	local errorcount=0 # Number of pids that finished with errors
+
+	local pidCount # number of given pids
+
+	IFS=';' read -a pidsArray <<< "$pids"
+	pidCount=${#pidsArray[@]}
+
+	while [ ${#pidsArray[@]} -gt 0 ]; do
+		newPidsArray=()
+		for pid in "${pidsArray[@]}"; do
+			if kill -0 $pid > /dev/null 2>&1; then
+				newPidsArray+=($pid)
+			else
+				wait $pid
+				result=$?
+				if [ $result -ne 0 ]; then
+					errorcount=$((errorcount+1))
+					Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG"
+				fi
+			fi
+		done
+
+		Spinner
+		exec_time=$(($SECONDS - $seconds_begin))
+		if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then
+			if [ $log_ttime -ne $exec_time ]; then
+				log_ttime=$exec_time
+				Logger "Current tasks still running with pids [${pidsArray[@]}]." "NOTICE"
+			fi
+		fi
+
+		if [ $exec_time -gt $soft_max_time ]; then
+			if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
+				Logger "Max soft execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]." "WARN"
+				soft_alert=1
+				SendAlert
+
+			fi
+			if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
+				Logger "Max hard execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]. Stopping task execution." "ERROR"
+				KillChilds $pid
+				if [ $? == 0 ]; then
+					Logger "Task stopped successfully" "NOTICE"
+					#return 0
+				else
+					errrorcount=$((errorcount+1))
+					#return 1
+				fi
+			fi
+		fi
+
+		pidsArray=("${newPidsArray[@]}")
+		sleep $SLEEP_TIME
+	done
+
+	Logger "${FUNCNAME[0]} ended for [$caller_name] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG"	#__WITH_PARANOIA_DEBUG
+	if [ $exit_on_error == true ] && [ $errorcount -gt 0 ]; then
+		Logger "Stopping execution." "CRITICAL"
+		exit 1337
+	else
+		return $errorcount
+	fi
+}
+
+function WaitForCompletion {
+	local pid="${1}" # pid to wait for
+	local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
+	local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
+	local caller_name="${4}" # Who called this function
+	Logger "${FUNCNAME[0]} called by [$caller_name]" "PARANOIA_DEBUG"	#__WITH_PARANOIA_DEBUG
+	__CheckArguments 4 $# ${FUNCNAME[0]} "$@"				#__WITH_PARANOIA_DEBUG
+
+	local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
+	local log_time=0 # local time instance for comparaison
+
+	local seconds_begin=$SECONDS # Seconds since the beginning of the script
+	local exec_time=0 # Seconds since the beginning of this function
+
+	local retval=0 # return value of monitored pid process
+
+	while eval "$PROCESS_TEST_CMD" > /dev/null
+	do
+		Spinner
+		if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then
+			if [ $log_time -ne $SECONDS ]; then
+				log_time=$SECONDS
+				Logger "Current task still running." "NOTICE"
+			fi
+		fi
+		if [ $SECONDS -gt $soft_max_time ]; then
+			if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then
+				Logger "Max soft execution time exceeded for script." "WARN"
+				soft_alert=1
+				SendAlert
+			fi
+			if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then
+				Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR"
+				KillChilds $pid
+				if [ $? == 0 ]; then
+					Logger "Task stopped successfully" "NOTICE"
+					return 0
+				else
+					return 1
+				fi
+			fi
+		fi
+		sleep $SLEEP_TIME
+	done
+	wait $pid
+	retval=$?
+	Logger "${FUNCNAME[0]} ended for [$caller_name] with status $retval." "PARANOIA_DEBUG"	#__WITH_PARANOIA_DEBUG
+	return $retval
+}
+
+#### MINIMAL-FUNCTION-SET END ####
+
 # obsolete, use StripQuotes
 function SedStripQuotes {
         echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
@@ -730,186 +886,6 @@ function GetRemoteOS {
 	fi
 }
 
-function WaitForPids {
-	# Takes a list of pids separated by space as argument, and waits until all pids are finished
-       local errors=0
-
-        while [ "$#" -gt 0 ]; do
-                for pid in "$@"; do
-                        shift
-                        if kill -0 "$pid" > /dev/null 2>&1; then
-                                Logger "[$pid] is alive." "DEBUG"
-                                set -- "$@" "$pid"
-                        else
-                                wait "$pid"
-                                result=$?
-                                if [ $result -eq 0 ]; then
-                                        Logger "[$pid] exited okay with [$result]" "DEBUG"
-                                else
-                                        errors=$((errors+1))
-                                        Logger "[$pid] exited with bad status [$result]." "WARN"
-                                fi
-                        fi
-                done
-                sleep $SLEEP_TIME
-        done
-	return $errors
-}
-
-
-function WaitForTaskCompletion {
-	local pids="${1}" # list of pids to wait for, separated by a semicolon
-	local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
-	local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
-	local caller_name="${4}" # Who called this function
-	local should_exit="${5}" # If true, the function exits on failure
-	Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG"	#__WITH_PARANOIA_DEBUG
-	#__CheckArguments 4 $# ${FUNCNAME[0]} "$@"				#__WITH_PARANOIA_DEBUG
-
-	local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
-	local log_ttime=0 # local time instance for comparaison
-
-	local seconds_begin=$SECONDS # Seconds since the beginning of the script
-	local exec_time=0 # Seconds since the beginning of this function
-
-	local retval=0 # return value of monitored pid process
-
-	local pid
-	local new_pids
-	local result
-
-	IFS=';' read -r -a pidarray <<< "$pids"
-
-	while [ ${#pidarray[@]} -gt 0 ]; do
-		newarray=""
-		for index in ${!pidarray[@]}; do
-			pid="${pidarray[index]}"
-			echo "run  for $pid"
-			if kill -0 $pid > /dev/null 2>&1; then
-				echo "pid [$pid] is running."
-				newarray+=$pid
-			else
-				wait "$pid"
-				result=$?
-				echo $result
-				if [ $result -eq 0 ]; then
-					echo "pid [$pid] is finished with exit code 0."
-				else
-					echo "pid [$pid] is finished with exit code $result."
-				fi
-			fi
-		done
-		pidarray=$newarray
-		sleep .05
- 	done
-}
-
-sleep 1 &
-pids=$!
-sleep 5 &
-pids="$pids;$!"
-echo "Waiting for pid $pids"
-#sleep 5 &
-#pids="$pids;$!"
-
-WaitForTaskCompletion $pids 0 0 "caller" 1
-
-echo "done"
-exit
-
-function old {
-
-
-
-
-	#TODO: test on FreeBSD, MacOS X and msys / cygwin
-	while kill -0 "$pid" > /dev/null 2>&1
-	do
-		Spinner
-		exec_time=$(($SECONDS - $seconds_begin))
-		if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then
-			if [ $log_ttime -ne $exec_time ]; then
-				log_ttime=$exec_time
-				Logger "Current task still running with pid [$pid]." "NOTICE"
-			fi
-		fi
-		if [ $exec_time -gt $soft_max_time ]; then
-			if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
-				Logger "Max soft execution time exceeded for task [$caller_name] with pid [$pid]." "WARN"
-				soft_alert=1
-				SendAlert
-
-			fi
-			if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
-				Logger "Max hard execution time exceeded for task [$caller_name] with pid [$pid]. Stopping task execution." "ERROR"
-				KillChilds $pid
-				if [ $? == 0 ]; then
-					Logger "Task stopped successfully" "NOTICE"
-					return 0
-				else
-					return 1
-				fi
-			fi
-		fi
-		sleep $SLEEP_TIME
-	done
-	wait $pid
-	retval=$?
-	Logger "${FUNCNAME[0]} ended for [$caller_name] with status $retval." "PARANOIA_DEBUG"	#__WITH_PARANOIA_DEBUG
-	return $retval
-}
-
-function WaitForCompletion {
-	local pid="${1}" # pid to wait for
-	local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
-	local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
-	local caller_name="${4}" # Who called this function
-	Logger "${FUNCNAME[0]} called by [$caller_name]" "PARANOIA_DEBUG"	#__WITH_PARANOIA_DEBUG
-	__CheckArguments 4 $# ${FUNCNAME[0]} "$@"				#__WITH_PARANOIA_DEBUG
-
-	local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
-	local log_time=0 # local time instance for comparaison
-
-	local seconds_begin=$SECONDS # Seconds since the beginning of the script
-	local exec_time=0 # Seconds since the beginning of this function
-
-	local retval=0 # return value of monitored pid process
-
-	#TODO: test on FreeBSD, MacOS X and msys / cygwin
-	while kill -0 "$pid" > /dev/null 2>&1
-	do
-		Spinner
-		if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then
-			if [ $log_time -ne $SECONDS ]; then
-				log_time=$SECONDS
-				Logger "Current task still running." "NOTICE"
-			fi
-		fi
-		if [ $SECONDS -gt $soft_max_time ]; then
-			if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then
-				Logger "Max soft execution time exceeded for script." "WARN"
-				soft_alert=1
-				SendAlert
-			fi
-			if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then
-				Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR"
-				KillChilds $pid
-				if [ $? == 0 ]; then
-					Logger "Task stopped successfully" "NOTICE"
-					return 0
-				else
-					return 1
-				fi
-			fi
-		fi
-		sleep $SLEEP_TIME
-	done
-	wait $pid
-	retval=$?
-	Logger "${FUNCNAME[0]} ended for [$caller_name] with status $retval." "PARANOIA_DEBUG"	#__WITH_PARANOIA_DEBUG
-	return $retval
-}
-
 function RunLocalCommand {
 	local command="${1}" # Command to run
 	local hard_max_time="${2}" # Max time to wait for command to compleet
@@ -979,24 +955,38 @@ function RunRemoteCommand {
 function RunBeforeHook {
 	__CheckArguments 0 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
 
+	local pids=
+
 	if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then
-		RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE
+		RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE &
+		pids="$!"
 	fi
 
 	if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then
-		RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE
+		RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE &
+		pids="$pids;$!"
+	fi
+	if [ "$pids" != "" ]; then
+		WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false
 	fi
 }
 
 function RunAfterHook {
 	__CheckArguments 0 $# ${FUNCNAME[0]} "$@"	#__WITH_PARANOIA_DEBUG
 
+	local pids
+
 	if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then
-		RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER
+		RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER &
+		pids="$!"
 	fi
 
 	if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then
-		RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER
+		RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER &
+		pids="$pids;$!"
+	fi
+	if [ "$pids" != "" ]; then
+		WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false
 	fi
 }
 
@@ -1120,8 +1110,6 @@ function RsyncPatternsFromAdd {
         local pattern_from="${2}"
 	__CheckArguments 2 $# ${FUNCNAME[0]} "$@"    #__WITH_PARANOIA_DEBUG
 
-	local pattern_from=
-
         ## Check if the exclude list has a full path, and if not, add the config file path if there is one
         if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
                 pattern_from="$(dirname $CONFIG_FILE)/$pattern_from"
@@ -1290,7 +1278,6 @@ function InitLocalOSSettings {
                 FIND_CMD=$(dirname $BASH)/find
                 # PROCESS_TEST_CMD assumes there is a variable $pid
 		# Tested on MSYS and cygwin
-		#TODO: remove PROCESS_TEST_CMD if kill -0 works
                 PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid'
                 PING_CMD='$SYSTEMROOT\system32\ping -n 2'
         else
diff --git a/obackup.sh b/obackup.sh
index 518b815..e07ae59 100755
--- a/obackup.sh
+++ b/obackup.sh
@@ -8,17 +8,20 @@ PROGRAM_VERSION=2.1-dev
 PROGRAM_BUILD=2016080602
 IS_STABLE=yes
 
-## FUNC_BUILD=2016072703
+#### MINIMAL-FUNCTION-SET BEGIN ####
+
+## FUNC_BUILD=2016080601
 ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
 
+#TODO: set _LOGGER_PREFIX in other apps, specially for osync daemon mode
+#TODO: set _LOGGER_STDERR in other apps
+
 ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
 if ! type "$BASH" > /dev/null; then
 	echo "Please run this script only with bash shell. Tested on bash >= 3.2"
 	exit 127
 fi
 
-#### obackup & osync specific code BEGIN ####
-
 ## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log will not be unnecessary big.
 KEEP_LOGGING=1801
 
@@ -28,13 +31,12 @@ export LC_ALL=C
 # Standard alert mail body
 MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
 
-#### obackup & osync specific code END ####
-
-#### MINIMAL-FUNCTION-SET BEGIN ####
-
-# Environment variables
+# Environment variables that can be overriden by programs
 _DRYRUN=0
 _SILENT=0
+_LOGGER_PREFIX="date"
+_LOGGER_STDERR=0
+
 
 # Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags
 ERROR_ALERT=0
@@ -44,7 +46,7 @@ WARN_ALERT=0
 ## allow debugging from command line with _DEBUG=yes
 if [ ! "$_DEBUG" == "yes" ]; then
 	_DEBUG=no
-	SLEEP_TIME=.001 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys
+	SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys
 	_VERBOSE=0
 else
 	SLEEP_TIME=1
@@ -85,7 +87,8 @@ ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log"
 
 
 function Dummy {
-	sleep .1
+
+	sleep $SLEEP_TIME
 }
 
 # Sub function of Logger
@@ -93,11 +96,11 @@ function _Logger {
 	local svalue="${1}" # What to log to stdout
 	local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value
 	local evalue="${3}" # What to log to stderr
+
 	echo -e "$lvalue" >> "$LOG_FILE"
 
-	# <OSYNC SPECIFIC> Special case in daemon mode where systemctl doesn't need double timestamps
-	if [ "$sync_on_changes" == "1" ]; then
-		cat <<< "$evalue" 1>&2	# Log to stderr in daemon mode
+	if [ "$_LOGGER_STDERR" -eq 1 ]; then
+		cat <<< "$evalue" 1>&2
 	elif [ "$_SILENT" -eq 0 ]; then
 		echo -e "$svalue"
 	fi
@@ -108,13 +111,13 @@ function Logger {
 	local value="${1}" # Sentence to log (in double quotes)
 	local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL
 
-	# <OSYNC SPECIFIC> Special case in daemon mode we should timestamp instead of counting seconds
-	if [ "$sync_on_changes" == "1" ]; then
+	if [ "$_LOGGER_PREFIX" == "time" ]; then
+		prefix="TIME: $SECONDS - "
+	elif [ "$_LOGGER_PREFIX" == "date" ]; then
 		prefix="$(date) - "
 	else
-		prefix="TIME: $SECONDS - "
+		prefix=""
 	fi
-	# </OSYNC SPECIFIC>
 
 	if [ "$level" == "CRITICAL" ]; then
 		_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value"
@@ -147,6 +150,7 @@ function _QuickLogger {
 	local value="${1}"
 	local destination="${2}" # Destination: stdout, log, both
 
+
 	if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then
 		echo -e "$(date) - $value" >> "$LOG_FILE"
 	elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then
@@ -158,6 +162,7 @@ function _QuickLogger {
 function QuickLogger {
 	local value="${1}"
 
+
 	if [ "$_SILENT" -eq 1 ]; then
 		_QuickLogger "$value" "log"
 	else
@@ -167,17 +172,17 @@ function QuickLogger {
 
 # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
 function KillChilds {
-	local pid="${1}"
+	local pid="${1}" # Parent pid to kill
 	local self="${2:-false}"
 
+
 	if children="$(pgrep -P "$pid")"; then
 		for child in $children; do
 			KillChilds "$child" true
 		done
 	fi
-
-	# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
-	if ( [ "$self" == true ] && eval $PROCESS_TEST_CMD > /dev/null 2>&1); then
+		# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
+	if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then
 		Logger "Sending SIGTERM to process [$pid]." "DEBUG"
 		kill -s SIGTERM "$pid"
 		if [ $? != 0 ]; then
@@ -188,13 +193,30 @@ function KillChilds {
 				Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
 				return 1
 			fi
+		else
+			return 0
 		fi
-		return 0
 	else
 		return 0
 	fi
 }
 
+function KillAllChilds {
+	local pids="${1}" # List of parent pids to kill separated by semi-colon
+
+
+	local errorcount=0
+
+	IFS=';' read -a pidsArray <<< "$pids"
+	for pid in "${pidsArray[@]}"; do
+		KillChilds $pid
+		if [ $? != 0 ]; then
+			errorcount=$((errorcount+1))
+			fi
+	done
+	return $errorcount
+}
+
 # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
 function SendAlert {
 
@@ -335,7 +357,7 @@ function SendAlert {
 		fi
 	fi
 
-	# If function has not returned 0 yet, assume it's critical that no alert can be sent
+	# If function has not returned 0 yet, assume it is critical that no alert can be sent
 	Logger "Cannot send alert (neither mutt, mail, sendmail, mailsend, sendemail or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue
 
 	# Delete tmp log file
@@ -469,7 +491,7 @@ function SendEmail {
 		fi
 	fi
 
-	# If function has not returned 0 yet, assume it's critical that no alert can be sent
+	# If function has not returned 0 yet, assume it is critical that no alert can be sent
 	Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue
 }
 
@@ -501,8 +523,6 @@ function LoadConfigFile {
 	CONFIG_FILE="$config_file"
 }
 
-#### MINIMAL-FUNCTION-SET END ####
-
 function Spinner {
 	if [ $_SILENT -eq 1 ]; then
 		return 0
@@ -536,6 +556,133 @@ function Spinner {
 	esac
 }
 
+function WaitForTaskCompletion {
+	local pids="${1}" # pids to wait for, separated by semi-colon
+	local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
+	local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
+	local caller_name="${4}" # Who called this function
+	local exit_on_error="${5:-false}" # Should the function exit on subprocess errors
+
+
+	local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
+	local log_ttime=0 # local time instance for comparaison
+
+	local seconds_begin=$SECONDS # Seconds since the beginning of the script
+	local exec_time=0 # Seconds since the beginning of this function
+
+	local retval=0 # return value of monitored pid process
+	local errorcount=0 # Number of pids that finished with errors
+
+	local pidCount # number of given pids
+
+	IFS=';' read -a pidsArray <<< "$pids"
+	pidCount=${#pidsArray[@]}
+
+	while [ ${#pidsArray[@]} -gt 0 ]; do
+		newPidsArray=()
+		for pid in "${pidsArray[@]}"; do
+			if kill -0 $pid > /dev/null 2>&1; then
+				newPidsArray+=($pid)
+			else
+				wait $pid
+				result=$?
+				if [ $result -ne 0 ]; then
+					errorcount=$((errorcount+1))
+					Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG"
+				fi
+			fi
+		done
+
+		Spinner
+		exec_time=$(($SECONDS - $seconds_begin))
+		if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then
+			if [ $log_ttime -ne $exec_time ]; then
+				log_ttime=$exec_time
+				Logger "Current tasks still running with pids [${pidsArray[@]}]." "NOTICE"
+			fi
+		fi
+
+		if [ $exec_time -gt $soft_max_time ]; then
+			if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
+				Logger "Max soft execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]." "WARN"
+				soft_alert=1
+				SendAlert
+
+			fi
+			if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
+				Logger "Max hard execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]. Stopping task execution." "ERROR"
+				KillChilds $pid
+				if [ $? == 0 ]; then
+					Logger "Task stopped successfully" "NOTICE"
+					#return 0
+				else
+					errrorcount=$((errorcount+1))
+					#return 1
+				fi
+			fi
+		fi
+
+		pidsArray=("${newPidsArray[@]}")
+		sleep $SLEEP_TIME
+	done
+
+	if [ $exit_on_error == true ] && [ $errorcount -gt 0 ]; then
+		Logger "Stopping execution." "CRITICAL"
+		exit 1337
+	else
+		return $errorcount
+	fi
+}
+
+function WaitForCompletion {
+	local pid="${1}" # pid to wait for
+	local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
+	local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
+	local caller_name="${4}" # Who called this function
+
+	local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
+	local log_time=0 # local time instance for comparaison
+
+	local seconds_begin=$SECONDS # Seconds since the beginning of the script
+	local exec_time=0 # Seconds since the beginning of this function
+
+	local retval=0 # return value of monitored pid process
+
+	while eval "$PROCESS_TEST_CMD" > /dev/null
+	do
+		Spinner
+		if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then
+			if [ $log_time -ne $SECONDS ]; then
+				log_time=$SECONDS
+				Logger "Current task still running." "NOTICE"
+			fi
+		fi
+		if [ $SECONDS -gt $soft_max_time ]; then
+			if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then
+				Logger "Max soft execution time exceeded for script." "WARN"
+				soft_alert=1
+				SendAlert
+			fi
+			if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then
+				Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR"
+				KillChilds $pid
+				if [ $? == 0 ]; then
+					Logger "Task stopped successfully" "NOTICE"
+					return 0
+				else
+					return 1
+				fi
+			fi
+		fi
+		sleep $SLEEP_TIME
+	done
+	wait $pid
+	retval=$?
+	return $retval
+}
+
+#### MINIMAL-FUNCTION-SET END ####
+
 # obsolete, use StripQuotes
 function SedStripQuotes {
         echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
@@ -713,180 +860,6 @@ function GetRemoteOS {
 	fi
 }
 
-function WaitForPids {
-	# Takes a list of pids separated by space as argument, and waits until all pids are finished
-       local errors=0
-
-        while [ "$#" -gt 0 ]; do
-                for pid in "$@"; do
-                        shift
-                        if kill -0 "$pid" > /dev/null 2>&1; then
-                                Logger "[$pid] is alive." "DEBUG"
-                                set -- "$@" "$pid"
-                        else
-                                wait "$pid"
-                                result=$?
-                                if [ $result -eq 0 ]; then
-                                        Logger "[$pid] exited okay with [$result]" "DEBUG"
-                                else
-                                        errors=$((errors+1))
-                                        Logger "[$pid] exited with bad status [$result]." "WARN"
-                                fi
-                        fi
-                done
-                sleep $SLEEP_TIME
-        done
-	return $errors
-}
-
-
-function WaitForTaskCompletion {
-	local pids="${1}" # list of pids to wait for, separated by a semicolon
-	local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
-	local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
-	local caller_name="${4}" # Who called this function
-	local should_exit="${5}" # If true, the function exits on failure
-
-	local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
-	local log_ttime=0 # local time instance for comparaison
-
-	local seconds_begin=$SECONDS # Seconds since the beginning of the script
-	local exec_time=0 # Seconds since the beginning of this function
-
-	local retval=0 # return value of monitored pid process
-
-	local pid
-	local new_pids
-	local result
-
-	IFS=';' read -r -a pidarray <<< "$pids"
-
-	while [ ${#pidarray[@]} -gt 0 ]; do
-		newarray=""
-		for index in ${!pidarray[@]}; do
-			pid="${pidarray[index]}"
-			echo "run  for $pid"
-			if kill -0 $pid > /dev/null 2>&1; then
-				echo "pid [$pid] is running."
-				newarray+=$pid
-			else
-				wait "$pid"
-				result=$?
-				echo $result
-				if [ $result -eq 0 ]; then
-					echo "pid [$pid] is finished with exit code 0."
-				else
-					echo "pid [$pid] is finished with exit code $result."
-				fi
-			fi
-		done
-		pidarray=$newarray
-		sleep .05
- 	done
-}
-
-sleep 1 &
-pids=$!
-sleep 5 &
-pids="$pids;$!"
-echo "Waiting for pid $pids"
-#sleep 5 &
-#pids="$pids;$!"
-
-WaitForTaskCompletion $pids 0 0 "caller" 1
-
-echo "done"
-exit
-
-function old {
-
-
-
-
-	#TODO: test on FreeBSD, MacOS X and msys / cygwin
-	while kill -0 "$pid" > /dev/null 2>&1
-	do
-		Spinner
-		exec_time=$(($SECONDS - $seconds_begin))
-		if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then
-			if [ $log_ttime -ne $exec_time ]; then
-				log_ttime=$exec_time
-				Logger "Current task still running with pid [$pid]." "NOTICE"
-			fi
-		fi
-		if [ $exec_time -gt $soft_max_time ]; then
-			if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
-				Logger "Max soft execution time exceeded for task [$caller_name] with pid [$pid]." "WARN"
-				soft_alert=1
-				SendAlert
-
-			fi
-			if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
-				Logger "Max hard execution time exceeded for task [$caller_name] with pid [$pid]. Stopping task execution." "ERROR"
-				KillChilds $pid
-				if [ $? == 0 ]; then
-					Logger "Task stopped successfully" "NOTICE"
-					return 0
-				else
-					return 1
-				fi
-			fi
-		fi
-		sleep $SLEEP_TIME
-	done
-	wait $pid
-	retval=$?
-	return $retval
-}
-
-function WaitForCompletion {
-	local pid="${1}" # pid to wait for
-	local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
-	local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
-	local caller_name="${4}" # Who called this function
-
-	local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
-	local log_time=0 # local time instance for comparaison
-
-	local seconds_begin=$SECONDS # Seconds since the beginning of the script
-	local exec_time=0 # Seconds since the beginning of this function
-
-	local retval=0 # return value of monitored pid process
-
-	#TODO: test on FreeBSD, MacOS X and msys / cygwin
-	while kill -0 "$pid" > /dev/null 2>&1
-	do
-		Spinner
-		if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then
-			if [ $log_time -ne $SECONDS ]; then
-				log_time=$SECONDS
-				Logger "Current task still running." "NOTICE"
-			fi
-		fi
-		if [ $SECONDS -gt $soft_max_time ]; then
-			if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then
-				Logger "Max soft execution time exceeded for script." "WARN"
-				soft_alert=1
-				SendAlert
-			fi
-			if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then
-				Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR"
-				KillChilds $pid
-				if [ $? == 0 ]; then
-					Logger "Task stopped successfully" "NOTICE"
-					return 0
-				else
-					return 1
-				fi
-			fi
-		fi
-		sleep $SLEEP_TIME
-	done
-	wait $pid
-	retval=$?
-	return $retval
-}
-
 function RunLocalCommand {
 	local command="${1}" # Command to run
 	local hard_max_time="${2}" # Max time to wait for command to compleet
@@ -953,23 +926,37 @@ function RunRemoteCommand {
 
 function RunBeforeHook {
 
+	local pids=
+
 	if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then
-		RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE
+		RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE &
+		pids="$!"
 	fi
 
 	if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then
-		RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE
+		RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE &
+		pids="$pids;$!"
+	fi
+	if [ "$pids" != "" ]; then
+		WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false
 	fi
 }
 
 function RunAfterHook {
 
+	local pids
+
 	if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then
-		RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER
+		RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER &
+		pids="$!"
 	fi
 
 	if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then
-		RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER
+		RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER &
+		pids="$pids;$!"
+	fi
+	if [ "$pids" != "" ]; then
+		WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false
 	fi
 }
 
@@ -1051,8 +1038,6 @@ function RsyncPatternsFromAdd {
         local pattern_type="${1}"
         local pattern_from="${2}"
 
-	local pattern_from=
-
         ## Check if the exclude list has a full path, and if not, add the config file path if there is one
         if [ "$(basename $pattern_from)" == "$pattern_from" ]; then
                 pattern_from="$(dirname $CONFIG_FILE)/$pattern_from"
@@ -1217,7 +1202,6 @@ function InitLocalOSSettings {
                 FIND_CMD=$(dirname $BASH)/find
                 # PROCESS_TEST_CMD assumes there is a variable $pid
 		# Tested on MSYS and cygwin
-		#TODO: remove PROCESS_TEST_CMD if kill -0 works
                 PROCESS_TEST_CMD='ps -a | awk "{\$1=\$1}\$1" | awk "{print \$1}" | grep $pid'
                 PING_CMD='$SYSTEMROOT\system32\ping -n 2'
         else