Rebuilt targets for v2.1-beta3

CHANGELOG.md

@@ -9,7 +9,14 @@ KNOWN ISSUES
 CHANGELOG
 ---------
 
+20 Jun 2017: obackup v2.1 beta3 released
+----------------------------------------
+
+- Fixed regression where some commandline arguments weren't honored anymore since 2.1 beta1 (--delete, --stats, --dontgetsize)
+- Fixed commandline arguments aren't checked against valid list
+
 14 Mar 2017: obackup v2.1 beta2 released
+----------------------------------------
 
 - Fixed remote commands can be run on local runs and obviously fail
 - Uninstall leaves ssh_filter if needed by other programs

README.md

@@ -1,7 +1,8 @@
-# obackup [![Build Status](https://travis-ci.org/deajan/obackup.svg?branch=master)](https://travis-ci.org/deajan/obackup) [![GitHub Release](https://img.shields.io/github/release/deajan/obackup.svg?label=Latest)](https://github.com/deajan/obackup/releases/latest)
+# obackup [![Build Status](https://travis-ci.org/deajan/obackup.svg?branch=master)](https://travis-ci.org/deajan/obackup) [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![GitHub Release](https://img.shields.io/github/release/deajan/obackup.svg?label=Latest)](https://github.com/deajan/obackup/releases/latest)
-
 
 A robust file & database backup script that works for local and remote push or pull backups via ssh.
+Designed to backup multiple subdirectories with a timeslot for each.
+Supports encryption while still using rsync to lower transfered data (see advantages and caveats below).
 
 ## About
 
@@ -34,7 +35,7 @@ You may disable this behavior in the config file.
 You can download the latest obackup script from authors website.
 You may also clone the following git which will maybe have some more recent builds.
 
-    $ git clone -b "v2.0-maint" git://github.com/deajan/obackup.git
+    $ git clone -b "v2.1-maint" git://github.com/deajan/obackup.git
     $ cd obackup
     $ ./install.sh
 

dev/bootstrap.sh

@@ -1,30 +1,64 @@
 #!/usr/bin/env bash
 
-## dev pre-processor bootstrap rev 2016121302
+## dev pre-processor bootstrap rev 2017062001
 ## Yeah !!! A really tech sounding name... In fact it's just include emulation in bash
 
+function Usage {
+	echo "$0 - Quick and dirty preprocessor for including ofunctions into programs"
+	echo "Creates and executes $0.tmp.sh"
+	echo "Usage:"
+	echo ""
+	echo "$0 --program=osync|osync_target_helper|obackup|pmocr [options to pass to program]"
+}
+
+
 if [ ! -f "./merge.sh" ]; then
 	echo "Plrase run bootstrap.sh from osync/dev directory."
 	exit 1
 fi
 
+bootstrapProgram=""
+opts=""
 outputFileName="$0"
 
-source "merge.sh"
+for i in "$@"; do
-__PREPROCESSOR_PROGRAM=obackup
+        case $i in
-__PREPROCESSOR_Constants
+                --program=*)
+                bootstrapProgram="${i##*=}"
+		;;
+		*)
+		opts=$opts" $i"
+		;;
+	esac
+done
 
-cp "n_$__PREPROCESSOR_PROGRAM.sh" "$outputFileName.tmp.sh"
+if [ "$bootstrapProgram" == "" ]; then
+	Usage
+	exit 128
+else
+	source "merge.sh"
+
+	__PREPROCESSOR_PROGRAM=$bootstrapProgram
+	__PREPROCESSOR_PROGRAM_EXEC="n_$bootstrapProgram.sh"
+	__PREPROCESSOR_Constants
+
+	if [ ! -f "$__PREPROCESSOR_PROGRAM_EXEC" ]; then
+		echo "Cannot find file [n_$bootstrapProgram.sh]."
+		exit 1
+	fi
+fi
+
+cp "$__PREPROCESSOR_PROGRAM_EXEC" "$outputFileName.tmp.sh"
 if [ $? != 0 ]; then
-	echo "Cannot copy original file [n_$__PREPROCESSOR_PROGRAM.sh] to [$outputFileName.tmp.sh]."
+	echo "Cannot copy original file [$__PREPROCESSOR_PROGRAM_EXEC] to [$outputFileName.tmp.sh]."
 	exit 1
 fi
 for subset in "${__PREPROCESSOR_SUBSETS[@]}"; do
 	__PREPROCESSOR_MergeSubset "$subset" "${subset//SUBSET/SUBSET END}" "ofunctions.sh" "$outputFileName.tmp.sh"
 done
-chmod +x "$0.tmp.sh"
+chmod +x "$outputFileName.tmp.sh"
 if [ $? != 0 ]; then
-	echo "Cannot make [$outputFileName] executable.."
+	echo "Cannot make [$outputFileName] executable."
 	exit 1
 fi
 
@@ -33,4 +67,4 @@ if type termux-fix-shebang > /dev/null 2>&1; then
 	termux-fix-shebang "$outputFileName.tmp.sh"
 fi
 
-"$outputFileName.tmp.sh" "$@"
+"$outputFileName.tmp.sh" $opts

dev/common_batch.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 SUBPROGRAM=[prgname]
 PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
-AUTHOR="(L) 2013-2016 by Orsiris de Jong"
+AUTHOR="(L) 2013-2017 by Orsiris de Jong"
 CONTACT="http://www.netpower.fr - ozy@netpower.fr"
 PROGRAM_BUILD=2016120401
 
@@ -119,15 +119,15 @@ function Batch {
 				fi
 			done
 			runList=("${runAgainList[@]}")
-			runs=$(($runs + 1))
+			runs=$((runs + 1))
 		done
 	fi
 }
 
 function Usage {
 	echo "$PROGRAM $PROGRAM_BUILD"
-	echo $AUTHOR
+	echo "$AUTHOR"
-	echo $CONTACT
+	echo "$CONTACT"
 	echo ""
 	echo "Batch script to sequentially run osync or obackup instances and rerun failed ones."
 	echo "Usage: $PROGRAM.sh [OPTIONS] [$SUBPROGRAM OPTIONS]"

dev/common_install.sh

@@ -12,7 +12,7 @@ PROGRAM_BINARY=$PROGRAM".sh"
 PROGRAM_BATCH=$PROGRAM"-batch.sh"
 SSH_FILTER="ssh_filter.sh"
 
-SCRIPT_BUILD=2017031402
+SCRIPT_BUILD=2017041701
 
 ## osync / obackup / pmocr / zsnap install script
 ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
@@ -210,7 +210,9 @@ function CopyProgram {
 function CopyServiceFiles {
 	if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
 		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true
-		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true
+		if [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM_USER" ]; then
+			CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true
+		fi
 
 		QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
 		QuickLogger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR."
@@ -219,9 +221,9 @@ function CopyServiceFiles {
 	elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then
 		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true
 
-		QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
+		QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]."
-		QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]."
+		QuickLogger "Can be activated with [service $SERVICE_FILE_INIT start]."
-		QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]."
+		QuickLogger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]."
 	else
 		QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation."
 	fi
@@ -263,7 +265,11 @@ function RemoveFile {
 
 function RemoveAll {
 	RemoveFile "$BIN_DIR/$PROGRAM_BINARY"
-	RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
+
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
+		RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
+	fi
+
 	if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then		# Check if any other program requiring ssh filter is present before removal
 		RemoveFile "$BIN_DIR/$SSH_FILTER"
 	else
@@ -281,6 +287,7 @@ function Usage {
 	echo "options:"
 	echo "--silent		Will log and bypass user interaction."
 	echo "--no-stats	Used with --silent in order to refuse sending anonymous install stats."
+	echo "--remove          Remove the program."
 	exit 127
 }
 
@@ -323,7 +330,9 @@ else
 	CreateDir "$BIN_DIR"
 	CopyExampleFiles
 	CopyProgram
-	CopyServiceFiles
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "pmocr" ]; then
+		CopyServiceFiles
+	fi
 	QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
 	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
 		QuickLogger ""

dev/debug_obackup.sh

@@ -6,8 +6,8 @@
 PROGRAM="obackup"
 AUTHOR="(C) 2013-2017 by Orsiris de Jong"
 CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
-PROGRAM_VERSION=2.1-beta2
+PROGRAM_VERSION=2.1-beta3
-PROGRAM_BUILD=2017021001
+PROGRAM_BUILD=2017062004
 IS_STABLE=no
 
 #### Execution order					#__WITH_PARANOIA_DEBUG
@@ -35,8 +35,8 @@ IS_STABLE=no
 #	FilesBackup					#__WITH_PARANOIA_DEBUG
 
 
-_OFUNCTIONS_VERSION=2.1-RC3+dev
+_OFUNCTIONS_VERSION=2.1.4-rc1
-_OFUNCTIONS_BUILD=2017031401
+_OFUNCTIONS_BUILD=2017060903
 _OFUNCTIONS_BOOTSTRAP=true
 
 ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@@ -61,6 +61,9 @@ fi
 ## Correct output of sort command (language agnostic sorting)
 export LC_ALL=C
 
+## Default umask for file creation
+umask 0077
+
 # Standard alert mail body
 MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
 
@@ -97,6 +100,8 @@ if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as envi
 fi
 
 SCRIPT_PID=$$
+
+# TODO: Check if %N works on MacOS
 TSTAMP=$(date '+%Y%m%dT%H%M%S.%N')
 
 LOCAL_USER=$(whoami)
@@ -288,7 +293,7 @@ function Logger {
 		return
 	elif [ "$level" == "VERBOSE" ]; then
 		if [ $_LOGGER_VERBOSE == true ]; then
-			_Logger "$prefix:$value" "$prefix$value"
+			_Logger "$prefix($level):$value" "$prefix$value"
 		fi
 		return
 	elif [ "$level" == "ALWAYS" ]; then
@@ -339,15 +344,29 @@ function KillChilds {
 	local pid="${1}" # Parent pid to kill childs
 	local self="${2:-false}" # Should parent be killed too ?
 
-	# Warning: pgrep does not exist in cygwin, have this checked in CheckEnvironment
+	# Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
-	if children="$(pgrep -P "$pid")"; then
+	if [ $(IsNumeric "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
-		for child in $children; do
+		Logger "Bogus pid given [$pid]." "CRITICAL"
-			Logger "Launching KillChilds \"$child\" true" "DEBUG"	#__WITH_PARANOIA_DEBUG
+		return 1
-			KillChilds "$child" true
-		done
 	fi
-		# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
+
+	if kill -0 "$pid" > /dev/null 2>&1; then
+		# Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
+		if children="$(pgrep -P "$pid")"; then
+			if [[ "$pid" == *"$children"* ]]; then
+				Logger "Bogus pgrep implementation." "CRITICAL"
+				children="${children/$pid/}"
+			fi
+			for child in $children; do
+				Logger "Launching KillChilds \"$child\" true" "DEBUG"	#__WITH_PARANOIA_DEBUG
+				KillChilds "$child" true
+			done
+		fi
+	fi
+
+	# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
 	if [ "$self" == true ]; then
+		# We need to check for pid again because it may have disappeared after recursive function call
 		if kill -0 "$pid" > /dev/null 2>&1; then
 			kill -s TERM "$pid"
 			Logger "Sent SIGTERM to process [$pid]." "DEBUG"
@@ -410,7 +429,6 @@ function SendAlert {
 
 	eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
 	if [ $? != 0 ]; then
-		Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
 		attachment=false
 	else
 		attachment=true
@@ -657,6 +675,22 @@ function Spinner {
 	fi
 }
 
+function _PerfProfiler {												#__WITH_PARANOIA_DEBUG
+	local perfString												#__WITH_PARANOIA_DEBUG
+															#__WITH_PARANOIA_DEBUG
+	perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan)					#__WITH_PARANOIA_DEBUG
+															#__WITH_PARANOIA_DEBUG
+	for i in $(pgrep -P $$); do											#__WITH_PARANOIA_DEBUG
+		perfString="$perfString\n"$(ps -p $i -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan | tail -1)	#__WITH_PARANOIA_DEBUG
+	done														#__WITH_PARANOIA_DEBUG
+															#__WITH_PARANOIA_DEBUG
+	if type iostat > /dev/null 2>&1; then										#__WITH_PARANOIA_DEBUG
+		perfString="$perfString\n"$(iostat)									#__WITH_PARANOIA_DEBUG
+	fi														#__WITH_PARANOIA_DEBUG
+															#__WITH_PARANOIA_DEBUG
+	Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG"								#__WITH_PARANOIA_DEBUG
+}															#__WITH_PARANOIA_DEBUG
+
 
 # Time control function for background processes, suitable for multiple synchronous processes
 # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
@@ -789,6 +823,11 @@ function WaitForTaskCompletion {
 		pidsArray=("${newPidsArray[@]}")
 		# Trivial wait time for bash to not eat up all CPU
 		sleep $sleepTime
+
+		if [ "$_PERF_PROFILER" == "yes" ]; then				##__WITH_PARANOIA_DEBUG
+			_PerfProfiler						##__WITH_PARANOIA_DEBUG
+		fi								##__WITH_PARANOIA_DEBUG
+
 	done
 
 	Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG"	#__WITH_PARANOIA_DEBUG
@@ -954,6 +993,10 @@ function ParallelExec {
 
 		# Trivial wait time for bash to not eat up all CPU
 		sleep $sleepTime
+
+		if [ "$_PERF_PROFILER" == "yes" ]; then				##__WITH_PARANOIA_DEBUG
+			_PerfProfiler						##__WITH_PARANOIA_DEBUG
+		fi								##__WITH_PARANOIA_DEBUG
 	done
 
 	return $errorCount
@@ -1139,7 +1182,7 @@ function GetLocalOS {
 		*"BSD"*)
 		LOCAL_OS="BSD"
 		;;
-		*"MINGW32"*|*"MSYS"*)
+		*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
 		LOCAL_OS="msys"
 		;;
 		*"CYGWIN"*)
@@ -1165,9 +1208,6 @@ function GetLocalOS {
 		exit 1
 		;;
 	esac
-	if [ "$_OFUNCTIONS_VERSION" != "" ]; then
-		Logger "Local OS: [$localOsVar]." "DEBUG"
-	fi
 
 	# Get linux versions
 	if [ -f "/etc/os-release" ]; then
@@ -1177,8 +1217,68 @@ function GetLocalOS {
 
 	# Add a global variable for statistics in installer
 	LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
+
+	if [ "$_OFUNCTIONS_VERSION" != "" ]; then
+		Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG"
+	fi
 }
 
+#__BEGIN_WITH_PARANOIA_DEBUG
+function __CheckArguments {
+	# Checks the number of arguments of a function and raises an error if some are missing
+
+	if [ "$_DEBUG" == "yes" ]; then
+		local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
+		local numberOfGivenArguments="${2}" # Number of arguments that have been passed
+
+		local minArgs
+		local maxArgs
+
+		# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
+		# If any of the arguments contains spaces, bash things there are two aguments
+		# In order to avoid this, we need to iterate over ${3} and count
+
+		callerName="${FUNCNAME[1]}"
+
+		local iterate=3
+		local fetchArguments=true
+		local argList=""
+		local countedArguments
+		while [ $fetchArguments == true ]; do
+			cmd='argument=${'$iterate'}'
+			eval $cmd
+			if [ "$argument" == "" ]; then
+				fetchArguments=false
+			else
+				argList="$argList[Argument $((iterate-2)): $argument] "
+				iterate=$((iterate+1))
+			fi
+		done
+
+		countedArguments=$((iterate-3))
+
+		if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
+			minArgs=$numberOfArguments
+			maxArgs=$numberOfArguments
+		else
+			IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
+		fi
+
+		Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
+
+		if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
+			Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
+			Logger "$callerName arguments: $argList" "ERROR"
+		else
+			if [ ! -z "$argList" ]; then
+				Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
+			fi
+		fi
+	fi
+}
+
+#__END_WITH_PARANOIA_DEBUG
+
 
 function GetRemoteOS {
 	__CheckArguments 0 $# "$@"	#__WITH_PARANOIA_DEBUG
@@ -1189,13 +1289,16 @@ function GetRemoteOS {
 
 	local remoteOsVar
 
-$SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
+$SSH_CMD env LC_ALL=C env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
+
 
 function GetOs {
 	local localOsVar
 	local localOsName
 	local localOsVer
 
+	local osInfo="/etc/os-release"
+
 	# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
 	if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
 		localOsVar="BusyBox"
@@ -1214,9 +1317,11 @@ function GetOs {
 		fi
 	fi
 	# Get linux versions
-	if [ -f "/etc/os-release" ]; then
+	if [ -f "$osInfo" ]; then
-		localOsName=$(GetConfFileValue "/etc/os-release" "NAME")
+		localOsName=$(grep "^NAME=" "$osInfo")
-		localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION")
+                localOsName="${localOsName##*=}"
+		localOsVer=$(grep "^VERSION=" "$osInfo")
+		localOsVer="${localOsVer##*=}"
 	fi
 
 	echo "$localOsVar ($localOsName $localOsVer)"
@@ -1238,7 +1343,7 @@ ENDSSH
 			*"BSD"*)
 			REMOTE_OS="BSD"
 			;;
-			*"MINGW32"*|*"MSYS"*)
+			*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
 			REMOTE_OS="msys"
 			;;
 			*"CYGWIN"*)
@@ -1323,7 +1428,7 @@ function RunRemoteCommand {
 	fi
 
 	Logger "Running command [$command] on remote host." "NOTICE"
-	cmd=$SSH_CMD' "env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
+	cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
 	Logger "cmd: $cmd" "DEBUG"
 	eval "$cmd" &
 	WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
@@ -1434,62 +1539,6 @@ function CheckConnectivity3rdPartyHosts {
 	fi											#__WITH_PARANOIA_DEBUG
 }
 
-#__BEGIN_WITH_PARANOIA_DEBUG
-function __CheckArguments {
-	# Checks the number of arguments of a function and raises an error if some are missing
-
-	if [ "$_DEBUG" == "yes" ]; then
-		local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
-		local numberOfGivenArguments="${2}" # Number of arguments that have been passed
-
-		local minArgs
-		local maxArgs
-
-		# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
-		# If any of the arguments contains spaces, bash things there are two aguments
-		# In order to avoid this, we need to iterate over ${3} and count
-
-		callerName="${FUNCNAME[1]}"
-
-		local iterate=3
-		local fetchArguments=true
-		local argList=""
-		local countedArguments
-		while [ $fetchArguments == true ]; do
-			cmd='argument=${'$iterate'}'
-			eval $cmd
-			if [ "$argument" == "" ]; then
-				fetchArguments=false
-			else
-				argList="$argList[Argument $((iterate-2)): $argument] "
-				iterate=$((iterate+1))
-			fi
-		done
-
-		countedArguments=$((iterate-3))
-
-		if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
-			minArgs=$numberOfArguments
-			maxArgs=$numberOfArguments
-		else
-			IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
-		fi
-
-		Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
-
-		if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
-			Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
-			Logger "$callerName arguments: $argList" "ERROR"
-		else
-			if [ ! -z "$argList" ]; then
-				Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
-			fi
-		fi
-	fi
-}
-
-#__END_WITH_PARANOIA_DEBUG
-
 function RsyncPatternsAdd {
 	local patternType="${1}"	# exclude or include
 	local pattern="${2}"
@@ -1620,13 +1669,13 @@ function PostInit {
 
 	# Define remote commands
 	if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then
-		SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
+		SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
-		SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
+		SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
-		RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
+		RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
 	elif [ -f "$SSH_PASSWORD_FILE" ]; then
-		SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
+		SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
-		SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -P $REMOTE_PORT"
+		SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT"
-		RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS -p $REMOTE_PORT"
+		RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT"
 	else
 		SSH_PASSWORD=""
 		SSH_CMD=""
@@ -1664,23 +1713,22 @@ function SetCompression {
 			COMPRESSION_PROGRAM="| pigz -c$compressionString"
 			COMPRESSION_EXTENSION=.gz
 			# obackup specific
-			if [ "$LOCAL_OS" != "MacOSX" ]; then
+			COMPRESSION_OPTIONS=--rsyncable
-				COMPRESSION_OPTIONS=--rsyncable
-			fi
 		elif type gzip > /dev/null 2>&1
 		then
 			COMPRESSION_PROGRAM="| gzip -c$compressionString"
 			COMPRESSION_EXTENSION=.gz
 			# obackup specific
-			if [ "$LOCAL_OS" != "MacOSX" ]; then
+			COMPRESSION_OPTIONS=--rsyncable
-				COMPRESSION_OPTIONS=--rsyncable
-			fi
 		else
 			COMPRESSION_PROGRAM=
 			COMPRESSION_EXTENSION=
 		fi
 	fi
-	ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
+
+	if [ ".${ALERT_LOG_FILE##*.}" != "$COMPRESSION_EXTENSION" ]; then
+		ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
+	fi
 }
 
 function InitLocalOSDependingSettings {
@@ -1844,35 +1892,46 @@ function ParentPid {
 
 # Neat version compare function found at http://stackoverflow.com/a/4025065/2635443
 # Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2
-vercomp () {
+function VerComp () {
-    if [[ $1 == $2 ]]
+	if [ "$1" == "" ] || [ "$2" == "" ]; then
-    then
+		Logger "Bogus Vercomp values [$1] and [$2]." "WARN"
-        return 0
+		return 1
-    fi
+	fi
-    local IFS=.
+
-    local i ver1=($1) ver2=($2)
+	if [[ $1 == $2 ]]
-    # fill empty fields in ver1 with zeros
+		then
-    for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
+			echo 0
-    do
+		return
-        ver1[i]=0
+	fi
-    done
+
-    for ((i=0; i<${#ver1[@]}; i++))
+	local IFS=.
-    do
+	local i ver1=($1) ver2=($2)
-        if [[ -z ${ver2[i]} ]]
+	# fill empty fields in ver1 with zeros
-        then
+	for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
-            # fill empty fields in ver2 with zeros
+	do
-            ver2[i]=0
+        	ver1[i]=0
-        fi
+	done
-        if ((10#${ver1[i]} > 10#${ver2[i]}))
+	for ((i=0; i<${#ver1[@]}; i++))
-        then
+	do
-            return 1
+		if [[ -z ${ver2[i]} ]]
-        fi
+		then
-        if ((10#${ver1[i]} < 10#${ver2[i]}))
+			# fill empty fields in ver2 with zeros
-        then
+			ver2[i]=0
-            return 2
+		fi
-        fi
+		if ((10#${ver1[i]} > 10#${ver2[i]}))
-    done
+		then
-    return 0
+			echo 1
+			return
+        	fi
+        	if ((10#${ver1[i]} < 10#${ver2[i]}))
+        	then
+			echo 2
+            		return
+        	fi
+    	done
+
+    	echo 0
+	return
 }
 
 function GetConfFileValue () {
@@ -1905,6 +1964,29 @@ function SetConfFileValue () {
         fi
 }
 
+# Function can replace [ -f /some/file* ] tests
+# Modified version of http://stackoverflow.com/a/6364244/2635443
+function WildcardFileExists () {
+        local file="${1}"
+        local exists=0
+
+        for f in $file; do
+                ## Check if the glob gets expanded to existing files.
+                ## If not, f here will be exactly the pattern above
+                ## and the exists test will evaluate to false.
+                if [ -e "$f" ]; then
+                        exists=1
+                        break
+                fi
+        done
+
+        if [ $exists -eq 1 ]; then
+                echo 1
+        else
+                echo 0
+        fi
+}
+
 # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap
 _OFUNCTIONS_BOOTSTRAP=true
 [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1
@@ -4219,23 +4301,23 @@ function Init {
 		REMOTE_HOST=${hosturi%%:*}
 	fi
 
-	## Add update to default RSYNC_ARGS
+	## Add update to default RSYNC_TYPE_ARGS
-	RSYNC_ARGS=$RSYNC_ARGS" -u"
+	RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" -u"
 
 	if [ $_LOGGER_VERBOSE == true ]; then
-		RSYNC_ARGS=$RSYNC_ARGS" -i"
+		RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" -i"
 	fi
 
 	if [ "$DELETE_VANISHED_FILES" == "yes" ]; then
-		RSYNC_ARGS=$RSYNC_ARGS" --delete"
+		RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --delete"
 	fi
 
 	if [ $stats == true ]; then
-		RSYNC_ARGS=$RSYNC_ARGS" --stats"
+		RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --stats"
 	fi
 
 	## Fix for symlink to directories on target cannot get updated
-	RSYNC_ARGS=$RSYNC_ARGS" --force"
+	RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --force"
 }
 
 function Main {
@@ -4324,12 +4406,16 @@ function Usage {
 _DRYRUN=false
 no_maxtime=false
 stats=false
-PARTIAL=no
+partial_transfers=false
+delete_vanished=false
+dont_get_backup_size=false
 _DECRYPT_MODE=false
 DECRYPT_PATH=""
 _ENCRYPT_MODE=false
 
 function GetCommandlineArguments {
+	local isFirstArgument=true
+
 	if [ $# -eq 0 ]; then
 		Usage
 	fi
@@ -4349,16 +4435,16 @@ function GetCommandlineArguments {
 			stats=false
 			;;
 			--partial)
-			PARTIAL="yes"
+			partial_transfers=true
 			;;
 			--no-maxtime)
 			no_maxtime=true
 			;;
 			--delete)
-			DELETE_VANISHED_FILES="yes"
+			delete_vanished=true
 			;;
 			--dontgetsize)
-			GET_BACKUP_SIZE="no"
+			dont_get_backup_size=true
 			;;
 			--help|-h|--version|-v)
 			Usage
@@ -4394,7 +4480,15 @@ function GetCommandlineArguments {
 			if [ $(IsNumeric $PARALLEL_ENCRYPTION_PROCESSES) -ne 1 ]; then
 				Logger "Bogus --parallel value. Using only one CPU." "WARN"
 			fi
+			;;
+			*)
+			if [ $isFirstArgument == false ]; then
+				Logger "Unknown option '$i'" "CRITICAL"
+				Usage
+			fi
+			;;
 		esac
+		isFirstArgument=false
 	done
 }
 
@@ -4436,6 +4530,27 @@ else
 	Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
 fi
 
+
+if [ $no_maxtime == true ]; then
+	SOFT_MAX_EXEC_TIME_DB_TASK=0
+	SOFT_MAX_EXEC_TIME_FILE_TASK=0
+	HARD_MAX_EXEC_TIME_DB_TASK=0
+	HARD_MAX_EXEC_TIME_FILE_TASK=0
+	HARD_MAX_EXEC_TIME_TOTAL=0
+fi
+
+if [ $partial_transfers == true ]; then
+	PARTIAL="yes"
+fi
+
+if [ $delete_vanished == true ]; then
+	DELETE_VANISHED_FILES="yes"
+fi
+
+if [ $dont_get_backup_size == true ]; then
+	GET_BACKUP_SIZE="no"
+fi
+
 if [ "$IS_STABLE" != "yes" ]; then
 	Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN"
 fi
@@ -4456,13 +4571,5 @@ PostInit
 CheckCurrentConfig
 GetRemoteOS
 InitRemoteOSDependingSettings
-
-if [ $no_maxtime == true ]; then
-	SOFT_MAX_EXEC_TIME_DB_TASK=0
-	SOFT_MAX_EXEC_TIME_FILE_TASK=0
-	HARD_MAX_EXEC_TIME_DB_TASK=0
-	HARD_MAX_EXEC_TIME_FILE_TASK=0
-	HARD_MAX_EXEC_TIME_TOTAL=0
-fi
 RunBeforeHook
 Main

dev/merge.sh

@@ -1,12 +1,18 @@
 #!/usr/bin/env bash
 
-## MERGE 2017031301
+## MERGE 2017061901
 
 ## Merges ofunctions.sh and n_program.sh into program.sh
 ## Adds installer
 
+function Usage {
+	echo "Merges ofunctions.sh and n_program.sh into debug_program.sh and ../program.sh"
+	echo "Usage"
+	echo "$0 osync|obackup|pmocr"
+}
+
 function __PREPROCESSOR_Merge {
-	PROGRAM=obackup
+	local PROGRAM="$1"
 
 	VERSION=$(grep "PROGRAM_VERSION=" n_$PROGRAM.sh)
 	VERSION=${VERSION#*=}
@@ -24,8 +30,7 @@ function __PREPROCESSOR_Merge {
 		__PREPROCESSOR_MergeSubset "$subset" "${subset//SUBSET/SUBSET END}" "ofunctions.sh" "debug_$PROGRAM.sh"
 	done
 
-	__PREPROCESSOR_CleanDebug
+	__PREPROCESSOR_CleanDebug "$PROGRAM"
-	__PREPROCESSOR_CopyCommons
 	rm -f tmp_$PROGRAM.sh
 	if [ $? != 0 ]; then
 		QuickLogger "Cannot remove tmp_$PROGRAM.sh"
@@ -51,6 +56,7 @@ function __PREPROCESSOR_Constants {
 	'#### UrlEncode SUBSET ####'
 	'#### HumanToNumeric SUBSET ####'
 	'#### ArrayContains SUBSET ####'
+	'#### VerComp SUBSET ####'
 	'#### GetConfFileValue SUBSET ####'
 	'#### SetConfFileValue SUBSET ####'
 	)
@@ -103,6 +109,8 @@ function __PREPROCESSOR_MergeSubset {
 }
 
 function __PREPROCESSOR_CleanDebug {
+	local PROGRAM="$1"
+
 	sed '/'$PARANOIA_DEBUG_BEGIN'/,/'$PARANOIA_DEBUG_END'/d' debug_$PROGRAM.sh | grep -v "$PARANOIA_DEBUG_LINE" > ../$PROGRAM.sh
 	if [ $? != 0 ]; then
 		QuickLogger "Cannot remove PARANOIA_DEBUG code from standard build."
@@ -126,6 +134,8 @@ function __PREPROCESSOR_CleanDebug {
 }
 
 function __PREPROCESSOR_CopyCommons {
+	local PROGRAM="$1"
+
 	sed "s/\[prgname\]/$PROGRAM/g" common_install.sh > ../install.sh
 	if [ $? != 0 ]; then
 		QuickLogger "Cannot assemble install."
@@ -171,5 +181,19 @@ function __PREPROCESSOR_CopyCommons {
 
 # If sourced don't do anything
 if [ "$(basename $0)" == "merge.sh" ]; then
-	__PREPROCESSOR_Merge
+	if [ "$1" == "osync" ]; then
+
+		__PREPROCESSOR_Merge osync
+		__PREPROCESSOR_Merge osync_target_helper
+		__PREPROCESSOR_CopyCommons osync
+	elif [ "$1" == "obackup" ]; then
+		__PREPROCESSOR_Merge obackup
+		__PREPROCESSOR_CopyCommons obackup
+	elif [ "$1" == "pmocr" ]; then
+		__PREPROCESSOR_Merge pmocr
+		__PREPROCESSOR_CopyCommons pmocr
+	else
+		echo "No valid program given."
+		exit 1
+	fi
 fi

dev/n_obackup.sh

@@ -6,8 +6,8 @@
 PROGRAM="obackup"
 AUTHOR="(C) 2013-2017 by Orsiris de Jong"
 CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
-PROGRAM_VERSION=2.1-beta2
+PROGRAM_VERSION=2.1-beta3
-PROGRAM_BUILD=2017021001
+PROGRAM_BUILD=2017062004
 IS_STABLE=no
 
 #### Execution order					#__WITH_PARANOIA_DEBUG
@@ -1784,23 +1784,23 @@ function Init {
                 REMOTE_HOST=${hosturi%%:*}
 	fi
 
-	## Add update to default RSYNC_ARGS
+	## Add update to default RSYNC_TYPE_ARGS
-	RSYNC_ARGS=$RSYNC_ARGS" -u"
+	RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" -u"
 
 	if [ $_LOGGER_VERBOSE == true ]; then
-		RSYNC_ARGS=$RSYNC_ARGS" -i"
+		RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" -i"
 	fi
 
 	if [ "$DELETE_VANISHED_FILES" == "yes" ]; then
-		RSYNC_ARGS=$RSYNC_ARGS" --delete"
+		RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --delete"
 	fi
 
 	if [ $stats == true ]; then
-		RSYNC_ARGS=$RSYNC_ARGS" --stats"
+		RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --stats"
 	fi
 
 	## Fix for symlink to directories on target cannot get updated
-	RSYNC_ARGS=$RSYNC_ARGS" --force"
+	RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --force"
 }
 
 function Main {
@@ -1889,12 +1889,16 @@ function Usage {
 _DRYRUN=false
 no_maxtime=false
 stats=false
-PARTIAL=no
+partial_transfers=false
+delete_vanished=false
+dont_get_backup_size=false
 _DECRYPT_MODE=false
 DECRYPT_PATH=""
 _ENCRYPT_MODE=false
 
 function GetCommandlineArguments {
+	local isFirstArgument=true
+
 	if [ $# -eq 0 ]; then
 		Usage
 	fi
@@ -1914,16 +1918,16 @@ function GetCommandlineArguments {
 			stats=false
 			;;
 			--partial)
-			PARTIAL="yes"
+			partial_transfers=true
 			;;
 			--no-maxtime)
 			no_maxtime=true
 			;;
 			--delete)
-			DELETE_VANISHED_FILES="yes"
+			delete_vanished=true
 			;;
 			--dontgetsize)
-			GET_BACKUP_SIZE="no"
+			dont_get_backup_size=true
 			;;
 			--help|-h|--version|-v)
 			Usage
@@ -1959,7 +1963,15 @@ function GetCommandlineArguments {
 			if [ $(IsNumeric $PARALLEL_ENCRYPTION_PROCESSES) -ne 1 ]; then
 				Logger "Bogus --parallel value. Using only one CPU." "WARN"
 			fi
+			;;
+			*)
+			if [ $isFirstArgument == false ]; then
+				Logger "Unknown option '$i'" "CRITICAL"
+				Usage
+			fi
+			;;
 		esac
+		isFirstArgument=false
 	done
 }
 
@@ -2001,6 +2013,27 @@ else
 	Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
 fi
 
+
+if [ $no_maxtime == true ]; then
+	SOFT_MAX_EXEC_TIME_DB_TASK=0
+	SOFT_MAX_EXEC_TIME_FILE_TASK=0
+	HARD_MAX_EXEC_TIME_DB_TASK=0
+	HARD_MAX_EXEC_TIME_FILE_TASK=0
+	HARD_MAX_EXEC_TIME_TOTAL=0
+fi
+
+if [ $partial_transfers == true ]; then
+	PARTIAL="yes"
+fi
+
+if [ $delete_vanished == true ]; then
+	DELETE_VANISHED_FILES="yes"
+fi
+
+if [ $dont_get_backup_size == true ]; then
+	GET_BACKUP_SIZE="no"
+fi
+
 if [ "$IS_STABLE" != "yes" ]; then
 	Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN"
 fi
@@ -2021,13 +2054,5 @@ PostInit
 CheckCurrentConfig
 GetRemoteOS
 InitRemoteOSDependingSettings
-
-if [ $no_maxtime == true ]; then
-	SOFT_MAX_EXEC_TIME_DB_TASK=0
-	SOFT_MAX_EXEC_TIME_FILE_TASK=0
-	HARD_MAX_EXEC_TIME_DB_TASK=0
-	HARD_MAX_EXEC_TIME_FILE_TASK=0
-	HARD_MAX_EXEC_TIME_TOTAL=0
-fi
 RunBeforeHook
 Main

dev/ofunctions.sh

@@ -2,8 +2,8 @@
 #### OFUNCTIONS FULL SUBSET ####
 #### OFUNCTIONS MINI SUBSET ####
 
-_OFUNCTIONS_VERSION=2.1-RC3+dev
+_OFUNCTIONS_VERSION=2.1.4-rc1
-_OFUNCTIONS_BUILD=2017031401
+_OFUNCTIONS_BUILD=2017060903
 #### _OFUNCTIONS_BOOTSTRAP SUBSET ####
 _OFUNCTIONS_BOOTSTRAP=true
 #### _OFUNCTIONS_BOOTSTRAP SUBSET END ####
@@ -30,6 +30,9 @@ fi
 ## Correct output of sort command (language agnostic sorting)
 export LC_ALL=C
 
+## Default umask for file creation
+umask 0077
+
 # Standard alert mail body
 MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
 
@@ -68,6 +71,8 @@ fi
 #### DEBUG SUBSET END ####
 
 SCRIPT_PID=$$
+
+# TODO: Check if %N works on MacOS
 TSTAMP=$(date '+%Y%m%dT%H%M%S.%N')
 
 LOCAL_USER=$(whoami)
@@ -261,7 +266,7 @@ function Logger {
 		return
 	elif [ "$level" == "VERBOSE" ]; then
 		if [ $_LOGGER_VERBOSE == true ]; then
-			_Logger "$prefix:$value" "$prefix$value"
+			_Logger "$prefix($level):$value" "$prefix$value"
 		fi
 		return
 	elif [ "$level" == "ALWAYS" ]; then
@@ -314,15 +319,29 @@ function KillChilds {
 	local pid="${1}" # Parent pid to kill childs
 	local self="${2:-false}" # Should parent be killed too ?
 
-	# Warning: pgrep does not exist in cygwin, have this checked in CheckEnvironment
+	# Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
-	if children="$(pgrep -P "$pid")"; then
+	if [ $(IsNumeric "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
-		for child in $children; do
+		Logger "Bogus pid given [$pid]." "CRITICAL"
-			Logger "Launching KillChilds \"$child\" true" "DEBUG"	#__WITH_PARANOIA_DEBUG
+		return 1
-			KillChilds "$child" true
-		done
 	fi
-		# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
+
+	if kill -0 "$pid" > /dev/null 2>&1; then
+		# Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
+		if children="$(pgrep -P "$pid")"; then
+			if [[ "$pid" == *"$children"* ]]; then
+				Logger "Bogus pgrep implementation." "CRITICAL"
+				children="${children/$pid/}"
+			fi
+			for child in $children; do
+				Logger "Launching KillChilds \"$child\" true" "DEBUG"	#__WITH_PARANOIA_DEBUG
+				KillChilds "$child" true
+			done
+		fi
+	fi
+
+	# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
 	if [ "$self" == true ]; then
+		# We need to check for pid again because it may have disappeared after recursive function call
 		if kill -0 "$pid" > /dev/null 2>&1; then
 			kill -s TERM "$pid"
 			Logger "Sent SIGTERM to process [$pid]." "DEBUG"
@@ -385,7 +404,6 @@ function SendAlert {
 
 	eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
 	if [ $? != 0 ]; then
-		Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
 		attachment=false
 	else
 		attachment=true
@@ -634,6 +652,22 @@ function Spinner {
 	fi
 }
 
+function _PerfProfiler {												#__WITH_PARANOIA_DEBUG
+	local perfString												#__WITH_PARANOIA_DEBUG
+															#__WITH_PARANOIA_DEBUG
+	perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan)					#__WITH_PARANOIA_DEBUG
+															#__WITH_PARANOIA_DEBUG
+	for i in $(pgrep -P $$); do											#__WITH_PARANOIA_DEBUG
+		perfString="$perfString\n"$(ps -p $i -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan | tail -1)	#__WITH_PARANOIA_DEBUG
+	done														#__WITH_PARANOIA_DEBUG
+															#__WITH_PARANOIA_DEBUG
+	if type iostat > /dev/null 2>&1; then										#__WITH_PARANOIA_DEBUG
+		perfString="$perfString\n"$(iostat)									#__WITH_PARANOIA_DEBUG
+	fi														#__WITH_PARANOIA_DEBUG
+															#__WITH_PARANOIA_DEBUG
+	Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG"								#__WITH_PARANOIA_DEBUG
+}															#__WITH_PARANOIA_DEBUG
+
 
 # Time control function for background processes, suitable for multiple synchronous processes
 # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
@@ -766,6 +800,11 @@ function WaitForTaskCompletion {
 		pidsArray=("${newPidsArray[@]}")
 		# Trivial wait time for bash to not eat up all CPU
 		sleep $sleepTime
+
+		if [ "$_PERF_PROFILER" == "yes" ]; then				##__WITH_PARANOIA_DEBUG
+			_PerfProfiler						##__WITH_PARANOIA_DEBUG
+		fi								##__WITH_PARANOIA_DEBUG
+
 	done
 
 	Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG"	#__WITH_PARANOIA_DEBUG
@@ -931,6 +970,10 @@ function ParallelExec {
 
 		# Trivial wait time for bash to not eat up all CPU
 		sleep $sleepTime
+
+		if [ "$_PERF_PROFILER" == "yes" ]; then				##__WITH_PARANOIA_DEBUG
+			_PerfProfiler						##__WITH_PARANOIA_DEBUG
+		fi								##__WITH_PARANOIA_DEBUG
 	done
 
 	return $errorCount
@@ -1125,7 +1168,7 @@ function GetLocalOS {
 		*"BSD"*)
 		LOCAL_OS="BSD"
 		;;
-		*"MINGW32"*|*"MSYS"*)
+		*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
 		LOCAL_OS="msys"
 		;;
 		*"CYGWIN"*)
@@ -1151,9 +1194,6 @@ function GetLocalOS {
 		exit 1
 		;;
 	esac
-	if [ "$_OFUNCTIONS_VERSION" != "" ]; then
-		Logger "Local OS: [$localOsVar]." "DEBUG"
-	fi
 
 	# Get linux versions
 	if [ -f "/etc/os-release" ]; then
@@ -1163,9 +1203,69 @@ function GetLocalOS {
 
 	# Add a global variable for statistics in installer
 	LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
+
+	if [ "$_OFUNCTIONS_VERSION" != "" ]; then
+		Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG"
+	fi
 }
 #### GetLocalOS SUBSET END ####
 
+#__BEGIN_WITH_PARANOIA_DEBUG
+function __CheckArguments {
+	# Checks the number of arguments of a function and raises an error if some are missing
+
+	if [ "$_DEBUG" == "yes" ]; then
+		local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
+		local numberOfGivenArguments="${2}" # Number of arguments that have been passed
+
+		local minArgs
+		local maxArgs
+
+		# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
+		# If any of the arguments contains spaces, bash things there are two aguments
+		# In order to avoid this, we need to iterate over ${3} and count
+
+		callerName="${FUNCNAME[1]}"
+
+		local iterate=3
+		local fetchArguments=true
+		local argList=""
+		local countedArguments
+		while [ $fetchArguments == true ]; do
+			cmd='argument=${'$iterate'}'
+			eval $cmd
+			if [ "$argument" == "" ]; then
+				fetchArguments=false
+			else
+				argList="$argList[Argument $((iterate-2)): $argument] "
+				iterate=$((iterate+1))
+			fi
+		done
+
+		countedArguments=$((iterate-3))
+
+		if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
+			minArgs=$numberOfArguments
+			maxArgs=$numberOfArguments
+		else
+			IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
+		fi
+
+		Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
+
+		if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
+			Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
+			Logger "$callerName arguments: $argList" "ERROR"
+		else
+			if [ ! -z "$argList" ]; then
+				Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
+			fi
+		fi
+	fi
+}
+
+#__END_WITH_PARANOIA_DEBUG
+
 #### OFUNCTIONS MINI SUBSET END ####
 
 function GetRemoteOS {
@@ -1177,13 +1277,16 @@ function GetRemoteOS {
 
 	local remoteOsVar
 
-$SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
+$SSH_CMD env LC_ALL=C env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
+
 
 function GetOs {
 	local localOsVar
 	local localOsName
 	local localOsVer
 
+	local osInfo="/etc/os-release"
+
 	# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
 	if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
 		localOsVar="BusyBox"
@@ -1202,9 +1305,11 @@ function GetOs {
 		fi
 	fi
 	# Get linux versions
-	if [ -f "/etc/os-release" ]; then
+	if [ -f "$osInfo" ]; then
-		localOsName=$(GetConfFileValue "/etc/os-release" "NAME")
+		localOsName=$(grep "^NAME=" "$osInfo")
-		localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION")
+                localOsName="${localOsName##*=}"
+		localOsVer=$(grep "^VERSION=" "$osInfo")
+		localOsVer="${localOsVer##*=}"
 	fi
 
 	echo "$localOsVar ($localOsName $localOsVer)"
@@ -1226,7 +1331,7 @@ ENDSSH
 			*"BSD"*)
 			REMOTE_OS="BSD"
 			;;
-			*"MINGW32"*|*"MSYS"*)
+			*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
 			REMOTE_OS="msys"
 			;;
 			*"CYGWIN"*)
@@ -1311,7 +1416,7 @@ function RunRemoteCommand {
 	fi
 
 	Logger "Running command [$command] on remote host." "NOTICE"
-	cmd=$SSH_CMD' "env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
+	cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
 	Logger "cmd: $cmd" "DEBUG"
 	eval "$cmd" &
 	WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
@@ -1422,62 +1527,6 @@ function CheckConnectivity3rdPartyHosts {
 	fi											#__WITH_PARANOIA_DEBUG
 }
 
-#__BEGIN_WITH_PARANOIA_DEBUG
-function __CheckArguments {
-	# Checks the number of arguments of a function and raises an error if some are missing
-
-	if [ "$_DEBUG" == "yes" ]; then
-		local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
-		local numberOfGivenArguments="${2}" # Number of arguments that have been passed
-
-		local minArgs
-		local maxArgs
-
-		# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
-		# If any of the arguments contains spaces, bash things there are two aguments
-		# In order to avoid this, we need to iterate over ${3} and count
-
-		callerName="${FUNCNAME[1]}"
-
-		local iterate=3
-		local fetchArguments=true
-		local argList=""
-		local countedArguments
-		while [ $fetchArguments == true ]; do
-			cmd='argument=${'$iterate'}'
-			eval $cmd
-			if [ "$argument" == "" ]; then
-				fetchArguments=false
-			else
-				argList="$argList[Argument $((iterate-2)): $argument] "
-				iterate=$((iterate+1))
-			fi
-		done
-
-		countedArguments=$((iterate-3))
-
-		if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
-			minArgs=$numberOfArguments
-			maxArgs=$numberOfArguments
-		else
-			IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
-		fi
-
-		Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
-
-		if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
-			Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
-			Logger "$callerName arguments: $argList" "ERROR"
-		else
-			if [ ! -z "$argList" ]; then
-				Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
-			fi
-		fi
-	fi
-}
-
-#__END_WITH_PARANOIA_DEBUG
-
 function RsyncPatternsAdd {
 	local patternType="${1}"	# exclude or include
 	local pattern="${2}"
@@ -1608,13 +1657,13 @@ function PostInit {
 
 	# Define remote commands
 	if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then
-		SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
+		SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
-		SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
+		SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
-		RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
+		RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
 	elif [ -f "$SSH_PASSWORD_FILE" ]; then
-		SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
+		SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
-		SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -P $REMOTE_PORT"
+		SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT"
-		RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS -p $REMOTE_PORT"
+		RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT"
 	else
 		SSH_PASSWORD=""
 		SSH_CMD=""
@@ -1652,23 +1701,22 @@ function SetCompression {
 			COMPRESSION_PROGRAM="| pigz -c$compressionString"
 			COMPRESSION_EXTENSION=.gz
 			# obackup specific
-			if [ "$LOCAL_OS" != "MacOSX" ]; then
+			COMPRESSION_OPTIONS=--rsyncable
-				COMPRESSION_OPTIONS=--rsyncable
-			fi
 		elif type gzip > /dev/null 2>&1
 		then
 			COMPRESSION_PROGRAM="| gzip -c$compressionString"
 			COMPRESSION_EXTENSION=.gz
 			# obackup specific
-			if [ "$LOCAL_OS" != "MacOSX" ]; then
+			COMPRESSION_OPTIONS=--rsyncable
-				COMPRESSION_OPTIONS=--rsyncable
-			fi
 		else
 			COMPRESSION_PROGRAM=
 			COMPRESSION_EXTENSION=
 		fi
 	fi
-	ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
+
+	if [ ".${ALERT_LOG_FILE##*.}" != "$COMPRESSION_EXTENSION" ]; then
+		ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
+	fi
 }
 
 function InitLocalOSDependingSettings {
@@ -1830,38 +1878,51 @@ function ParentPid {
 	fi
 }
 
+#### VerComp SUBSET ####
 # Neat version compare function found at http://stackoverflow.com/a/4025065/2635443
 # Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2
-vercomp () {
+function VerComp () {
-    if [[ $1 == $2 ]]
+	if [ "$1" == "" ] || [ "$2" == "" ]; then
-    then
+		Logger "Bogus Vercomp values [$1] and [$2]." "WARN"
-        return 0
+		return 1
-    fi
+	fi
-    local IFS=.
+
-    local i ver1=($1) ver2=($2)
+	if [[ $1 == $2 ]]
-    # fill empty fields in ver1 with zeros
+		then
-    for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
+			echo 0
-    do
+		return
-        ver1[i]=0
+	fi
-    done
+
-    for ((i=0; i<${#ver1[@]}; i++))
+	local IFS=.
-    do
+	local i ver1=($1) ver2=($2)
-        if [[ -z ${ver2[i]} ]]
+	# fill empty fields in ver1 with zeros
-        then
+	for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
-            # fill empty fields in ver2 with zeros
+	do
-            ver2[i]=0
+        	ver1[i]=0
-        fi
+	done
-        if ((10#${ver1[i]} > 10#${ver2[i]}))
+	for ((i=0; i<${#ver1[@]}; i++))
-        then
+	do
-            return 1
+		if [[ -z ${ver2[i]} ]]
-        fi
+		then
-        if ((10#${ver1[i]} < 10#${ver2[i]}))
+			# fill empty fields in ver2 with zeros
-        then
+			ver2[i]=0
-            return 2
+		fi
-        fi
+		if ((10#${ver1[i]} > 10#${ver2[i]}))
-    done
+		then
-    return 0
+			echo 1
+			return
+        	fi
+        	if ((10#${ver1[i]} < 10#${ver2[i]}))
+        	then
+			echo 2
+            		return
+        	fi
+    	done
+
+    	echo 0
+	return
 }
+#### VerComp SUBSET END ####
 
 #### GetConfFileValue SUBSET ####
 function GetConfFileValue () {
@@ -1897,4 +1958,27 @@ function SetConfFileValue () {
 }
 #### SetConfFileValue SUBSET END ####
 
+# Function can replace [ -f /some/file* ] tests
+# Modified version of http://stackoverflow.com/a/6364244/2635443
+function WildcardFileExists () {
+        local file="${1}"
+        local exists=0
+
+        for f in $file; do
+                ## Check if the glob gets expanded to existing files.
+                ## If not, f here will be exactly the pattern above
+                ## and the exists test will evaluate to false.
+                if [ -e "$f" ]; then
+                        exists=1
+                        break
+                fi
+        done
+
+        if [ $exists -eq 1 ]; then
+                echo 1
+        else
+                echo 0
+        fi
+}
+
 #### OFUNCTIONS FULL SUBSET END ####

dev/tests/run_tests.sh

@@ -2,7 +2,7 @@
 
 #TODO Encrypted Pull runs on F25 fail for decryption
 
-## obackup basic tests suite 2017020903
+## obackup basic tests suite 2017040801
 
 OBACKUP_DIR="$(pwd)"
 OBACKUP_DIR=${OBACKUP_DIR%%/dev*}
@@ -153,6 +153,10 @@ function oneTimeSetUp () {
 	START_TIME=$SECONDS
 
 	source "$DEV_DIR/ofunctions.sh"
+
+	# Set default umask because ofunctions set 0077
+	umask 0022
+
 	GetLocalOS
 
 	echo "Detected OS: $LOCAL_OS"
@@ -293,7 +297,7 @@ function setUp () {
 
 function test_Merge () {
 	cd "$DEV_DIR"
-	./merge.sh
+	./merge.sh obackup
 	assertEquals "Merging code" "0" $?
 
 	cd "$OBACKUP_DIR"

install.sh

@@ -12,7 +12,7 @@ PROGRAM_BINARY=$PROGRAM".sh"
 PROGRAM_BATCH=$PROGRAM"-batch.sh"
 SSH_FILTER="ssh_filter.sh"
 
-SCRIPT_BUILD=2017031402
+SCRIPT_BUILD=2017041701
 
 ## osync / obackup / pmocr / zsnap install script
 ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
@@ -122,7 +122,7 @@ function GetLocalOS {
 		*"BSD"*)
 		LOCAL_OS="BSD"
 		;;
-		*"MINGW32"*|*"MSYS"*)
+		*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
 		LOCAL_OS="msys"
 		;;
 		*"CYGWIN"*)
@@ -148,9 +148,6 @@ function GetLocalOS {
 		exit 1
 		;;
 	esac
-	if [ "$_OFUNCTIONS_VERSION" != "" ]; then
-		Logger "Local OS: [$localOsVar]." "DEBUG"
-	fi
 
 	# Get linux versions
 	if [ -f "/etc/os-release" ]; then
@@ -160,6 +157,10 @@ function GetLocalOS {
 
 	# Add a global variable for statistics in installer
 	LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
+
+	if [ "$_OFUNCTIONS_VERSION" != "" ]; then
+		Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG"
+	fi
 }
 function GetConfFileValue () {
         local file="${1}"
@@ -331,7 +332,9 @@ function CopyProgram {
 function CopyServiceFiles {
 	if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
 		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true
-		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true
+		if [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM_USER" ]; then
+			CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true
+		fi
 
 		QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
 		QuickLogger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR."
@@ -340,9 +343,9 @@ function CopyServiceFiles {
 	elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then
 		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true
 
-		QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
+		QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]."
-		QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]."
+		QuickLogger "Can be activated with [service $SERVICE_FILE_INIT start]."
-		QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]."
+		QuickLogger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]."
 	else
 		QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation."
 	fi
@@ -384,7 +387,11 @@ function RemoveFile {
 
 function RemoveAll {
 	RemoveFile "$BIN_DIR/$PROGRAM_BINARY"
-	RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
+
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
+		RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
+	fi
+
 	if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then		# Check if any other program requiring ssh filter is present before removal
 		RemoveFile "$BIN_DIR/$SSH_FILTER"
 	else
@@ -402,6 +409,7 @@ function Usage {
 	echo "options:"
 	echo "--silent		Will log and bypass user interaction."
 	echo "--no-stats	Used with --silent in order to refuse sending anonymous install stats."
+	echo "--remove          Remove the program."
 	exit 127
 }
 
@@ -444,7 +452,9 @@ else
 	CreateDir "$BIN_DIR"
 	CopyExampleFiles
 	CopyProgram
-	CopyServiceFiles
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "pmocr" ]; then
+		CopyServiceFiles
+	fi
 	QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
 	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
 		QuickLogger ""

obackup-batch.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 SUBPROGRAM=obackup
 PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
-AUTHOR="(L) 2013-2016 by Orsiris de Jong"
+AUTHOR="(L) 2013-2017 by Orsiris de Jong"
 CONTACT="http://www.netpower.fr - ozy@netpower.fr"
 PROGRAM_BUILD=2016120401
 
@@ -119,15 +119,15 @@ function Batch {
 				fi
 			done
 			runList=("${runAgainList[@]}")
-			runs=$(($runs + 1))
+			runs=$((runs + 1))
 		done
 	fi
 }
 
 function Usage {
 	echo "$PROGRAM $PROGRAM_BUILD"
-	echo $AUTHOR
+	echo "$AUTHOR"
-	echo $CONTACT
+	echo "$CONTACT"
 	echo ""
 	echo "Batch script to sequentially run osync or obackup instances and rerun failed ones."
 	echo "Usage: $PROGRAM.sh [OPTIONS] [$SUBPROGRAM OPTIONS]"

obackup.sh

@@ -6,14 +6,14 @@
 PROGRAM="obackup"
 AUTHOR="(C) 2013-2017 by Orsiris de Jong"
 CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
-PROGRAM_VERSION=2.1-beta2
+PROGRAM_VERSION=2.1-beta3
-PROGRAM_BUILD=2017021001
+PROGRAM_BUILD=2017062004
 IS_STABLE=no
 
 
 
-_OFUNCTIONS_VERSION=2.1-RC3+dev
+_OFUNCTIONS_VERSION=2.1.4-rc1
-_OFUNCTIONS_BUILD=2017031401
+_OFUNCTIONS_BUILD=2017060903
 _OFUNCTIONS_BOOTSTRAP=true
 
 ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@@ -38,6 +38,9 @@ fi
 ## Correct output of sort command (language agnostic sorting)
 export LC_ALL=C
 
+## Default umask for file creation
+umask 0077
+
 # Standard alert mail body
 MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
 
@@ -70,6 +73,8 @@ if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as envi
 fi
 
 SCRIPT_PID=$$
+
+# TODO: Check if %N works on MacOS
 TSTAMP=$(date '+%Y%m%dT%H%M%S.%N')
 
 LOCAL_USER=$(whoami)
@@ -255,7 +260,7 @@ function Logger {
 		return
 	elif [ "$level" == "VERBOSE" ]; then
 		if [ $_LOGGER_VERBOSE == true ]; then
-			_Logger "$prefix:$value" "$prefix$value"
+			_Logger "$prefix($level):$value" "$prefix$value"
 		fi
 		return
 	elif [ "$level" == "ALWAYS" ]; then
@@ -301,14 +306,28 @@ function KillChilds {
 	local pid="${1}" # Parent pid to kill childs
 	local self="${2:-false}" # Should parent be killed too ?
 
-	# Warning: pgrep does not exist in cygwin, have this checked in CheckEnvironment
+	# Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
-	if children="$(pgrep -P "$pid")"; then
+	if [ $(IsNumeric "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
-		for child in $children; do
+		Logger "Bogus pid given [$pid]." "CRITICAL"
-			KillChilds "$child" true
+		return 1
-		done
 	fi
-		# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
+
+	if kill -0 "$pid" > /dev/null 2>&1; then
+		# Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
+		if children="$(pgrep -P "$pid")"; then
+			if [[ "$pid" == *"$children"* ]]; then
+				Logger "Bogus pgrep implementation." "CRITICAL"
+				children="${children/$pid/}"
+			fi
+			for child in $children; do
+				KillChilds "$child" true
+			done
+		fi
+	fi
+
+	# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
 	if [ "$self" == true ]; then
+		# We need to check for pid again because it may have disappeared after recursive function call
 		if kill -0 "$pid" > /dev/null 2>&1; then
 			kill -s TERM "$pid"
 			Logger "Sent SIGTERM to process [$pid]." "DEBUG"
@@ -369,7 +388,6 @@ function SendAlert {
 
 	eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
 	if [ $? != 0 ]; then
-		Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
 		attachment=false
 	else
 		attachment=true
@@ -615,6 +633,7 @@ function Spinner {
 }
 
 
+
 # Time control function for background processes, suitable for multiple synchronous processes
 # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
 # Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached
@@ -739,6 +758,8 @@ function WaitForTaskCompletion {
 		pidsArray=("${newPidsArray[@]}")
 		# Trivial wait time for bash to not eat up all CPU
 		sleep $sleepTime
+
+
 	done
 
 
@@ -897,6 +918,7 @@ function ParallelExec {
 
 		# Trivial wait time for bash to not eat up all CPU
 		sleep $sleepTime
+
 	done
 
 	return $errorCount
@@ -1081,7 +1103,7 @@ function GetLocalOS {
 		*"BSD"*)
 		LOCAL_OS="BSD"
 		;;
-		*"MINGW32"*|*"MSYS"*)
+		*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
 		LOCAL_OS="msys"
 		;;
 		*"CYGWIN"*)
@@ -1107,9 +1129,6 @@ function GetLocalOS {
 		exit 1
 		;;
 	esac
-	if [ "$_OFUNCTIONS_VERSION" != "" ]; then
-		Logger "Local OS: [$localOsVar]." "DEBUG"
-	fi
 
 	# Get linux versions
 	if [ -f "/etc/os-release" ]; then
@@ -1119,9 +1138,14 @@ function GetLocalOS {
 
 	# Add a global variable for statistics in installer
 	LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
+
+	if [ "$_OFUNCTIONS_VERSION" != "" ]; then
+		Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG"
+	fi
 }
 
 
+
 function GetRemoteOS {
 
 	if [ "$REMOTE_OPERATION" != "yes" ]; then
@@ -1130,13 +1154,16 @@ function GetRemoteOS {
 
 	local remoteOsVar
 
-$SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
+$SSH_CMD env LC_ALL=C env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
+
 
 function GetOs {
 	local localOsVar
 	local localOsName
 	local localOsVer
 
+	local osInfo="/etc/os-release"
+
 	# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
 	if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
 		localOsVar="BusyBox"
@@ -1155,9 +1182,11 @@ function GetOs {
 		fi
 	fi
 	# Get linux versions
-	if [ -f "/etc/os-release" ]; then
+	if [ -f "$osInfo" ]; then
-		localOsName=$(GetConfFileValue "/etc/os-release" "NAME")
+		localOsName=$(grep "^NAME=" "$osInfo")
-		localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION")
+                localOsName="${localOsName##*=}"
+		localOsVer=$(grep "^VERSION=" "$osInfo")
+		localOsVer="${localOsVer##*=}"
 	fi
 
 	echo "$localOsVar ($localOsName $localOsVer)"
@@ -1179,7 +1208,7 @@ ENDSSH
 			*"BSD"*)
 			REMOTE_OS="BSD"
 			;;
-			*"MINGW32"*|*"MSYS"*)
+			*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
 			REMOTE_OS="msys"
 			;;
 			*"CYGWIN"*)
@@ -1262,7 +1291,7 @@ function RunRemoteCommand {
 	fi
 
 	Logger "Running command [$command] on remote host." "NOTICE"
-	cmd=$SSH_CMD' "env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
+	cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
 	Logger "cmd: $cmd" "DEBUG"
 	eval "$cmd" &
 	WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
@@ -1365,7 +1394,6 @@ function CheckConnectivity3rdPartyHosts {
 		fi
 }
 
-
 function RsyncPatternsAdd {
 	local patternType="${1}"	# exclude or include
 	local pattern="${2}"
@@ -1491,13 +1519,13 @@ function PostInit {
 
 	# Define remote commands
 	if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then
-		SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
+		SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
-		SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
+		SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
-		RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
+		RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
 	elif [ -f "$SSH_PASSWORD_FILE" ]; then
-		SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
+		SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
-		SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -P $REMOTE_PORT"
+		SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT"
-		RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS -p $REMOTE_PORT"
+		RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT"
 	else
 		SSH_PASSWORD=""
 		SSH_CMD=""
@@ -1535,23 +1563,22 @@ function SetCompression {
 			COMPRESSION_PROGRAM="| pigz -c$compressionString"
 			COMPRESSION_EXTENSION=.gz
 			# obackup specific
-			if [ "$LOCAL_OS" != "MacOSX" ]; then
+			COMPRESSION_OPTIONS=--rsyncable
-				COMPRESSION_OPTIONS=--rsyncable
-			fi
 		elif type gzip > /dev/null 2>&1
 		then
 			COMPRESSION_PROGRAM="| gzip -c$compressionString"
 			COMPRESSION_EXTENSION=.gz
 			# obackup specific
-			if [ "$LOCAL_OS" != "MacOSX" ]; then
+			COMPRESSION_OPTIONS=--rsyncable
-				COMPRESSION_OPTIONS=--rsyncable
-			fi
 		else
 			COMPRESSION_PROGRAM=
 			COMPRESSION_EXTENSION=
 		fi
 	fi
-	ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
+
+	if [ ".${ALERT_LOG_FILE##*.}" != "$COMPRESSION_EXTENSION" ]; then
+		ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
+	fi
 }
 
 function InitLocalOSDependingSettings {
@@ -1713,35 +1740,46 @@ function ParentPid {
 
 # Neat version compare function found at http://stackoverflow.com/a/4025065/2635443
 # Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2
-vercomp () {
+function VerComp () {
-    if [[ $1 == $2 ]]
+	if [ "$1" == "" ] || [ "$2" == "" ]; then
-    then
+		Logger "Bogus Vercomp values [$1] and [$2]." "WARN"
-        return 0
+		return 1
-    fi
+	fi
-    local IFS=.
+
-    local i ver1=($1) ver2=($2)
+	if [[ $1 == $2 ]]
-    # fill empty fields in ver1 with zeros
+		then
-    for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
+			echo 0
-    do
+		return
-        ver1[i]=0
+	fi
-    done
+
-    for ((i=0; i<${#ver1[@]}; i++))
+	local IFS=.
-    do
+	local i ver1=($1) ver2=($2)
-        if [[ -z ${ver2[i]} ]]
+	# fill empty fields in ver1 with zeros
-        then
+	for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
-            # fill empty fields in ver2 with zeros
+	do
-            ver2[i]=0
+        	ver1[i]=0
-        fi
+	done
-        if ((10#${ver1[i]} > 10#${ver2[i]}))
+	for ((i=0; i<${#ver1[@]}; i++))
-        then
+	do
-            return 1
+		if [[ -z ${ver2[i]} ]]
-        fi
+		then
-        if ((10#${ver1[i]} < 10#${ver2[i]}))
+			# fill empty fields in ver2 with zeros
-        then
+			ver2[i]=0
-            return 2
+		fi
-        fi
+		if ((10#${ver1[i]} > 10#${ver2[i]}))
-    done
+		then
-    return 0
+			echo 1
+			return
+        	fi
+        	if ((10#${ver1[i]} < 10#${ver2[i]}))
+        	then
+			echo 2
+            		return
+        	fi
+    	done
+
+    	echo 0
+	return
 }
 
 function GetConfFileValue () {
@@ -1774,6 +1812,29 @@ function SetConfFileValue () {
         fi
 }
 
+# Function can replace [ -f /some/file* ] tests
+# Modified version of http://stackoverflow.com/a/6364244/2635443
+function WildcardFileExists () {
+        local file="${1}"
+        local exists=0
+
+        for f in $file; do
+                ## Check if the glob gets expanded to existing files.
+                ## If not, f here will be exactly the pattern above
+                ## and the exists test will evaluate to false.
+                if [ -e "$f" ]; then
+                        exists=1
+                        break
+                fi
+        done
+
+        if [ $exists -eq 1 ]; then
+                echo 1
+        else
+                echo 0
+        fi
+}
+
 # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap
 _OFUNCTIONS_BOOTSTRAP=true
 [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1
@@ -4011,23 +4072,23 @@ function Init {
 		REMOTE_HOST=${hosturi%%:*}
 	fi
 
-	## Add update to default RSYNC_ARGS
+	## Add update to default RSYNC_TYPE_ARGS
-	RSYNC_ARGS=$RSYNC_ARGS" -u"
+	RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" -u"
 
 	if [ $_LOGGER_VERBOSE == true ]; then
-		RSYNC_ARGS=$RSYNC_ARGS" -i"
+		RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" -i"
 	fi
 
 	if [ "$DELETE_VANISHED_FILES" == "yes" ]; then
-		RSYNC_ARGS=$RSYNC_ARGS" --delete"
+		RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --delete"
 	fi
 
 	if [ $stats == true ]; then
-		RSYNC_ARGS=$RSYNC_ARGS" --stats"
+		RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --stats"
 	fi
 
 	## Fix for symlink to directories on target cannot get updated
-	RSYNC_ARGS=$RSYNC_ARGS" --force"
+	RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --force"
 }
 
 function Main {
@@ -4114,12 +4175,16 @@ function Usage {
 _DRYRUN=false
 no_maxtime=false
 stats=false
-PARTIAL=no
+partial_transfers=false
+delete_vanished=false
+dont_get_backup_size=false
 _DECRYPT_MODE=false
 DECRYPT_PATH=""
 _ENCRYPT_MODE=false
 
 function GetCommandlineArguments {
+	local isFirstArgument=true
+
 	if [ $# -eq 0 ]; then
 		Usage
 	fi
@@ -4139,16 +4204,16 @@ function GetCommandlineArguments {
 			stats=false
 			;;
 			--partial)
-			PARTIAL="yes"
+			partial_transfers=true
 			;;
 			--no-maxtime)
 			no_maxtime=true
 			;;
 			--delete)
-			DELETE_VANISHED_FILES="yes"
+			delete_vanished=true
 			;;
 			--dontgetsize)
-			GET_BACKUP_SIZE="no"
+			dont_get_backup_size=true
 			;;
 			--help|-h|--version|-v)
 			Usage
@@ -4184,7 +4249,15 @@ function GetCommandlineArguments {
 			if [ $(IsNumeric $PARALLEL_ENCRYPTION_PROCESSES) -ne 1 ]; then
 				Logger "Bogus --parallel value. Using only one CPU." "WARN"
 			fi
+			;;
+			*)
+			if [ $isFirstArgument == false ]; then
+				Logger "Unknown option '$i'" "CRITICAL"
+				Usage
+			fi
+			;;
 		esac
+		isFirstArgument=false
 	done
 }
 
@@ -4226,6 +4299,27 @@ else
 	Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
 fi
 
+
+if [ $no_maxtime == true ]; then
+	SOFT_MAX_EXEC_TIME_DB_TASK=0
+	SOFT_MAX_EXEC_TIME_FILE_TASK=0
+	HARD_MAX_EXEC_TIME_DB_TASK=0
+	HARD_MAX_EXEC_TIME_FILE_TASK=0
+	HARD_MAX_EXEC_TIME_TOTAL=0
+fi
+
+if [ $partial_transfers == true ]; then
+	PARTIAL="yes"
+fi
+
+if [ $delete_vanished == true ]; then
+	DELETE_VANISHED_FILES="yes"
+fi
+
+if [ $dont_get_backup_size == true ]; then
+	GET_BACKUP_SIZE="no"
+fi
+
 if [ "$IS_STABLE" != "yes" ]; then
 	Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN"
 fi
@@ -4246,13 +4340,5 @@ PostInit
 CheckCurrentConfig
 GetRemoteOS
 InitRemoteOSDependingSettings
-
-if [ $no_maxtime == true ]; then
-	SOFT_MAX_EXEC_TIME_DB_TASK=0
-	SOFT_MAX_EXEC_TIME_FILE_TASK=0
-	HARD_MAX_EXEC_TIME_DB_TASK=0
-	HARD_MAX_EXEC_TIME_FILE_TASK=0
-	HARD_MAX_EXEC_TIME_TOTAL=0
-fi
 RunBeforeHook
 Main