Rebuilt targets for v2.1-beta3

This reverts commit 52161b3faf.

.travis.yml

@@ -0,0 +1,18 @@
+language:
+    bash
+
+services:
+    mysql
+
+os:
+    linux
+    osx
+
+sudo:
+    required
+
+before_script:
+    mysql -e 'CREATE DATABASE travistest;'
+
+script:
+    TRAVIS_RUN=true dev/tests/run_tests.sh

CHANGELOG.md

@@ -2,16 +2,115 @@ KNOWN ISSUES
 ------------
 
 - Backup size check does not honor rsync exclude patterns
+- Encryption does not honor rsync exclude patterns
 - Bandwidth parameter is ignored for SQL backups
 - Missing symlink support when run from MSYS environment
-- Mysqldump errors aren't taken in account
 
 CHANGELOG
 ---------
 
-README: FreeBSD execution needs mailer (not found), sudo missing, bash needed, sed missing (see if StripQuotes mandatory)
+20 Jun 2017: obackup v2.1 beta3 released
+----------------------------------------
+
+- Fixed regression where some commandline arguments weren't honored anymore since 2.1 beta1 (--delete, --stats, --dontgetsize)
+- Fixed commandline arguments aren't checked against valid list
+
+14 Mar 2017: obackup v2.1 beta2 released
+----------------------------------------
+
+- Fixed remote commands can be run on local runs and obviously fail
+- Uninstall leaves ssh_filter if needed by other programs
+- Logger now obfuscates _REMOTE_TOKEN
+- Improved sudo privilege run
+- Brand new ssh filter from osync project
+- Better installer with --remove option from osync project
+- Updated ofunctions from osync project
+	- Fixes UTF-8 escaped characters in log files due to LC_ALL=C
+	- Optional MAIL_BODY_CHARSET so destination mails aren't sent as UTF-8 anymore depending on systems
+- Minor fixes
+
+04 Jan 2017: obackup v2.1 beta1 released
+----------------------------------------
+
+- Fixed wrong file size fetched remotely since v2.1 rewrite
+- Fixed missing databases in manual list fails to trigger an alert
+- Improved support for GPG ver >= 2.1
+- Added encryption / decryption parallel execution support
+- Improved compatibility for RotateCopies
+- Unit tests now run on CentOS 5,6
+- Added optional rsync arguments configuration value
+- Forcec bash usage on remote connections in order to be FreeBSD 11 compatible
+- Spinner is less prone to move logging on screen
+- Fixed another random error involving warns and errors triggered by earlier runs with same PID flag files
+- Adde more preflight checks (pgrep presence)
+- Added --no-prefix, --error-only and --summary switches
+- Updated installer from osync
+- Updated merge.sh script to handle includes
+- Improved remote logging
+- Simplified osync-batch runner (internally and for user)
+	- Better filename handling
+	- Easier to read log output
+        - Always passes --silent to obackup
+        - All options that do not belong to obackup-batch are automatically passed to obackup
+- Improved installer OS detection
+- Fixed upgrade script cannot update header on BSD / MacOS X
+- Fixed SendEmail function on MacOS X
+- Fixed MAX_SOFT_EXEC_TIME_PER_XX_TASK not enforced bug introduced with newer ofunctions from v2.1
+- PRESERVE_ACL and PRESERVE_XATTR are ignored when local or remote OS is MacOS or msys or Cygwin
+- Fixed PRESERVE_EXECUTABILITY was ommited volontary on MacOS X because of rsync syntax
+- merge.sh is now BSD and Mac compatible
+- Unit tests are now BSD and Mac compatible
+- Local runs should not check for remote connectivity
+- Fixed error alerts cannot be triggered from subprocesses
+- Fixed error flags 
+- Faster remote OS detection
+- Added busybox (and Android Termux) support
+	- More portable file size functions
+	- More portable compression program commands
+	- More paranoia checks
+	- Added busybox sendmail support
+	- Added tls and ssl support for sendmail
+- Added ssh password file support
+- Added unit tests
+	- Added basic unit tests for all three operation modes
+	- Added process management function tests
+	- Added file rotation tests
+	- Added upgrade script test
+	- Added encryption tests
+	- Added missing files / databases test
+	- Added timed execution tests
+- Implemented backup encryption using GPG (see documentation for advantages and caveats)
+	- Backup encrypted but still use differential engine :)
+- Database backup improvements
+	- Added mysqldump options to config file
+- Improved unit tests
+- Added more preflight checks
+- Logs sent by mail are easier to read
+	- Better subject (currently running or finished run)
+	- Fixed bogus double log sent in alert mails
+	- Only current run log is now sent
+	- Alert sending is now triggered after last action
+- Made unix signals posix compliant
+- Improved upgrade script
+	- Upgrade script now updates header
+	- Can add any missing value now
+	- Added encrpytion support
+- Fixed problem with spaces in directories to backup (again !)
+- Added options to ignore permissions, ownership and groups
+- Improved batch runner
+	- Batch runner works for directories and direct paths
+	- Fixed batch runner does not rerun obackup on warnings only
+	- Code compliance
+	- More clear semantic
+- Made keep logging value configurable and not mandatory
+- Fixed handling of processes in uninterruptible sleep state
+- Code cleanup
+- Refactored waiting functions
+- Fixed double RunAfterHook launch
 
 06 Aug 2016: obackup v2.0 released
+----------------------------------
+
 - Made logging begin before remote checks for sanity purposes
 - RunAfterCommands can get executed when trapquit
 - Improved process killing and process time control
@@ -38,6 +137,8 @@ README: FreeBSD execution needs mailer (not found), sudo missing, bash needed, s
 - A long list of minor improvements and bug fixes
 
 v0-1.x - Jan 2013 - Oct 2015
+----------------------------
+
 - New function to kill child processes
 - Fixed no_maxtime not honored
 - Improved some logging, also added highlighting to stdout errors
@@ -96,7 +197,10 @@ v0-1.x - Jan 2013 - Oct 2015
 - Improved OS detection and added prelimnary MacOS X support
 - Improved execution hook logs
 - Improved RunLocalCommand execution hook
-- 02 Nov. 2013: v1.84 RC3
+
+02 Nov. 2013: obackup v1.84RC3 released
+---------------------------------------
+
 - Updated documentation
 - Minor rewrites in recursive backup code
 - Added base directory files backup for recursive directories backup
@@ -123,7 +227,10 @@ v0-1.x - Jan 2013 - Oct 2015
 - Improved dryrun output
 - Improved remote connecivity detection 
 - Fixed a typo in configuration file
-- 18 Aug. 2013: Now v1.84 RC2
+
+18 Aug. 2013: obackup v1.84RC2 released
+---------------------------------------
+
 - Added possibility to change default logfile
 - Simplified dryrun (removed dryrun function and merged it with main function)
 - Simplified Init function
@@ -135,14 +242,17 @@ v0-1.x - Jan 2013 - Oct 2015
 - Added --verbose switch (will add databases list,  rsync commands, and file backup list)
 - Improved task execution checks and more code cleanup
 - Fixed CleanUp function if DEBUG=yes, also function is now launched from TrapQuit
-- 16 Jul. 2013: version tagged as v1.84 RC1
+
+16 Jul. 2013: obackup v1.84RC1 released
+---------------------------------------
+
 - Code cleanup
 - Uploaded first documentation
 - Fixed an issue with RotateBackups
 - Updated obackup to log failed ssh command results
 - Updated ssh command filter to log failed commands
 - Updated ssh command filter to accept personalized commands
-- 23 Jun. 2013 v 1.84 RC1 approaching
+- 23 Jun. 2013: v1.84 RC1 approaching
 - Added ssh commands filter, updated documentation
 - Rewrote local space check function
 - Added ability to run another executable than rsync (see documentation on sudo execution)
@@ -152,5 +262,8 @@ v0-1.x - Jan 2013 - Oct 2015
 - Updated command line argument --silent processing
 - Added remote before and after command execution hook
 - Added local before and after command execution hook
-- 14 Jun 2013
+
+14 Jun 2013
+-----------
+
 - Initial public release, fully functionnal

README.md

@@ -1,7 +1,8 @@
-obackup
-=======
+# obackup [![Build Status](https://travis-ci.org/deajan/obackup.svg?branch=master)](https://travis-ci.org/deajan/obackup) [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![GitHub Release](https://img.shields.io/github/release/deajan/obackup.svg?label=Latest)](https://github.com/deajan/obackup/releases/latest)
 
 A robust file & database backup script that works for local and remote push or pull backups via ssh.
+Designed to backup multiple subdirectories with a timeslot for each.
+Supports encryption while still using rsync to lower transfered data (see advantages and caveats below).
 
 ## About
 
@@ -34,7 +35,7 @@ You may disable this behavior in the config file.
 You can download the latest obackup script from authors website.
 You may also clone the following git which will maybe have some more recent builds.
 
-    $ git clone -b "v2.0" git://github.com/deajan/obackup.git
+    $ git clone -b "v2.1-maint" git://github.com/deajan/obackup.git
     $ cd obackup
     $ ./install.sh
 
@@ -69,6 +70,10 @@ All backup activity is logged to "/var/log/obackup_backupname.log" or current di
 
 You may mix "--silent" and "--verbose" parameters to output verbose input only in the log files.
 
+## Troubleshooting
+
+Whenever you may encounter rsync zombie processes and/or in unterruptible sleep state processes, you should force unmounting network drives obackup is supposed to deal with.
+
 ## Final words
 
 Backup tasks aren't always reliable, connectivity loss, insufficient disk space, hacked servers with tons of unusefull stuff to backup... Anything can happen.

dev/bootstrap.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+## dev pre-processor bootstrap rev 2017062001
+## Yeah !!! A really tech sounding name... In fact it's just include emulation in bash
+
+function Usage {
+	echo "$0 - Quick and dirty preprocessor for including ofunctions into programs"
+	echo "Creates and executes $0.tmp.sh"
+	echo "Usage:"
+	echo ""
+	echo "$0 --program=osync|osync_target_helper|obackup|pmocr [options to pass to program]"
+}
+
+
+if [ ! -f "./merge.sh" ]; then
+	echo "Plrase run bootstrap.sh from osync/dev directory."
+	exit 1
+fi
+
+bootstrapProgram=""
+opts=""
+outputFileName="$0"
+
+for i in "$@"; do
+        case $i in
+                --program=*)
+                bootstrapProgram="${i##*=}"
+		;;
+		*)
+		opts=$opts" $i"
+		;;
+	esac
+done
+
+if [ "$bootstrapProgram" == "" ]; then
+	Usage
+	exit 128
+else
+	source "merge.sh"
+
+	__PREPROCESSOR_PROGRAM=$bootstrapProgram
+	__PREPROCESSOR_PROGRAM_EXEC="n_$bootstrapProgram.sh"
+	__PREPROCESSOR_Constants
+
+	if [ ! -f "$__PREPROCESSOR_PROGRAM_EXEC" ]; then
+		echo "Cannot find file [n_$bootstrapProgram.sh]."
+		exit 1
+	fi
+fi
+
+cp "$__PREPROCESSOR_PROGRAM_EXEC" "$outputFileName.tmp.sh"
+if [ $? != 0 ]; then
+	echo "Cannot copy original file [$__PREPROCESSOR_PROGRAM_EXEC] to [$outputFileName.tmp.sh]."
+	exit 1
+fi
+for subset in "${__PREPROCESSOR_SUBSETS[@]}"; do
+	__PREPROCESSOR_MergeSubset "$subset" "${subset//SUBSET/SUBSET END}" "ofunctions.sh" "$outputFileName.tmp.sh"
+done
+chmod +x "$outputFileName.tmp.sh"
+if [ $? != 0 ]; then
+	echo "Cannot make [$outputFileName] executable."
+	exit 1
+fi
+
+# Termux fix
+if type termux-fix-shebang > /dev/null 2>&1; then
+	termux-fix-shebang "$outputFileName.tmp.sh"
+fi
+
+"$outputFileName.tmp.sh" $opts

dev/common_batch.sh

@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+SUBPROGRAM=[prgname]
+PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
+AUTHOR="(L) 2013-2017 by Orsiris de Jong"
+CONTACT="http://www.netpower.fr - ozy@netpower.fr"
+PROGRAM_BUILD=2016120401
+
+## Runs an osync /obackup instance for every conf file found
+## If an instance fails, run it again if time permits
+
+if ! type "$BASH" > /dev/null; then
+	echo "Please run this script only with bash shell. Tested on bash >= 3.2"
+	exit 127
+fi
+
+## If maximum execution time is not reached, failed instances will be rerun. Max exec time is in seconds. Example is set to 10 hours.
+MAX_EXECUTION_TIME=36000
+
+## Specifies the number of total runs an instance may get
+MAX_RUNS=3
+
+## Log file path
+if [ -w /var/log ]; then
+	LOG_FILE=/var/log/$SUBPROGRAM-batch.log
+else
+	LOG_FILE=./$SUBPROGRAM-batch.log
+fi
+
+# No need to edit under this line ##############################################################
+
+function _logger {
+	local value="${1}" # What to log
+	echo -e "$value" >> "$LOG_FILE"
+}
+
+function Logger {
+	local value="${1}" # What to log
+	local level="${2}" # Log level: DEBUG, NOTICE, WARN, ERROR, CRITIAL
+
+	prefix="$(date) - "
+
+	if [ "$level" == "CRITICAL" ]; then
+		_logger "$prefix\e[41m$value\e[0m"
+	elif [ "$level" == "ERROR" ]; then
+		_logger "$prefix\e[91m$value\e[0m"
+	elif [ "$level" == "WARN" ]; then
+		_logger "$prefix\e[93m$value\e[0m"
+	elif [ "$level" == "NOTICE" ]; then
+		_logger "$prefix$value"
+	elif [ "$level" == "DEBUG" ]; then
+		if [ "$DEBUG" == "yes" ]; then
+			_logger "$prefix$value"
+		fi
+	else
+		_logger "\e[41mLogger function called without proper loglevel.\e[0m"
+		_logger "$prefix$value"
+	fi
+}
+
+function CheckEnvironment {
+	## osync / obackup executable full path can be set here if it cannot be found on the system
+	if ! type $SUBPROGRAM.sh > /dev/null 2>&1
+	then
+		if [ -f /usr/local/bin/$SUBPROGRAM.sh ]
+		then
+			SUBPROGRAM_EXECUTABLE=/usr/local/bin/$SUBPROGRAM.sh
+		else
+			Logger "Could not find [/usr/local/bin/$SUBPROGRAM.sh]" "CRITICAL"
+			( >&2 echo "Could not find [/usr/local/bin/$SUBPROGRAM.sh]" )
+			exit 1
+		fi
+	else
+		SUBPROGRAM_EXECUTABLE=$(type -p $SUBPROGRAM.sh)
+	fi
+
+	if [ "$CONF_FILE_PATH" == "" ]; then
+		Usage
+	fi
+}
+
+function Batch {
+	local runs=1 # Number of batch runs
+	local runList # Actual conf file list to run
+	local runAgainList # List of failed conf files sto run again
+
+	local confFile
+	local result
+
+	local i
+
+	# Using -e because find will accept directories or files
+	if [ ! -e "$CONF_FILE_PATH" ]; then
+		Logger "Cannot find conf file path [$CONF_FILE_PATH]." "CRITICAL"
+		Usage
+	else
+		# Ugly hack to read files into an array while preserving special characters
+		runList=()
+		while IFS= read -d $'\0' -r file; do runList+=("$file"); done < <(find "$CONF_FILE_PATH" -maxdepth 1 -iname "*.conf" -print0)
+
+		while ([ $MAX_EXECUTION_TIME -gt $SECONDS ] || [ $MAX_EXECUTION_TIME -eq 0 ]) && [ "${#runList[@]}" -gt 0 ] && [ $runs -le $MAX_RUNS ]; do
+			runAgainList=()
+			Logger "Sequential run n°$runs of $SUBPROGRAM instances for:" "NOTICE"
+			for confFile in "${runList[@]}"; do
+				Logger "$(basename $confFile)" "NOTICE"
+			done
+			for confFile in "${runList[@]}"; do
+				$SUBPROGRAM_EXECUTABLE "$confFile" --silent $opts &
+				wait $!
+				result=$?
+				if [ $result != 0 ]; then
+					if [ $result == 1 ] || [ $result == 128 ]; then # Do not handle exit code 128 because it is already handled here
+						Logger "Instance $(basename $confFile) failed with exit code [$result]." "ERROR"
+						runAgainList+=("$confFile")
+					elif [ $result == 2 ]; then
+						Logger "Instance $(basename $confFile) finished with warnings." "WARN"
+					fi
+				else
+					Logger "Instance $(basename $confFile) succeed." "NOTICE"
+				fi
+			done
+			runList=("${runAgainList[@]}")
+			runs=$((runs + 1))
+		done
+	fi
+}
+
+function Usage {
+	echo "$PROGRAM $PROGRAM_BUILD"
+	echo "$AUTHOR"
+	echo "$CONTACT"
+	echo ""
+	echo "Batch script to sequentially run osync or obackup instances and rerun failed ones."
+	echo "Usage: $PROGRAM.sh [OPTIONS] [$SUBPROGRAM OPTIONS]"
+	echo ""
+	echo "[OPTIONS]"
+	echo "--path=/path/to/conf      Path to osync / obackup conf files, defaults to /etc/osync or /etc/obackup"
+	echo "--max-runs=X              Number of max runs per instance, (defaults to 3)"
+	echo "--max-exec-time=X         Retry failed instances only if max execution time not reached (defaults to 36000 seconds). Set to 0 to bypass execution time check"
+	echo "[$SUBPROGRAM OPTIONS]"
+	echo "Specify whatever options $PROGRAM accepts. Example"
+	echo "$PROGRAM.sh --path=/etc/$SUBPROGRAM --no-maxtime"
+	echo ""
+	echo "No output will be written to stdout/stderr."
+	echo "Verify log file in [$LOG_FILE]."
+	exit 128
+}
+
+opts=""
+for i in "$@"
+do
+	case $i in
+		--path=*)
+		CONF_FILE_PATH=${i##*=}
+		;;
+		--max-runs=*)
+		MAX_RUNS=${i##*=}
+		;;
+		--max-exec-time=*)
+		MAX_EXECUTION_TIME=${i##*=}
+		;;
+		--help|-h|-?)
+		Usage
+		;;
+		*)
+		opts="$opts$i "
+		;;
+	esac
+done
+
+CheckEnvironment
+Logger "$(date) $SUBPROGRAM batch run" "NOTICE"
+Batch

dev/common_install.sh

@@ -0,0 +1,360 @@
+#!/usr/bin/env bash
+
+## Installer script suitable for osync / obackup / pmocr
+
+include #### _OFUNCTIONS_BOOTSTRAP SUBSET ####
+
+PROGRAM=[prgname]
+
+PROGRAM_VERSION=$(grep "PROGRAM_VERSION=" $PROGRAM.sh)
+PROGRAM_VERSION=${PROGRAM_VERSION#*=}
+PROGRAM_BINARY=$PROGRAM".sh"
+PROGRAM_BATCH=$PROGRAM"-batch.sh"
+SSH_FILTER="ssh_filter.sh"
+
+SCRIPT_BUILD=2017041701
+
+## osync / obackup / pmocr / zsnap install script
+## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
+## Please adapt this to fit your distro needs
+
+# Get current install.sh path from http://stackoverflow.com/a/246128/2635443
+SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+CONF_DIR=$FAKEROOT/etc/$PROGRAM
+BIN_DIR="$FAKEROOT/usr/local/bin"
+SERVICE_DIR_INIT=$FAKEROOT/etc/init.d
+# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora
+SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system
+SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user
+
+if [ "$PROGRAM" == "osync" ]; then
+	SERVICE_NAME="osync-srv"
+elif [ "$PROGRAM" == "pmocr" ]; then
+	SERVICE_NAME="pmocr-srv"
+fi
+
+SERVICE_FILE_INIT="$SERVICE_NAME"
+SERVICE_FILE_SYSTEMD_SYSTEM="$SERVICE_NAME@.service"
+SERVICE_FILE_SYSTEMD_USER="$SERVICE_NAME@.service.user"
+
+## Generic code
+
+## Default log file
+if [ -w "$FAKEROOT/var/log" ]; then
+	LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log"
+elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
+	LOG_FILE="$HOME/$PROGRAM-install.log"
+else
+	LOG_FILE="./$PROGRAM-install.log"
+fi
+
+include #### QuickLogger SUBSET ####
+include #### UrlEncode SUBSET ####
+include #### GetLocalOS SUBSET ####
+include #### GetConfFileValue SUBSET ####
+
+function SetLocalOSSettings {
+	USER=root
+
+	# LOCAL_OS and LOCAL_OS_FULL are global variables set at GetLocalOS
+
+	case $LOCAL_OS in
+		*"BSD"*)
+		GROUP=wheel
+		;;
+		*"MacOSX"*)
+		GROUP=admin
+		;;
+		*"msys"*|*"Cygwin"*)
+		USER=""
+		GROUP=""
+		;;
+		*)
+		GROUP=root
+		;;
+	esac
+
+	if [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "BusyBox" ]; then
+		QuickLogger "Cannot be installed on [$LOCAL_OS]. Please use $PROGRAM.sh directly."
+		exit 1
+	fi
+
+	if ([ "$USER" != "" ] && [ "$(whoami)" != "$USER" ] && [ "$FAKEROOT" == "" ]); then
+		QuickLogger "Must be run as $USER."
+		exit 1
+	fi
+
+	OS=$(UrlEncode "$LOCAL_OS_FULL")
+}
+
+function GetInit {
+	if [ -f /sbin/init ]; then
+		if file /sbin/init | grep systemd > /dev/null; then
+			init="systemd"
+		else
+			init="initV"
+		fi
+	else
+		QuickLogger "Can't detect initV or systemd. Service files won't be installed. You can still run $PROGRAM manually or via cron."
+		init="none"
+	fi
+}
+
+function CreateDir {
+	local dir="${1}"
+
+	if [ ! -d "$dir" ]; then
+		mkdir "$dir"
+		if [ $? == 0 ]; then
+			QuickLogger "Created directory [$dir]."
+		else
+			QuickLogger "Cannot create directory [$dir]."
+			exit 1
+		fi
+	fi
+}
+
+function CopyFile {
+	local sourcePath="${1}"
+	local destPath="${2}"
+	local fileName="${3}"
+	local fileMod="${4}"
+	local fileUser="${5}"
+	local fileGroup="${6}"
+	local overwrite="${7:-false}"
+
+	local userGroup=""
+	local oldFileName
+
+	if [ -f "$destPath/$fileName" ] && [ $overwrite == false ]; then
+		oldFileName="$fileName"
+		fileName="$oldFileName.new"
+		cp "$sourcePath/$oldFileName" "$destPath/$fileName"
+	else
+		cp "$sourcePath/$fileName" "$destPath"
+	fi
+
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot copy [$fileName] to [$destPath]. Make sure to run install script in the directory containing all other files."
+		QuickLogger "Also make sure you have permissions to write to [$BIN_DIR]."
+		exit 1
+	else
+		QuickLogger "Copied [$fileName] to [$destPath]."
+		if [ "$fileMod" != "" ]; then
+			chmod "$fileMod" "$destPath/$fileName"
+			if [ $? != 0 ]; then
+				QuickLogger "Cannot set file permissions of [$destPath/$fileName] to [$fileMod]."
+				exit 1
+			else
+				QuickLogger "Set file permissions to [$fileMod] on [$destPath/$fileName]."
+			fi
+		fi
+
+		if [ "$fileUser" != "" ]; then
+			userGroup="$fileUser"
+
+			if [ "$fileGroup" != "" ]; then
+				userGroup="$userGroup"":$fileGroup"
+			fi
+
+			chown "$userGroup" "$destPath/$fileName"
+			if [ $? != 0 ]; then
+				QuickLogger "Could not set file ownership on [$destPath/$fileName] to [$userGroup]."
+				exit 1
+			else
+				QuickLogger "Set file ownership on [$destPath/$fileName] to [$userGroup]."
+			fi
+		fi
+	fi
+}
+
+function CopyExampleFiles {
+	exampleFiles=()
+	exampleFiles[0]="sync.conf.example"		# osync
+	exampleFiles[1]="host_backup.conf.example"	# obackup
+	exampleFiles[2]="exclude.list.example"		# osync & obackup
+	exampleFiles[3]="snapshot.conf.example"		# zsnap
+	exampleFiles[4]="default.conf"			# pmocr
+
+	for file in "${exampleFiles[@]}"; do
+		if [ -f "$SCRIPT_PATH/$file" ]; then
+			CopyFile "$SCRIPT_PATH" "$CONF_DIR" "$file" "" "" "" false
+		fi
+	done
+}
+
+function CopyProgram {
+	binFiles=()
+	binFiles[0]="$PROGRAM_BINARY"
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
+		binFiles[1]="$PROGRAM_BATCH"
+		binFiles[2]="$SSH_FILTER"
+	fi
+
+	local user=""
+	local group=""
+
+	if ([ "$USER" != "" ] && [ "$FAKEROOT" == "" ]); then
+		user="$USER"
+	fi
+	if ([ "$GROUP" != "" ] && [ "$FAKEROOT" == "" ]); then
+		group="$GROUP"
+	fi
+
+	for file in "${binFiles[@]}"; do
+		CopyFile "$SCRIPT_PATH" "$BIN_DIR" "$file" 755 "$user" "$group" true
+	done
+}
+
+function CopyServiceFiles {
+	if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
+		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true
+		if [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM_USER" ]; then
+			CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true
+		fi
+
+		QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
+		QuickLogger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR."
+		QuickLogger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]."
+		QuickLogger "In userland, active with [systemctl --user start $SERVICE_NAME@instance.conf]."
+	elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then
+		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true
+
+		QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]."
+		QuickLogger "Can be activated with [service $SERVICE_FILE_INIT start]."
+		QuickLogger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]."
+	else
+		QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation."
+	fi
+}
+
+function Statistics {
+	if type wget > /dev/null; then
+		wget -qO- "$STATS_LINK" > /dev/null 2>&1
+		if [ $? == 0 ]; then
+			return 0
+		fi
+	fi
+
+	if type curl > /dev/null; then
+		curl "$STATS_LINK" -o /dev/null > /dev/null 2>&1
+		if [ $? == 0 ]; then
+			return 0
+		fi
+	fi
+
+	QuickLogger "Neiter wget nor curl could be used for. Cannot run statistics. Use the provided link please."
+	return 1
+}
+
+function RemoveFile {
+	local file="${1}"
+
+	if [ -f "$file" ]; then
+		rm -f "$file"
+		if [ $? != 0 ]; then
+			QuickLogger "Could not remove file [$file]."
+		else
+			QuickLogger "Removed file [$file]."
+		fi
+	else
+		QuickLogger "File [$file] not found. Skipping."
+	fi
+}
+
+function RemoveAll {
+	RemoveFile "$BIN_DIR/$PROGRAM_BINARY"
+
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
+		RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
+	fi
+
+	if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then		# Check if any other program requiring ssh filter is present before removal
+		RemoveFile "$BIN_DIR/$SSH_FILTER"
+	else
+		QuickLogger "Skipping removal of [$BIN_DIR/$SSH_FILTER] because other programs present that need it."
+	fi
+	RemoveFile "$SERVICE_DIR_SYSTEMD_SYSTEM/$SERVICE_FILE_SYSTEMD_SYSTEM"
+	RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM"
+	RemoveFile "$SERVICE_DIR_INIT/$SERVICE_FILE_INIT"
+
+	QuickLogger "Skipping configuration files in [$CONF_DIR]. You may remove this directory manually."
+}
+
+function Usage {
+	echo "Installs $PROGRAM into $BIN_DIR"
+	echo "options:"
+	echo "--silent		Will log and bypass user interaction."
+	echo "--no-stats	Used with --silent in order to refuse sending anonymous install stats."
+	echo "--remove          Remove the program."
+	exit 127
+}
+
+_LOGGER_SILENT=false
+_STATS=1
+ACTION="install"
+
+for i in "$@"
+do
+	case $i in
+		--silent)
+		_LOGGER_SILENT=true
+		;;
+		--no-stats)
+		_STATS=0
+		;;
+		--remove)
+		ACTION="uninstall"
+		;;
+		--help|-h|-?)
+		Usage
+	esac
+done
+
+if [ "$FAKEROOT" != "" ]; then
+	mkdir -p "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_USER" "$BIN_DIR"
+fi
+
+GetLocalOS
+SetLocalOSSettings
+GetInit
+
+STATS_LINK="http://instcount.netpower.fr?program=$PROGRAM&version=$PROGRAM_VERSION&os=$OS&action=$ACTION"
+
+if [ "$ACTION" == "uninstall" ]; then
+	RemoveAll
+	QuickLogger "$PROGRAM uninstalled."
+else
+	CreateDir "$CONF_DIR"
+	CreateDir "$BIN_DIR"
+	CopyExampleFiles
+	CopyProgram
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "pmocr" ]; then
+		CopyServiceFiles
+	fi
+	QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
+		QuickLogger ""
+		QuickLogger "If connecting remotely, consider setup ssh filter to enhance security."
+		QuickLogger ""
+	fi
+fi
+
+if [ $_STATS -eq 1 ]; then
+	if [ $_LOGGER_SILENT == true ]; then
+		Statistics
+	else
+		QuickLogger "In order to make usage statistics, the script would like to connect to $STATS_LINK"
+		read -r -p "No data except those in the url will be send. Allow [Y/n] " response
+		case $response in
+			[nN])
+			exit
+			;;
+			*)
+			Statistics
+			exit $?
+			;;
+		esac
+	fi
+fi

dev/merge.sh

@@ -1,71 +1,199 @@
 #!/usr/bin/env bash
 
-## MERGE 2016080601
+## MERGE 2017061901
 
 ## Merges ofunctions.sh and n_program.sh into program.sh
 ## Adds installer
 
-PROGRAM=obackup
-VERSION=$(grep "PROGRAM_VERSION=" n_$PROGRAM.sh)
-VERSION=${VERSION#*=}
-
-PARANOIA_DEBUG_LINE="__WITH_PARANOIA_DEBUG"
-PARANOIA_DEBUG_BEGIN="#__BEGIN_WITH_PARANOIA_DEBUG"
-PARANOIA_DEBUG_END="#__END_WITH_PARANOIA_DEBUG"
-MINIMUM_FUNCTION_BEGIN="#### MINIMAL-FUNCTION-SET BEGIN ####"
-MINIMUM_FUNCTION_END="#### MINIMAL-FUNCTION-SET END ####"
-
-function Unexpand {
-	unexpand n_$PROGRAM.sh > tmp_$PROGRAM.sh
+function Usage {
+	echo "Merges ofunctions.sh and n_program.sh into debug_program.sh and ../program.sh"
+	echo "Usage"
+	echo "$0 osync|obackup|pmocr"
 }
 
-function MergeAll {
+function __PREPROCESSOR_Merge {
+	local PROGRAM="$1"
 
-	sed "/source \"\.\/ofunctions.sh\"/r ofunctions.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' > debug_$PROGRAM.sh
-	chmod +x debug_$PROGRAM.sh
+	VERSION=$(grep "PROGRAM_VERSION=" n_$PROGRAM.sh)
+	VERSION=${VERSION#*=}
+	__PREPROCESSOR_Constants
+
+	source "ofunctions.sh"
+	if [ $? != 0 ]; then
+		echo "Please run $0 in dev directory with ofunctions.sh"
+		exit 1
+	fi
+
+	__PREPROCESSOR_Unexpand "n_$PROGRAM.sh" "debug_$PROGRAM.sh"
+
+	for subset in "${__PREPROCESSOR_SUBSETS[@]}"; do
+		__PREPROCESSOR_MergeSubset "$subset" "${subset//SUBSET/SUBSET END}" "ofunctions.sh" "debug_$PROGRAM.sh"
+	done
+
+	__PREPROCESSOR_CleanDebug "$PROGRAM"
+	rm -f tmp_$PROGRAM.sh
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot remove tmp_$PROGRAM.sh"
+		exit 1
+	fi
 }
 
-function MergeMinimum {
-        sed -n "/$MINIMUM_FUNCTION_BEGIN/,/$MINIMUM_FUNCTION_END/p" ofunctions.sh > tmp_minimal.sh
-        sed "/source \"\.\/ofunctions.sh\"/r tmp_minimal.sh" tmp_$PROGRAM.sh | grep -v 'source "./ofunctions.sh"' | grep -v "$PARANOIA_DEBUG_LINE" > debug_$PROGRAM.sh
-	rm -f tmp_minimal.sh
-        chmod +x debug_$PROGRAM.sh
+function __PREPROCESSOR_Constants {
+	PARANOIA_DEBUG_LINE="#__WITH_PARANOIA_DEBUG"
+	PARANOIA_DEBUG_BEGIN="#__BEGIN_WITH_PARANOIA_DEBUG"
+	PARANOIA_DEBUG_END="#__END_WITH_PARANOIA_DEBUG"
+
+	__PREPROCESSOR_SUBSETS=(
+	'#### OFUNCTIONS FULL SUBSET ####'
+	'#### OFUNCTIONS MINI SUBSET ####'
+	'#### _OFUNCTIONS_BOOTSTRAP SUBSET ####'
+	'#### DEBUG SUBSET ####'
+	'#### TrapError SUBSET ####'
+	'#### RemoteLogger SUBSET ####'
+	'#### QuickLogger SUBSET ####'
+	'#### GetLocalOS SUBSET ####'
+	'#### IsInteger SUBSET ####'
+	'#### UrlEncode SUBSET ####'
+	'#### HumanToNumeric SUBSET ####'
+	'#### ArrayContains SUBSET ####'
+	'#### VerComp SUBSET ####'
+	'#### GetConfFileValue SUBSET ####'
+	'#### SetConfFileValue SUBSET ####'
+	)
 }
 
+function __PREPROCESSOR_Unexpand {
+	local source="${1}"
+	local destination="${2}"
 
-function CleanDebug {
-
-# sed explanation
-#/pattern1/{         # if pattern1 is found
-#    p               # print it
-#    :a              # loop
-#        N           # and accumulate lines
-#    /pattern2/!ba   # until pattern2 is found
-#    s/.*\n//        # delete the part before pattern2
-#}
-#p
-
-	sed -n '/'$PARANOIA_DEBUG_BEGIN'/{p; :a; N; /'$PARANOIA_DEBUG_END'/!ba; s/.*\n//}; p' debug_$PROGRAM.sh | grep -v "$PARANOIA_DEBUG_LINE" > ../$PROGRAM.sh
-	chmod +x ../$PROGRAM.sh
+	unexpand "$source" > "$destination"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot unexpand [$source] to [$destination]."
+		exit 1
+	fi
 }
 
-function CopyCommons {
-	sed "s/\[prgname\]/$PROGRAM/g" common_install.sh > ../tmp_install.sh
-	sed "s/\[version\]/$VERSION/g" ../tmp_install.sh > ../install.sh
+function __PREPROCESSOR_MergeSubset {
+	local subsetBegin="${1}"
+	local subsetEnd="${2}"
+	local subsetFile="${3}"
+	local mergedFile="${4}"
+
+	sed -n "/$subsetBegin/,/$subsetEnd/p" "$subsetFile" > "$subsetFile.$subsetBegin"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot sed subset [$subsetBegin -- $subsetEnd] in [$subsetFile]."
+		exit 1
+	fi
+	sed "/include $subsetBegin/r $subsetFile.$subsetBegin" "$mergedFile" | grep -v -E "$subsetBegin\$|$subsetEnd\$" > "$mergedFile.tmp"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot add subset [$subsetBegin] to [$mergedFile]."
+		exit 1
+	fi
+	rm -f "$subsetFile.$subsetBegin"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot remove temporary subset [$subsetFile.$subsetBegin]."
+		exit 1
+	fi
+
+	rm -f "$mergedFile"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot remove merged original file [$mergedFile]."
+		exit 1
+	fi
+
+	mv "$mergedFile.tmp" "$mergedFile"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot move merged tmp file to original [$mergedFile]."
+		exit 1
+	fi
+}
+
+function __PREPROCESSOR_CleanDebug {
+	local PROGRAM="$1"
+
+	sed '/'$PARANOIA_DEBUG_BEGIN'/,/'$PARANOIA_DEBUG_END'/d' debug_$PROGRAM.sh | grep -v "$PARANOIA_DEBUG_LINE" > ../$PROGRAM.sh
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot remove PARANOIA_DEBUG code from standard build."
+		exit 1
+	fi
+
+	chmod +x "debug_$PROGRAM.sh"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot chmod debug_$PROGRAM.sh"
+		exit 1
+	else
+		QuickLogger "Prepared ./debug_$PROGRAM.sh"
+	fi
+	chmod +x "../$PROGRAM.sh"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot chmod $PROGRAM.sh"
+		exit 1
+	else
+		QuickLogger "Prepared ../$PROGRAM.sh"
+	fi
+}
+
+function __PREPROCESSOR_CopyCommons {
+	local PROGRAM="$1"
+
+	sed "s/\[prgname\]/$PROGRAM/g" common_install.sh > ../install.sh
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot assemble install."
+		exit 1
+	fi
+
+	for subset in "${__PREPROCESSOR_SUBSETS[@]}"; do
+		__PREPROCESSOR_MergeSubset "$subset" "${subset//SUBSET/SUBSET END}" "ofunctions.sh" "../install.sh"
+	done
+
+	#sed "s/\[version\]/$VERSION/g" ../tmp_install.sh > ../install.sh
+	#if [ $? != 0 ]; then
+	#	QuickLogger "Cannot change install version."
+	#	exit 1
+	#fi
 	if [ -f "common_batch.sh" ]; then
 		sed "s/\[prgname\]/$PROGRAM/g" common_batch.sh > ../$PROGRAM-batch.sh
+		if [ $? != 0 ]; then
+			QuickLogger "Cannot assemble batch runner."
+			exit 1
+		fi
+		chmod +x ../$PROGRAM-batch.sh
+		if [ $? != 0 ]; then
+			QuickLogger "Cannot chmod $PROGRAM-batch.sh"
+			exit 1
+		else
+			QuickLogger "Prepared ../$PROGRAM-batch.sh"
+		fi
 	fi
 	chmod +x ../install.sh
-	chmod +x ../$PROGRAM-batch.sh
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot chmod install.sh"
+		exit 1
+	else
+		QuickLogger "Prepared ../install.sh"
+	fi
 	rm -f ../tmp_install.sh
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot chmod $PROGRAM.sh"
+		exit 1
+	fi
 }
 
-Unexpand
-if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
-	MergeAll
-else
-	MergeMinimum
+# If sourced don't do anything
+if [ "$(basename $0)" == "merge.sh" ]; then
+	if [ "$1" == "osync" ]; then
+
+		__PREPROCESSOR_Merge osync
+		__PREPROCESSOR_Merge osync_target_helper
+		__PREPROCESSOR_CopyCommons osync
+	elif [ "$1" == "obackup" ]; then
+		__PREPROCESSOR_Merge obackup
+		__PREPROCESSOR_CopyCommons obackup
+	elif [ "$1" == "pmocr" ]; then
+		__PREPROCESSOR_Merge pmocr
+		__PREPROCESSOR_CopyCommons pmocr
+	else
+		echo "No valid program given."
+		exit 1
+	fi
 fi
-CleanDebug
-CopyCommons
-rm -f tmp_$PROGRAM.sh

dev/shellcheck.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+#SC1090 = not following non constants source
+#SC1091 = not following source
+#SC2086 = quoting errors (shellcheck is way too picky about quoting)
+#SC2120 = only for debug version
+
+shellcheck -e SC1090,SC1091,SC2086,SC2119,SC2120 $1

dev/tests/conf/local.conf

@@ -0,0 +1,228 @@
+#!/usr/bin/env bash
+
+###### obackup - Local or Remote, push or pull backup script for files & mysql databases
+###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr)
+###### obackup v2.1x config file rev 2016081701
+
+###### GENERAL BACKUP OPTIONS
+
+## Backup identification string.
+INSTANCE_ID="local-test"
+
+## Log file location. Leaving this empty will create log file at /var/log/obackup.INSTANCE_ID.log (or current directory if /var/log doesn't exist).
+LOGFILE=""
+
+## Elements to backup
+SQL_BACKUP=yes
+FILE_BACKUP=yes
+
+## Backups can be done local, pulled from another server or pushed to a backup server. Available options are [local,pull,push].
+## Pulled backups are the safest option, as the backup server contains the RSA key and cannot be compromised by another server.
+BACKUP_TYPE=local
+
+###### BACKUP STORAGE
+
+## Storage paths of the backups (absolute paths of the local or remote system)
+SQL_STORAGE="${HOME}/obackup-storage/sql-local"
+FILE_STORAGE="${HOME}/obackup-storage/files-local"
+
+## Encryption
+ENCRYPTION=yes
+
+## Backup encryption needs a temporary storage space in order to encrypt files before sending them (absolute paths of the local or remote system)
+CRYPT_STORAGE="${HOME}/obackup-storage/crypt-local"
+ 
+## GPG recipient (pubkey for this recipient must exist, see gpg2 --list-keys or gpg --list-keys
+GPG_RECIPIENT="John Doe"
+
+## Create backup directories if they do not exist
+CREATE_DIRS=yes
+
+## Keep absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
+## You should leave this enabled if you intend to use 'backup task division' functionality of oBackup, or everything will end up in the same directory.
+KEEP_ABSOLUTE_PATHS=yes
+
+## Generate an alert if backup size is lower than given value in Kb (this can also help identifying empty mount dirs).
+BACKUP_SIZE_MINIMUM=1024
+
+## Check backup size before proceeding
+GET_BACKUP_SIZE=yes
+
+## Generate an alert if storage free space is lower than given value in Kb.
+## Keep in mind that disabling backup file size test will only test min space against SQL backup size.
+SQL_WARN_MIN_SPACE=1048576
+FILE_WARN_MIN_SPACE=1048576
+
+###### REMOTE ONLY OPTIONS
+
+## In case of pulled or pushed backups, remote system URI needs to be supplied.
+REMOTE_SYSTEM_URI="ssh://backupuser@remote.system.tld:22/"
+
+## You can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
+SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa"
+
+## ssh compression should be used unless your remote connection is good enough (LAN)
+SSH_COMPRESSION=yes
+
+## Ignore ssh known hosts verification. DANGER WILL ROBINSON DANGER: This can lead to security risks. Only enable if you know what you're doing.
+SSH_IGNORE_KNOWN_HOSTS=no
+
+## Remote rsync executable path. Leave this empty in most cases
+RSYNC_REMOTE_PATH=""
+
+## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
+REMOTE_HOST_PING=yes
+
+## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
+REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
+
+## If enabled, commands will be executed as superuser on remote side. See documentation for /etc/sudoers configuration ("find", "du", "tee" and "rsync" need to be allowed). Requiretty needs to be disabled.
+SUDO_EXEC=no
+
+###### DATABASE SPECIFIC OPTIONS
+
+## Database backup user
+SQL_USER=root
+
+## Enabling the following option will save all databases on local or remote given SQL instance except the ones specified in the exclude list.
+## Every found database will be backed up as separate backup task.
+DATABASES_ALL=yes
+DATABASES_ALL_EXCLUDE_LIST="test;information_schema;zarafa_prod"
+## Alternatively, if DATABASES_ALL=no, you can specify a list of databases to backup separated by spaces.
+DATABASES_LIST="mysql"
+
+## Max backup execution time per Database task. Soft max exec time generates a warning only. Hard max exec time generates a warning and stops current backup task.
+## If a task gets stopped, next one in the task list gets executed. Time is specified in seconds.
+SOFT_MAX_EXEC_TIME_DB_TASK=3600
+HARD_MAX_EXEC_TIME_DB_TASK=7200
+
+## mysqldump options (ex: --extended-insert, --single-transaction, --quick...). See MySQL / MariaDB manual
+## default option: --opt
+MYSQLDUMP_OPTIONS="--opt --single-transaction"
+
+## Preferred SQL dump compression. Compression methods can be xz, lzma, pigz or gzip (will fallback from xz to gzip depending if available)
+## Generally, level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
+COMPRESSION_LEVEL=3
+
+###### FILES SPECIFIC OPTIONS
+
+## File backups are divided in tasks. Every directory in DIRECTORY_LIST will be processed as a unique task.
+## Every subdirectory of each directory in RECURSIVE_DIRECTORY_LIST will be processed as a unique task.
+## Example: RECURSIVE_DIRECTORY_LIST="/home;/var" will create backup tasks tasks "/home/dir1, "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/something".
+## You can exclude directories from the avove backup task creation, ex: avoid backing up "/home/dir2" by adding it to RECURSIVE_EXCLUDE_LIST.
+
+## Directories backup list. List of semicolon separated directories that will be backed up.
+DIRECTORY_LIST="${HOME}/obackup-testdata/testData"
+RECURSIVE_DIRECTORY_LIST="${HOME}/obackup-testdata/testDataRecursive"
+RECURSIVE_EXCLUDE_LIST="${HOME}/obackup-testdata/testDataRecursive/Excluded"
+
+## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns)
+RSYNC_PATTERN_FIRST=include
+
+## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work).
+## Paths are relative to sync dirs. List elements are separated by a semicolon.
+RSYNC_INCLUDE_PATTERN=""
+RSYNC_EXCLUDE_PATTERN="*.ded"
+#RSYNC_EXCLUDE_PATTERN="tmp;archives"
+
+## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
+## This file has to be in the same directory as the config file
+## Paths are relative to sync dirs. One element per line.
+RSYNC_INCLUDE_FROM=""
+RSYNC_EXCLUDE_FROM=""
+#RSYNC_EXCLUDE_FROM="exclude.list"
+
+## List separator char. You may set an alternative separator char for your directories lists above.
+PATH_SEPARATOR_CHAR=";"
+
+## Preserve basic linux permissions
+PRESERVE_PERMISSIONS=yes
+PRESERVE_OWNER=yes
+PRESERVE_GROUP=yes
+## On MACOS X, does not work and will be ignored 
+PRESERVE_EXECUTABILITY=yes
+
+## Preserve ACLS. Make sure source and target FS can hold same ACLs or you'll get loads of errors.
+PRESERVE_ACL=no
+## Preserve Xattr. MAke sure source and target FS can hold same Xattr or you'll get loads of errors.
+PRESERVE_XATTR=no
+
+## Transforms symlinks into referent files/dirs
+COPY_SYMLINKS=yes
+## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root.
+KEEP_DIRLINKS=yes
+## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
+PRESERVE_HARDLINKS=no
+
+
+## Let RSYNC compress file transfers. Do not use this on local-local backup schemes. Also, this is not useful if SSH compression is enabled.
+RSYNC_COMPRESS=no
+
+## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
+SOFT_MAX_EXEC_TIME_FILE_TASK=3600
+HARD_MAX_EXEC_TIME_FILE_TASK=7200
+
+## Keep partial uploads that can be resumed on next run, experimental feature
+PARTIAL=no
+
+## Delete files on destination that vanished from source. Do not turn this on unless you enabled backup rotation  or a snapshotting FS like zfs to keep those vanished files on the destination.
+DELETE_VANISHED_FILES=no
+
+## Use delta copy algortithm (usefull when local paths are network drives), defaults to yes
+DELTA_COPIES=yes
+
+## Bandwidth limit Kbytes / second for file backups. Leave 0 to disable limitation.
+BANDWIDTH=0
+
+## Paranoia option. Don't change this unless you read the documentation.
+RSYNC_EXECUTABLE=rsync
+
+###### ALERT OPTIONS
+
+## Alert email addresses separated by a space character
+DESTINATION_MAILS=""
+
+## Windows specific (msys / cygwin environment) only mail options (used with mailsend.exe from muquit, http://github.com/muquit/mailsend or from sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail/
+SENDER_MAIL="alert@your.system.tld"
+SMTP_SERVER=smtp.your.isp.tld
+SMTP_PORT=25
+# encryption can be tls, ssl or none
+SMTP_ENCRYPTION=none
+SMTP_USER=
+SMTP_PASSWORD=
+
+###### GENERAL BACKUP OPTIONS
+
+## Max execution time of whole backup process. Soft max exec time generates a warning only.
+## Hard max exec time generates a warning and stops the whole backup execution.
+SOFT_MAX_EXEC_TIME_TOTAL=30000
+HARD_MAX_EXEC_TIME_TOTAL=36000
+
+## Log a message every KEEP_LOGGING seconds just to know the task is still alive
+KEEP_LOGGING=1801
+
+## Backup Rotation. You may rotate backups if you don't use snapshots on your backup server.
+ROTATE_SQL_BACKUPS=yes
+ROTATE_SQL_COPIES=7
+ROTATE_FILE_BACKUPS=yes
+ROTATE_FILE_COPIES=7
+
+###### EXECUTION HOOKS
+
+## Commands can will be run before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set).
+## This is useful to make a snapshot before backing up data, or even handle snapshots of backed up data.
+LOCAL_RUN_BEFORE_CMD=""
+LOCAL_RUN_AFTER_CMD=""
+
+REMOTE_RUN_BEFORE_CMD=""
+REMOTE_RUN_AFTER_CMD=""
+
+## Max execution time of commands before they get force killed. Leave 0 if you don't want this to happen. Time is specified in seconds.
+MAX_EXEC_TIME_PER_CMD_BEFORE=0
+MAX_EXEC_TIME_PER_CMD_AFTER=0
+
+## Stops whole backup execution if one of the above commands fail
+STOP_ON_CMD_ERROR=no
+
+## Run local and remote after backup cmd's even on failure
+RUN_AFTER_CMD_ON_ERROR=yes

dev/tests/conf/max-exec-time.conf

@@ -0,0 +1,222 @@
+#!/usr/bin/env bash
+
+###### obackup - Local or Remote, push or pull backup script for files & mysql databases
+###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr)
+###### obackup v2.1x config file rev 2016081701
+
+###### GENERAL BACKUP OPTIONS
+
+## Backup identification string.
+INSTANCE_ID="local-test"
+
+## Log file location. Leaving this empty will create log file at /var/log/obackup.INSTANCE_ID.log (or current directory if /var/log doesn't exist).
+LOGFILE=""
+
+## Elements to backup
+SQL_BACKUP=yes
+FILE_BACKUP=yes
+
+## Backups can be done local, pulled from another server or pushed to a backup server. Available options are [local,pull,push].
+## Pulled backups are the safest option, as the backup server contains the RSA key and cannot be compromised by another server.
+BACKUP_TYPE=local
+
+###### BACKUP STORAGE
+
+## Storage paths of the backups (absolute paths of the local or remote system)
+SQL_STORAGE="${HOME}/obackup-storage/sql"
+FILE_STORAGE="${HOME}/obackup-storage/files"
+
+## Backup encryption using GPG and duplicity. Feature not ready yet.
+ENCRYPTION=no
+
+## Create backup directories if they do not exist
+CREATE_DIRS=yes
+
+## Keep absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
+## You should leave this enabled if you intend to use 'backup task division' functionality of oBackup, or everything will end up in the same directory.
+KEEP_ABSOLUTE_PATHS=yes
+
+## Generate an alert if backup size is lower than given value in Kb (this can also help identifying empty mount dirs).
+BACKUP_SIZE_MINIMUM=1024
+
+## Check backup size before proceeding
+GET_BACKUP_SIZE=yes
+
+## Generate an alert if storage free space is lower than given value in Kb.
+## Keep in mind that disabling backup file size test will only test min space against SQL backup size.
+SQL_WARN_MIN_SPACE=1048576
+FILE_WARN_MIN_SPACE=1048576
+
+###### REMOTE ONLY OPTIONS
+
+## In case of pulled or pushed backups, remote system URI needs to be supplied.
+REMOTE_SYSTEM_URI="ssh://backupuser@remote.system.tld:22/"
+
+## You can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
+SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa"
+
+## ssh compression should be used unless your remote connection is good enough (LAN)
+SSH_COMPRESSION=yes
+
+## Ignore ssh known hosts verification. DANGER WILL ROBINSON DANGER: This can lead to security risks. Only enable if you know what you're doing.
+SSH_IGNORE_KNOWN_HOSTS=no
+
+## Remote rsync executable path. Leave this empty in most cases
+RSYNC_REMOTE_PATH=""
+
+## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
+REMOTE_HOST_PING=yes
+
+## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
+REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
+
+## If enabled, commands will be executed as superuser on remote side. See documentation for /etc/sudoers configuration ("find", "du", "tee" and "rsync" need to be allowed). Requiretty needs to be disabled.
+SUDO_EXEC=no
+
+###### DATABASE SPECIFIC OPTIONS
+
+## Database backup user
+SQL_USER=root
+
+## Enabling the following option will save all databases on local or remote given SQL instance except the ones specified in the exclude list.
+## Every found database will be backed up as separate backup task.
+DATABASES_ALL=yes
+DATABASES_ALL_EXCLUDE_LIST="test;information_schema"
+## Alternatively, if DATABASES_ALL=no, you can specify a list of databases to backup separated by spaces.
+DATABASES_LIST="mysql"
+
+## Max backup execution time per Database task. Soft max exec time generates a warning only. Hard max exec time generates a warning and stops current backup task.
+## If a task gets stopped, next one in the task list gets executed. Time is specified in seconds.
+SOFT_MAX_EXEC_TIME_DB_TASK=1000
+HARD_MAX_EXEC_TIME_DB_TASK=1000
+
+## mysqldump options (ex: --extended-insert, --single-transaction, --quick...). See MySQL / MariaDB manual
+## default option: --opt
+MYSQLDUMP_OPTIONS="--opt --single-transaction"
+
+## Preferred SQL dump compression. Compression methods can be xz, lzma, pigz or gzip (will fallback from xz to gzip depending if available)
+## Generally, level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
+COMPRESSION_LEVEL=3
+
+###### FILES SPECIFIC OPTIONS
+
+## File backups are divided in tasks. Every directory in DIRECTORY_LIST will be processed as a unique task.
+## Every subdirectory of each directory in RECURSIVE_DIRECTORY_LIST will be processed as a unique task.
+## Example: RECURSIVE_DIRECTORY_LIST="/home;/var" will create backup tasks tasks "/home/dir1, "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/something".
+## You can exclude directories from the avove backup task creation, ex: avoid backing up "/home/dir2" by adding it to RECURSIVE_EXCLUDE_LIST.
+
+## Directories backup list. List of semicolon separated directories that will be backed up.
+DIRECTORY_LIST="${HOME}/obackup-testdata/testData"
+RECURSIVE_DIRECTORY_LIST="${HOME}/obackup-testdata/testDataRecursive"
+RECURSIVE_EXCLUDE_LIST="${HOME}/obackup-testdata/testDataRecursive/Excluded"
+
+## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns)
+RSYNC_PATTERN_FIRST=include
+
+## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work).
+## Paths are relative to sync dirs. List elements are separated by a semicolon.
+RSYNC_INCLUDE_PATTERN=""
+RSYNC_EXCLUDE_PATTERN="*.ded"
+#RSYNC_EXCLUDE_PATTERN="tmp;archives"
+
+## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
+## This file has to be in the same directory as the config file
+## Paths are relative to sync dirs. One element per line.
+RSYNC_INCLUDE_FROM=""
+RSYNC_EXCLUDE_FROM=""
+#RSYNC_EXCLUDE_FROM="exclude.list"
+
+## List separator char. You may set an alternative separator char for your directories lists above.
+PATH_SEPARATOR_CHAR=";"
+
+## Preserve basic linux permissions
+PRESERVE_PERMISSIONS=yes
+PRESERVE_OWNER=yes
+PRESERVE_GROUP=yes
+## On MACOS X, does not work and will be ignored 
+PRESERVE_EXECUTABILITY=yes
+
+## Preserve ACLS. Make sure source and target FS can hold same ACLs or you'll get loads of errors.
+PRESERVE_ACL=no
+## Preserve Xattr. MAke sure source and target FS can hold same Xattr or you'll get loads of errors.
+PRESERVE_XATTR=no
+
+## Transforms symlinks into referent files/dirs
+COPY_SYMLINKS=yes
+## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root.
+KEEP_DIRLINKS=yes
+## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
+PRESERVE_HARDLINKS=no
+
+
+## Let RSYNC compress file transfers. Do not use this on local-local backup schemes. Also, this is not useful if SSH compression is enabled.
+RSYNC_COMPRESS=no
+
+## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
+SOFT_MAX_EXEC_TIME_FILE_TASK=1000
+HARD_MAX_EXEC_TIME_FILE_TASK=1000
+
+## Keep partial uploads that can be resumed on next run, experimental feature
+PARTIAL=no
+
+## Delete files on destination that vanished from source. Do not turn this on unless you enabled backup rotation  or a snapshotting FS like zfs to keep those vanished files on the destination.
+DELETE_VANISHED_FILES=no
+
+## Use delta copy algortithm (usefull when local paths are network drives), defaults to yes
+DELTA_COPIES=yes
+
+## Bandwidth limit Kbytes / second for file backups. Leave 0 to disable limitation.
+BANDWIDTH=0
+
+## Paranoia option. Don't change this unless you read the documentation.
+RSYNC_EXECUTABLE=rsync
+
+###### ALERT OPTIONS
+
+## Alert email addresses separated by a space character
+DESTINATION_MAILS=""
+
+## Windows specific (msys / cygwin environment) only mail options (used with mailsend.exe from muquit, http://github.com/muquit/mailsend or from sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail/
+SENDER_MAIL="alert@your.system.tld"
+SMTP_SERVER=smtp.your.isp.tld
+SMTP_PORT=25
+# encryption can be tls, ssl or none
+SMTP_ENCRYPTION=none
+SMTP_USER=
+SMTP_PASSWORD=
+
+###### GENERAL BACKUP OPTIONS
+
+## Max execution time of whole backup process. Soft max exec time generates a warning only.
+## Hard max exec time generates a warning and stops the whole backup execution.
+SOFT_MAX_EXEC_TIME_TOTAL=1000
+HARD_MAX_EXEC_TIME_TOTAL=1
+
+## Log a message every KEEP_LOGGING seconds just to know the task is still alive
+KEEP_LOGGING=1801
+
+## Backup Rotation. You may rotate backups if you don't use snapshots on your backup server.
+ROTATE_SQL_BACKUPS=no
+ROTATE_SQL_COPIES=7
+ROTATE_FILE_BACKUPS=no
+ROTATE_FILE_COPIES=7
+
+###### EXECUTION HOOKS
+
+## Commands can will be run before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set).
+## This is useful to make a snapshot before backing up data, or even handle snapshots of backed up data.
+LOCAL_RUN_BEFORE_CMD=""
+LOCAL_RUN_AFTER_CMD=""
+
+REMOTE_RUN_BEFORE_CMD=""
+REMOTE_RUN_AFTER_CMD=""
+
+## Max execution time of commands before they get force killed. Leave 0 if you don't want this to happen. Time is specified in seconds.
+MAX_EXEC_TIME_PER_CMD_BEFORE=0
+MAX_EXEC_TIME_PER_CMD_AFTER=0
+
+## Stops whole backup execution if one of the above commands fail
+STOP_ON_CMD_ERROR=no
+
+## Run local and remote after backup cmd's even on failure
+RUN_AFTER_CMD_ON_ERROR=yes

dev/tests/conf/old.conf

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+###### Remote (or local) backup script for files & databases
+###### (C) 2013 by Ozy de Jong (www.badministrateur.com)
+###### Config file rev 0408201301
+
+## Backup identification, any string you want
+BACKUP_ID="really-old-config-file"
+
+## General backup options
+BACKUP_SQL=yes
+BACKUP_FILES=yes
+
+## Local storage paths
+LOCAL_SQL_STORAGE="${HOME}/obackup-storage/sql-old"
+LOCAL_FILE_STORAGE="${HOME}/obackup-storage/files-old"
+## Keep the absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
+## You should leave this enabled if you use recursive directories backup lists or they'll all end in the same path.
+LOCAL_STORAGE_KEEP_ABSOLUTE_PATHS=yes
+## Generate an alert if backup size is lower than given value in Kb, useful for local mounted backups.
+BACKUP_SIZE_MINIMUM=1024
+## Generate an alert if local storage free space is lower than given value in Kb.
+LOCAL_STORAGE_WARN_MIN_SPACE=1048576
+
+## If enabled, file backups will be processed with sudo command. See documentation for /etc/sudoers configuration ("find", "du" and "rsync" need to be allowed). Requiretty needs to be disabled.
+SUDO_EXEC=no
+## Paranoia option. Don't change this unless you read the documentation and still feel concerned about security issues.
+RSYNC_EXECUTABLE=rsync
+
+## Remote options (will make backups of remote system through ssh tunnel, public RSA key need to be put into /home/.ssh/authorized_keys in remote users home directory)
+REMOTE_BACKUP=yes
+SSH_RSA_PRIVATE_KEY=${HOME}/.ssh/id_rsa_local
+REMOTE_USER=root
+REMOTE_HOST=localhost
+REMOTE_PORT=22
+## ssh compression should be used unless your remote connection is good enough (LAN)
+SSH_COMPRESSION=yes
+## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
+REMOTE_HOST_PING=yes
+## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
+REMOTE_3RD_PARTY_HOST="www.kernel.org www.google.com"
+
+## Databases options
+SQL_USER=root
+## Save all databases except the ones specified in the exlude list. Every found database will be backed up as separate task (see documentation for explanation about tasks)
+DATABASES_ALL=yes
+DATABASES_ALL_EXCLUDE_LIST="test;information_schema;zarafa_prod"
+# Alternatively, you can specifiy a manual list of databases to backup separated by spaces
+DATABASES_LIST=""
+## Max backup execution time per DB task. Soft is warning only. Hard is warning, stopping backup task and processing next one. Time is specified in seconds
+SOFT_MAX_EXEC_TIME_DB_TASK=3600
+HARD_MAX_EXEC_TIME_DB_TASK=7200
+## Preferred sql dump compression. Can be set to xz, lzma or gzip.
+## Generally, xz level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
+COMPRESSION_PROGRAM=xz
+COMPRESSION_LEVEL=3
+## Dump compression should be done on remote side but can also be done locally to lower remote system usage (will take more bandwidth, check for ssh compression)
+COMPRESSION_REMOTE=yes
+
+## Path separator. You can set whatever seperator you want in your directories list below. You may change this in case you have some esoteric filenames (try to use unconventional separators like | ).
+PATH_SEPARATOR_CHAR=";"
+## File backup lists. Double quoted directory list separated by the $PATH_SEPARATOR_CHAR. Every directory will be processed as task (see documentation for explanation about tasks)
+DIRECTORIES_SIMPLE_LIST="${HOME}/obackup-testdata/testData"
+## Recurse directory list separated by the $PATH_SEPARATOR_CHAR. Will create a backup task per subdirectory (one level only), eg RECURSE_LIST="/home /var" will create tasks "/home/dir1", "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/whatever"
+DIRECTORIES_RECURSE_LIST="${HOME}/obackup-testdata/testDataRecursive"
+## You can optionally exclude directories from RECURSE_LIST tasks, eg on the above example you could exclude /home/dir2 by adding it to RECURSE_EXCLUDE_LIST
+DIRECTORIES_RECURSE_EXCLUDE_LIST="${HOME}/obackup-testdata/testDataRecursive/Excluded"
+## Be aware that every recurse list will have it's own root (exclude pattern is relative from /home/web for /home/web/{recursedir})
+RSYNC_EXCLUDE_PATTERN="*.ded"
+## Preserve ACLS. Make sure target FS can hold ACLs or you'll get loads of errors.
+PRESERVE_ACL=no
+## Preserve Xattr
+PRESERVE_XATTR=no
+## Let RSYNC compress file transfers. Do not use if you already enabled SSH compression.
+RSYNC_COMPRESS=yes
+## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
+SOFT_MAX_EXEC_TIME_FILE_TASK=3600
+HARD_MAX_EXEC_TIME_FILE_TASK=7200
+
+## Alert email adresses separated by a space character
+DESTINATION_MAILS="your@mail.tld"
+
+## Max execution time of whole backup process. Soft is warning only. Hard is warning and stopping whole backup process.
+SOFT_MAX_EXEC_TIME_TOTAL=30000
+HARD_MAX_EXEC_TIME_TOTAL=36000
+
+## Backup Rotation in case you don't use a snapshot aware file system like zfs or btrfs to perform a snapshot before every backup
+ROTATE_BACKUPS=yes
+ROTATE_COPIES=7
+
+## Commands that will be run  before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set to yes). Very usefull to initiate snapshots.
+## Set max execution time to 0 if you want these commands not to get stopped, else set a value in seconds after which execution will be stopped.
+LOCAL_RUN_BEFORE_CMD=""
+LOCAL_RUN_AFTER_CMD=""
+
+REMOTE_RUN_BEFORE_CMD=""
+REMOTE_RUN_AFTER_CMD=""
+
+MAX_EXEC_TIME_PER_CMD_BEFORE=0
+MAX_EXEC_TIME_PER_CMD_AFTER=0
+

dev/tests/conf/pull.conf

@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+
+###### obackup - Local or Remote, push or pull backup script for files & mysql databases
+###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr)
+###### obackup v2.1x config file rev 2016081701
+
+###### GENERAL BACKUP OPTIONS
+
+## Backup identification string.
+INSTANCE_ID="pull-test"
+
+## Log file location. Leaving this empty will create log file at /var/log/obackup.INSTANCE_ID.log (or current directory if /var/log doesn't exist).
+LOGFILE=""
+
+## Elements to backup
+SQL_BACKUP=yes
+FILE_BACKUP=yes
+
+## Backups can be done local, pulled from another server or pushed to a backup server. Available options are [local,pull,push].
+## Pulled backups are the safest option, as the backup server contains the RSA key and cannot be compromised by another server.
+BACKUP_TYPE=pull
+
+###### BACKUP STORAGE
+
+## Storage paths of the backups (absolute paths of the local or remote system)
+SQL_STORAGE="${HOME}/obackup-storage/sql-pull"
+FILE_STORAGE="${HOME}/obackup-storage/files-pull"
+
+## Encryption
+ENCRYPTION=no
+
+## Backup encryption needs a temporary storage space in order to encrypt files before sending them (absolute paths of the local or remote system)
+CRYPT_STORAGE="${HOME}/obackup-storage/crypt-pull"
+
+## GPG recipient (pubkey for this recipient must exist, see gpg2 --list-keys or gpg --list-keys
+GPG_RECIPIENT="John Doe"
+
+## Create backup directories if they do not exist
+CREATE_DIRS=yes
+
+## Keep absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
+## You should leave this enabled if you intend to use 'backup task division' functionality of oBackup, or everything will end up in the same directory.
+KEEP_ABSOLUTE_PATHS=yes
+
+## Generate an alert if backup size is lower than given value in Kb (this can also help identifying empty mount dirs).
+BACKUP_SIZE_MINIMUM=1024
+
+## Check backup size before proceeding
+GET_BACKUP_SIZE=yes
+
+## Generate an alert if storage free space is lower than given value in Kb.
+## Keep in mind that disabling backup file size test will only test min space against SQL backup size.
+SQL_WARN_MIN_SPACE=1048576
+FILE_WARN_MIN_SPACE=1048576
+
+###### REMOTE ONLY OPTIONS
+
+## In case of pulled or pushed backups, remote system URI needs to be supplied.
+REMOTE_SYSTEM_URI="ssh://root@localhost:49999/"
+
+## You can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
+SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa_local"
+
+## Alternatively, you may specify an SSH password file (less secure). Needs sshpass utility installed.
+SSH_PASSWORD_FILE=""
+
+_REMOTE_TOKEN=SomeAlphaNumericToken9
+
+## ssh compression should be used unless your remote connection is good enough (LAN)
+SSH_COMPRESSION=yes
+
+## Ignore ssh known hosts verification. DANGER WILL ROBINSON DANGER: This can lead to security risks. Only enable if you know what you're doing.
+SSH_IGNORE_KNOWN_HOSTS=no
+
+## Remote rsync executable path. Leave this empty in most cases
+RSYNC_REMOTE_PATH=""
+
+## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
+REMOTE_HOST_PING=yes
+
+## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
+REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
+
+## If enabled, commands will be executed as superuser on remote side. See documentation for /etc/sudoers configuration ("find", "du", "tee" and "rsync" need to be allowed). Requiretty needs to be disabled.
+SUDO_EXEC=no
+
+###### DATABASE SPECIFIC OPTIONS
+
+## Database backup user
+SQL_USER=root
+
+## Enabling the following option will save all databases on local or remote given SQL instance except the ones specified in the exclude list.
+## Every found database will be backed up as separate backup task.
+DATABASES_ALL=yes
+DATABASES_ALL_EXCLUDE_LIST="test;information_schema;zarafa_prod"
+## Alternatively, if DATABASES_ALL=no, you can specify a list of databases to backup separated by spaces.
+DATABASES_LIST=mysql
+
+## Max backup execution time per Database task. Soft max exec time generates a warning only. Hard max exec time generates a warning and stops current backup task.
+## If a task gets stopped, next one in the task list gets executed. Time is specified in seconds.
+SOFT_MAX_EXEC_TIME_DB_TASK=3600
+HARD_MAX_EXEC_TIME_DB_TASK=7200
+
+## mysqldump options (ex: --extended-insert, --single-transaction, --quick...). See MySQL / MariaDB manual
+## default option: --opt
+MYSQLDUMP_OPTIONS="--opt --single-transaction"
+
+## Preferred SQL dump compression. Compression methods can be xz, lzma, pigz or gzip (will fallback from xz to gzip depending if available)
+## Generally, level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
+COMPRESSION_LEVEL=3
+
+###### FILES SPECIFIC OPTIONS
+
+## File backups are divided in tasks. Every directory in DIRECTORY_LIST will be processed as a unique task.
+## Every subdirectory of each directory in RECURSIVE_DIRECTORY_LIST will be processed as a unique task.
+## Example: RECURSIVE_DIRECTORY_LIST="/home;/var" will create backup tasks tasks "/home/dir1, "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/something".
+## You can exclude directories from the avove backup task creation, ex: avoid backing up "/home/dir2" by adding it to RECURSIVE_EXCLUDE_LIST.
+
+## Directories backup list. List of semicolon separated directories that will be backed up.
+DIRECTORY_LIST=/root/obackup-testdata/testData
+RECURSIVE_DIRECTORY_LIST=/root/obackup-testdata/testDataRecursive
+RECURSIVE_EXCLUDE_LIST="${HOME}/obackup-testdata/testDataRecursive/Excluded"
+
+## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns)
+RSYNC_PATTERN_FIRST=include
+
+## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work).
+## Paths are relative to sync dirs. List elements are separated by a semicolon.
+RSYNC_INCLUDE_PATTERN=""
+RSYNC_EXCLUDE_PATTERN="*.ded"
+#RSYNC_EXCLUDE_PATTERN="tmp;archives"
+
+## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
+## This file has to be in the same directory as the config file
+## Paths are relative to sync dirs. One element per line.
+RSYNC_INCLUDE_FROM=""
+RSYNC_EXCLUDE_FROM=""
+#RSYNC_EXCLUDE_FROM="exclude.list"
+
+## List separator char. You may set an alternative separator char for your directories lists above.
+PATH_SEPARATOR_CHAR=";"
+
+## Preserve basic linux permissions
+PRESERVE_PERMISSIONS=yes
+PRESERVE_OWNER=yes
+PRESERVE_GROUP=yes
+## On MACOS X, does not work and will be ignored 
+PRESERVE_EXECUTABILITY=yes
+
+## Preserve ACLS. Make sure source and target FS can hold same ACLs or you'll get loads of errors.
+PRESERVE_ACL=no
+## Preserve Xattr. MAke sure source and target FS can hold same Xattr or you'll get loads of errors.
+PRESERVE_XATTR=no
+
+## Transforms symlinks into referent files/dirs
+COPY_SYMLINKS=yes
+## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root.
+KEEP_DIRLINKS=yes
+## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
+PRESERVE_HARDLINKS=no
+
+
+## Let RSYNC compress file transfers. Do not use this on local-local backup schemes. Also, this is not useful if SSH compression is enabled.
+RSYNC_COMPRESS=no
+
+## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
+SOFT_MAX_EXEC_TIME_FILE_TASK=3600
+HARD_MAX_EXEC_TIME_FILE_TASK=7200
+
+## Keep partial uploads that can be resumed on next run, experimental feature
+PARTIAL=no
+
+## Delete files on destination that vanished from source. Do not turn this on unless you enabled backup rotation  or a snapshotting FS like zfs to keep those vanished files on the destination.
+DELETE_VANISHED_FILES=no
+
+## Use delta copy algortithm (usefull when local paths are network drives), defaults to yes
+DELTA_COPIES=yes
+
+## Bandwidth limit Kbytes / second for file backups. Leave 0 to disable limitation.
+BANDWIDTH=0
+
+## Paranoia option. Don't change this unless you read the documentation.
+RSYNC_EXECUTABLE=rsync
+
+###### ALERT OPTIONS
+
+## Alert email addresses separated by a space character
+DESTINATION_MAILS=""
+
+## Windows specific (msys / cygwin environment) only mail options (used with mailsend.exe from muquit, http://github.com/muquit/mailsend or from sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail/
+SENDER_MAIL="alert@your.system.tld"
+SMTP_SERVER=smtp.your.isp.tld
+SMTP_PORT=25
+# encryption can be tls, ssl or none
+SMTP_ENCRYPTION=none
+SMTP_USER=
+SMTP_PASSWORD=
+
+###### GENERAL BACKUP OPTIONS
+
+## Max execution time of whole backup process. Soft max exec time generates a warning only.
+## Hard max exec time generates a warning and stops the whole backup execution.
+SOFT_MAX_EXEC_TIME_TOTAL=30000
+HARD_MAX_EXEC_TIME_TOTAL=36000
+
+## Log a message every KEEP_LOGGING seconds just to know the task is still alive
+KEEP_LOGGING=1801
+
+## Backup Rotation. You may rotate backups if you don't use snapshots on your backup server.
+ROTATE_SQL_BACKUPS=yes
+ROTATE_SQL_COPIES=7
+ROTATE_FILE_BACKUPS=yes
+ROTATE_FILE_COPIES=7
+
+###### EXECUTION HOOKS
+
+## Commands can will be run before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set).
+## This is useful to make a snapshot before backing up data, or even handle snapshots of backed up data.
+LOCAL_RUN_BEFORE_CMD=""
+LOCAL_RUN_AFTER_CMD=""
+
+REMOTE_RUN_BEFORE_CMD=""
+REMOTE_RUN_AFTER_CMD=""
+
+## Max execution time of commands before they get force killed. Leave 0 if you don't want this to happen. Time is specified in seconds.
+MAX_EXEC_TIME_PER_CMD_BEFORE=0
+MAX_EXEC_TIME_PER_CMD_AFTER=0
+
+## Stops whole backup execution if one of the above commands fail
+STOP_ON_CMD_ERROR=no
+
+## Run local and remote after backup cmd's even on failure
+RUN_AFTER_CMD_ON_ERROR=yes

dev/tests/conf/push.conf

@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+
+###### obackup - Local or Remote, push or pull backup script for files & mysql databases
+###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr)
+###### obackup v2.1x config file rev 2016081701
+
+###### GENERAL BACKUP OPTIONS
+
+## Backup identification string.
+INSTANCE_ID="push-test"
+
+## Log file location. Leaving this empty will create log file at /var/log/obackup.INSTANCE_ID.log (or current directory if /var/log doesn't exist).
+LOGFILE=""
+
+## Elements to backup
+SQL_BACKUP=yes
+FILE_BACKUP=yes
+
+## Backups can be done local, pulled from another server or pushed to a backup server. Available options are [local,pull,push].
+## Pulled backups are the safest option, as the backup server contains the RSA key and cannot be compromised by another server.
+BACKUP_TYPE=push
+
+###### BACKUP STORAGE
+
+## Storage paths of the backups (absolute paths of the local or remote system)
+SQL_STORAGE="${HOME}/obackup-storage/sql-push"
+FILE_STORAGE="${HOME}/obackup-storage/files-push"
+
+## Encryption
+ENCRYPTION=no
+
+## Backup encryption needs a temporary storage space in order to encrypt files before sending them (absolute paths of the local or remote system)
+CRYPT_STORAGE="${HOME}/obackup-storage/crypt-push"
+
+## GPG recipient (pubkey for this recipient must exist, see gpg2 --list-keys or gpg --list-keys
+GPG_RECIPIENT="John Doe"
+
+## Create backup directories if they do not exist
+CREATE_DIRS=yes
+
+## Keep absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
+## You should leave this enabled if you intend to use 'backup task division' functionality of oBackup, or everything will end up in the same directory.
+KEEP_ABSOLUTE_PATHS=yes
+
+## Generate an alert if backup size is lower than given value in Kb (this can also help identifying empty mount dirs).
+BACKUP_SIZE_MINIMUM=1024
+
+## Check backup size before proceeding
+GET_BACKUP_SIZE=yes
+
+## Generate an alert if storage free space is lower than given value in Kb.
+## Keep in mind that disabling backup file size test will only test min space against SQL backup size.
+SQL_WARN_MIN_SPACE=1048576
+FILE_WARN_MIN_SPACE=1048576
+
+###### REMOTE ONLY OPTIONS
+
+## In case of pulled or pushed backups, remote system URI needs to be supplied.
+REMOTE_SYSTEM_URI="ssh://root@localhost:49999/"
+
+## You can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
+SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa_local"
+
+## Alternatively, you may specify an SSH password file (less secure). Needs sshpass utility installed.
+SSH_PASSWORD_FILE=""
+
+_REMOTE_TOKEN=SomeAlphaNumericToken9
+
+## ssh compression should be used unless your remote connection is good enough (LAN)
+SSH_COMPRESSION=yes
+
+## Ignore ssh known hosts verification. DANGER WILL ROBINSON DANGER: This can lead to security risks. Only enable if you know what you're doing.
+SSH_IGNORE_KNOWN_HOSTS=no
+
+## Remote rsync executable path. Leave this empty in most cases
+RSYNC_REMOTE_PATH=""
+
+## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
+REMOTE_HOST_PING=yes
+
+## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
+REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
+
+## If enabled, commands will be executed as superuser on remote side. See documentation for /etc/sudoers configuration ("find", "du", "tee" and "rsync" need to be allowed). Requiretty needs to be disabled.
+SUDO_EXEC=no
+
+###### DATABASE SPECIFIC OPTIONS
+
+## Database backup user
+SQL_USER=root
+
+## Enabling the following option will save all databases on local or remote given SQL instance except the ones specified in the exclude list.
+## Every found database will be backed up as separate backup task.
+DATABASES_ALL=yes
+DATABASES_ALL_EXCLUDE_LIST="test;information_schema;zarafa_prod"
+## Alternatively, if DATABASES_ALL=no, you can specify a list of databases to backup separated by spaces.
+DATABASES_LIST=mysql
+
+## Max backup execution time per Database task. Soft max exec time generates a warning only. Hard max exec time generates a warning and stops current backup task.
+## If a task gets stopped, next one in the task list gets executed. Time is specified in seconds.
+SOFT_MAX_EXEC_TIME_DB_TASK=3600
+HARD_MAX_EXEC_TIME_DB_TASK=7200
+
+## mysqldump options (ex: --extended-insert, --single-transaction, --quick...). See MySQL / MariaDB manual
+## default option: --opt
+MYSQLDUMP_OPTIONS="--opt --single-transaction"
+
+## Preferred SQL dump compression. Compression methods can be xz, lzma, pigz or gzip (will fallback from xz to gzip depending if available)
+## Generally, level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
+COMPRESSION_LEVEL=3
+
+###### FILES SPECIFIC OPTIONS
+
+## File backups are divided in tasks. Every directory in DIRECTORY_LIST will be processed as a unique task.
+## Every subdirectory of each directory in RECURSIVE_DIRECTORY_LIST will be processed as a unique task.
+## Example: RECURSIVE_DIRECTORY_LIST="/home;/var" will create backup tasks tasks "/home/dir1, "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/something".
+## You can exclude directories from the avove backup task creation, ex: avoid backing up "/home/dir2" by adding it to RECURSIVE_EXCLUDE_LIST.
+
+## Directories backup list. List of semicolon separated directories that will be backed up.
+DIRECTORY_LIST=/root/obackup-testdata/testData
+RECURSIVE_DIRECTORY_LIST=/root/obackup-testdata/testDataRecursive
+RECURSIVE_EXCLUDE_LIST="${HOME}/obackup-testdata/testDataRecursive/Excluded"
+
+## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns)
+RSYNC_PATTERN_FIRST=include
+
+## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work).
+## Paths are relative to sync dirs. List elements are separated by a semicolon.
+RSYNC_INCLUDE_PATTERN=""
+RSYNC_EXCLUDE_PATTERN="*.ded"
+#RSYNC_EXCLUDE_PATTERN="tmp;archives"
+
+## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
+## This file has to be in the same directory as the config file
+## Paths are relative to sync dirs. One element per line.
+RSYNC_INCLUDE_FROM=""
+RSYNC_EXCLUDE_FROM=""
+#RSYNC_EXCLUDE_FROM="exclude.list"
+
+## List separator char. You may set an alternative separator char for your directories lists above.
+PATH_SEPARATOR_CHAR=";"
+
+## Preserve basic linux permissions
+PRESERVE_PERMISSIONS=yes
+PRESERVE_OWNER=yes
+PRESERVE_GROUP=yes
+## On MACOS X, does not work and will be ignored 
+PRESERVE_EXECUTABILITY=yes
+
+## Preserve ACLS. Make sure source and target FS can hold same ACLs or you'll get loads of errors.
+PRESERVE_ACL=no
+## Preserve Xattr. MAke sure source and target FS can hold same Xattr or you'll get loads of errors.
+PRESERVE_XATTR=no
+
+## Transforms symlinks into referent files/dirs
+COPY_SYMLINKS=yes
+## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root.
+KEEP_DIRLINKS=yes
+## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
+PRESERVE_HARDLINKS=no
+
+
+## Let RSYNC compress file transfers. Do not use this on local-local backup schemes. Also, this is not useful if SSH compression is enabled.
+RSYNC_COMPRESS=no
+
+## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
+SOFT_MAX_EXEC_TIME_FILE_TASK=3600
+HARD_MAX_EXEC_TIME_FILE_TASK=7200
+
+## Keep partial uploads that can be resumed on next run, experimental feature
+PARTIAL=no
+
+## Delete files on destination that vanished from source. Do not turn this on unless you enabled backup rotation  or a snapshotting FS like zfs to keep those vanished files on the destination.
+DELETE_VANISHED_FILES=no
+
+## Use delta copy algortithm (usefull when local paths are network drives), defaults to yes
+DELTA_COPIES=yes
+
+## Bandwidth limit Kbytes / second for file backups. Leave 0 to disable limitation.
+BANDWIDTH=0
+
+## Paranoia option. Don't change this unless you read the documentation.
+RSYNC_EXECUTABLE=rsync
+
+###### ALERT OPTIONS
+
+## Alert email addresses separated by a space character
+DESTINATION_MAILS=""
+
+## Windows specific (msys / cygwin environment) only mail options (used with mailsend.exe from muquit, http://github.com/muquit/mailsend or from sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail/
+SENDER_MAIL="alert@your.system.tld"
+SMTP_SERVER=smtp.your.isp.tld
+SMTP_PORT=25
+# encryption can be tls, ssl or none
+SMTP_ENCRYPTION=none
+SMTP_USER=
+SMTP_PASSWORD=
+
+###### GENERAL BACKUP OPTIONS
+
+## Max execution time of whole backup process. Soft max exec time generates a warning only.
+## Hard max exec time generates a warning and stops the whole backup execution.
+SOFT_MAX_EXEC_TIME_TOTAL=30000
+HARD_MAX_EXEC_TIME_TOTAL=36000
+
+## Log a message every KEEP_LOGGING seconds just to know the task is still alive
+KEEP_LOGGING=1801
+
+## Backup Rotation. You may rotate backups if you don't use snapshots on your backup server.
+ROTATE_SQL_BACKUPS=yes
+ROTATE_SQL_COPIES=7
+ROTATE_FILE_BACKUPS=yes
+ROTATE_FILE_COPIES=7
+
+###### EXECUTION HOOKS
+
+## Commands can will be run before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set).
+## This is useful to make a snapshot before backing up data, or even handle snapshots of backed up data.
+LOCAL_RUN_BEFORE_CMD=""
+LOCAL_RUN_AFTER_CMD=""
+
+REMOTE_RUN_BEFORE_CMD=""
+REMOTE_RUN_AFTER_CMD=""
+
+## Max execution time of commands before they get force killed. Leave 0 if you don't want this to happen. Time is specified in seconds.
+MAX_EXEC_TIME_PER_CMD_BEFORE=0
+MAX_EXEC_TIME_PER_CMD_AFTER=0
+
+## Stops whole backup execution if one of the above commands fail
+STOP_ON_CMD_ERROR=no
+
+## Run local and remote after backup cmd's even on failure
+RUN_AFTER_CMD_ON_ERROR=yes

dev/tests/run_tests.sh

@@ -0,0 +1,930 @@
+#!/usr/bin/env bash
+
+#TODO Encrypted Pull runs on F25 fail for decryption
+
+## obackup basic tests suite 2017040801
+
+OBACKUP_DIR="$(pwd)"
+OBACKUP_DIR=${OBACKUP_DIR%%/dev*}
+DEV_DIR="$OBACKUP_DIR/dev"
+TESTS_DIR="$DEV_DIR/tests"
+
+CONF_DIR="$TESTS_DIR/conf"
+LOCAL_CONF="local.conf"
+PULL_CONF="pull.conf"
+PUSH_CONF="push.conf"
+
+OLD_CONF="old.conf"
+TMP_OLD_CONF="tmp.old.conf"
+MAX_EXEC_CONF="max-exec-time.conf"
+
+OBACKUP_EXECUTABLE="obackup.sh"
+OBACKUP_DEV_EXECUTABLE="dev/n_obackup.sh"
+OBACKUP_UPGRADE="upgrade-v1.x-2.1x.sh"
+TMP_FILE="$DEV_DIR/tmp"
+
+SOURCE_DIR="${HOME}/obackup-testdata"
+TARGET_DIR="${HOME}/obackup-storage"
+
+TARGET_DIR_SQL_LOCAL="$TARGET_DIR/sql-local"
+TARGET_DIR_FILE_LOCAL="$TARGET_DIR/files-local"
+TARGET_DIR_CRYPT_LOCAL="$TARGET_DIR/crypt-local"
+
+TARGET_DIR_SQL_PULL="$TARGET_DIR/sql-pull"
+TARGET_DIR_FILE_PULL="$TARGET_DIR/files-pull"
+TARGET_DIR_CRYPT_PULL="$TARGET_DIR/crypt-pull"
+
+TARGET_DIR_SQL_PUSH="$TARGET_DIR/sql-push"
+TARGET_DIR_FILE_PUSH="$TARGET_DIR/files-push"
+TARGET_DIR_CRYPT_PUSH="$TARGET_DIR/crypt-push"
+
+
+SIMPLE_DIR="testData"
+RECURSIVE_DIR="testDataRecursive"
+
+S_DIR_1="dir rect ory"
+R_EXCLUDED_DIR="Excluded"
+R_DIR_1="a"
+R_DIR_2="b"
+R_DIR_3="c d"
+
+S_FILE_1="some file"
+R_FILE_1="file_1"
+R_FILE_2="file 2"
+R_FILE_3="file 3"
+N_FILE_1="non recurse file"
+
+EXCLUDED_FILE="exclu.ded"
+
+DATABASE_1="mysql.sql.xz"
+DATABASE_2="performance_schema.sql.xz"
+DATABASE_EXCLUDED="information_schema.sql.xz"
+
+CRYPT_EXTENSION=".obackup.gpg"
+ROTATE_1_EXTENSION=".obackup.1"
+
+PASSFILE="passfile"
+CRYPT_TESTFILE="testfile"
+
+# Later populated variables
+OBACKUP_VERSION=2.x
+OBACKUP_MIN_VERSION=x
+OBACKUP_IS_STABLE=maybe
+
+function SetupSSH {
+        echo -e  'y\n'| ssh-keygen -t rsa -b 2048 -N "" -f "${HOME}/.ssh/id_rsa_local"
+        if ! grep "$(cat ${HOME}/.ssh/id_rsa_local.pub)" "${HOME}/.ssh/authorized_keys"; then
+		echo "from=\"*\",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command=\"/usr/local/bin/ssh_filter.sh SomeAlphaNumericToken9\" $(cat ${HOME}/.ssh/id_rsa_local.pub)" >> "${HOME}/.ssh/authorized_keys"
+        fi
+	chmod 600 "${HOME}/.ssh/authorized_keys"
+
+        # Add localhost to known hosts so self connect works
+        if [ -z "$(ssh-keygen -F localhost)" ]; then
+                ssh-keyscan -H localhost >> "${HOME}/.ssh/known_hosts"
+        fi
+
+        # Update remote conf files with SSH port
+        sed -i.tmp 's#ssh://.*@localhost:[0-9]*/#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/#' "$CONF_DIR/$PULL_CONF"
+        sed -i.tmp 's#ssh://.*@localhost:[0-9]*/#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/#' "$CONF_DIR/$PUSH_CONF"
+}
+
+function RemoveSSH {
+        local pubkey
+
+        if [ -f "${HOME}/.ssh/id_rsa_local" ]; then
+
+                pubkey=$(cat "${HOME}/.ssh/id_rsa_local.pub")
+		sed -i.bak "s|.*$pubkey.*||g" "${HOME}/.ssh/authorized_keys"
+                rm -f "${HOME}/.ssh/{id_rsa_local.pub,id_rsa_local}"
+        fi
+}
+
+function SetupGPG {
+	if type gpg2 > /dev/null; then
+		CRYPT_TOOL=gpg2
+	elif type gpg > /dev/null; then
+		CRYPT_TOOL=gpg
+	else
+		echo "No gpg support"
+		assertEquals "Failed to detect gpg" "1" $?
+		return
+	fi
+
+	echo "Crypt tool=$CRYPT_TOOL"
+
+	if ! $CRYPT_TOOL --list-keys | grep "John Doe" > /dev/null; then
+
+		cat >gpgcommand <<EOF
+%echo Generating a GPG Key
+Key-Type: RSA
+Key-Length: 4096
+Name-Real: John Doe
+Name-Comment: obackup-test-key
+Name-Email: john@example.com
+Expire-Date: 0
+Passphrase: PassPhrase123
+# Do a commit here, so that we can later print "done" :-)
+%commit
+%echo done
+EOF
+
+		if type apt-get > /dev/null 2>&1; then
+			sudo apt-get install rng-tools
+		fi
+
+		# Setup fast entropy
+		if type rngd > /dev/null 2>&1; then
+			$SUDO_CMD rngd -r /dev/urandom
+		else
+			echo "No rngd support"
+		fi
+
+		$CRYPT_TOOL --batch --gen-key gpgcommand
+		echo "Currently owned $CRYPT_TOOL keys"
+		echo $($CRYPT_TOOL --list-keys)
+		rm -f gpgcommand
+
+	fi
+
+	echo "PassPhrase123" > "$TESTS_DIR/$PASSFILE"
+}
+
+function oneTimeSetUp () {
+	START_TIME=$SECONDS
+
+	source "$DEV_DIR/ofunctions.sh"
+
+	# Set default umask because ofunctions set 0077
+	umask 0022
+
+	GetLocalOS
+
+	echo "Detected OS: $LOCAL_OS"
+
+        # Set some travis related changes
+        if [ "$TRAVIS_RUN" == true ]; then
+        echo "Running with travis settings"
+                REMOTE_USER="travis"
+		RHOST_PING="no"
+                SetConfFileValue "$CONF_DIR/$PULL_CONF" "REMOTE_3RD_PARTY_HOSTS" ""
+                SetConfFileValue "$CONF_DIR/$PUSH_CONF" "REMOTE_3RD_PARTY_HOSTS" ""
+		# Config value didn't have S at the end in old files
+                SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_3RD_PARTY_HOST" ""
+
+                SetConfFileValue "$CONF_DIR/$PULL_CONF" "REMOTE_HOST_PING" "no"
+                SetConfFileValue "$CONF_DIR/$PUSH_CONF" "REMOTE_HOST_PING" "no"
+                SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_HOST_PING" "no"
+        else
+            	echo "Running with local settings"
+                REMOTE_USER="root"
+		RHOST_PING="yes"
+                SetConfFileValue "$CONF_DIR/$PULL_CONF" "REMOTE_3RD_PARTY_HOSTS" "\"www.kernel.org www.google.com\""
+                SetConfFileValue "$CONF_DIR/$PUSH_CONF" "REMOTE_3RD_PARTY_HOSTS" "\"www.kernel.org www.google.com\""
+		# Config value didn't have S at the end in old files
+                SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_3RD_PARTY_HOST" "\"www.kernel.org www.google.com\""
+
+                SetConfFileValue "$CONF_DIR/$PULL_CONF" "REMOTE_HOST_PING" "yes"
+                SetConfFileValue "$CONF_DIR/$PUSH_CONF" "REMOTE_HOST_PING" "yes"
+                SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_HOST_PING" "yes"
+        fi
+
+        # Get default ssh port from env
+        if [ "$SSH_PORT" == "" ]; then
+                SSH_PORT=22
+        fi
+
+
+	#TODO: Assuming that macos has the same syntax than bsd here
+        if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then
+                SUDO_CMD=""
+        elif [ "$LOCAL_OS" == "BSD" ] || [ "$LOCAL_OS" == "MacOSX" ]; then
+                SUDO_CMD=""
+        else
+                SUDO_CMD="sudo"
+        fi
+
+
+	SetupGPG
+	if [ "$SKIP_REMOTE" != "yes" ]; then
+		SetupSSH
+	fi
+
+	# Get OBACKUP version
+        OBACKUP_VERSION=$(GetConfFileValue "$OBACKUP_DIR/$OBACKUP_DEV_EXECUTABLE" "PROGRAM_VERSION")
+        OBACKUP_VERSION="${OBACKUP_VERSION##*=}"
+        OBACKUP_MIN_VERSION="${OBACKUP_VERSION:2:1}"
+
+        OBACKUP_IS_STABLE=$(GetConfFileValue "$OBACKUP_DIR/$OBACKUP_DEV_EXECUTABLE" "IS_STABLE")
+
+        echo "Running with $OBACKUP_VERSION ($OBACKUP_MIN_VERSION) STABLE=$OBACKUP_IS_STABLE"
+
+	# Set basic values that could get changed later
+	for i in "$LOCAL_CONF" "$PULL_CONF" "$PUSH_CONF"; do
+		SetConfFileValue "$CONF_DIR/$i" "ENCRYPTION" "no"
+		SetConfFileValue "$CONF_DIR/$i" "DATABASES_ALL" "yes"
+		SetConfFileValue "$CONF_DIR/$i" "DATABASES_LIST" "mysql"
+		SetConfFileValue "$CONF_DIR/$i" "FILE_BACKUP" "yes"
+		SetConfFileValue "$CONF_DIR/$i" "DIRECTORY_LIST" "${HOME}/obackup-testdata/testData"
+		SetConfFileValue "$CONF_DIR/$i" "RECURSIVE_DIRECTORY_LIST" "${HOME}/obackup-testdata/testDataRecursive"
+		SetConfFileValue "$CONF_DIR/$i" "SQL_BACKUP" "yes"
+	done
+}
+
+function oneTimeTearDown () {
+	SetConfFileValue "$OBACKUP_DIR/$OBACKUP_EXECUTABLE" "IS_STABLE" "$OBACKUP_IS_STABLE"
+
+	RemoveSSH
+
+	#TODO: uncomment this when dev is done
+	#rm -rf "$SOURCE_DIR"
+	#rm -rf "$TARGET_DIR"
+	rm -f "$TMP_FILE"
+
+	cd "$OBACKUP_DIR"
+	$SUDO_CMD ./install.sh --remove --silent --no-stats
+	assertEquals "Uninstall failed" "0" $?
+
+        ELAPSED_TIME=$(($SECONDS - $START_TIME))
+        echo "It took $ELAPSED_TIME seconds to run these tests."
+}
+
+
+function setUp () {
+	rm -rf "$SOURCE_DIR"
+	rm -rf "$TARGET_DIR"
+
+	mkdir -p "$SOURCE_DIR/$SIMPLE_DIR/$S_DIR_1"
+	mkdir -p "$SOURCE_DIR/$RECURSIVE_DIR/$R_EXCLUDED_DIR"
+	mkdir -p "$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_1"
+	mkdir -p "$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_2"
+	mkdir -p "$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_3"
+
+	touch "$SOURCE_DIR/$SIMPLE_DIR/$S_DIR_1/$S_FILE_1"
+	touch "$SOURCE_DIR/$SIMPLE_DIR/$EXCLUDED_FILE"
+	touch "$SOURCE_DIR/$RECURSIVE_DIR/$N_FILE_1"
+	touch "$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_1/$R_FILE_1"
+	touch "$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_2/$R_FILE_2"
+	dd if=/dev/urandom of="$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_3/$R_FILE_3" bs=1048576 count=2
+	assertEquals "dd file creation" "0" $?
+	touch "$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_3/$EXCLUDED_FILE"
+
+	FilePresence=(
+	"$SOURCE_DIR/$SIMPLE_DIR/$S_DIR_1/$S_FILE_1"
+	"$SOURCE_DIR/$RECURSIVE_DIR/$N_FILE_1"
+	"$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_1/$R_FILE_1"
+	"$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_2/$R_FILE_2"
+	"$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_3/$R_FILE_3"
+	)
+
+	DatabasePresence=(
+	"$DATABASE_1"
+	"$DATABASE_2"
+	)
+
+	FileExcluded=(
+	"$SOURCE_DIR/$SIMPLE_DIR/$EXCLUDED_FILE"
+	"$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_3/$R_EXCLUDED_FILE"
+	)
+
+	DatabaseExcluded=(
+	"$DATABASE_EXCLUDED"
+	)
+
+	DirectoriesExcluded=(
+	"$RECURSIVE_DIR/$R_EXCLUDED_DIR"
+	)
+}
+
+function test_Merge () {
+	cd "$DEV_DIR"
+	./merge.sh obackup
+	assertEquals "Merging code" "0" $?
+
+	cd "$OBACKUP_DIR"
+	$SUDO_CMD ./install.sh --silent --no-stats
+	assertEquals "Install failed" "0" $?
+
+	# Set obackup version to stable while testing to avoid warning message
+        SetConfFileValue "$OBACKUP_DIR/$OBACKUP_EXECUTABLE" "IS_STABLE" "yes"
+}
+
+# Keep this function to check GPG behavior depending on OS. (GPG 2.1 / GPG 2.0x / GPG 1.4 don't behave the same way)
+function test_GPG () {
+	echo "Encrypting file"
+	$CRYPT_TOOL --out "$TESTS_DIR/$CRYPT_TESTFILE$CRYPT_EXTENSION" --recipient "John Doe" --batch --yes --encrypt "$TESTS_DIR/$PASSFILE"
+	assertEquals "Encrypt file" "0" $?
+
+
+        # Detect if GnuPG >= 2.1 that does not allow automatic pin entry anymore
+        cryptToolVersion=$($CRYPT_TOOL --version | head -1 | awk '{print $3}')
+        cryptToolMajorVersion=${cryptToolVersion%%.*}
+        cryptToolSubVersion=${cryptToolVersion#*.}
+        cryptToolSubVersion=${cryptToolSubVersion%.*}
+
+        if [ $cryptToolMajorVersion -eq 2 ] && [ $cryptToolSubVersion -ge 1 ]; then
+                additionalParameters="--pinentry-mode loopback"
+        fi
+
+	if [ "$CRYPT_TOOL" == "gpg2" ]; then
+                options="--batch --yes"
+        elif [ "$CRYPT_TOOL" == "gpg" ]; then
+                options="--no-use-agent --batch"
+        fi
+
+
+	echo "Decrypt using passphrase file"
+	$CRYPT_TOOL $options --out "$TESTS_DIR/$CRYPT_TESTFILE" --batch --yes $additionalParameters --passphrase-file="$TESTS_DIR/$PASSFILE" --decrypt "$TESTS_DIR/$CRYPT_TESTFILE$CRYPT_EXTENSION"
+	assertEquals "Decrypt file using passfile" "0" $?
+
+	echo "Decrypt using passphrase"
+	$CRYPT_TOOL $options --out "$TESTS_DIR/$CRYPT_TESTFILE" --batch --yes $additionalParameters --passphrase PassPhrase123 --decrypt "$TESTS_DIR/$CRYPT_TESTFILE$CRYPT_EXTENSION"
+	assertEquals "Decrypt file using passphrase" "0" $?
+
+	echo "Decrypt using passphrase file with cat"
+	$CRYPT_TOOL $options --out "$TESTS_DIR/$CRYPT_TESTFILE" --batch --yes $additionalParameters --passphrase $(cat "$TESTS_DIR/$PASSFILE") --decrypt "$TESTS_DIR/$CRYPT_TESTFILE$CRYPT_EXTENSION"
+	assertEquals "Decrypt file using passphrase" "0" $?
+}
+
+function test_LocalRun () {
+	SetConfFileValue "$CONF_DIR/$LOCAL_CONF" "ENCRYPTION" "no"
+
+	# Basic return code tests. Need to go deep into file presence testing
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$LOCAL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${FilePresence[@]}"; do
+		[ -f "$TARGET_DIR_FILE_LOCAL/$file" ]
+		assertEquals "File Presence [$TARGET_DIR_FILE_LOCAL/$file]" "0" $?
+	done
+
+	for file in "${FileExcluded[@]}"; do
+		[ -f "$TARGET_DIR_FILE_LOCAL/$file" ]
+		assertEquals "File Excluded [$TARGET_DIR_FILE_LOCAL/$file]" "1" $?
+	done
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_LOCAL/$file" ]
+		assertEquals "Database Presence [$TARGET_DIR_SQL_LOCAL/$file]" "0" $?
+	done
+
+	for file in "${DatabaseExcluded[@]}"; do
+		[ -f "$TARGET_DIR_SQL_LOCAL/$file" ]
+		assertEquals "Database Excluded [$TARGET_DIR_SQL_LOCAL/$file]" "1" $?
+	done
+
+	for directory in "${DirectoriesExcluded[@]}"; do
+		[ -d "$TARGET_DIR_FILE_LOCAL/$directory" ]
+		assertEquals "Directory Excluded [$TARGET_DIR_FILE_LOCAL/$directory]" "1" $?
+	done
+
+	diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-local/$SOURCE_DIR" | grep -i Exclu
+	[ $(diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-local/$SOURCE_DIR" | grep -i Exclu | wc -l) -eq 2 ]
+	assertEquals "Diff should only output excluded files" "0" $?
+
+	# Tests presence of rotated files
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$LOCAL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_LOCAL/$file$ROTATE_1_EXTENSION" ]
+		assertEquals "Database rotated Presence [$TARGET_DIR_SQL_LOCAL/$file]" "0" $?
+	done
+
+	[ -d "$TARGET_DIR_FILE_LOCAL/$(dirname $SOURCE_DIR)$ROTATE_1_EXTENSION" ]
+	assertEquals "File rotated Presence [$TARGET_DIR_FILE_LOCAL/$(dirname $SOURCE_DIR)$ROTATE_1_EXTENSION]" "0" $?
+
+}
+
+function test_PullRun () {
+	if [ "$SKIP_REMOTE" == "yes" ]; then
+		return 0
+	fi
+
+	SetConfFileValue "$CONF_DIR/$PULL_CONF" "ENCRYPTION" "no"
+
+	# Basic return code tests. Need to go deep into file presence testing
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PULL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${FilePresence[@]}"; do
+		[ -f "$TARGET_DIR_FILE_PULL/$file" ]
+		assertEquals "File Presence [$TARGET_DIR_FILE_PULL/$file]" "0" $?
+	done
+
+	for file in "${FileExcluded[@]}"; do
+		[ -f "$TARGET_DIR_FILE_PULL/$file" ]
+		assertEquals "File Excluded [$TARGET_DIR_FILE_PULL/$file]" "1" $?
+	done
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PULL/$file" ]
+		assertEquals "Database Presence [$TARGET_DIR_SQL_PULL/$file]" "0" $?
+	done
+
+	for file in "${DatabaseExcluded[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PULL/$file" ]
+		assertEquals "Database Excluded [$TARGET_DIR_SQL_PULL/$file]" "1" $?
+	done
+
+	for directory in "${DirectoriesExcluded[@]}"; do
+		[ -d "$TARGET_DIR_FILE_PULL/$directory" ]
+		assertEquals "Directory Excluded [$TARGET_DIR_FILE_PULL/$directory]" "1" $?
+	done
+
+	diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-pull/$SOURCE_DIR" | grep -i Exclu
+	[ $(diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-pull/$SOURCE_DIR" | grep -i Exclu | wc -l) -eq 2 ]
+	assertEquals "Diff should only output excluded files" "0" $?
+
+	# Tests presence of rotated files
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PULL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PULL/$file$ROTATE_1_EXTENSION" ]
+		assertEquals "Database rotated Presence [$TARGET_DIR_SQL_PULL/$file$ROTATE_1_EXTENSION]" "0" $?
+	done
+
+	[ -d "$TARGET_DIR_FILE_PULL/$(dirname $SOURCE_DIR)$ROTATE_1_EXTENSION" ]
+	assertEquals "File rotated Presence [$TARGET_DIR_FILE_PULL/$(dirname $SOURCE_DIR)$ROTATE_1_EXTENSION]" "0" $?
+
+}
+
+function test_PushRun () {
+	if [ "$SKIP_REMOTE" == "yes" ]; then
+		return 0
+	fi
+
+	SetConfFileValue "$CONF_DIR/$PUSH_CONF" "ENCRYPTION" "no"
+
+	# Basic return code tests. Need to go deep into file presence testing
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PUSH_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${FilePresence[@]}"; do
+		[ -f "$TARGET_DIR_FILE_PUSH/$file" ]
+		assertEquals "File Presence [$TARGET_DIR_FILE_PUSH/$file]" "0" $?
+	done
+
+	for file in "${FileExcluded[@]}"; do
+		[ -f "$TARGET_DIR_FILE_PUSH/$file" ]
+		assertEquals "File Excluded [$TARGET_DIR_FILE_PUSH/$file]" "1" $?
+	done
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PUSH/$file" ]
+		assertEquals "Database Presence [$TARGET_DIR_SQL_PUSH/$file]" "0" $?
+	done
+
+	for file in "${DatabaseExcluded[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PUSH/$file" ]
+		assertEquals "Database Excluded [$TARGET_DIR_SQL_PUSH/$file]" "1" $?
+	done
+
+	for directory in "${DirectoriesExcluded[@]}"; do
+		[ -d "$TARGET_DIR_FILE_PUSH/$directory" ]
+		assertEquals "Directory Excluded [$TARGET_DIR_FILE_PUSH/$directory]" "1" $?
+	done
+
+	diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-push/$SOURCE_DIR" | grep -i Exclu
+	[ $(diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-push/$SOURCE_DIR" | grep -i Exclu | wc -l) -eq 2 ]
+	assertEquals "Diff should only output excluded files" "0" $?
+
+	# Tests presence of rotated files
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PUSH_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PUSH/$file$ROTATE_1_EXTENSION" ]
+		assertEquals "Database rotated Presence [$TARGET_DIR_SQL_PUSH/$file$ROTATE_1_EXTENSION]" "0" $?
+	done
+
+	[ -d "$TARGET_DIR_FILE_PUSH/$(dirname $SOURCE_DIR)$ROTATE_1_EXTENSION" ]
+	assertEquals "File rotated Presence [$TARGET_DIR_FILE_PUSH/$(dirname $SOURCE_DIR)$ROTATE_1_EXTENSION]" "0" $?
+
+}
+
+function test_EncryptLocalRun () {
+	SetConfFileValue "$CONF_DIR/$LOCAL_CONF" "ENCRYPTION" "yes"
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$LOCAL_CONF"
+
+	for file in "${FilePresence[@]}"; do
+		[ -f "$TARGET_DIR_CRYPT_LOCAL/$file$CRYPT_EXTENSION" ]
+		assertEquals "File Presence [$TARGET_DIR_CRYPT_LOCAL/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+# TODO: Exclusion lists don't work with encrypted files yet
+#	for file in "${FileExcluded[@]}"; do
+#		[ -f "$TARGET_DIR_CRYPT_LOCAL/$file$CRYPT_EXTENSION" ]
+#		assertEquals "File Excluded [$TARGET_DIR_CRYPT_LOCAL/$file$CRYPT_EXTENSION]" "1" $?
+#	done
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_LOCAL/$file$CRYPT_EXTENSION" ]
+		assertEquals "Database Presence [$TARGET_DIR_SQL_LOCAL/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+#	for file in "${DatabaseExcluded[@]}"; do
+#		[ -f "$TARGET_DIR_SQL_LOCAL/$file$CRYPT_EXTENSION" ]
+#		assertEquals "Database Excluded [$TARGET_DIR_SQL_LOCAL/$file$CRYPT_EXTENSION]" "1" $?
+#	done
+
+#	for directory in "${DirectoriesExcluded[@]}"; do
+#		[ -d "$TARGET_DIR_CRYPT_LOCAL/$directory" ]
+#		assertEquals "Directory Excluded [$TARGET_DIR_CRYPT_LOCAL/$directory]" "1" $?
+#	done
+
+	diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-local/$SOURCE_DIR" | grep -i Exclu
+	[ $(diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-local/$SOURCE_DIR" | grep -i Exclu | wc -l) -eq 5 ]
+	assertEquals "Diff should only output excluded files" "0" $?
+
+	# Tests presence of rotated files
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$LOCAL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_LOCAL/$file$CRYPT_EXTENSION$ROTATE_1_EXTENSION" ]
+		assertEquals "Database rotated Presence [$TARGET_DIR_SQL_LOCAL/$file$CRYPT_EXTENSION$ROTATE_1_EXTENSION]" "0" $?
+	done
+
+	[ -d "$TARGET_DIR_CRYPT_LOCAL/$(dirname $SOURCE_DIR)$CRYPT_EXTENSION$ROTATE_1_EXTENSION" ]
+	assertEquals "File rotated Presence [$TARGET_DIR_CRYPT_LOCAL/$(dirname $SOURCE_DIR)$CRYPT_EXTENSION$ROTATE_1_EXTENSION]" "0" $?
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE --decrypt="$TARGET_DIR_SQL_LOCAL" --passphrase-file="$TESTS_DIR/$PASSFILE"
+	assertEquals "Decrypt sql storage in [$TARGET_DIR_SQL_LOCAL]" "0" $?
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE --decrypt="$TARGET_DIR_CRYPT_LOCAL" --passphrase-file="$TESTS_DIR/$PASSFILE"
+	assertEquals "Decrypt file storage in [$TARGET_DIR_CRYPT_LOCAL]" "0" $?
+
+	SetConfFileValue "$CONF_DIR/$LOCAL_CONF" "ENCRYPTION" "no"
+}
+
+function test_EncryptPullRun () {
+	if [ "$SKIP_REMOTE" == "yes" ]; then
+		return 0
+	fi
+
+	# Basic return code tests. Need to go deep into file presence testing
+	SetConfFileValue "$CONF_DIR/$PULL_CONF" "ENCRYPTION" "yes"
+
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PULL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${FilePresence[@]}"; do
+		[ -f "$TARGET_DIR_CRYPT_PULL/$file$CRYPT_EXTENSION" ]
+		assertEquals "File Presence [$TARGET_DIR_CRYPT_PULL/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+#	for file in "${FileExcluded[@]}"; do
+#		[ -f "$TARGET_DIR_CRYPT_PULL/$file$CRYPT_EXTENSION" ]
+#		assertEquals "File Excluded [$TARGET_DIR_CRYPT_PULL/$file$CRYPT_EXTENSION]" "1" $?
+#	done
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PULL/$file$CRYPT_EXTENSION" ]
+		assertEquals "Database Presence [$TARGET_DIR_SQL_PULL/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+#	for file in "${DatabaseExcluded[@]}"; do
+#		[ -f "$TARGET_DIR_SQL_PULL/$file$CRYPT_EXTENSION" ]
+#		assertEquals "Database Excluded [$TARGET_DIR_SQL_PULL/$file$CRYPT_EXTENSION]" "1" $?
+#	done
+
+#	for directory in "${DirectoriesExcluded[@]}"; do
+#		[ -d "$TARGET_DIR_FILE_PULL/$directory" ]
+#		assertEquals "Directory Excluded [$TARGET_DIR_FILE_PULL/$directory]" "1" $?
+#	done
+
+	# Only excluded files should be listed here
+	diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-pull/$SOURCE_DIR" | grep -i Exclu
+	[ $(diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-pull/$SOURCE_DIR" | grep -i Exclu | wc -l) -eq 2 ]
+	assertEquals "Diff should only output excluded files" "0" $?
+
+	# Tests presence of rotated files
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PULL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PULL/$file$CRYPT_EXTENSION$ROTATE_1_EXTENSION" ]
+		assertEquals "Database rotated Presence [$TARGET_DIR_SQL_PULL/$file$CRYPT_EXTENSION$ROTATE_1_EXTENSION]" "0" $?
+	done
+
+	[ -d "$TARGET_DIR_FILE_CRYPT/$(dirname $SOURCE_DIR)$CRYPT_EXTENSION$ROTATE_1_EXTENSION" ]
+	assertEquals "File rotated Presence [$TARGET_DIR_FILE_CRYPT/$(dirname $SOURCE_DIR)$CRYPT_EXTENSION$ROTATE_1_EXTENSION]" "0" $?
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE --decrypt="$TARGET_DIR_SQL_PULL" --passphrase-file="$TESTS_DIR/$PASSFILE"
+	assertEquals "Decrypt sql storage in [$TARGET_DIR_SQL_PULL]" "0" $?
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE --decrypt="$TARGET_DIR_CRYPT_PULL" --passphrase-file="$TESTS_DIR/$PASSFILE"
+	assertEquals "Decrypt file storage in [$TARGET_DIR_CRYPT_PULL]" "0" $?
+
+	SetConfFileValue "$CONF_DIR/$PULL_CONF" "ENCRYPTION" "no"
+}
+
+function test_EncryptPushRun () {
+	if [ "$SKIP_REMOTE" == "yes" ]; then
+		return 0
+	fi
+
+	# Basic return code tests. Need to go deep into file presence testing
+	SetConfFileValue "$CONF_DIR/$PUSH_CONF" "ENCRYPTION" "yes"
+
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PUSH_CONF"
+	assertEquals "Return code" "0" $?
+
+	# Same here, why do we check for crypt extension in file_push instead of file_crypt
+	for file in "${FilePresence[@]}"; do
+		[ -f "$TARGET_DIR_FILE_PUSH/$file$CRYPT_EXTENSION" ]
+		assertEquals "File Presence [$TARGET_DIR_FILE_PUSH/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+#	for file in "${FileExcluded[@]}"; do
+#		[ -f "$TARGET_DIR_FILE_PUSH/$file$CRYPT_EXTENSION" ]
+#		assertEquals "File Excluded [$TARGET_DIR_FILE_PUSH/$file$CRYPT_EXTENSION]" "1" $?
+#	done
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PUSH/$file$CRYPT_EXTENSION" ]
+		assertEquals "Database Presence [$TARGET_DIR_SQL_PUSH/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+#	for file in "${DatabaseExcluded[@]}"; do
+#		[ -f "$TARGET_DIR_SQL_PUSH/$file$CRYPT_EXTENSION" ]
+#		assertEquals "Database Excluded [$TARGET_DIR_SQL_PUSH/$file$CRYPT_EXTENSION]" "1" $?
+#	done
+
+#	for directory in "${DirectoriesExcluded[@]}"; do
+#		[ -d "$TARGET_DIR_FILE_PUSH/$directory" ]
+#		assertEquals "Directory Excluded [$TARGET_DIR_FILE_PUSH/$directory]" "1" $?
+#	done
+
+	diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-push/$SOURCE_DIR" | grep -i Exclu
+	[ $(diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-push/$SOURCE_DIR" | grep -i Exclu | wc -l) -eq 5 ]
+	assertEquals "Diff should only output excluded files" "0" $?
+	# Tests presence of rotated files
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PUSH_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PUSH/$file$CRYPT_EXTENSION$ROTATE_1_EXTENSION" ]
+		assertEquals "Database rotated Presence [$TARGET_DIR_SQL_PUSH/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+	[ -d "$TARGET_DIR_FILE_PUSH/$(dirname $SOURCE_DIR)$CRYPT_EXTENSION$ROTATE_1_EXTENSION" ]
+	assertEquals "File rotated Presence [$TARGET_DIR_FILE_PUSH/$(dirname $SOURCE_DIR)$CRYPT_EXTENSION$ROTATE_1_EXTENSION]" "0" $?
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE --decrypt="$TARGET_DIR_SQL_PUSH" --passphrase-file="$TESTS_DIR/$PASSFILE" --verbose
+	assertEquals "Decrypt sql storage in [$TARGET_DIR_SQL_PUSH]" "0" $?
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE --decrypt="$TARGET_DIR_FILE_PUSH" --passphrase-file="$TESTS_DIR/$PASSFILE"
+	assertEquals "Decrypt file storage in [$TARGET_DIR_FILE_PUSH]" "0" $?
+
+	SetConfFileValue "$CONF_DIR/$PUSH_CONF" "ENCRYPTION" "no"
+}
+
+function test_missing_databases () {
+	cd "$OBACKUP_DIR"
+
+	# Prepare files for missing databases
+	for i in "$LOCAL_CONF" "$PUSH_CONF" "$PULL_CONF"; do
+		SetConfFileValue "$CONF_DIR/$i" "DATABASES_ALL" "no"
+		SetConfFileValue "$CONF_DIR/$i" "DATABASES_LIST" "\"zorglub;mysql\""
+		SetConfFileValue "$CONF_DIR/$i" "SQL_BACKUP" "yes"
+		SetConfFileValue "$CONF_DIR/$i" "FILE_BACKUP" "no"
+
+		REMOTE_HOST=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$i"
+		assertEquals "Missing databases should trigger error with [$i]" "1" $?
+
+		SetConfFileValue "$CONF_DIR/$i" "DATABASES_ALL" "yes"
+		SetConfFileValue "$CONF_DIR/$i" "DATABASES_LIST" "mysql"
+		SetConfFileValue "$CONF_DIR/$i" "FILE_BACKUP" "yes"
+
+	done
+
+	for i in "$LOCAL_CONF" "$PUSH_CONF" "$PULL_CONF"; do
+		SetConfFileValue "$CONF_DIR/$i" "DIRECTORY_LIST" "${HOME}/obackup-testdata/nonPresentData"
+		SetConfFileValue "$CONF_DIR/$i" "RECURSIVE_DIRECTORY_LIST" "${HOME}/obackup-testdata/nonPresentDataRecursive"
+		SetConfFileValue "$CONF_DIR/$i" "SQL_BACKUP" "no"
+		SetConfFileValue "$CONF_DIR/$i" "FILE_BACKUP" "yes"
+
+		REMOTE_HOST=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$i"
+		assertEquals "Missing files should trigger error with [$i]" "1" $?
+		echo "glob"
+		return
+		echo "nope"
+		SetConfFileValue "$CONF_DIR/$i" "DIRECTORY_LIST" "${HOME}/obackup-testdata/testData"
+		SetConfFileValue "$CONF_DIR/$i" "RECURSIVE_DIRECTORY_LIST" "${HOME}/obackup-testdata/testDataRecursive"
+		SetConfFileValue "$CONF_DIR/$i" "SQL_BACKUP" "yes"
+	done
+}
+
+function test_timed_execution () {
+	cd "$OBACKUP_DIR"
+
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_DB_TASK" 1
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_TOTAL" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_TOTAL" 1000
+
+	SLEEP_TIME=2 REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$MAX_EXEC_CONF"
+	assertEquals "Soft max exec time db reached in obackup Return code" "2" $?
+
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_DB_TASK" 1
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_TOTAL" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_TOTAL" 1000
+
+	SLEEP_TIME=2 REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$MAX_EXEC_CONF"
+	assertEquals "Hard max exec time db reached in obackup Return code" "1" $?
+
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_FILE_TASK" 1
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_TOTAL" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_TOTAL" 1000
+
+	SLEEP_TIME=2 REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$MAX_EXEC_CONF"
+	assertEquals "Soft max exec time file reached in obackup Return code" "2" $?
+
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_FILE_TASK" 1
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_TOTAL" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_TOTAL" 1000
+
+	SLEEP_TIME=2 REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$MAX_EXEC_CONF"
+	assertEquals "Hard max exec time file reached in obackup Return code" "1" $?
+
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_TOTAL" 1
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_TOTAL" 1000
+
+	SLEEP_TIME=1.5 REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$MAX_EXEC_CONF"
+	assertEquals "Soft max exec time total reached in obackup Return code" "2" $?
+
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_TOTAL" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_TOTAL" 1
+
+	SLEEP_TIME=2 REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$MAX_EXEC_CONF"
+	assertEquals "Hard max exec time total reached in obackup Return code" "1" $?
+}
+
+function test_WaitForTaskCompletion () {
+	local pids
+	# Standard wait
+	sleep 1 &
+	pids="$!"
+	sleep 2 &
+	pids="$pids;$!"
+	WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
+	assertEquals "WaitForTaskCompletion test 1" "0" $?
+
+	# Standard wait with warning
+	sleep 2 &
+	pids="$!"
+	sleep 5 &
+	pids="$pids;$!"
+
+	WaitForTaskCompletion $pids 3 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
+	assertEquals "WaitForTaskCompletion test 2" "0" $?
+
+	# Both pids are killed
+	sleep 5 &
+	pids="$!"
+	sleep 5 &
+	pids="$pids;$!"
+
+	WaitForTaskCompletion $pids 0 2 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
+	assertEquals "WaitForTaskCompletion test 3" "2" $?
+
+	# One of two pids are killed
+	sleep 2 &
+	pids="$!"
+	sleep 10 &
+	pids="$pids;$!"
+
+	WaitForTaskCompletion $pids 0 3 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
+	assertEquals "WaitForTaskCompletion test 4" "1" $?
+
+	# Count since script begin, the following should output two warnings and both pids should get killed
+	sleep 20 &
+	pids="$!"
+	sleep 20 &
+	pids="$pids;$!"
+
+	WaitForTaskCompletion $pids 3 5 $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
+	assertEquals "WaitForTaskCompletion test 5" "2" $?
+}
+
+function test_ParallelExec () {
+	local cmd
+
+	# Test if parallelExec works correctly in array mode
+
+	cmd="sleep 2;sleep 2;sleep 2;sleep 2"
+	ParallelExec 4 "$cmd"
+	assertEquals "ParallelExec test 1" "0" $?
+
+	cmd="sleep 2;du /none;sleep 2"
+	ParallelExec 2 "$cmd"
+	assertEquals "ParallelExec test 2" "1" $?
+
+	cmd="sleep 4;du /none;sleep 3;du /none;sleep 2"
+	ParallelExec 3 "$cmd"
+	assertEquals "ParallelExec test 3" "2" $?
+
+	# Test if parallelExec works correctly in file mode
+
+	echo "sleep 2" > "$TMP_FILE"
+	echo "sleep 2" >> "$TMP_FILE"
+	echo "sleep 2" >> "$TMP_FILE"
+	echo "sleep 2" >> "$TMP_FILE"
+	ParallelExec 4 "$TMP_FILE" true
+	assertEquals "ParallelExec test 4" "0" $?
+
+	echo "sleep 2" > "$TMP_FILE"
+	echo "du /nome" >> "$TMP_FILE"
+	echo "sleep 2" >> "$TMP_FILE"
+	ParallelExec 2 "$TMP_FILE" true
+	assertEquals "ParallelExec test 5" "1" $?
+
+	echo "sleep 4" > "$TMP_FILE"
+	echo "du /none" >> "$TMP_FILE"
+	echo "sleep 3" >> "$TMP_FILE"
+	echo "du /none" >> "$TMP_FILE"
+	echo "sleep 2" >> "$TMP_FILE"
+	ParallelExec 3 "$TMP_FILE" true
+	assertEquals "ParallelExec test 6" "2" $?
+
+	#function ParallelExec $numberOfProcesses $commandsArg $readFromFile $softTime $HardTime $sleepTime $keepLogging $counting $Spinner $noError $callerName
+	# Test if parallelExec works correctly in array mode with full  time control
+
+	cmd="sleep 5;sleep 5;sleep 5;sleep 5;sleep 5"
+	ParallelExec 4 "$cmd" false 1 0 .05 3600 true true false ${FUNCNAME[0]}
+	assertEquals "ParallelExec full test 1" "0" $?
+
+	cmd="sleep 2;du /none;sleep 2;sleep 2;sleep 4"
+	ParallelExec 2 "$cmd" false 0 0 .1 2 true false false ${FUNCNAME[0]}
+	assertEquals "ParallelExec full test 2" "1" $?
+
+	cmd="sleep 4;du /none;sleep 3;du /none;sleep 2"
+	ParallelExec 3 "$cmd" false 1 2 .05 7000 true true false ${FUNCNAME[0]}
+	assertNotEquals "ParallelExec full test 3" "0" $?
+
+}
+
+function test_UpgradeConfPullRun () {
+
+	# Basic return code tests. Need to go deep into file presence testing
+	cd "$OBACKUP_DIR"
+
+
+	# Make a security copy of the old config file
+	cp "$CONF_DIR/$OLD_CONF" "$CONF_DIR/$TMP_OLD_CONF"
+
+	./$OBACKUP_UPGRADE "$CONF_DIR/$TMP_OLD_CONF"
+	assertEquals "Conf file upgrade" "0" $?
+
+        # Update remote conf files with SSH port
+        sed -i.tmp 's#ssh://.*@localhost:[0-9]*/#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/#' "$CONF_DIR/$TMP_OLD_CONF"
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$TMP_OLD_CONF"
+	assertEquals "Upgraded conf file execution test" "0" $?
+
+	rm -f "$CONF_DIR/$TMP_OLD_CONF"
+	rm -f "$CONF_DIR/$TMP_OLD_CONF.save"
+}
+
+. "$TESTS_DIR/shunit2/shunit2"
+

dev/tests/shunit2/shunit2_test.sh

@@ -0,0 +1,124 @@
+#! /bin/sh
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2008 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit test suite runner.
+#
+# This script runs all the unit tests that can be found, and generates a nice
+# report of the tests.
+
+MY_NAME=`basename $0`
+MY_PATH=`dirname $0`
+
+PREFIX='shunit2_test_'
+SHELLS='/bin/sh /bin/bash /bin/dash /bin/ksh /bin/pdksh /bin/zsh'
+TESTS=''
+for test in ${PREFIX}[a-z]*.sh; do
+  TESTS="${TESTS} ${test}"
+done
+
+# load common unit test functions
+. ../lib/versions
+. ./shunit2_test_helpers
+
+usage()
+{
+  echo "usage: ${MY_NAME} [-e key=val ...] [-s shell(s)] [-t test(s)]"
+}
+
+env=''
+
+# process command line flags
+while getopts 'e:hs:t:' opt; do
+  case ${opt} in
+    e)  # set an environment variable
+      key=`expr "${OPTARG}" : '\([^=]*\)='`
+      val=`expr "${OPTARG}" : '[^=]*=\(.*\)'`
+      if [ -z "${key}" -o -z "${val}" ]; then
+        usage
+        exit 1
+      fi
+      eval "${key}='${val}'"
+      export ${key}
+      env="${env:+${env} }${key}"
+      ;;
+    h) usage; exit 0 ;;  # output help
+    s) shells=${OPTARG} ;;  # list of shells to run
+    t) tests=${OPTARG} ;;  # list of tests to run
+    *) usage; exit 1 ;;
+  esac
+done
+shift `expr ${OPTIND} - 1`
+
+# fill shells and/or tests
+shells=${shells:-${SHELLS}}
+tests=${tests:-${TESTS}}
+
+# error checking
+if [ -z "${tests}" ]; then
+  th_error 'no tests found to run; exiting'
+  exit 1
+fi
+
+cat <<EOF
+#------------------------------------------------------------------------------
+# System data
+#
+
+# test run info
+shells: ${shells}
+tests: ${tests}
+EOF
+for key in ${env}; do
+  eval "echo \"${key}=\$${key}\""
+done
+echo
+
+# output system data
+echo "# system info"
+echo "$ date"
+date
+echo
+
+echo "$ uname -mprsv"
+uname -mprsv
+
+#
+# run tests
+#
+
+for shell in ${shells}; do
+  echo
+
+  # check for existance of shell
+  if [ ! -x ${shell} ]; then
+    th_warn "unable to run tests with the ${shell} shell"
+    continue
+  fi
+
+  cat <<EOF
+
+#------------------------------------------------------------------------------
+# Running the test suite with ${shell}
+#
+EOF
+
+  SHUNIT_SHELL=${shell}  # pass shell onto tests
+  shell_name=`basename ${shell}`
+  shell_version=`versions_shellVersion "${shell}"`
+
+  echo "shell name: ${shell_name}"
+  echo "shell version: ${shell_version}"
+
+  # execute the tests
+  for suite in ${tests}; do
+    suiteName=`expr "${suite}" : "${PREFIX}\(.*\).sh"`
+    echo
+    echo "--- Executing the '${suiteName}' test suite ---"
+    ( exec ${shell} ./${suite} 2>&1; )
+  done
+done

dev/tests/shunit2/shunit2_test_asserts.sh

@@ -0,0 +1,206 @@
+#! /bin/sh
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2008 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+#
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit test for assert functions
+
+# load test helpers
+. ./shunit2_test_helpers
+
+#------------------------------------------------------------------------------
+# suite tests
+#
+
+commonEqualsSame()
+{
+  fn=$1
+
+  ( ${fn} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'equal' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} "${MSG}" 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'equal; with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} 'abc def' 'abc def' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'equal with spaces' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'not equal' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} '' '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'null values' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} arg1 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+commonNotEqualsSame()
+{
+  fn=$1
+
+  ( ${fn} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not same' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} "${MSG}" 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not same, with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'same' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} '' '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'null values' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} arg1 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+testAssertEquals()
+{
+  commonEqualsSame 'assertEquals'
+}
+
+testAssertNotEquals()
+{
+  commonNotEqualsSame 'assertNotEquals'
+}
+
+testAssertSame()
+{
+  commonEqualsSame 'assertSame'
+}
+
+testAssertNotSame()
+{
+  commonNotEqualsSame 'assertNotSame'
+}
+
+testAssertNull()
+{
+  ( assertNull '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'null' $? "${stdoutF}" "${stderrF}"
+
+  ( assertNull "${MSG}" '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'null, with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( assertNull 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'not null' $? "${stdoutF}" "${stderrF}"
+
+  ( assertNull >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( assertNull arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+testAssertNotNull()
+{
+  ( assertNotNull 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not null' $? "${stdoutF}" "${stderrF}"
+
+  ( assertNotNull "${MSG}" 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not null, with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( assertNotNull 'x"b' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not null, with double-quote' $? \
+      "${stdoutF}" "${stderrF}"
+
+  ( assertNotNull "x'b" >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not null, with single-quote' $? \
+      "${stdoutF}" "${stderrF}"
+
+  ( assertNotNull 'x$b' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not null, with dollar' $? \
+      "${stdoutF}" "${stderrF}"
+
+  ( assertNotNull 'x`b' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not null, with backtick' $? \
+      "${stdoutF}" "${stderrF}"
+
+  ( assertNotNull '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'null' $? "${stdoutF}" "${stderrF}"
+
+  # there is no test for too few arguments as $1 might actually be null
+
+  ( assertNotNull arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+testAssertTrue()
+{
+  ( assertTrue 0 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'true' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue "${MSG}" 0 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'true, with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue '[ 0 -eq 0 ]' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'true condition' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue 1 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'false' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue '[ 0 -eq 1 ]' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'false condition' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'null' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+testAssertFalse()
+{
+  ( assertFalse 1 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'false' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse "${MSG}" 1 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'false, with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse '[ 0 -eq 1 ]' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'false condition' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse 0 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'true' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse '[ 0 -eq 0 ]' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'true condition' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'true condition' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+#------------------------------------------------------------------------------
+# suite functions
+#
+
+oneTimeSetUp()
+{
+  th_oneTimeSetUp
+
+  MSG='This is a test message'
+}
+
+# load and run shUnit2
+[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0
+. ${TH_SHUNIT}

dev/tests/shunit2/shunit2_test_failures.sh

@@ -0,0 +1,86 @@
+#! /bin/sh
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2008 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+#
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit test for failure functions
+
+# load common unit-test functions
+. ./shunit2_test_helpers
+
+#-----------------------------------------------------------------------------
+# suite tests
+#
+
+testFail()
+{
+  ( fail >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'fail' $? "${stdoutF}" "${stderrF}"
+
+  ( fail "${MSG}" >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'fail with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( fail arg1 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+testFailNotEquals()
+{
+  ( failNotEquals 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'same' $? "${stdoutF}" "${stderrF}"
+
+  ( failNotEquals "${MSG}" 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'same with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( failNotEquals 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'not same' $? "${stdoutF}" "${stderrF}"
+
+  ( failNotEquals '' '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'null values' $? "${stdoutF}" "${stderrF}"
+
+  ( failNotEquals >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( failNotEquals arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+testFailSame()
+{
+  ( failSame 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'same' $? "${stdoutF}" "${stderrF}"
+
+  ( failSame "${MSG}" 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'same with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( failSame 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'not same' $? "${stdoutF}" "${stderrF}"
+
+  ( failSame '' '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'null values' $? "${stdoutF}" "${stderrF}"
+
+  ( failSame >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( failSame arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+#-----------------------------------------------------------------------------
+# suite functions
+#
+
+oneTimeSetUp()
+{
+  th_oneTimeSetUp
+
+  MSG='This is a test message'
+}
+
+# load and run shUnit2
+[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0
+. ${TH_SHUNIT}

dev/tests/shunit2/shunit2_test_helpers

@@ -0,0 +1,229 @@
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2008 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+#
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit test common functions
+
+# treat unset variables as an error when performing parameter expansion
+set -u
+
+# set shwordsplit for zsh
+[ -n "${ZSH_VERSION:-}" ] && setopt shwordsplit
+
+#
+# constants
+#
+
+# path to shUnit2 library. can be overridden by setting SHUNIT_INC
+TH_SHUNIT=${SHUNIT_INC:-./shunit2}
+
+# configure debugging. set the DEBUG environment variable to any
+# non-empty value to enable debug output, or TRACE to enable trace
+# output.
+TRACE=${TRACE:+'th_trace '}
+[ -n "${TRACE}" ] && DEBUG=1
+[ -z "${TRACE}" ] && TRACE=':'
+
+DEBUG=${DEBUG:+'th_debug '}
+[ -z "${DEBUG}" ] && DEBUG=':'
+
+#
+# variables
+#
+
+th_RANDOM=0
+
+#
+# functions
+#
+
+# message functions
+th_trace() { echo "${MY_NAME}:TRACE $@" >&2; }
+th_debug() { echo "${MY_NAME}:DEBUG $@" >&2; }
+th_info() { echo "${MY_NAME}:INFO $@" >&2; }
+th_warn() { echo "${MY_NAME}:WARN $@" >&2; }
+th_error() { echo "${MY_NAME}:ERROR $@" >&2; }
+th_fatal() { echo "${MY_NAME}:FATAL $@" >&2; }
+
+# output subtest name
+th_subtest() { echo " $@" >&2; }
+
+th_oneTimeSetUp()
+{
+  # these files will be cleaned up automatically by shUnit2
+  stdoutF="${SHUNIT_TMPDIR}/stdout"
+  stderrF="${SHUNIT_TMPDIR}/stderr"
+  returnF="${SHUNIT_TMPDIR}/return"
+  expectedF="${SHUNIT_TMPDIR}/expected"
+}
+
+# generate a random number
+th_generateRandom()
+{
+  tfgr_random=${th_RANDOM}
+
+  while [ "${tfgr_random}" = "${th_RANDOM}" ]; do
+    if [ -n "${RANDOM:-}" ]; then
+      # $RANDOM works
+      tfgr_random=${RANDOM}${RANDOM}${RANDOM}$$
+    elif [ -r '/dev/urandom' ]; then
+      tfgr_random=`od -vAn -N4 -tu4 </dev/urandom |sed 's/^[^0-9]*//'`
+    else
+      tfgr_date=`date '+%H%M%S'`
+      tfgr_random=`expr ${tfgr_date} \* $$`
+      unset tfgr_date
+    fi
+    [ "${tfgr_random}" = "${th_RANDOM}" ] && sleep 1
+  done
+
+  th_RANDOM=${tfgr_random}
+  unset tfgr_random
+}
+
+# this section returns the data section from the specified section of a file. a
+# datasection is defined by a [header], one or more lines of data, and then a
+# blank line.
+th_getDataSect()
+{
+  th_sgrep "\\[$1\\]" "$2" |sed '1d'
+}
+
+# this function greps a section from a file. a section is defined as a group of
+# lines preceeded and followed by blank lines.
+th_sgrep()
+{
+  th_pattern_=$1
+  shift
+
+  sed -e '/./{H;$!d;}' -e "x;/${th_pattern_}/"'!d;' $@ |sed '1d'
+
+  unset th_pattern_
+}
+
+# Custom assert that checks for true return value (0), and no output to STDOUT
+# or STDERR. If a non-zero return value is encountered, the output of STDERR
+# will be output.
+#
+# Args:
+#  th_test_: string: name of the subtest
+#  th_rtrn_: integer: the return value of the subtest performed
+#  th_stdout_: string: filename where stdout was redirected to
+#  th_stderr_: string: filename where stderr was redirected to
+th_assertTrueWithNoOutput()
+{
+  th_test_=$1
+  th_rtrn_=$2
+  th_stdout_=$3
+  th_stderr_=$4
+
+  assertTrue "${th_test_}; expected return value of zero" ${th_rtrn_}
+  [ ${th_rtrn_} -ne ${SHUNIT_TRUE} ] && cat "${th_stderr_}"
+  assertFalse "${th_test_}; expected no output to STDOUT" \
+      "[ -s '${th_stdout_}' ]"
+  assertFalse "${th_test_}; expected no output to STDERR" \
+      "[ -s '${th_stderr_}' ]"
+
+  unset th_test_ th_rtrn_ th_stdout_ th_stderr_
+}
+
+# Custom assert that checks for non-zero return value, output to STDOUT, but no
+# output to STDERR.
+#
+# Args:
+#  th_test_: string: name of the subtest
+#  th_rtrn_: integer: the return value of the subtest performed
+#  th_stdout_: string: filename where stdout was redirected to
+#  th_stderr_: string: filename where stderr was redirected to
+th_assertFalseWithOutput()
+{
+  th_test_=$1
+  th_rtrn_=$2
+  th_stdout_=$3
+  th_stderr_=$4
+
+  assertFalse "${th_test_}; expected non-zero return value" ${th_rtrn_}
+  assertTrue "${th_test_}; expected output to STDOUT" \
+      "[ -s '${th_stdout_}' ]"
+  assertFalse "${th_test_}; expected no output to STDERR" \
+      "[ -s '${th_stderr_}' ]"
+  [ -s "${th_stdout_}" -a ! -s "${th_stderr_}" ] || \
+      _th_showOutput ${SHUNIT_FALSE} "${th_stdout_}" "${th_stderr_}"
+
+  unset th_test_ th_rtrn_ th_stdout_ th_stderr_
+}
+
+# Custom assert that checks for non-zero return value, no output to STDOUT, but
+# output to STDERR.
+#
+# Args:
+#  th_test_: string: name of the subtest
+#  th_rtrn_: integer: the return value of the subtest performed
+#  th_stdout_: string: filename where stdout was redirected to
+#  th_stderr_: string: filename where stderr was redirected to
+th_assertFalseWithError()
+{
+  th_test_=$1
+  th_rtrn_=$2
+  th_stdout_=$3
+  th_stderr_=$4
+
+  assertFalse "${th_test_}; expected non-zero return value" ${th_rtrn_}
+  assertFalse "${th_test_}; expected no output to STDOUT" \
+      "[ -s '${th_stdout_}' ]"
+  assertTrue "${th_test_}; expected output to STDERR" \
+      "[ -s '${th_stderr_}' ]"
+  [ ! -s "${th_stdout_}" -a -s "${th_stderr_}" ] || \
+      _th_showOutput ${SHUNIT_FALSE} "${th_stdout_}" "${th_stderr_}"
+
+  unset th_test_ th_rtrn_ th_stdout_ th_stderr_
+}
+
+# Some shells, zsh on Solaris in particular, return immediately from a sub-shell
+# when a non-zero return value is encountered. To properly catch these values,
+# they are either written to disk, or recognized as an error the file is empty.
+th_clearReturn() { cp /dev/null "${returnF}"; }
+th_queryReturn()
+{
+  if [ -s "${returnF}" ]; then
+    th_return=`cat "${returnF}"`
+  else
+    th_return=${SHUNIT_ERROR}
+  fi
+}
+
+# Providing external and internal calls to the showOutput helper function.
+th_showOutput() { _th_showOutput $@; }
+_th_showOutput()
+{
+  _th_return_=$1
+  _th_stdout_=$2
+  _th_stderr_=$3
+
+  isSkipping
+  if [ $? -eq ${SHUNIT_FALSE} -a ${_th_return_} != ${SHUNIT_TRUE} ]; then
+    if [ -n "${_th_stdout_}" -a -s "${_th_stdout_}" ]; then
+      echo '>>> STDOUT' >&2
+      cat "${_th_stdout_}" >&2
+    fi
+    if [ -n "${_th_stderr_}" -a -s "${_th_stderr_}" ]; then
+      echo '>>> STDERR' >&2
+      cat "${_th_stderr_}" >&2
+    fi
+    if [ -n "${_th_stdout_}" -o -n "${_th_stderr_}" ]; then
+      echo '<<< end output' >&2
+    fi
+  fi
+
+  unset _th_return_ _th_stdout_ _th_stderr_
+}
+
+#
+# main
+#
+
+${TRACE} 'trace output enabled'
+${DEBUG} 'debug output enabled'

dev/tests/shunit2/shunit2_test_macros.sh

@@ -0,0 +1,246 @@
+#! /bin/sh
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2008 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit test for macros.
+
+# load test helpers
+. ./shunit2_test_helpers
+
+#------------------------------------------------------------------------------
+# suite tests
+#
+
+testAssertEquals()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_EQUALS_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_EQUALS_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_EQUALS_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_EQUALS_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testAssertNotEquals()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_NOT_EQUALS_} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NOT_EQUALS_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_NOT_EQUALS_} '"some msg"' 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NOT_EQUALS_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testSame()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_SAME_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_SAME_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_SAME_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_SAME_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testNotSame()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_NOT_SAME_} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NOT_SAME_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_NOT_SAME_} '"some msg"' 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NOT_SAME_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testNull()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_NULL_} 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NULL_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_NULL_} '"some msg"' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NULL_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testNotNull()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_NOT_NULL_} '' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NOT_NULL_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_NOT_NULL_} '"some msg"' '""' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NOT_NULL_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stdoutF}" "${stderrF}" >&2
+}
+
+testAssertTrue()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_TRUE_} ${SHUNIT_FALSE} >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_TRUE_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+
+  ( ${_ASSERT_TRUE_} '"some msg"' ${SHUNIT_FALSE} >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_TRUE_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testAssertFalse()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_FALSE_} ${SHUNIT_TRUE} >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_FALSE_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_FALSE_} '"some msg"' ${SHUNIT_TRUE} >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_FALSE_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testFail()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_FAIL_} >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_FAIL_} '"some msg"' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testFailNotEquals()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_FAIL_NOT_EQUALS_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_NOT_EQUALS_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_FAIL_NOT_EQUALS_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_NOT_EQUALS_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testFailSame()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_FAIL_SAME_} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_SAME_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_FAIL_SAME_} '"some msg"' 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_SAME_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testFailNotSame()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_FAIL_NOT_SAME_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_NOT_SAME_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_FAIL_NOT_SAME_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_NOT_SAME_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+#------------------------------------------------------------------------------
+# suite functions
+#
+
+oneTimeSetUp()
+{
+  th_oneTimeSetUp
+}
+
+# load and run shUnit2
+[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0
+. ${TH_SHUNIT}

dev/tests/shunit2/shunit2_test_misc.sh

@@ -0,0 +1,160 @@
+#! /bin/sh
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2008 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+#
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit tests of miscellaneous things
+
+# load test helpers
+. ./shunit2_test_helpers
+
+#------------------------------------------------------------------------------
+# suite tests
+#
+
+# Note: the test script is prefixed with '#' chars so that shUnit2 does not
+# incorrectly interpret the embedded functions as real functions.
+testUnboundVariable()
+{
+  unittestF="${SHUNIT_TMPDIR}/unittest"
+  sed 's/^#//' >"${unittestF}" <<EOF
+## treat unset variables as an error when performing parameter expansion
+#set -u
+#
+#boom() { x=\$1; }  # this function goes boom if no parameters are passed!
+#test_boom()
+#{
+#   assertEquals 1 1
+#   boom  # No parameter given
+#   assertEquals 0 \$?
+#}
+#. ${TH_SHUNIT}
+EOF
+  ( exec ${SHUNIT_SHELL:-sh} "${unittestF}" >"${stdoutF}" 2>"${stderrF}" )
+  assertFalse 'expected a non-zero exit value' $?
+  grep '^ASSERT:Unknown failure' "${stdoutF}" >/dev/null
+  assertTrue 'assert message was not generated' $?
+  grep '^Ran [0-9]* test' "${stdoutF}" >/dev/null
+  assertTrue 'test count message was not generated' $?
+  grep '^FAILED' "${stdoutF}" >/dev/null
+  assertTrue 'failure message was not generated' $?
+}
+
+testIssue7()
+{
+  ( assertEquals 'Some message.' 1 2 >"${stdoutF}" 2>"${stderrF}" )
+  diff "${stdoutF}" - >/dev/null <<EOF
+ASSERT:Some message. expected:<1> but was:<2>
+EOF
+  rtrn=$?
+  assertEquals ${SHUNIT_TRUE} ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testPrepForSourcing()
+{
+  assertEquals '/abc' `_shunit_prepForSourcing '/abc'`
+  assertEquals './abc' `_shunit_prepForSourcing './abc'`
+  assertEquals './abc' `_shunit_prepForSourcing 'abc'`
+}
+
+testEscapeCharInStr()
+{
+  actual=`_shunit_escapeCharInStr '\' ''`
+  assertEquals '' "${actual}"
+  assertEquals 'abc\\' `_shunit_escapeCharInStr '\' 'abc\'`
+  assertEquals 'abc\\def' `_shunit_escapeCharInStr '\' 'abc\def'`
+  assertEquals '\\def' `_shunit_escapeCharInStr '\' '\def'`
+
+  actual=`_shunit_escapeCharInStr '"' ''`
+  assertEquals '' "${actual}"
+  assertEquals 'abc\"' `_shunit_escapeCharInStr '"' 'abc"'`
+  assertEquals 'abc\"def' `_shunit_escapeCharInStr '"' 'abc"def'`
+  assertEquals '\"def' `_shunit_escapeCharInStr '"' '"def'`
+
+  actual=`_shunit_escapeCharInStr '$' ''`
+  assertEquals '' "${actual}"
+  assertEquals 'abc\$' `_shunit_escapeCharInStr '$' 'abc$'`
+  assertEquals 'abc\$def' `_shunit_escapeCharInStr '$' 'abc$def'`
+  assertEquals '\$def' `_shunit_escapeCharInStr '$' '$def'`
+
+#  actual=`_shunit_escapeCharInStr "'" ''`
+#  assertEquals '' "${actual}"
+#  assertEquals "abc\\'" `_shunit_escapeCharInStr "'" "abc'"`
+#  assertEquals "abc\\'def" `_shunit_escapeCharInStr "'" "abc'def"`
+#  assertEquals "\\'def" `_shunit_escapeCharInStr "'" "'def"`
+
+#  # must put the backtick in a variable so the shell doesn't misinterpret it
+#  # while inside a backticked sequence (e.g. `echo '`'` would fail).
+#  backtick='`'
+#  actual=`_shunit_escapeCharInStr ${backtick} ''`
+#  assertEquals '' "${actual}"
+#  assertEquals '\`abc' \
+#      `_shunit_escapeCharInStr "${backtick}" ${backtick}'abc'`
+#  assertEquals 'abc\`' \
+#      `_shunit_escapeCharInStr "${backtick}" 'abc'${backtick}`
+#  assertEquals 'abc\`def' \
+#      `_shunit_escapeCharInStr "${backtick}" 'abc'${backtick}'def'`
+}
+
+testEscapeCharInStr_specialChars()
+{
+  # make sure our forward slash doesn't upset sed
+  assertEquals '/' `_shunit_escapeCharInStr '\' '/'`
+
+  # some shells escape these differently
+  #assertEquals '\\a' `_shunit_escapeCharInStr '\' '\a'`
+  #assertEquals '\\b' `_shunit_escapeCharInStr '\' '\b'`
+}
+
+# Test the various ways of declaring functions.
+#
+# Prefixing (then stripping) with comment symbol so these functions aren't
+# treated as real functions by shUnit2.
+testExtractTestFunctions()
+{
+  f="${SHUNIT_TMPDIR}/extract_test_functions"
+  sed 's/^#//' <<EOF >"${f}"
+#testABC() { echo 'ABC'; }
+#test_def() {
+#  echo 'def'
+#}
+#testG3 ()
+#{
+#  echo 'G3'
+#}
+#function test4() { echo '4'; }
+#	test5() { echo '5'; }
+#some_test_function() { echo 'some func'; }
+#func_with_test_vars() {
+#  testVariable=1234
+#}
+EOF
+
+  actual=`_shunit_extractTestFunctions "${f}"`
+  assertEquals 'testABC test_def testG3 test4 test5' "${actual}"
+}
+
+#------------------------------------------------------------------------------
+# suite functions
+#
+
+setUp()
+{
+  for f in ${expectedF} ${stdoutF} ${stderrF}; do
+    cp /dev/null ${f}
+  done
+}
+
+oneTimeSetUp()
+{
+  th_oneTimeSetUp
+}
+
+# load and run shUnit2
+[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0
+. ${TH_SHUNIT}

dev/tests/shunit2/shunit2_test_standalone.sh

@@ -0,0 +1,41 @@
+#! /bin/sh
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2010 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit test for standalone operation.
+#
+# This unit test is purely to test that calling shunit2 directly, while passing
+# the name of a unit test script, works. When run, this script determines if it
+# is running as a standalone program, and calls main() if it is.
+
+ARGV0=`basename "$0"`
+
+# load test helpers
+. ./shunit2_test_helpers
+
+#------------------------------------------------------------------------------
+# suite tests
+#
+
+testStandalone()
+{
+  assertTrue ${SHUNIT_TRUE}
+}
+
+#------------------------------------------------------------------------------
+# main
+#
+
+main()
+{
+  ${TH_SHUNIT} "${ARGV0}"
+}
+
+# are we running as a standalone?
+if [ "${ARGV0}" = 'shunit2_test_standalone.sh' ]; then
+  if [ $# -gt 0 ]; then main "$@"; else main; fi
+fi

host_backup.conf.example

@@ -2,7 +2,7 @@
 
 ###### obackup - Local or Remote, push or pull backup script for files & mysql databases
 ###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr)
-###### obackup v2.x config file rev 2016052501
+###### obackup v2.1x config file rev 2017020901
 
 ###### GENERAL BACKUP OPTIONS
 
@@ -22,13 +22,25 @@ BACKUP_TYPE=local
 
 ###### BACKUP STORAGE
 
-## Storage paths of the backups (absolute paths of the local or remote system)
+## Storage paths of the backups (absolute paths of the local or remote system). Please use ${HOME} instead of ~ if needed.
 SQL_STORAGE="/home/storage/backup/sql"
 FILE_STORAGE="/home/storage/backup/files"
 
-## Backup encryption using GPG and duplicity. Feature not ready yet.
+## Backup encryption using GPG and rsync.
+## Push backups get encrypted locally in CRYPT_STORAGE before they are sent to the remote system
+## Local and pull backups get encrypted after backup, in CRYPT_STORAGE
 ENCRYPTION=no
 
+## Backup encryption needs a temporary storage space in order to encrypt files before sending them (absolute paths of the local or remote system)
+## In case of a pull backup, an encrypted copy of FILE_BACKUP goes here
+CRYPT_STORAGE=/home/storage/backup/crpyt
+
+## GPG recipient (pubkey for this recipient must exist, see gpg2 --list-keys or gpg --list-keys
+GPG_RECIPIENT="John Doe"
+
+## Use n CPUs for encryption / decryption where n is an integer. Defaults to 1
+PARALLEL_ENCRYPTION_PROCESSES=
+
 ## Create backup directories if they do not exist
 CREATE_DIRS=yes
 
@@ -53,12 +65,19 @@ FILE_WARN_MIN_SPACE=1048576
 REMOTE_SYSTEM_URI="ssh://backupuser@remote.system.tld:22/"
 
 ## You can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
-SSH_RSA_PRIVATE_KEY="/root/.ssh/id_rsa"
+SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa"
+
+## Alternatively, you may specify an SSH password file (less secure). Needs sshpass utility installed.
+SSH_PASSWORD_FILE=""
+
+## When using ssh filter, you must specify a remote token matching the one setup in authorized_keys
+_REMOTE_TOKEN=SomeAlphaNumericToken9
 
 ## ssh compression should be used unless your remote connection is good enough (LAN)
 SSH_COMPRESSION=yes
 
 ## Ignore ssh known hosts verification. DANGER WILL ROBINSON DANGER: This can lead to security risks. Only enable if you know what you're doing.
+## Works on Redhat / CentOS, doesn't work on Debian / Ubunutu
 SSH_IGNORE_KNOWN_HOSTS=no
 
 ## Remote rsync executable path. Leave this empty in most cases
@@ -75,14 +94,14 @@ SUDO_EXEC=no
 
 ###### DATABASE SPECIFIC OPTIONS
 
-## Database backup user
-SQL_USER=backupuser
+## Database backup user (should be the same you are running obackup with)
+SQL_USER=root
 
 ## Enabling the following option will save all databases on local or remote given SQL instance except the ones specified in the exclude list.
 ## Every found database will be backed up as separate backup task.
 DATABASES_ALL=yes
-DATABASES_ALL_EXCLUDE_LIST="test"
-## Alternatively, if DATABASES_ALL=no, you can specify a list of databases to backup separated by spaces.
+DATABASES_ALL_EXCLUDE_LIST="test;mysql"
+## Alternatively, if DATABASES_ALL=no, you can specify a list of databases to backup separated by semi-colons.
 #DATABASES_LIST="somedatabase"
 
 ## Max backup execution time per Database task. Soft max exec time generates a warning only. Hard max exec time generates a warning and stops current backup task.
@@ -90,8 +109,13 @@ DATABASES_ALL_EXCLUDE_LIST="test"
 SOFT_MAX_EXEC_TIME_DB_TASK=3600
 HARD_MAX_EXEC_TIME_DB_TASK=7200
 
+## mysqldump options (ex: --extended-insert, --single-transaction, --quick...). See MySQL / MariaDB manual
+## default option: --opt
+MYSQLDUMP_OPTIONS="--opt --single-transaction"
+
 ## Preferred SQL dump compression. Compression methods can be xz, lzma, pigz or gzip (will fallback from xz to gzip depending if available)
 ## Generally, level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
+## If you use encryption, compression will only bring small benefits as GPG already has pretty good compression included
 COMPRESSION_LEVEL=3
 
 ###### FILES SPECIFIC OPTIONS
@@ -100,6 +124,7 @@ COMPRESSION_LEVEL=3
 ## Every subdirectory of each directory in RECURSIVE_DIRECTORY_LIST will be processed as a unique task.
 ## Example: RECURSIVE_DIRECTORY_LIST="/home;/var" will create backup tasks tasks "/home/dir1, "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/something".
 ## You can exclude directories from the avove backup task creation, ex: avoid backing up "/home/dir2" by adding it to RECURSIVE_EXCLUDE_LIST.
+## Please use ${HOME} instead of ~ if needed.
 
 ## Directories backup list. List of semicolon separated directories that will be backed up.
 DIRECTORY_LIST="/var/named"
@@ -125,6 +150,17 @@ RSYNC_EXCLUDE_FROM=""
 ## List separator char. You may set an alternative separator char for your directories lists above.
 PATH_SEPARATOR_CHAR=";"
 
+## Optional arguments passed to rsync executable. The following are already managed by the program and shoul never be passed here
+## -rltD -n -P -o -g --executability -A -X -zz -L -K -H -8 -u -i --stats --checksum --bwlimit --partial --partial-dir --exclude --exclude-from --include--from --no-whole-file --whole-file --list-only
+RSYNC_OPTIONAL_ARGS=""
+
+## Preserve basic linux permissions
+PRESERVE_PERMISSIONS=yes
+PRESERVE_OWNER=yes
+PRESERVE_GROUP=yes
+## On MACOS X, does not work and will be ignored 
+PRESERVE_EXECUTABILITY=yes
+
 ## Preserve ACLS. Make sure source and target FS can hold same ACLs or you'll get loads of errors.
 PRESERVE_ACL=no
 ## Preserve Xattr. MAke sure source and target FS can hold same Xattr or you'll get loads of errors.
@@ -165,7 +201,12 @@ RSYNC_EXECUTABLE=rsync
 ## Alert email addresses separated by a space character
 DESTINATION_MAILS="your@mail.address"
 
-## Windows specific (msys / cygwin environment) only mail options (used with mailsend.exe from muquit, http://github.com/muquit/mailsend or from sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail/
+## Optional change of mail body encoding (using iconv)
+## By default, all mails are sent in UTF-8 format without header (because of maximum compatibility of all platforms)
+## You may specify an optional encoding here (like "ISO-8859-1" or whatever iconv can handle)
+MAIL_BODY_CHARSET=""
+
+## Environment specific mail options (used with busybox sendemail, mailsend.exe from muquit, http://github.com/muquit/mailsend or sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail)
 SENDER_MAIL="alert@your.system.tld"
 SMTP_SERVER=smtp.your.isp.tld
 SMTP_PORT=25
@@ -181,6 +222,9 @@ SMTP_PASSWORD=
 SOFT_MAX_EXEC_TIME_TOTAL=30000
 HARD_MAX_EXEC_TIME_TOTAL=36000
 
+## Log a message every KEEP_LOGGING seconds just to know the task is still alive
+KEEP_LOGGING=1801
+
 ## Backup Rotation. You may rotate backups if you don't use snapshots on your backup server.
 ROTATE_SQL_BACKUPS=no
 ROTATE_SQL_COPIES=7

install.sh

@@ -1,115 +1,213 @@
 #!/usr/bin/env bash
 
+## Installer script suitable for osync / obackup / pmocr
+
+_OFUNCTIONS_BOOTSTRAP=true
+
 PROGRAM=obackup
-PROGRAM_VERSION=2.0
+
+PROGRAM_VERSION=$(grep "PROGRAM_VERSION=" $PROGRAM.sh)
+PROGRAM_VERSION=${PROGRAM_VERSION#*=}
 PROGRAM_BINARY=$PROGRAM".sh"
 PROGRAM_BATCH=$PROGRAM"-batch.sh"
-SCRIPT_BUILD=2016052601
+SSH_FILTER="ssh_filter.sh"
+
+SCRIPT_BUILD=2017041701
 
 ## osync / obackup / pmocr / zsnap install script
-## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8 & 10
+## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
 ## Please adapt this to fit your distro needs
 
-#TODO: silent mode and no stats mode
+# Get current install.sh path from http://stackoverflow.com/a/246128/2635443
+SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-CONF_DIR=/etc/$PROGRAM
-BIN_DIR=/usr/local/bin
-SERVICE_DIR_INIT=/etc/init.d
+CONF_DIR=$FAKEROOT/etc/$PROGRAM
+BIN_DIR="$FAKEROOT/usr/local/bin"
+SERVICE_DIR_INIT=$FAKEROOT/etc/init.d
 # Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora
-SERVICE_DIR_SYSTEMD_SYSTEM=/lib/systemd/system
-SERVICE_DIR_SYSTEMD_USER=/etc/systemd/user
+SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system
+SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user
 
-## osync specific code
-OSYNC_SERVICE_FILE_INIT="osync-srv"
-OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM="osync-srv@.service"
-OSYNC_SERVICE_FILE_SYSTEMD_USER="osync-srv@.service.user"
+if [ "$PROGRAM" == "osync" ]; then
+	SERVICE_NAME="osync-srv"
+elif [ "$PROGRAM" == "pmocr" ]; then
+	SERVICE_NAME="pmocr-srv"
+fi
 
-## pmocr specfic code
-PMOCR_SERVICE_FILE_INIT="pmocr-srv"
-PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM="pmocr-srv.service"
+SERVICE_FILE_INIT="$SERVICE_NAME"
+SERVICE_FILE_SYSTEMD_SYSTEM="$SERVICE_NAME@.service"
+SERVICE_FILE_SYSTEMD_USER="$SERVICE_NAME@.service.user"
 
 ## Generic code
 
 ## Default log file
-if [ -w /var/log ]; then
-        LOG_FILE="/var/log/$PROGRAM-install.log"
+if [ -w "$FAKEROOT/var/log" ]; then
+	LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log"
 elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
-        LOG_FILE="$HOME/$PROGRAM-install.log"
+	LOG_FILE="$HOME/$PROGRAM-install.log"
 else
-    	LOG_FILE="./$PROGRAM-install.log"
+	LOG_FILE="./$PROGRAM-install.log"
 fi
 
-# Generic quick logging function
+# QuickLogger subfunction, can be called directly
 function _QuickLogger {
-        local value="${1}"
-       	local destination="${2}" # Destination: stdout, log, both
+	local value="${1}"
+	local destination="${2}" # Destination: stdout, log, both
 
-        if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then
-                echo -e "$(date) - $value" >> "$LOG_FILE"
-        elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then
-                echo -e "$value"
-       	fi
+	if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then
+		echo -e "$(date) - $value" >> "$LOG_FILE"
+	elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then
+		echo -e "$value"
+	fi
 }
 
+# Generic quick logging function
 function QuickLogger {
 	local value="${1}"
 
-	if [ "$_SILENT" -eq 1 ]; then
+	if [ "$_LOGGER_SILENT" == true ]; then
 		_QuickLogger "$value" "log"
 	else
 		_QuickLogger "$value" "stdout"
 	fi
 }
+## from https://gist.github.com/cdown/1163649
+function UrlEncode {
+	local length="${#1}"
 
-function urlencode() {
-    # urlencode <string>
-
-    local LANG=C
-    local length="${#1}"
-    for (( i = 0; i < length; i++ )); do
-        local c="${1:i:1}"
-        case $c in
-            [a-zA-Z0-9.~_-]) printf "$c" ;;
-            *) printf '%%%02X' "'$c" ;; 
-        esac
-    done
+	local LANG=C
+	for (( i = 0; i < length; i++ )); do
+		local c="${1:i:1}"
+		case $c in
+			[a-zA-Z0-9.~_-])
+			printf "$c"
+			;;
+			*)
+			printf '%%%02X' "'$c"
+			;;
+		esac
+	done
 }
+function GetLocalOS {
+	local localOsVar
+	local localOsName
+	local localOsVer
 
-function SetOSSettings {
-	USER=root
-
-	local local_os_var
-
-	local_os_var="$(uname -spio 2>&1)"
-	if [ $? != 0 ]; then
-		local_os_var="$(uname -v 2>&1)"
-		if [ $? != 0 ]; then
-			local_os_var="$(uname)"
+	# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
+	if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
+		localOsVar="BusyBox"
+	else
+		# Detecting the special ubuntu userland in Windows 10 bash
+		if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
+			localOsVar="Microsoft"
+		else
+			localOsVar="$(uname -spior 2>&1)"
+			if [ $? != 0 ]; then
+				localOsVar="$(uname -v 2>&1)"
+				if [ $? != 0 ]; then
+					localOsVar="$(uname)"
+				fi
+			fi
 		fi
 	fi
 
-	case $local_os_var in
+	case $localOsVar in
+		# Android uname contains both linux and android, keep it before linux entry
+		*"Android"*)
+		LOCAL_OS="Android"
+		;;
+		*"Linux"*)
+		LOCAL_OS="Linux"
+		;;
+		*"BSD"*)
+		LOCAL_OS="BSD"
+		;;
+		*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
+		LOCAL_OS="msys"
+		;;
+		*"CYGWIN"*)
+		LOCAL_OS="Cygwin"
+		;;
+		*"Microsoft"*)
+		LOCAL_OS="WinNT10"
+		;;
+		*"Darwin"*)
+		LOCAL_OS="MacOSX"
+		;;
+		*"BusyBox"*)
+		LOCAL_OS="BusyBox"
+		;;
+		*)
+		if [ "$IGNORE_OS_TYPE" == "yes" ]; then
+			Logger "Running on unknown local OS [$localOsVar]." "WARN"
+			return
+		fi
+		if [ "$_OFUNCTIONS_VERSION" != "" ]; then
+			Logger "Running on >> $localOsVar << not supported. Please report to the author." "ERROR"
+		fi
+		exit 1
+		;;
+	esac
+
+	# Get linux versions
+	if [ -f "/etc/os-release" ]; then
+		localOsName=$(GetConfFileValue "/etc/os-release" "NAME")
+		localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION")
+	fi
+
+	# Add a global variable for statistics in installer
+	LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
+
+	if [ "$_OFUNCTIONS_VERSION" != "" ]; then
+		Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG"
+	fi
+}
+function GetConfFileValue () {
+        local file="${1}"
+        local name="${2}"
+        local value
+
+        value=$(grep "^$name=" "$file")
+        if [ $? == 0 ]; then
+                value="${value##*=}"
+                echo "$value"
+        else
+		Logger "Cannot get value for [$name] in config file [$file]." "ERROR"
+        fi
+}
+
+function SetLocalOSSettings {
+	USER=root
+
+	# LOCAL_OS and LOCAL_OS_FULL are global variables set at GetLocalOS
+
+	case $LOCAL_OS in
 		*"BSD"*)
 		GROUP=wheel
 		;;
-		*"Darwin"*)
+		*"MacOSX"*)
 		GROUP=admin
 		;;
+		*"msys"*|*"Cygwin"*)
+		USER=""
+		GROUP=""
+		;;
 		*)
 		GROUP=root
 		;;
-		*"MINGW32"*|*"CYGWIN"*)
-		USER=""
-		GROUP=""
-		;;
 	esac
 
-	if ([ "$USER" != "" ] && [ "$(whoami)" != "$USER" ]); then
-	  	QuickLogger "Must be run as $USER."
+	if [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "BusyBox" ]; then
+		QuickLogger "Cannot be installed on [$LOCAL_OS]. Please use $PROGRAM.sh directly."
 		exit 1
 	fi
 
-	OS=$(urlencode "$local_os_var")
+	if ([ "$USER" != "" ] && [ "$(whoami)" != "$USER" ] && [ "$FAKEROOT" == "" ]); then
+		QuickLogger "Must be run as $USER."
+		exit 1
+	fi
+
+	OS=$(UrlEncode "$LOCAL_OS_FULL")
 }
 
 function GetInit {
@@ -125,137 +223,185 @@ function GetInit {
 	fi
 }
 
-function CreateConfDir {
-	if [ ! -d "$CONF_DIR" ]; then
-		mkdir "$CONF_DIR"
+function CreateDir {
+	local dir="${1}"
+
+	if [ ! -d "$dir" ]; then
+		mkdir "$dir"
 		if [ $? == 0 ]; then
-			QuickLogger "Created directory [$CONF_DIR]."
+			QuickLogger "Created directory [$dir]."
 		else
-			QuickLogger "Cannot create directory [$CONF_DIR]."
+			QuickLogger "Cannot create directory [$dir]."
 			exit 1
 		fi
+	fi
+}
+
+function CopyFile {
+	local sourcePath="${1}"
+	local destPath="${2}"
+	local fileName="${3}"
+	local fileMod="${4}"
+	local fileUser="${5}"
+	local fileGroup="${6}"
+	local overwrite="${7:-false}"
+
+	local userGroup=""
+	local oldFileName
+
+	if [ -f "$destPath/$fileName" ] && [ $overwrite == false ]; then
+		oldFileName="$fileName"
+		fileName="$oldFileName.new"
+		cp "$sourcePath/$oldFileName" "$destPath/$fileName"
 	else
-		QuickLogger "Config directory [$CONF_DIR] exists."
+		cp "$sourcePath/$fileName" "$destPath"
+	fi
+
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot copy [$fileName] to [$destPath]. Make sure to run install script in the directory containing all other files."
+		QuickLogger "Also make sure you have permissions to write to [$BIN_DIR]."
+		exit 1
+	else
+		QuickLogger "Copied [$fileName] to [$destPath]."
+		if [ "$fileMod" != "" ]; then
+			chmod "$fileMod" "$destPath/$fileName"
+			if [ $? != 0 ]; then
+				QuickLogger "Cannot set file permissions of [$destPath/$fileName] to [$fileMod]."
+				exit 1
+			else
+				QuickLogger "Set file permissions to [$fileMod] on [$destPath/$fileName]."
+			fi
+		fi
+
+		if [ "$fileUser" != "" ]; then
+			userGroup="$fileUser"
+
+			if [ "$fileGroup" != "" ]; then
+				userGroup="$userGroup"":$fileGroup"
+			fi
+
+			chown "$userGroup" "$destPath/$fileName"
+			if [ $? != 0 ]; then
+				QuickLogger "Could not set file ownership on [$destPath/$fileName] to [$userGroup]."
+				exit 1
+			else
+				QuickLogger "Set file ownership on [$destPath/$fileName] to [$userGroup]."
+			fi
+		fi
 	fi
 }
 
 function CopyExampleFiles {
-	if [ -f "./sync.conf.example" ]; then
-		cp "./sync.conf.example" "/etc/$PROGRAM/sync.conf.example"
-	fi
+	exampleFiles=()
+	exampleFiles[0]="sync.conf.example"		# osync
+	exampleFiles[1]="host_backup.conf.example"	# obackup
+	exampleFiles[2]="exclude.list.example"		# osync & obackup
+	exampleFiles[3]="snapshot.conf.example"		# zsnap
+	exampleFiles[4]="default.conf"			# pmocr
 
-	if [ -f "./host_backup.conf.example" ]; then
-		cp "./host_backup.conf.example" "/etc/$PROGRAM/host_backup.conf.example"
-	fi
-
-	if [ -f "./exlude.list.example" ]; then
-		cp "./exclude.list.example" "/etc/$PROGRAM"
-	fi
-
-	if [ -f "./snapshot.conf.example" ]; then
-		cp "./snapshot.conf.example" "/etc/$PROGRAM/snapshot.conf.example"
-	fi
+	for file in "${exampleFiles[@]}"; do
+		if [ -f "$SCRIPT_PATH/$file" ]; then
+			CopyFile "$SCRIPT_PATH" "$CONF_DIR" "$file" "" "" "" false
+		fi
+	done
 }
 
 function CopyProgram {
-	cp "./$PROGRAM_BINARY" "$BIN_DIR"
-	if [ $? != 0 ]; then
-		QuickLogger "Cannot copy $PROGRAM_BINARY to [$BIN_DIR]. Make sure to run install script in the directory containing all other files."
-		QuickLogger "Also make sure you have permissions to write to [$BIN_DIR]."
-		exit 1
-	else
-		chmod 755 "$BIN_DIR/$PROGRAM_BINARY"
-		QuickLogger "Copied $PROGRAM_BINARY to [$BIN_DIR]."
+	binFiles=()
+	binFiles[0]="$PROGRAM_BINARY"
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
+		binFiles[1]="$PROGRAM_BATCH"
+		binFiles[2]="$SSH_FILTER"
 	fi
 
-	if [ -f "./$PROGRAM_BATCH" ]; then
-		cp "./$PROGRAM_BATCH" "$BIN_DIR"
-		if [ $? != 0 ]; then
-			QuickLogger "Cannot copy $PROGRAM_BATCH to [$BIN_DIR]."
-		else
-			chmod 755 "$BIN_DIR/$PROGRAM_BATCH"
-			QuickLogger "Copied $PROGRAM_BATCH to [$BIN_DIR]."
-		fi
+	local user=""
+	local group=""
+
+	if ([ "$USER" != "" ] && [ "$FAKEROOT" == "" ]); then
+		user="$USER"
+	fi
+	if ([ "$GROUP" != "" ] && [ "$FAKEROOT" == "" ]); then
+		group="$GROUP"
 	fi
 
-	if [  -f "./ssh_filter.sh" ]; then
-		cp "./ssh_filter.sh" "$BIN_DIR"
-		if [ $? != 0 ]; then
-			QuickLogger "Cannot copy ssh_filter.sh to [$BIN_DIR]."
-		else
-			chmod 755 "$BIN_DIR/ssh_filter.sh"
-			if ([ "$USER" != "" ] && [ "$GROUP" != "" ]); then
-				chown $USER:$GROUP "$BIN_DIR/ssh_filter.sh"
-			fi
-			QuickLogger "Copied ssh_filter.sh to [$BIN_DIR]."
-		fi
-	fi
+	for file in "${binFiles[@]}"; do
+		CopyFile "$SCRIPT_PATH" "$BIN_DIR" "$file" 755 "$user" "$group" true
+	done
 }
 
 function CopyServiceFiles {
-	# OSYNC SPECIFIC
-	if ([ "$init" == "systemd" ] && [ -f "./$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
-		cp "./$OSYNC_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM" && cp "./$OSYNC_SERVICE_FILE_SYSTEMD_USER" "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM"
-		if [ $? != 0 ]; then
-			QuickLogger "Cannot copy the systemd file to [$SERVICE_DIR_SYSTEMD_SYSTEM] or [$SERVICE_DIR_SYSTEMD_USER]."
-		else
-			QuickLogger "Created osync-srv service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
-			QuickLogger "Can be activated with [systemctl start osync-srv@instance.conf] where instance.conf is the name of the config file in /etc/osync."
-			QuickLogger "Can be enabled on boot with [systemctl enable osync-srv@instance.conf]."
-			QuickLogger "In userland, active with [systemctl --user start osync-srv@instance.conf]."
+	if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
+		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true
+		if [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM_USER" ]; then
+			CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true
 		fi
-	elif ([ "$init" == "initV" ] && [ -f "./$OSYNC_SERVICE_FILE_INIT" ]); then
-		cp "./$OSYNC_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT"
-		if [ $? != 0 ]; then
-			QuickLogger "Cannot copy osync-srv to [$SERVICE_DIR_INIT]."
-		else
-			chmod 755 "$SERVICE_DIR_INIT/$OSYNC_SERVICE_FILE_INIT"
-			QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
-			QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]."
-			QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]."
-		fi
-	fi
 
-	# PMOCR SPECIFIC
-	if ([ "$init" == "systemd" ] && [ -f "./$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
-		cp "./$PMOCR_SERVICE_FILE_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_SYSTEM"
-		if [ $? != 0 ]; then
-			QuickLogger "Cannot copy the systemd file to [$SERVICE_DIR_SYSTEMD_SYSTEM] or [$SERVICE_DIR_SYSTEMD_USER]."
-		else
-			QuickLogger "Created pmocr-srv service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
-			QuickLogger "Can be activated with [systemctl start pmocr-srv] after configuring file options in [$BIN_DIR/$PROGRAM]."
-			QuickLogger "Can be enabled on boot with [systemctl enable pmocr-srv]."
-		fi
-	elif ([ "$init" == "initV" ] && [ -f "./$PMOCR_SERVICE_FILE_INIT" ]); then
-		cp "./$PMOCR_SERVICE_FILE_INIT" "$SERVICE_DIR_INIT"
-		if [ $? != 0 ]; then
-			QuickLogger "Cannot copy pmoct-srv to [$SERVICE_DIR_INIT]."
-		else
-			chmod 755 "$SERVICE_DIR_INIT/$PMOCR_SERVICE_FILE_INIT"
-			QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
-			QuickLogger "Can be activated with [service $PMOCR_SERVICE_FILE_INIT start]."
-			QuickLogger "Can be enabled on boot with [chkconfig $PMOCR_SERVICE_FILE_INIT on]."
-		fi
+		QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
+		QuickLogger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR."
+		QuickLogger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]."
+		QuickLogger "In userland, active with [systemctl --user start $SERVICE_NAME@instance.conf]."
+	elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then
+		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true
+
+		QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]."
+		QuickLogger "Can be activated with [service $SERVICE_FILE_INIT start]."
+		QuickLogger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]."
+	else
+		QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation."
 	fi
 }
 
 function Statistics {
-        if type wget > /dev/null; then
-                wget -qO- "$STATS_LINK" > /dev/null 2>&1
-                if [ $? == 0 ]; then
-                        return 0
-                fi
+	if type wget > /dev/null; then
+		wget -qO- "$STATS_LINK" > /dev/null 2>&1
+		if [ $? == 0 ]; then
+			return 0
+		fi
 	fi
 
-        if type curl > /dev/null; then
-                curl "$STATS_LINK" -o /dev/null > /dev/null 2>&1
-                if [ $? == 0 ]; then
-                        return 0
-                fi
+	if type curl > /dev/null; then
+		curl "$STATS_LINK" -o /dev/null > /dev/null 2>&1
+		if [ $? == 0 ]; then
+			return 0
+		fi
 	fi
 
-       	QuickLogger "Neiter wget nor curl could be used for. Cannot run statistics. Use the provided link please."
-        return 1
+	QuickLogger "Neiter wget nor curl could be used for. Cannot run statistics. Use the provided link please."
+	return 1
+}
+
+function RemoveFile {
+	local file="${1}"
+
+	if [ -f "$file" ]; then
+		rm -f "$file"
+		if [ $? != 0 ]; then
+			QuickLogger "Could not remove file [$file]."
+		else
+			QuickLogger "Removed file [$file]."
+		fi
+	else
+		QuickLogger "File [$file] not found. Skipping."
+	fi
+}
+
+function RemoveAll {
+	RemoveFile "$BIN_DIR/$PROGRAM_BINARY"
+
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
+		RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
+	fi
+
+	if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then		# Check if any other program requiring ssh filter is present before removal
+		RemoveFile "$BIN_DIR/$SSH_FILTER"
+	else
+		QuickLogger "Skipping removal of [$BIN_DIR/$SSH_FILTER] because other programs present that need it."
+	fi
+	RemoveFile "$SERVICE_DIR_SYSTEMD_SYSTEM/$SERVICE_FILE_SYSTEMD_SYSTEM"
+	RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM"
+	RemoveFile "$SERVICE_DIR_INIT/$SERVICE_FILE_INIT"
+
+	QuickLogger "Skipping configuration files in [$CONF_DIR]. You may remove this directory manually."
 }
 
 function Usage {
@@ -263,41 +409,66 @@ function Usage {
 	echo "options:"
 	echo "--silent		Will log and bypass user interaction."
 	echo "--no-stats	Used with --silent in order to refuse sending anonymous install stats."
+	echo "--remove          Remove the program."
 	exit 127
 }
 
-_SILENT=0
+_LOGGER_SILENT=false
 _STATS=1
+ACTION="install"
+
 for i in "$@"
 do
 	case $i in
 		--silent)
-		_SILENT=1
+		_LOGGER_SILENT=true
 		;;
 		--no-stats)
 		_STATS=0
 		;;
+		--remove)
+		ACTION="uninstall"
+		;;
 		--help|-h|-?)
 		Usage
 	esac
 done
 
-SetOSSettings
-CreateConfDir
-CopyExampleFiles
-CopyProgram
+if [ "$FAKEROOT" != "" ]; then
+	mkdir -p "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_USER" "$BIN_DIR"
+fi
+
+GetLocalOS
+SetLocalOSSettings
 GetInit
-CopyServiceFiles
 
-STATS_LINK="http://instcount.netpower.fr?program=$PROGRAM&version=$PROGRAM_VERSION&os=$OS"
+STATS_LINK="http://instcount.netpower.fr?program=$PROGRAM&version=$PROGRAM_VERSION&os=$OS&action=$ACTION"
+
+if [ "$ACTION" == "uninstall" ]; then
+	RemoveAll
+	QuickLogger "$PROGRAM uninstalled."
+else
+	CreateDir "$CONF_DIR"
+	CreateDir "$BIN_DIR"
+	CopyExampleFiles
+	CopyProgram
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "pmocr" ]; then
+		CopyServiceFiles
+	fi
+	QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
+		QuickLogger ""
+		QuickLogger "If connecting remotely, consider setup ssh filter to enhance security."
+		QuickLogger ""
+	fi
+fi
 
-QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
 if [ $_STATS -eq 1 ]; then
-	if [ $_SILENT -eq 1 ]; then
+	if [ $_LOGGER_SILENT == true ]; then
 		Statistics
 	else
-		QuickLogger "In order to make install statistics, the script would like to connect to $STATS_LINK"
-		read -r -p "No data except those in the url will be send. Allow [Y/n]" response
+		QuickLogger "In order to make usage statistics, the script would like to connect to $STATS_LINK"
+		read -r -p "No data except those in the url will be send. Allow [Y/n] " response
 		case $response in
 			[nN])
 			exit

obackup-batch.sh

@@ -1,21 +1,23 @@
 #!/usr/bin/env bash
 SUBPROGRAM=obackup
 PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
-AUTHOR="(L) 2013-2016 by Orsiris de Jong"
+AUTHOR="(L) 2013-2017 by Orsiris de Jong"
 CONTACT="http://www.netpower.fr - ozy@netpower.fr"
-PROGRAM_BUILD=2016052501
+PROGRAM_BUILD=2016120401
 
 ## Runs an osync /obackup instance for every conf file found
 ## If an instance fails, run it again if time permits
 
-## Configuration file path. The path where all the osync / obackup conf files are, usually /etc/osync or /etc/obackup
-CONF_FILE_PATH=/etc/$SUBPROGRAM
+if ! type "$BASH" > /dev/null; then
+	echo "Please run this script only with bash shell. Tested on bash >= 3.2"
+	exit 127
+fi
 
 ## If maximum execution time is not reached, failed instances will be rerun. Max exec time is in seconds. Example is set to 10 hours.
 MAX_EXECUTION_TIME=36000
 
-## Specifies the number of reruns an instance may get
-MAX_RERUNS=3
+## Specifies the number of total runs an instance may get
+MAX_RUNS=3
 
 ## Log file path
 if [ -w /var/log ]; then
@@ -63,72 +65,83 @@ function CheckEnvironment {
 		then
 			SUBPROGRAM_EXECUTABLE=/usr/local/bin/$SUBPROGRAM.sh
 		else
-			Logger "Could not find $SUBPROGRAM.sh" "CRITICAL"
+			Logger "Could not find [/usr/local/bin/$SUBPROGRAM.sh]" "CRITICAL"
+			( >&2 echo "Could not find [/usr/local/bin/$SUBPROGRAM.sh]" )
 			exit 1
 		fi
 	else
 		SUBPROGRAM_EXECUTABLE=$(type -p $SUBPROGRAM.sh)
 	fi
 
-	## Check for CONF_FILE_PATH
-	if [ ! -d "$CONF_FILE_PATH" ]; then
-		Logger "Cannot find conf file path $CONF_FILE_PATH" "CRITICAL"
+	if [ "$CONF_FILE_PATH" == "" ]; then
 		Usage
 	fi
 }
 
 function Batch {
-	## Get list of .conf files
-	for i in $CONF_FILE_PATH/*.conf
-	do
-		if [ "$RUN" == "" ]; then
-			RUN="$i"
-		else
-			RUN=$RUN" $i"
-		fi
-	done
+	local runs=1 # Number of batch runs
+	local runList # Actual conf file list to run
+	local runAgainList # List of failed conf files sto run again
 
-	RERUNS=0
-	while ([ $MAX_EXECUTION_TIME -gt $SECONDS ] || [ $MAX_EXECUTION_TIME -eq 0 ]) && [ "$RUN" != "" ] && [ $MAX_RERUNS -gt $RERUNS ]
-	do
-		Logger "$SUBPROGRAM instances will be run for: $RUN" "NOTICE"
-		for i in $RUN
-		do
-			$SUBPROGRAM_EXECUTABLE "$i" $opts &
-			wait $!
-			if [ $? != 0 ]; then
-				Logger "Run instance $(basename $i) failed" "ERROR"
-				if [ "$RUN_AGAIN" == "" ]; then
-					RUN_AGAIN="$i"
+	local confFile
+	local result
+
+	local i
+
+	# Using -e because find will accept directories or files
+	if [ ! -e "$CONF_FILE_PATH" ]; then
+		Logger "Cannot find conf file path [$CONF_FILE_PATH]." "CRITICAL"
+		Usage
+	else
+		# Ugly hack to read files into an array while preserving special characters
+		runList=()
+		while IFS= read -d $'\0' -r file; do runList+=("$file"); done < <(find "$CONF_FILE_PATH" -maxdepth 1 -iname "*.conf" -print0)
+
+		while ([ $MAX_EXECUTION_TIME -gt $SECONDS ] || [ $MAX_EXECUTION_TIME -eq 0 ]) && [ "${#runList[@]}" -gt 0 ] && [ $runs -le $MAX_RUNS ]; do
+			runAgainList=()
+			Logger "Sequential run n°$runs of $SUBPROGRAM instances for:" "NOTICE"
+			for confFile in "${runList[@]}"; do
+				Logger "$(basename $confFile)" "NOTICE"
+			done
+			for confFile in "${runList[@]}"; do
+				$SUBPROGRAM_EXECUTABLE "$confFile" --silent $opts &
+				wait $!
+				result=$?
+				if [ $result != 0 ]; then
+					if [ $result == 1 ] || [ $result == 128 ]; then # Do not handle exit code 128 because it is already handled here
+						Logger "Instance $(basename $confFile) failed with exit code [$result]." "ERROR"
+						runAgainList+=("$confFile")
+					elif [ $result == 2 ]; then
+						Logger "Instance $(basename $confFile) finished with warnings." "WARN"
+					fi
 				else
-					RUN_AGAIN=$RUN_AGAIN" $i"
+					Logger "Instance $(basename $confFile) succeed." "NOTICE"
 				fi
-			else
-				Logger "Run instance $(basename $i) succeed." "NOTICE"
-			fi
+			done
+			runList=("${runAgainList[@]}")
+			runs=$((runs + 1))
 		done
-		RUN="$RUN_AGAIN"
-		RUN_AGAIN=""
-		RERUNS=$(($RERUNS + 1))
-	done
+	fi
 }
 
 function Usage {
 	echo "$PROGRAM $PROGRAM_BUILD"
-	echo $AUTHOR
-	echo $CONTACT
+	echo "$AUTHOR"
+	echo "$CONTACT"
 	echo ""
 	echo "Batch script to sequentially run osync or obackup instances and rerun failed ones."
-	echo "Usage: $SUBPROGRAM-batch.sh [OPTIONS]"
+	echo "Usage: $PROGRAM.sh [OPTIONS] [$SUBPROGRAM OPTIONS]"
 	echo ""
 	echo "[OPTIONS]"
 	echo "--path=/path/to/conf      Path to osync / obackup conf files, defaults to /etc/osync or /etc/obackup"
-	echo "--max-reruns=X            Number of runs  max for failed instances, (defaults to 3)"
-	echo "--max-exec-time=X         Retry failed instances only if max execution time not reached (defaults to 36000 seconds). Set to 0 to bypass execution time check."
-	echo "--no-maxtime		Run osync / obackup without honoring conf file defined timeouts"
-	echo "--dry                     Will run osync / obackup without actually doing anything; just testing"
-	echo "--silent                  Will run osync / obackup without any output to stdout, used for cron jobs"
-	echo "--verbose                 Increases output"
+	echo "--max-runs=X              Number of max runs per instance, (defaults to 3)"
+	echo "--max-exec-time=X         Retry failed instances only if max execution time not reached (defaults to 36000 seconds). Set to 0 to bypass execution time check"
+	echo "[$SUBPROGRAM OPTIONS]"
+	echo "Specify whatever options $PROGRAM accepts. Example"
+	echo "$PROGRAM.sh --path=/etc/$SUBPROGRAM --no-maxtime"
+	echo ""
+	echo "No output will be written to stdout/stderr."
+	echo "Verify log file in [$LOG_FILE]."
 	exit 128
 }
 
@@ -136,23 +149,11 @@ opts=""
 for i in "$@"
 do
 	case $i in
-		--silent)
-		opts=$opts" --silent"
-		;;
-		--dry)
-		opts=$opts" --dry"
-		;;
-		--verbose)
-		opts=$opts" --verbose"
-		;;
-		--no-maxtime)
-		opts=$opts" --no-maxtime"
-		;;
 		--path=*)
 		CONF_FILE_PATH=${i##*=}
 		;;
-		--max-reruns=*)
-		MAX_RERUNS=${i##*=}
+		--max-runs=*)
+		MAX_RUNS=${i##*=}
 		;;
 		--max-exec-time=*)
 		MAX_EXECUTION_TIME=${i##*=}
@@ -161,8 +162,7 @@ do
 		Usage
 		;;
 		*)
-		Logger "Unknown param '$i'" "CRITICAL"
-		Usage
+		opts="$opts$i "
 		;;
 	esac
 done

ssh_filter.sh

@@ -5,102 +5,49 @@
 ##### It will filter the commands that can be run remotely via ssh.
 ##### Please chmod 755 and chown root:root this file
 
-##### Obackup needed commands: rsync find du mysql mysqldump (sudo)
-##### Osync needed commands: rsync find du echo mkdir rm if df (sudo)
-SCRIPT_BUILD=2016031401
+##### Any command that has env _REMOTE_TOKEN= with the corresponding token in it will be run
+##### Any other command will return a "syntax error"
+##### For details, see ssh_filter.log
 
-## If enabled, execution of "sudo" command will be allowed.
+SCRIPT_BUILD=2017020802
+
+## Allow sudo
 SUDO_EXEC=yes
-## Paranoia option. Don't change this unless you read the documentation and still feel concerned about security issues.
-RSYNC_EXECUTABLE=rsync
-## Enable other commands, useful for remote execution hooks like remotely creating snapshots.
-CMD1=""
-CMD2=""
-CMD3=""
 
-LOG_FILE=~/.ssh/ssh_filter.log
+## Log all valid commands too
+_DEBUG=no
+
+## Set remote token in authorized_keys
+if [ "$1" != "" ]; then
+	_REMOTE_TOKEN="${1}"
+fi
+
+LOG_FILE="${HOME}/.ssh/ssh_filter.log"
 
 function Log {
 	DATE=$(date)
-	echo "$DATE - $1" >> $LOG_FILE
+	echo "$DATE - $1" >> "$LOG_FILE"
 }
 
 function Go {
+	if [ "$_DEBUG" == "yes" ]; then
+		Log "Executing [$SSH_ORIGINAL_COMMAND]."
+	fi
 	eval "$SSH_ORIGINAL_COMMAND"
 }
 
-case ${SSH_ORIGINAL_COMMAND%% *} in
-	"$RSYNC_EXECUTABLE")
-	Go ;;
-	"echo")
-	Go ;;
-	"find")
-	Go ;;
-	"du")
-	Go ;;
-	"mkdir")
-	Go ;;
-	"rm")
-	Go ;;
-	"df")
-	Go ;;
-	"mv")
-	Go ;;
-	"$CMD1")
-	if [ "$CMD1" != "" ]; then
-		Go
-	fi
-	;;
-	"$CMD2")
-	if [ "$CMD2" != "" ]; then
-		Go
-	fi
-	;;
-	"$CMD3")
-	if [ "$CMD3" != "" ]; then
-		Go
-	fi
-	;;
-	"sudo")
-	if [ "$SUDO_EXEC" == "yes" ]; then
-		if [[ "$SSH_ORIGINAL_COMMAND" == "sudo $RSYNC_EXECUTABLE"* ]]; then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo du"* ]]; then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo find"* ]]; then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo mkdir"* ]]; then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo rm"* ]]; then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo echo"* ]]; then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo df"* ]]; then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo mv"* ]]; then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo $CMD1"* ]]; then
-			if [ "$CMD1" != "" ]; then
-			Go
-			fi
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo $CMD2"* ]]; then
-			if [ "$CMD2" != "" ]; then
-			Go
-			fi
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo $CMD3"* ]]; then
-			if [ "$CMD3" != "" ]; then
-			Go
-			fi
-		else
-			Log "Command [$SSH_ORIGINAL_COMMAND] not allowed."
+case "${SSH_ORIGINAL_COMMAND}" in
+	*"env _REMOTE_TOKEN=$_REMOTE_TOKEN"*)
+		if [ "$SUDO_EXEC" != "yes" ] && [[ $SSH_ORIGINAL_COMMAND == *"sudo "* ]]; then
+			Log "Command [$SSH_ORIGINAL_COMMAND] contains sudo which is not allowed."
+			echo "Syntax error unexpected end of file"
 			exit 1
 		fi
-	else
-		Log "Command [$SSH_ORIGINAL_COMMAND] not allowed. sudo not enabled."
-		exit 1
-	fi
+	Go
 	;;
 	*)
 	Log "Command [$SSH_ORIGINAL_COMMAND] not allowed."
+	echo "Syntax error near unexpected token"
 	exit 1
+	;;
 esac

upgrade-v1.x-2.1x.sh

@@ -1,19 +1,199 @@
 #!/usr/bin/env bash
 
-PROGRAM="obackup config file upgrade script"
+PROGRAM="obackup.upgrade"
 SUBPROGRAM="obackup"
-AUTHOR="(C) 2015 by Orsiris \"Ozy\" de Jong"
+AUTHOR="(C) 2016 by Orsiris de Jong"
 CONTACT="http://www.netpower.fr/obacup - ozy@netpower.fr"
 OLD_PROGRAM_VERSION="v1.x"
-NEW_PROGRAM_VERSION="v2.x"
-PROGRAM_BUILD=2016041201
+NEW_PROGRAM_VERSION="v2.1x"
+CONFIG_FILE_VERSION=2017020901
+PROGRAM_BUILD=2016113001
 
-## type -p does not work on platforms other than linux (bash). If if does not work, always as$
 if ! type "$BASH" > /dev/null; then
         echo "Please run this script only with bash shell. Tested on bash >= 3.2"
         exit 127
 fi
 
+# Defines all keywords / value sets in obackup configuration files
+# bash does not support two dimensional arrays, so we declare two arrays:
+# ${KEYWORDS[index]}=${VALUES[index]}
+
+KEYWORDS=(
+INSTANCE_ID
+LOGFILE
+SQL_BACKUP
+FILE_BACKUP
+BACKUP_TYPE
+SQL_STORAGE
+FILE_STORAGE
+ENCRYPTION
+CRYPT_STORAGE
+GPG_RECIPIENT
+PARALLEL_ENCRYPTION_PROCESSES
+CREATE_DIRS
+KEEP_ABSOLUTE_PATHS
+BACKUP_SIZE_MINIMUM
+GET_BACKUP_SIZE
+SQL_WARN_MIN_SPACE
+FILE_WARN_MIN_SPACE
+REMOTE_SYSTEM_URI
+SSH_RSA_PRIVATE_KEY
+SSH_PASSWORD_FILE
+_REMOTE_TOKEN
+SSH_COMPRESSION
+SSH_IGNORE_KNOWN_HOSTS
+RSYNC_REMOTE_PATH
+REMOTE_HOST_PING
+REMOTE_3RD_PARTY_HOSTS
+SUDO_EXEC
+SQL_USER
+DATABASES_ALL
+DATABASES_ALL_EXCLUDE_LIST
+DATABASES_LIST
+SOFT_MAX_EXEC_TIME_DB_TASK
+HARD_MAX_EXEC_TIME_DB_TASK
+MYSQLDUMP_OPTIONS
+COMPRESSION_LEVEL
+DIRECTORY_LIST
+RECURSIVE_DIRECTORY_LIST
+RECURSIVE_EXCLUDE_LIST
+RSYNC_PATTERN_FIRST
+RSYNC_INCLUDE_PATTERN
+RSYNC_EXCLUDE_PATTERN
+RSYNC_INCLUDE_FROM
+RSYNC_EXCLUDE_FROM
+PATH_SEPARATOR_CHAR
+RSYNC_OPTIONAL_ARGS
+PRESERVE_PERMISSIONS
+PRESERVE_OWNER
+PRESERVE_GROUP
+PRESERVE_EXECUTABILITY
+PRESERVE_ACL
+PRESERVE_XATTR
+COPY_SYMLINKS
+KEEP_DIRLINKS
+PRESERVE_HARDLINKS
+RSYNC_COMPRESS
+SOFT_MAX_EXEC_TIME_FILE_TASK
+HARD_MAX_EXEC_TIME_FILE_TASK
+PARTIAL
+DELETE_VANISHED_FILES
+DELTA_COPIES
+BANDWIDTH
+RSYNC_EXECUTABLE
+DESTINATION_MAILS
+MAIL_BODY_CHARSET
+SENDER_MAIL
+SMTP_SERVER
+SMTP_PORT
+SMTP_ENCRYPTION
+SMTP_USER
+SMTP_PASSWORD
+SOFT_MAX_EXEC_TIME_TOTAL
+HARD_MAX_EXEC_TIME_TOTAL
+KEEP_LOGGING
+ROTATE_SQL_BACKUPS
+ROTATE_SQL_COPIES
+ROTATE_FILE_BACKUPS
+ROTATE_FILE_COPIES
+LOCAL_RUN_BEFORE_CMD
+LOCAL_RUN_AFTER_CMD
+REMOTE_RUN_BEFORE_CMD
+REMOTE_RUN_AFTER_CMD
+MAX_EXEC_TIME_PER_CMD_BEFORE
+MAX_EXEC_TIME_PER_CMD_AFTER
+STOP_ON_CMD_ERROR
+RUN_AFTER_CMD_ON_ERROR
+)
+
+VALUES=(
+test-backup
+''
+yes
+yes
+local
+/home/storage/sql
+/home/storage/files
+no
+/home/storage/crypt
+'Your Name used with GPG signature'
+''
+yes
+yes
+1024
+yes
+1048576
+1048576
+ssh://backupuser@remote.system.tld:22/
+${HOME}/.ssh/id_rsa
+''
+SomeAlphaNumericToken9
+yes
+no
+''
+yes
+'www.kernel.org www.google.com'
+no
+root
+yes
+test
+''
+3600
+7200
+'--opt --single-transaction'
+3
+/some/path
+/home
+/home/backupuser\;/host/lost+found
+include
+''
+''
+''
+''
+\;
+''
+yes
+yes
+yes
+yes
+no
+no
+yes
+yes
+no
+no
+3600
+7200
+no
+no
+yes
+0
+rsync
+infrastructure@example.com
+''
+sender@example.com
+smtp.isp.tld
+25
+none
+''
+''
+30000
+36000
+1801
+no
+7
+no
+7
+''
+''
+''
+''
+0
+0
+no
+no
+)
+
 function Usage {
 	echo "$PROGRAM $PROGRAM_BUILD"
 	echo $AUTHOR
@@ -42,10 +222,10 @@ function LoadConfigFile {
 	fi
 }
 
-function RewriteConfigFiles {
+function RewriteOldConfigFiles {
 	local config_file="${1}"
 
-	if ((! grep "BACKUP_ID=" $config_file > /dev/null) && ( ! grep "INSTANCE_ID=" $config_file > /dev/null)); then
+	if ! grep "BACKUP_ID=" $config_file > /dev/null && ! grep "INSTANCE_ID=" $config_file > /dev/null; then
 		echo "File [$config_file] does not seem to be a obackup config file."
 		exit 1
 	fi
@@ -64,9 +244,7 @@ function RewriteConfigFiles {
 	sed -i'.tmp' 's/^BACKUP_FILES=/FILE_BACKUP=/g' "$config_file"
 	sed -i'.tmp' 's/^LOCAL_SQL_STORAGE=/SQL_STORAGE=/g' "$config_file"
 	sed -i'.tmp' 's/^LOCAL_FILE_STORAGE=/FILE_STORAGE=/g' "$config_file"
-	if ! grep "^ENCRYPTION=" "$config_file" > /dev/null; then
-		sed -i'.tmp' '/^FILE_STORAGE=*/a\'$'\n''ENCRYPTION=no\'$'\n''' "$config_file"
-	fi
+
 	sed -i'.tmp' 's/^DISABLE_GET_BACKUP_FILE_SIZE=no/GET_BACKUP_SIZE=yes/g' "$config_file"
 	sed -i'.tmp' 's/^DISABLE_GET_BACKUP_FILE_SIZE=yes/GET_BACKUP_SIZE=no/g' "$config_file"
 	sed -i'.tmp' 's/^LOCAL_STORAGE_KEEP_ABSOLUTE_PATHS=/KEEP_ABSOLUTE_PATHS=/g' "$config_file"
@@ -104,9 +282,9 @@ function RewriteConfigFiles {
 		REMOTE_SYSTEM_URI="ssh://$REMOTE_USER@$REMOTE_HOST:$REMOTE_PORT/"
 
 		sed -i'.tmp' 's#^REMOTE_BACKUP=yes#REMOTE_SYSTEM_URI='$REMOTE_SYSTEM_URI'#g' "$config_file"
-		sed -i'.tmp' '/^REMOTE_USER=*/d' "$config_file"
-		sed -i'.tmp' '/^REMOTE_HOST=*/d' "$config_file"
-		sed -i'.tmp' '/^REMOTE_PORT=*/d' "$config_file"
+		sed -i'.tmp' '/^REMOTE_USER==*/d' "$config_file"
+		sed -i'.tmp' '/^REMOTE_HOST==*/d' "$config_file"
+		sed -i'.tmp' '/^REMOTE_PORT==*/d' "$config_file"
 
 		sed -i'.tmp' '/^INSTANCE_ID=*/a\'$'\n''BACKUP_TYPE=pull\'$'\n''' "$config_file"
 	else
@@ -114,79 +292,36 @@ function RewriteConfigFiles {
 			sed -i'.tmp' '/^INSTANCE_ID=*/a\'$'\n''BACKUP_TYPE=local\'$'\n''' "$config_file"
 		fi
 	fi
+	sed -i'.tmp' 's/^REMOTE_3RD_PARTY_HOST=/REMOTE_3RD_PARTY_HOSTS=/g' "$config_file"
+}
 
-	# Add new config values from v1.1 if they don't exist
-	if ! grep "^ENCRYPTION=" "$config_file" > /dev/null; then
-		sed -i'.tmp' '/^FILE_STORAGE=*/a\'$'\n''ENCRYPTION=no\'$'\n''' "$config_file"
-	fi
+function AddMissingConfigOptions {
+	local config_file="${1}"
+	local counter=0
 
-	if ! grep "^CREATE_DIRS=" "$config_file" > /dev/null; then
-		sed -i'.tmp' '/^ENCRYPTION=*/a\'$'\n''CREATE_DIRS=yes\'$'\n''' "$config_file"
-	fi
+	while [ $counter -lt ${#KEYWORDS[@]} ]; do
+		if ! grep "^${KEYWORDS[$counter]}=" > /dev/null "$config_file"; then
+			echo "${KEYWORDS[$counter]} not found"
+			if [ $counter -gt 0 ]; then
+				sed -i'.tmp' '/^'${KEYWORDS[$((counter-1))]}'=*/a\'$'\n'${KEYWORDS[$counter]}'="'"${VALUES[$counter]}"'"\'$'\n''' "$config_file"
+				if [ $? -ne 0 ]; then
+					echo "Cannot add missing ${[KEYWORDS[$counter]}."
+					exit 1
+				fi
+			else
+				sed -i'.tmp' '/onfig file rev*/a\'$'\n'${KEYWORDS[$counter]}'="'"${VALUES[$counter]}"'"\'$'\n''' "$config_file"
+			fi
+			echo "Added missing ${KEYWORDS[$counter]} config option with default option [${VALUES[$counter]}]"
+		fi
+		counter=$((counter+1))
+	done
+}
 
-	if ! grep "^GET_BACKUP_SIZE=" "$config_file" > /dev/null; then
-		sed -i'.tmp' '/^BACKUP_SIZE_MINIMUM=*/a\'$'\n''GET_BACKUP_SIZE=yes\'$'\n''' "$config_file"
-	fi
+function UpdateConfigHeader {
+	local config_file="${1}"
 
-	if ! grep "^RSYNC_REMOTE_PATH=" "$config_file" > /dev/null; then
-		sed -i'.tmp' '/^SSH_COMPRESSION=*/a\'$'\n''RSYNC_REMOTE_PATH=\'$'\n''' "$config_file"
-	fi
-
-	if ! grep "^SSH_IGNORE_KNOWN_HOSTS=" "$config_file" > /dev/null; then
-		sed -i'.tmp' '/^SSH_COMPRESSION=*/a\'$'\n''SSH_IGNORE_KNOWN_HOSTS=no\'$'\n''' "$config_file"
-	fi
-
-	if ! grep "^REMOTE_HOST_PING=" "$config_file" > /dev/null; then
-		sed -i'.tmp' '/^RSYNC_REMOTE_PATH=*/a\'$'\n''REMOTE_HOST_PING=yes\'$'\n''' "$config_file"
-	fi
-
-	if ! grep "^COPY_SYMLINKS=" "$config_file" > /dev/null; then
-		sed -i'.tmp' '/^PRESERVE_XATTR=*/a\'$'\n''COPY_SYMLINKS=yes\'$'\n''' "$config_file"
-	fi
-
-	if ! grep "^KEEP_DIRLINKS=" "$config_file" > /dev/null; then
-		sed -i'.tmp' '/^COPY_SYMLINKS=*/a\'$'\n''KEEP_DIRLINKS=yes\'$'\n''' "$config_file"
-	fi
-
-	if ! grep "^PRESERVE_HARDLINKS=" "$config_file" > /dev/null; then
-		sed -i'.tmp' '/^KEEP_DIRLINKS=*/a\'$'\n''PRESERVE_HARDLINKS=no\'$'\n''' "$config_file"
-	fi
-
-	if ! grep "^RSYNC_PATTERN_FIRST=" "$config_file" > /dev/null; then
-		sed -i'.tmp' '/^LOGFILE=*/a\'$'\n''RSYNC_PATTERN_FIRST=include\'$'\n''' "$config_file"
-	fi
-
-	if ! grep "^RSYNC_INCLUDE_PATTERN=" "$config_file" > /dev/null; then
-	        sed -i'.tmp' '/^RSYNC_EXCLUDE_PATTERN=*/a\'$'\n''RSYNC_INCLUDE_PATTERN=""\'$'\n''' "$config_file"
-	fi
-
-	if ! grep "^RSYNC_INCLUDE_FROM=" "$config_file" > /dev/null; then
-	        sed -i'.tmp' '/^RSYNC_EXCLUDE_FROM=*/a\'$'\n''RSYNC_INCLUDE_FROM=""\'$'\n''' "$config_file"
-	fi
-
-        if ! grep "^PARTIAL=" "$config_file" > /dev/null; then
-                sed -i'.tmp' '/^HARD_MAX_EXEC_TIME_FILE_TASK==*/a\'$'\n''PARTIAL=no\'$'\n''' "$config_file"
-        fi
-
-	if ! grep "^DELETE_VANISHED_FILES=" "$config_file" > /dev/null; then
-		sed -i'.tmp' '/^PARTIAL=*/a\'$'\n''DELETE_VANISHED_FILES=no\'$'\n''' "$config_file"
-	fi
-
-	if ! grep "^DELTA_COPIES=" "$config_file" > /dev/null; then
-                sed -i'.tmp' '/^PARTIAL=*/a\'$'\n''DELTA_COPIES=yes\'$'\n''' "$config_file"
-        fi
-
-	if ! grep "^BANDWIDTH=" "$config_file" > /dev/null; then
-                sed -i'.tmp' '/^DELTA_COPIES=*/a\'$'\n''BANDWIDTH=0\'$'\n''' "$config_file"
-        fi
-
-	if ! grep "^STOP_ON_CMD_ERROR=" "$config_file" > /dev/null; then
-                sed -i'.tmp' '/^MAX_EXEC_TIME_PER_CMD_AFTER=*/a\'$'\n''STOP_ON_CMD_ERROR=no\'$'\n''' "$config_file"
-        fi
-
-	if ! grep "^RUN_AFTER_CMD_ON_ERROR=" "$config_file" > /dev/null; then
-                sed -i'.tmp' '/^STOP_ON_CMD_ERROR=*/a\'$'\n''RUN_AFTER_CMD_ON_ERROR=no\'$'\n''' "$config_file"
-        fi
+	# "onfig file rev" to deal with earlier variants of the file
+        sed -i'.tmp' 's/.*onfig file rev.*/##### '$SUBPROGRAM' config file rev '$CONFIG_FILE_VERSION' '$NEW_PROGRAM_VERSION'/' "$config_file"
 
 	rm -f "$config_file.tmp"
 }
@@ -196,7 +331,9 @@ if [ "$1" != "" ] && [ -f "$1" ] && [ -w "$1" ]; then
 	# Make sure there is no ending slash
 	CONF_FILE="${CONF_FILE%/}"
 	LoadConfigFile "$CONF_FILE"
-	RewriteConfigFiles "$CONF_FILE"
+	RewriteOldConfigFiles "$CONF_FILE"
+	AddMissingConfigOptions "$CONF_FILE"
+	UpdateConfigHeader "$CONF_FILE"
 else
 	Usage
 fi