Rebuilt targets

This reverts commit 52161b3faf.

.travis.yml

@@ -0,0 +1,18 @@
+language:
+    bash
+
+services:
+    mysql
+
+os:
+    linux
+    osx
+
+sudo:
+    required
+
+before_script:
+    mysql -e 'CREATE DATABASE travistest;'
+
+script:
+    TRAVIS_RUN=true dev/tests/run_tests.sh

CHANGELOG.md

@@ -1,25 +1,199 @@
-SHORT FUTURE IMPROVEMENTS
--------------------------
-
-- Rewrite rsync exclude patterns using \"pattern\" instead of escaped chars
-- Clean most of recursive task creation code
-
-FAR FUTURE IMPROVEMENTS
------------------------
-
-- (Secret world domination... Still need to get bald and get a cat)
-
 KNOWN ISSUES
 ------------
 
 - Backup size check does not honor rsync exclude patterns
+- Encryption does not honor rsync exclude patterns
 - Bandwidth parameter is ignored for SQL backups
 - Missing symlink support when run from MSYS environment
 
 CHANGELOG
 ---------
 
-- 02 Nov. 2013: v1.84 RC3
+14 Mar 2017: obackup v2.1 beta2 released
+
+- Fixed remote commands can be run on local runs and obviously fail
+- Uninstall leaves ssh_filter if needed by other programs
+- Logger now obfuscates _REMOTE_TOKEN
+- Improved sudo privilege run
+- Brand new ssh filter from osync project
+- Better installer with --remove option from osync project
+- Updated ofunctions from osync project
+	- Fixes UTF-8 escaped characters in log files due to LC_ALL=C
+	- Optional MAIL_BODY_CHARSET so destination mails aren't sent as UTF-8 anymore depending on systems
+- Minor fixes
+
+04 Jan 2017: obackup v2.1 beta1 released
+----------------------------------------
+
+- Fixed wrong file size fetched remotely since v2.1 rewrite
+- Fixed missing databases in manual list fails to trigger an alert
+- Improved support for GPG ver >= 2.1
+- Added encryption / decryption parallel execution support
+- Improved compatibility for RotateCopies
+- Unit tests now run on CentOS 5,6
+- Added optional rsync arguments configuration value
+- Forcec bash usage on remote connections in order to be FreeBSD 11 compatible
+- Spinner is less prone to move logging on screen
+- Fixed another random error involving warns and errors triggered by earlier runs with same PID flag files
+- Adde more preflight checks (pgrep presence)
+- Added --no-prefix, --error-only and --summary switches
+- Updated installer from osync
+- Updated merge.sh script to handle includes
+- Improved remote logging
+- Simplified osync-batch runner (internally and for user)
+	- Better filename handling
+	- Easier to read log output
+        - Always passes --silent to obackup
+        - All options that do not belong to obackup-batch are automatically passed to obackup
+- Improved installer OS detection
+- Fixed upgrade script cannot update header on BSD / MacOS X
+- Fixed SendEmail function on MacOS X
+- Fixed MAX_SOFT_EXEC_TIME_PER_XX_TASK not enforced bug introduced with newer ofunctions from v2.1
+- PRESERVE_ACL and PRESERVE_XATTR are ignored when local or remote OS is MacOS or msys or Cygwin
+- Fixed PRESERVE_EXECUTABILITY was ommited volontary on MacOS X because of rsync syntax
+- merge.sh is now BSD and Mac compatible
+- Unit tests are now BSD and Mac compatible
+- Local runs should not check for remote connectivity
+- Fixed error alerts cannot be triggered from subprocesses
+- Fixed error flags 
+- Faster remote OS detection
+- Added busybox (and Android Termux) support
+	- More portable file size functions
+	- More portable compression program commands
+	- More paranoia checks
+	- Added busybox sendmail support
+	- Added tls and ssl support for sendmail
+- Added ssh password file support
+- Added unit tests
+	- Added basic unit tests for all three operation modes
+	- Added process management function tests
+	- Added file rotation tests
+	- Added upgrade script test
+	- Added encryption tests
+	- Added missing files / databases test
+	- Added timed execution tests
+- Implemented backup encryption using GPG (see documentation for advantages and caveats)
+	- Backup encrypted but still use differential engine :)
+- Database backup improvements
+	- Added mysqldump options to config file
+- Improved unit tests
+- Added more preflight checks
+- Logs sent by mail are easier to read
+	- Better subject (currently running or finished run)
+	- Fixed bogus double log sent in alert mails
+	- Only current run log is now sent
+	- Alert sending is now triggered after last action
+- Made unix signals posix compliant
+- Improved upgrade script
+	- Upgrade script now updates header
+	- Can add any missing value now
+	- Added encrpytion support
+- Fixed problem with spaces in directories to backup (again !)
+- Added options to ignore permissions, ownership and groups
+- Improved batch runner
+	- Batch runner works for directories and direct paths
+	- Fixed batch runner does not rerun obackup on warnings only
+	- Code compliance
+	- More clear semantic
+- Made keep logging value configurable and not mandatory
+- Fixed handling of processes in uninterruptible sleep state
+- Code cleanup
+- Refactored waiting functions
+- Fixed double RunAfterHook launch
+
+06 Aug 2016: obackup v2.0 released
+----------------------------------
+
+- Made logging begin before remote checks for sanity purposes
+- RunAfterCommands can get executed when trapquit
+- Improved process killing and process time control
+- Added optional statistics for installer
+- Added an option to ignore knownhosts for ssh connections (use with caution, this can lead to a security issue)
+- Improved mail fallback
+- More logging enhancements
+- Improved upgrade script
+- Revamped rsync patterns to allow include and exclude patterns
+- Better SQL and file backup task separation (rotate copies and warnings are defined for sql and/or file)
+- Added reverse backup, now backups can be local, pushed or pulled to or from a remote system
+- Better fallback for SendAlert even if disk full
+- Added an alert email sent on warnings while backup script is running
+- Way better logging of errors in _GetDirectoriesSizeX, _BackupDatabaseX, _CreateStorageDirectoriesX
+- Added bogus config file checks & environment checks
+- Full code refactoring to use local and remote code once
+- Fully merged codebase with osync
+	- Added (much) more verbose debugging (and possibility to remove debug code to gain speed)
+	- Replace child_pid by $? directly, add a better sub process killer in TrapQuit
+	- Added some automatic checks in code, for _DEBUG mode (and _PARANOIA_DEBUG now)
+	- Improved Logging
+	- Updated obackup to be fully compliant with coding style
+- Fixed creation of bogus subdirectories in some cases
+- A long list of minor improvements and bug fixes
+
+v0-1.x - Jan 2013 - Oct 2015
+----------------------------
+
+- New function to kill child processes
+- Fixed no_maxtime not honored
+- Improved some logging, also added highlighting to stdout errors
+- Backported some fixes from Osync
+	- Small improvements on install script
+	- Copy ssh_filter.sh from osync project
+	- Small improvements in obackup-batch.sh time management
+- Quick and dirty hack to get the full last run log in SendAlert email
+- Added detection of obackup.sh script in obackup-batch.sh to overcome mising path in crontab
+- Moved command line arguments after config file load for allowing command line overrides
+- Added a config file option equivalent to --dontgetsize
+- Added basic install script from osync project
+- Added obackup-batch.sh from osync project to rerun failed backups in row
+- Delta copy algorithm is now used even for local copies (usefull for network drives), this can be overriden in config file
+- Added --dontgetsize parameter to backup huge systems immediatly
+- Fixed multiple keep logging messages since sleep time between commands has been lowered under a second
+- Create local subdirectories if not exist before running rsync (rsync doesn't handle mkdir -p)
+- Backported some fixes from Osync
+	- Lowered sleep time between commands
+	- Lowered debug sleep times
+	- Fixed a bug with exclude pattern globbing preventing multiple exludes
+	- Lowered default compression level for email alerts (for low end systems)
+- Prevent exclude pattern globbing before the pattern reaches the rsync cmd
+- Fixed some typos with ported code from osync rendering stats and partial downloads unusable
+- Added delete on destination option for files that vanished from source
+- Fixed ignoring compression level in conf file
+- Added experimental partial downloads support for rsync so big files can be resumed on slow links
+- Fixed dry mode sql backup output
+- Prevented triggering TrapError if there are no child processes to terminate on TrapQuit
+- Improved mysql debug logs
+- Prevent creation of backup-id less log file when DEBUG is set
+- WARNING: Default behavior is now to copy the referrent files and directories from symlinks (this can reach files outside the backup root)
+- Recursive directory search now includes symlinks (find -L option. -type d cannot be replaced by -xtype d because of portability issues with BSD)
+- Dry mode does not create target directories anymore
+- Dry mode also tries mysqldumps now (check for error messages being logged)
+- Added experimental partial download support
+- Added Rsync exclude files suppport from osync
+- Fixed another issue with existing symlinks to directories on target on non recursive backups
+- Fixed remaining rsync -E option preventing obackup to work correctly on MacOS X
+- Fixed an issue with existing symlinks to directories on target
+- Prevent changed IFS to make ping commands fail
+- Added RotateCopies execution time (spinner support)
+- redirect stderr for mysqldump to catch problems
+- Moved msys specific code to Init(Local|Remote)OSSettings except in TrapQuit that needs to work at any moment
+- Added support for multithreaded gzip (if pigz is installed)
+- Merged back changes from osync codebase
+	- Enhanced debugging
+	- Added language agnostic system command output
+	- Enhanced log sending
+	- Better handling of OS specific commands
+	- Improved WaitForTaskCompletion when DEBUG enabled or SILENT enabled
+	- Enhanced OS detection
+- More correct error message on remote connection failure
+- Gzipped logs are now deleted once sent
+- Fixed some typos (thanks to Pavel Kiryukhin)
+- Improved OS detection and added prelimnary MacOS X support
+- Improved execution hook logs
+- Improved RunLocalCommand execution hook
+
+02 Nov. 2013: obackup v1.84RC3 released
+---------------------------------------
+
 - Updated documentation
 - Minor rewrites in recursive backup code
 - Added base directory files backup for recursive directories backup
@@ -46,7 +220,10 @@ CHANGELOG
 - Improved dryrun output
 - Improved remote connecivity detection 
 - Fixed a typo in configuration file
-- 18 Aug. 2013: Now v1.84 RC2
+
+18 Aug. 2013: obackup v1.84RC2 released
+---------------------------------------
+
 - Added possibility to change default logfile
 - Simplified dryrun (removed dryrun function and merged it with main function)
 - Simplified Init function
@@ -58,14 +235,17 @@ CHANGELOG
 - Added --verbose switch (will add databases list,  rsync commands, and file backup list)
 - Improved task execution checks and more code cleanup
 - Fixed CleanUp function if DEBUG=yes, also function is now launched from TrapQuit
-- 16 Jul. 2013: version tagged as v1.84 RC1
+
+16 Jul. 2013: obackup v1.84RC1 released
+---------------------------------------
+
 - Code cleanup
 - Uploaded first documentation
 - Fixed an issue with RotateBackups
 - Updated obackup to log failed ssh command results
 - Updated ssh command filter to log failed commands
 - Updated ssh command filter to accept personalized commands
-- 23 Jun. 2013 v 1.84 RC1 approaching
+- 23 Jun. 2013: v1.84 RC1 approaching
 - Added ssh commands filter, updated documentation
 - Rewrote local space check function
 - Added ability to run another executable than rsync (see documentation on sudo execution)
@@ -75,5 +255,8 @@ CHANGELOG
 - Updated command line argument --silent processing
 - Added remote before and after command execution hook
 - Added local before and after command execution hook
-- 14 Jun 2013
+
+14 Jun 2013
+-----------
+
 - Initial public release, fully functionnal

CODING_STYLE.TXT

@@ -0,0 +1,147 @@
+Coding style used for my bash projects (v2.1 Oct 2015)
+
+++++++ Header
+
+Always use the following header
+
+----BEGIN HEADER
+#!/usr/bin/env bash
+
+PROGRAM="program-name" # Long description
+AUTHOR="(L) 20XX-20YY by Orsiris \"Ozy\" de Jong"
+CONTACT="http://www.example.com me@example.com"
+PROGRAM_BUILD=YYYYMMDDVV
+
+## Optional instructions
+----END HEADER
+
+Using bind style versionning:
+YYYYMMDDVV (Year, Month, Day, Revision): Example: 2015012402 = 2nd revision of 24 Jan 2015
+
+#!/usr/bin/env bash instead of #!/bin/bash
+
+Change old scripts with
+for i in $(grep -r '#!/bin/bash' * |cut -f1 -d':'); do sed -i 's&#!/bin/bash&#!/usr/bin/env bash&g' $i; done
+
+
+type instead of type -p for bash test (other shells don't know -p)
+++++++ Indentation
+
+Using tabs
+Transform old shell scripts using unexpand command
+
+++++++ Comments
+
+Some command # comment
+## Some comment on a new line
+################################################# Some separation
+
+++++++ Work comments
+
+Whenever there is some idea to postpone, use #TODO[-version]:[dev-name:] some remark
+A marker must be left where on the line a dev is working (when the work isn't finished). Marker is #WIP:dev-name: some remark
+dev-name is mandatory if more than one person is coding
+Example: #TODO-v2.1:deajan: need to do something
+
+++++++ Variables
+
+All local variables are lowercase, separated by _ (ex: low_wait)
+All global variables full upercase, separated by _ (ex: EXEC_TIME)
+All environment variables (verbose, silent, debug, etc) have prefix _ and are full upercase, separated by _ (ex: _PARANOIA_DEBUG)
+
+++++++ Functions
+
+Every word in a function begins with an uppercase (ex: SomeFunctionDoesThings)
+
+Define functions this way. Use sed ':a;N;$!ba;s/\n{\n/ {\n/g' to adapt when opening bracket is on a new line.
+
+function something {
+	
+}
+
+If function has some arguments, use local variable names that are more readable than $1...$n. Explain via comments what those variables contain if needed.
+
+function anotherthing {
+	local var_name="${1}"
+	local other_var_name="${2}" # This variable contains stuff
+}
+
+Functions should always have return status
+function thirdthing {
+	some_command
+	return $?
+}
+
+++++++ Sub functions
+
+When a function is a subroutine of another function, it is called _SomethingAsSubFunction
+
+++++++ Function argument check
+
+Bash does not provide any checks against missing function arguments. Also, missing quotes can lead to an inconsistent number of arguments.
+Every function call will be checked by __CheckArguments which takes the number of arguments, $# (the real number of args given), the parent function name and the parent function's arguments.
+__CheckArguments will trigger a critical error if number of arguments if incorrect. This will also prevent silent typo errors.
+Ex:
+
+function Something {
+	local some="${1}"
+	local other="${2}"
+	local args="${3}"
+	__CheckArguments 3 $# $FUNCNAME "$*"
+
+__CheckArguments will only trigger if script is called with DEBUG=yes
+Also, with PARANOIA_DEBUG=yes, __CheckArguments will recount all arguments given by "$*" and compare. This can mislead if arguments contain spaces.
+
+++++++ If statements
+
+If statements will be fully written (word "if" must be used). then is written on the same line.
+(Use sed ':a;N;$!ba;s/]\n\t*then/]; then/g' to convert files to this format... Replace "],new line, zero or more tabs, then" by "; then")
+if [ something ]; then
+	stuff
+else
+	other stuff
+fi
+
+++++++ Logging
+
+A logging function is available with the following levels of logging:
+
+- DEBUG: Only log this when DEBUG flas is set in program. Any command forged for eval should be logged by this.
+- NOTICE: Standard messages
+- WARN: Requires attention
+- ERROR: Program produced an error but continues execution
+- CRITICAL: Program execution is halted
+
+++++++ Eval
+
+Most commands should be logged to a tmp file.
+The basic way of doing is:
+
+cmd='"something '$somevar'" > some_file 2>&1'
+eval $cmd &
+WaitForTaskCompletion $! 0 0 $FUNCNAME
+
+Remote commands should exist as:
+
+cmd=$SSH_CMD' "some; commands \"'$VARIABLE'\" some; other; commands" > some_file 2>&1'
+
+++++++ File variables
+
+All eval cmd should exit their content to a file called "$RUNDIR/osync.$FUNCNAME.$SCRIPT_PID"
+Dots are used instead of '_' so variables can be separated with a forbidden char in variables, so they get detected.
+
+++++++ Finding code errors
+
+Use shellcheck.net now and then (ignore SC2086 in our case)
+
+Use a low tech approach to find uneven number of quotes per line
+
+tr -cd "'\n" < my_bash_file.sh | awk 'length%2==1 {print NR, $0}'
+tr -cd "\"\n" < my_bash_file.sh | awk 'length%2==1 {print NR, $0}'
+
+++++++ ofunctions
+
+As obackup and osync share alot of common functions, ofunctions.sh will host all shared code.
+Dev programs n_osync.sh and n_obackup.sh will source ofunctions.sh
+Release programs will still include ofunctions.sh in order to enhance ease of use.
+

LICENCE.TXT

@@ -1,4 +1,4 @@
-Copyright (c) 2013, Orsiris "Ozy" de Jong. ozy@netpower.fr
+Copyright (c) 2013-2016, Orsiris "Ozy" de Jong. ozy@netpower.fr
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

README.md

@@ -1,12 +1,11 @@
-obackup
+# obackup [![Build Status](https://travis-ci.org/deajan/obackup.svg?branch=master)](https://travis-ci.org/deajan/obackup) [![GitHub Release](https://img.shields.io/github/release/deajan/obackup.svg?label=Latest)](https://github.com/deajan/obackup/releases/latest)
-=======
 
-A small robust file & database backup script for local to local or remote to local backups via ssh.
+
-Works especially well for multiple virtualhost backups with 'backup divide task' functionnality.
+A robust file & database backup script that works for local and remote push or pull backups via ssh.
 
 ## About
 
-OBackup is designed from ground to make the backup process as reliable as possible.
+obackup is designed to make the backup process as reliable as possible.
 It divides the whole backup process into tasks, allowing each task to execute for a certain amount of time.
 If a task doesn't finish in time, it's stopped and the next task in list is processed.
 Before a task gets stopped, a first warning message is generated telling the task takes too long.
@@ -14,28 +13,32 @@ Every action gets logged, and if a warning has been generated, a task gets stopp
 
 Remote backups are initiated from the backup server instead of the production server, so hacked servers won't get ssh access to the backup server.
 
-OBackup can enumerate and backup all MariaDB / MySQL databases present on a server.
+obackup can enumerate and backup all MariaDB / MySQL databases present on a server.
 It can also enumarate all subdirectories of a given path and process them as separate tasks (usefull for multiple vhosts).
 It will do several checks before launching a backup like execution checks, dryruns, checking backup size and available local disk space.
 
-Obackup can execute local and remote commands before and after backup execution,
+obackup can execute local and remote commands before and after backup execution,
 thus providing an easy way to handle snapshots (see https://github.com/deajan/zsnap for a zfs snapshot management script).
 It may also rotate backups for you.
 
-As of today, obackup has been tested successfully on RHEL / CentOS 5, CentOS 6, Debian 6.0.7 and Linux Mint 14.
+As of today, obackup has been tested successfully on RHEL / CentOS 5, 6 and 7, Debian 6 and 7, Linux Mint 14 and 17, FreeBSD 8.3 and 10.3.
-Currently, Obackup also runs on FreeBSD and Windows MSYS environment, altough it is not fully tested yet.
+Currently, obackup also runs on MacOSX and Windows MSYS environment.
 
-Feel free to drop me a mail for limited support in my free time.
+## Warning
+
+Obackup now follows symlinks and treats them as the referent files / dirs, following symlinks even outside the backup root, which IMHO is more secure in terms of backups.
+You may disable this behavior in the config file.
 
 ## Installation
 
 You can download the latest obackup script from authors website.
 You may also clone the following git which will maybe have some more recent builds.
 
-    $ git clone git://github.com/deajan/obackup.git
+    $ git clone -b "v2.0-maint" git://github.com/deajan/obackup.git
-    $ chmod +x ./obackup.sh
+    $ cd obackup
+    $ ./install.sh
 
-Obackup needs to run with bash shell, using any other shell will most probably fail.  
+obackup needs to run with bash shell, using any other shell will most probably fail.  
 Once you have grabbed a copy, just edit the config file with your favorite text editor to setup your environment and you're ready to run.
 A detailled documentation can be found on the author's site.
 You can run multiple instances of obackup scripts with different backup environments. Just create another configuration file,
@@ -64,10 +67,16 @@ that's actually launched and it's result.
 One you're happy with a test run, you may run obackup as a cron task with the "--silent" parameter so output will not be written to stdout.
 All backup activity is logged to "/var/log/obackup_backupname.log" or current directory if /var/log is not writable.
 
+You may mix "--silent" and "--verbose" parameters to output verbose input only in the log files.
+
+## Troubleshooting
+
+Whenever you may encounter rsync zombie processes and/or in unterruptible sleep state processes, you should force unmounting network drives obackup is supposed to deal with.
+
 ## Final words
 
 Backup tasks aren't always reliable, connectivity loss, insufficient disk space, hacked servers with tons of unusefull stuff to backup... Anything can happen.
-Obackup will sent your a warning email for every issue it can handle.
+obackup will sent your a warning email for every issue it can handle.
 Nevertheless, you should assure yourself that your backup tasks will get done the way you meant it. Also, a backup isn't valuable until you're sure
 you can successfully restore. Try to restore your backups to check whether everything is okay. Backups will keep file permissions and owners,
 but may loose ACLs if destination file system won't handle them. 
@@ -75,7 +84,4 @@ but may loose ACLs if destination file system won't handle them.
 ## Author
 
 Feel free to mail me for limited support in my free time :)
-Orsiris "Ozy" de Jong | ozy@netpower.fr
+Orsiris de Jong | ozy@netpower.fr
-
-
- 

dev/bootstrap.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+## dev pre-processor bootstrap rev 2016121302
+## Yeah !!! A really tech sounding name... In fact it's just include emulation in bash
+
+if [ ! -f "./merge.sh" ]; then
+	echo "Plrase run bootstrap.sh from osync/dev directory."
+	exit 1
+fi
+
+outputFileName="$0"
+
+source "merge.sh"
+__PREPROCESSOR_PROGRAM=obackup
+__PREPROCESSOR_Constants
+
+cp "n_$__PREPROCESSOR_PROGRAM.sh" "$outputFileName.tmp.sh"
+if [ $? != 0 ]; then
+	echo "Cannot copy original file [n_$__PREPROCESSOR_PROGRAM.sh] to [$outputFileName.tmp.sh]."
+	exit 1
+fi
+for subset in "${__PREPROCESSOR_SUBSETS[@]}"; do
+	__PREPROCESSOR_MergeSubset "$subset" "${subset//SUBSET/SUBSET END}" "ofunctions.sh" "$outputFileName.tmp.sh"
+done
+chmod +x "$0.tmp.sh"
+if [ $? != 0 ]; then
+	echo "Cannot make [$outputFileName] executable.."
+	exit 1
+fi
+
+# Termux fix
+if type termux-fix-shebang > /dev/null 2>&1; then
+	termux-fix-shebang "$outputFileName.tmp.sh"
+fi
+
+"$outputFileName.tmp.sh" "$@"

dev/common_batch.sh

@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+SUBPROGRAM=[prgname]
+PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
+AUTHOR="(L) 2013-2016 by Orsiris de Jong"
+CONTACT="http://www.netpower.fr - ozy@netpower.fr"
+PROGRAM_BUILD=2016120401
+
+## Runs an osync /obackup instance for every conf file found
+## If an instance fails, run it again if time permits
+
+if ! type "$BASH" > /dev/null; then
+	echo "Please run this script only with bash shell. Tested on bash >= 3.2"
+	exit 127
+fi
+
+## If maximum execution time is not reached, failed instances will be rerun. Max exec time is in seconds. Example is set to 10 hours.
+MAX_EXECUTION_TIME=36000
+
+## Specifies the number of total runs an instance may get
+MAX_RUNS=3
+
+## Log file path
+if [ -w /var/log ]; then
+	LOG_FILE=/var/log/$SUBPROGRAM-batch.log
+else
+	LOG_FILE=./$SUBPROGRAM-batch.log
+fi
+
+# No need to edit under this line ##############################################################
+
+function _logger {
+	local value="${1}" # What to log
+	echo -e "$value" >> "$LOG_FILE"
+}
+
+function Logger {
+	local value="${1}" # What to log
+	local level="${2}" # Log level: DEBUG, NOTICE, WARN, ERROR, CRITIAL
+
+	prefix="$(date) - "
+
+	if [ "$level" == "CRITICAL" ]; then
+		_logger "$prefix\e[41m$value\e[0m"
+	elif [ "$level" == "ERROR" ]; then
+		_logger "$prefix\e[91m$value\e[0m"
+	elif [ "$level" == "WARN" ]; then
+		_logger "$prefix\e[93m$value\e[0m"
+	elif [ "$level" == "NOTICE" ]; then
+		_logger "$prefix$value"
+	elif [ "$level" == "DEBUG" ]; then
+		if [ "$DEBUG" == "yes" ]; then
+			_logger "$prefix$value"
+		fi
+	else
+		_logger "\e[41mLogger function called without proper loglevel.\e[0m"
+		_logger "$prefix$value"
+	fi
+}
+
+function CheckEnvironment {
+	## osync / obackup executable full path can be set here if it cannot be found on the system
+	if ! type $SUBPROGRAM.sh > /dev/null 2>&1
+	then
+		if [ -f /usr/local/bin/$SUBPROGRAM.sh ]
+		then
+			SUBPROGRAM_EXECUTABLE=/usr/local/bin/$SUBPROGRAM.sh
+		else
+			Logger "Could not find [/usr/local/bin/$SUBPROGRAM.sh]" "CRITICAL"
+			( >&2 echo "Could not find [/usr/local/bin/$SUBPROGRAM.sh]" )
+			exit 1
+		fi
+	else
+		SUBPROGRAM_EXECUTABLE=$(type -p $SUBPROGRAM.sh)
+	fi
+
+	if [ "$CONF_FILE_PATH" == "" ]; then
+		Usage
+	fi
+}
+
+function Batch {
+	local runs=1 # Number of batch runs
+	local runList # Actual conf file list to run
+	local runAgainList # List of failed conf files sto run again
+
+	local confFile
+	local result
+
+	local i
+
+	# Using -e because find will accept directories or files
+	if [ ! -e "$CONF_FILE_PATH" ]; then
+		Logger "Cannot find conf file path [$CONF_FILE_PATH]." "CRITICAL"
+		Usage
+	else
+		# Ugly hack to read files into an array while preserving special characters
+		runList=()
+		while IFS= read -d $'\0' -r file; do runList+=("$file"); done < <(find "$CONF_FILE_PATH" -maxdepth 1 -iname "*.conf" -print0)
+
+		while ([ $MAX_EXECUTION_TIME -gt $SECONDS ] || [ $MAX_EXECUTION_TIME -eq 0 ]) && [ "${#runList[@]}" -gt 0 ] && [ $runs -le $MAX_RUNS ]; do
+			runAgainList=()
+			Logger "Sequential run n°$runs of $SUBPROGRAM instances for:" "NOTICE"
+			for confFile in "${runList[@]}"; do
+				Logger "$(basename $confFile)" "NOTICE"
+			done
+			for confFile in "${runList[@]}"; do
+				$SUBPROGRAM_EXECUTABLE "$confFile" --silent $opts &
+				wait $!
+				result=$?
+				if [ $result != 0 ]; then
+					if [ $result == 1 ] || [ $result == 128 ]; then # Do not handle exit code 128 because it is already handled here
+						Logger "Instance $(basename $confFile) failed with exit code [$result]." "ERROR"
+						runAgainList+=("$confFile")
+					elif [ $result == 2 ]; then
+						Logger "Instance $(basename $confFile) finished with warnings." "WARN"
+					fi
+				else
+					Logger "Instance $(basename $confFile) succeed." "NOTICE"
+				fi
+			done
+			runList=("${runAgainList[@]}")
+			runs=$(($runs + 1))
+		done
+	fi
+}
+
+function Usage {
+	echo "$PROGRAM $PROGRAM_BUILD"
+	echo $AUTHOR
+	echo $CONTACT
+	echo ""
+	echo "Batch script to sequentially run osync or obackup instances and rerun failed ones."
+	echo "Usage: $PROGRAM.sh [OPTIONS] [$SUBPROGRAM OPTIONS]"
+	echo ""
+	echo "[OPTIONS]"
+	echo "--path=/path/to/conf      Path to osync / obackup conf files, defaults to /etc/osync or /etc/obackup"
+	echo "--max-runs=X              Number of max runs per instance, (defaults to 3)"
+	echo "--max-exec-time=X         Retry failed instances only if max execution time not reached (defaults to 36000 seconds). Set to 0 to bypass execution time check"
+	echo "[$SUBPROGRAM OPTIONS]"
+	echo "Specify whatever options $PROGRAM accepts. Example"
+	echo "$PROGRAM.sh --path=/etc/$SUBPROGRAM --no-maxtime"
+	echo ""
+	echo "No output will be written to stdout/stderr."
+	echo "Verify log file in [$LOG_FILE]."
+	exit 128
+}
+
+opts=""
+for i in "$@"
+do
+	case $i in
+		--path=*)
+		CONF_FILE_PATH=${i##*=}
+		;;
+		--max-runs=*)
+		MAX_RUNS=${i##*=}
+		;;
+		--max-exec-time=*)
+		MAX_EXECUTION_TIME=${i##*=}
+		;;
+		--help|-h|-?)
+		Usage
+		;;
+		*)
+		opts="$opts$i "
+		;;
+	esac
+done
+
+CheckEnvironment
+Logger "$(date) $SUBPROGRAM batch run" "NOTICE"
+Batch

dev/common_install.sh

@@ -0,0 +1,351 @@
+#!/usr/bin/env bash
+
+## Installer script suitable for osync / obackup / pmocr
+
+include #### _OFUNCTIONS_BOOTSTRAP SUBSET ####
+
+PROGRAM=[prgname]
+
+PROGRAM_VERSION=$(grep "PROGRAM_VERSION=" $PROGRAM.sh)
+PROGRAM_VERSION=${PROGRAM_VERSION#*=}
+PROGRAM_BINARY=$PROGRAM".sh"
+PROGRAM_BATCH=$PROGRAM"-batch.sh"
+SSH_FILTER="ssh_filter.sh"
+
+SCRIPT_BUILD=2017031402
+
+## osync / obackup / pmocr / zsnap install script
+## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
+## Please adapt this to fit your distro needs
+
+# Get current install.sh path from http://stackoverflow.com/a/246128/2635443
+SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+CONF_DIR=$FAKEROOT/etc/$PROGRAM
+BIN_DIR="$FAKEROOT/usr/local/bin"
+SERVICE_DIR_INIT=$FAKEROOT/etc/init.d
+# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora
+SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system
+SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user
+
+if [ "$PROGRAM" == "osync" ]; then
+	SERVICE_NAME="osync-srv"
+elif [ "$PROGRAM" == "pmocr" ]; then
+	SERVICE_NAME="pmocr-srv"
+fi
+
+SERVICE_FILE_INIT="$SERVICE_NAME"
+SERVICE_FILE_SYSTEMD_SYSTEM="$SERVICE_NAME@.service"
+SERVICE_FILE_SYSTEMD_USER="$SERVICE_NAME@.service.user"
+
+## Generic code
+
+## Default log file
+if [ -w "$FAKEROOT/var/log" ]; then
+	LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log"
+elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
+	LOG_FILE="$HOME/$PROGRAM-install.log"
+else
+	LOG_FILE="./$PROGRAM-install.log"
+fi
+
+include #### QuickLogger SUBSET ####
+include #### UrlEncode SUBSET ####
+include #### GetLocalOS SUBSET ####
+include #### GetConfFileValue SUBSET ####
+
+function SetLocalOSSettings {
+	USER=root
+
+	# LOCAL_OS and LOCAL_OS_FULL are global variables set at GetLocalOS
+
+	case $LOCAL_OS in
+		*"BSD"*)
+		GROUP=wheel
+		;;
+		*"MacOSX"*)
+		GROUP=admin
+		;;
+		*"msys"*|*"Cygwin"*)
+		USER=""
+		GROUP=""
+		;;
+		*)
+		GROUP=root
+		;;
+	esac
+
+	if [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "BusyBox" ]; then
+		QuickLogger "Cannot be installed on [$LOCAL_OS]. Please use $PROGRAM.sh directly."
+		exit 1
+	fi
+
+	if ([ "$USER" != "" ] && [ "$(whoami)" != "$USER" ] && [ "$FAKEROOT" == "" ]); then
+		QuickLogger "Must be run as $USER."
+		exit 1
+	fi
+
+	OS=$(UrlEncode "$LOCAL_OS_FULL")
+}
+
+function GetInit {
+	if [ -f /sbin/init ]; then
+		if file /sbin/init | grep systemd > /dev/null; then
+			init="systemd"
+		else
+			init="initV"
+		fi
+	else
+		QuickLogger "Can't detect initV or systemd. Service files won't be installed. You can still run $PROGRAM manually or via cron."
+		init="none"
+	fi
+}
+
+function CreateDir {
+	local dir="${1}"
+
+	if [ ! -d "$dir" ]; then
+		mkdir "$dir"
+		if [ $? == 0 ]; then
+			QuickLogger "Created directory [$dir]."
+		else
+			QuickLogger "Cannot create directory [$dir]."
+			exit 1
+		fi
+	fi
+}
+
+function CopyFile {
+	local sourcePath="${1}"
+	local destPath="${2}"
+	local fileName="${3}"
+	local fileMod="${4}"
+	local fileUser="${5}"
+	local fileGroup="${6}"
+	local overwrite="${7:-false}"
+
+	local userGroup=""
+	local oldFileName
+
+	if [ -f "$destPath/$fileName" ] && [ $overwrite == false ]; then
+		oldFileName="$fileName"
+		fileName="$oldFileName.new"
+		cp "$sourcePath/$oldFileName" "$destPath/$fileName"
+	else
+		cp "$sourcePath/$fileName" "$destPath"
+	fi
+
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot copy [$fileName] to [$destPath]. Make sure to run install script in the directory containing all other files."
+		QuickLogger "Also make sure you have permissions to write to [$BIN_DIR]."
+		exit 1
+	else
+		QuickLogger "Copied [$fileName] to [$destPath]."
+		if [ "$fileMod" != "" ]; then
+			chmod "$fileMod" "$destPath/$fileName"
+			if [ $? != 0 ]; then
+				QuickLogger "Cannot set file permissions of [$destPath/$fileName] to [$fileMod]."
+				exit 1
+			else
+				QuickLogger "Set file permissions to [$fileMod] on [$destPath/$fileName]."
+			fi
+		fi
+
+		if [ "$fileUser" != "" ]; then
+			userGroup="$fileUser"
+
+			if [ "$fileGroup" != "" ]; then
+				userGroup="$userGroup"":$fileGroup"
+			fi
+
+			chown "$userGroup" "$destPath/$fileName"
+			if [ $? != 0 ]; then
+				QuickLogger "Could not set file ownership on [$destPath/$fileName] to [$userGroup]."
+				exit 1
+			else
+				QuickLogger "Set file ownership on [$destPath/$fileName] to [$userGroup]."
+			fi
+		fi
+	fi
+}
+
+function CopyExampleFiles {
+	exampleFiles=()
+	exampleFiles[0]="sync.conf.example"		# osync
+	exampleFiles[1]="host_backup.conf.example"	# obackup
+	exampleFiles[2]="exclude.list.example"		# osync & obackup
+	exampleFiles[3]="snapshot.conf.example"		# zsnap
+	exampleFiles[4]="default.conf"			# pmocr
+
+	for file in "${exampleFiles[@]}"; do
+		if [ -f "$SCRIPT_PATH/$file" ]; then
+			CopyFile "$SCRIPT_PATH" "$CONF_DIR" "$file" "" "" "" false
+		fi
+	done
+}
+
+function CopyProgram {
+	binFiles=()
+	binFiles[0]="$PROGRAM_BINARY"
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
+		binFiles[1]="$PROGRAM_BATCH"
+		binFiles[2]="$SSH_FILTER"
+	fi
+
+	local user=""
+	local group=""
+
+	if ([ "$USER" != "" ] && [ "$FAKEROOT" == "" ]); then
+		user="$USER"
+	fi
+	if ([ "$GROUP" != "" ] && [ "$FAKEROOT" == "" ]); then
+		group="$GROUP"
+	fi
+
+	for file in "${binFiles[@]}"; do
+		CopyFile "$SCRIPT_PATH" "$BIN_DIR" "$file" 755 "$user" "$group" true
+	done
+}
+
+function CopyServiceFiles {
+	if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
+		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true
+		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true
+
+		QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
+		QuickLogger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR."
+		QuickLogger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]."
+		QuickLogger "In userland, active with [systemctl --user start $SERVICE_NAME@instance.conf]."
+	elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then
+		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true
+
+		QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
+		QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]."
+		QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]."
+	else
+		QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation."
+	fi
+}
+
+function Statistics {
+	if type wget > /dev/null; then
+		wget -qO- "$STATS_LINK" > /dev/null 2>&1
+		if [ $? == 0 ]; then
+			return 0
+		fi
+	fi
+
+	if type curl > /dev/null; then
+		curl "$STATS_LINK" -o /dev/null > /dev/null 2>&1
+		if [ $? == 0 ]; then
+			return 0
+		fi
+	fi
+
+	QuickLogger "Neiter wget nor curl could be used for. Cannot run statistics. Use the provided link please."
+	return 1
+}
+
+function RemoveFile {
+	local file="${1}"
+
+	if [ -f "$file" ]; then
+		rm -f "$file"
+		if [ $? != 0 ]; then
+			QuickLogger "Could not remove file [$file]."
+		else
+			QuickLogger "Removed file [$file]."
+		fi
+	else
+		QuickLogger "File [$file] not found. Skipping."
+	fi
+}
+
+function RemoveAll {
+	RemoveFile "$BIN_DIR/$PROGRAM_BINARY"
+	RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
+	if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then		# Check if any other program requiring ssh filter is present before removal
+		RemoveFile "$BIN_DIR/$SSH_FILTER"
+	else
+		QuickLogger "Skipping removal of [$BIN_DIR/$SSH_FILTER] because other programs present that need it."
+	fi
+	RemoveFile "$SERVICE_DIR_SYSTEMD_SYSTEM/$SERVICE_FILE_SYSTEMD_SYSTEM"
+	RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM"
+	RemoveFile "$SERVICE_DIR_INIT/$SERVICE_FILE_INIT"
+
+	QuickLogger "Skipping configuration files in [$CONF_DIR]. You may remove this directory manually."
+}
+
+function Usage {
+	echo "Installs $PROGRAM into $BIN_DIR"
+	echo "options:"
+	echo "--silent		Will log and bypass user interaction."
+	echo "--no-stats	Used with --silent in order to refuse sending anonymous install stats."
+	exit 127
+}
+
+_LOGGER_SILENT=false
+_STATS=1
+ACTION="install"
+
+for i in "$@"
+do
+	case $i in
+		--silent)
+		_LOGGER_SILENT=true
+		;;
+		--no-stats)
+		_STATS=0
+		;;
+		--remove)
+		ACTION="uninstall"
+		;;
+		--help|-h|-?)
+		Usage
+	esac
+done
+
+if [ "$FAKEROOT" != "" ]; then
+	mkdir -p "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_USER" "$BIN_DIR"
+fi
+
+GetLocalOS
+SetLocalOSSettings
+GetInit
+
+STATS_LINK="http://instcount.netpower.fr?program=$PROGRAM&version=$PROGRAM_VERSION&os=$OS&action=$ACTION"
+
+if [ "$ACTION" == "uninstall" ]; then
+	RemoveAll
+	QuickLogger "$PROGRAM uninstalled."
+else
+	CreateDir "$CONF_DIR"
+	CreateDir "$BIN_DIR"
+	CopyExampleFiles
+	CopyProgram
+	CopyServiceFiles
+	QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
+		QuickLogger ""
+		QuickLogger "If connecting remotely, consider setup ssh filter to enhance security."
+		QuickLogger ""
+	fi
+fi
+
+if [ $_STATS -eq 1 ]; then
+	if [ $_LOGGER_SILENT == true ]; then
+		Statistics
+	else
+		QuickLogger "In order to make usage statistics, the script would like to connect to $STATS_LINK"
+		read -r -p "No data except those in the url will be send. Allow [Y/n] " response
+		case $response in
+			[nN])
+			exit
+			;;
+			*)
+			Statistics
+			exit $?
+			;;
+		esac
+	fi
+fi

dev/merge.sh

@@ -0,0 +1,175 @@
+#!/usr/bin/env bash
+
+## MERGE 2017031301
+
+## Merges ofunctions.sh and n_program.sh into program.sh
+## Adds installer
+
+function __PREPROCESSOR_Merge {
+	PROGRAM=obackup
+
+	VERSION=$(grep "PROGRAM_VERSION=" n_$PROGRAM.sh)
+	VERSION=${VERSION#*=}
+	__PREPROCESSOR_Constants
+
+	source "ofunctions.sh"
+	if [ $? != 0 ]; then
+		echo "Please run $0 in dev directory with ofunctions.sh"
+		exit 1
+	fi
+
+	__PREPROCESSOR_Unexpand "n_$PROGRAM.sh" "debug_$PROGRAM.sh"
+
+	for subset in "${__PREPROCESSOR_SUBSETS[@]}"; do
+		__PREPROCESSOR_MergeSubset "$subset" "${subset//SUBSET/SUBSET END}" "ofunctions.sh" "debug_$PROGRAM.sh"
+	done
+
+	__PREPROCESSOR_CleanDebug
+	__PREPROCESSOR_CopyCommons
+	rm -f tmp_$PROGRAM.sh
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot remove tmp_$PROGRAM.sh"
+		exit 1
+	fi
+}
+
+function __PREPROCESSOR_Constants {
+	PARANOIA_DEBUG_LINE="#__WITH_PARANOIA_DEBUG"
+	PARANOIA_DEBUG_BEGIN="#__BEGIN_WITH_PARANOIA_DEBUG"
+	PARANOIA_DEBUG_END="#__END_WITH_PARANOIA_DEBUG"
+
+	__PREPROCESSOR_SUBSETS=(
+	'#### OFUNCTIONS FULL SUBSET ####'
+	'#### OFUNCTIONS MINI SUBSET ####'
+	'#### _OFUNCTIONS_BOOTSTRAP SUBSET ####'
+	'#### DEBUG SUBSET ####'
+	'#### TrapError SUBSET ####'
+	'#### RemoteLogger SUBSET ####'
+	'#### QuickLogger SUBSET ####'
+	'#### GetLocalOS SUBSET ####'
+	'#### IsInteger SUBSET ####'
+	'#### UrlEncode SUBSET ####'
+	'#### HumanToNumeric SUBSET ####'
+	'#### ArrayContains SUBSET ####'
+	'#### GetConfFileValue SUBSET ####'
+	'#### SetConfFileValue SUBSET ####'
+	)
+}
+
+function __PREPROCESSOR_Unexpand {
+	local source="${1}"
+	local destination="${2}"
+
+	unexpand "$source" > "$destination"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot unexpand [$source] to [$destination]."
+		exit 1
+	fi
+}
+
+function __PREPROCESSOR_MergeSubset {
+	local subsetBegin="${1}"
+	local subsetEnd="${2}"
+	local subsetFile="${3}"
+	local mergedFile="${4}"
+
+	sed -n "/$subsetBegin/,/$subsetEnd/p" "$subsetFile" > "$subsetFile.$subsetBegin"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot sed subset [$subsetBegin -- $subsetEnd] in [$subsetFile]."
+		exit 1
+	fi
+	sed "/include $subsetBegin/r $subsetFile.$subsetBegin" "$mergedFile" | grep -v -E "$subsetBegin\$|$subsetEnd\$" > "$mergedFile.tmp"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot add subset [$subsetBegin] to [$mergedFile]."
+		exit 1
+	fi
+	rm -f "$subsetFile.$subsetBegin"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot remove temporary subset [$subsetFile.$subsetBegin]."
+		exit 1
+	fi
+
+	rm -f "$mergedFile"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot remove merged original file [$mergedFile]."
+		exit 1
+	fi
+
+	mv "$mergedFile.tmp" "$mergedFile"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot move merged tmp file to original [$mergedFile]."
+		exit 1
+	fi
+}
+
+function __PREPROCESSOR_CleanDebug {
+	sed '/'$PARANOIA_DEBUG_BEGIN'/,/'$PARANOIA_DEBUG_END'/d' debug_$PROGRAM.sh | grep -v "$PARANOIA_DEBUG_LINE" > ../$PROGRAM.sh
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot remove PARANOIA_DEBUG code from standard build."
+		exit 1
+	fi
+
+	chmod +x "debug_$PROGRAM.sh"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot chmod debug_$PROGRAM.sh"
+		exit 1
+	else
+		QuickLogger "Prepared ./debug_$PROGRAM.sh"
+	fi
+	chmod +x "../$PROGRAM.sh"
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot chmod $PROGRAM.sh"
+		exit 1
+	else
+		QuickLogger "Prepared ../$PROGRAM.sh"
+	fi
+}
+
+function __PREPROCESSOR_CopyCommons {
+	sed "s/\[prgname\]/$PROGRAM/g" common_install.sh > ../install.sh
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot assemble install."
+		exit 1
+	fi
+
+	for subset in "${__PREPROCESSOR_SUBSETS[@]}"; do
+		__PREPROCESSOR_MergeSubset "$subset" "${subset//SUBSET/SUBSET END}" "ofunctions.sh" "../install.sh"
+	done
+
+	#sed "s/\[version\]/$VERSION/g" ../tmp_install.sh > ../install.sh
+	#if [ $? != 0 ]; then
+	#	QuickLogger "Cannot change install version."
+	#	exit 1
+	#fi
+	if [ -f "common_batch.sh" ]; then
+		sed "s/\[prgname\]/$PROGRAM/g" common_batch.sh > ../$PROGRAM-batch.sh
+		if [ $? != 0 ]; then
+			QuickLogger "Cannot assemble batch runner."
+			exit 1
+		fi
+		chmod +x ../$PROGRAM-batch.sh
+		if [ $? != 0 ]; then
+			QuickLogger "Cannot chmod $PROGRAM-batch.sh"
+			exit 1
+		else
+			QuickLogger "Prepared ../$PROGRAM-batch.sh"
+		fi
+	fi
+	chmod +x ../install.sh
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot chmod install.sh"
+		exit 1
+	else
+		QuickLogger "Prepared ../install.sh"
+	fi
+	rm -f ../tmp_install.sh
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot chmod $PROGRAM.sh"
+		exit 1
+	fi
+}
+
+# If sourced don't do anything
+if [ "$(basename $0)" == "merge.sh" ]; then
+	__PREPROCESSOR_Merge
+fi

dev/shellcheck.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+#SC1090 = not following non constants source
+#SC1091 = not following source
+#SC2086 = quoting errors (shellcheck is way too picky about quoting)
+#SC2120 = only for debug version
+
+shellcheck -e SC1090,SC1091,SC2086,SC2119,SC2120 $1

dev/tests/conf/local.conf

@@ -0,0 +1,228 @@
+#!/usr/bin/env bash
+
+###### obackup - Local or Remote, push or pull backup script for files & mysql databases
+###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr)
+###### obackup v2.1x config file rev 2016081701
+
+###### GENERAL BACKUP OPTIONS
+
+## Backup identification string.
+INSTANCE_ID="local-test"
+
+## Log file location. Leaving this empty will create log file at /var/log/obackup.INSTANCE_ID.log (or current directory if /var/log doesn't exist).
+LOGFILE=""
+
+## Elements to backup
+SQL_BACKUP=yes
+FILE_BACKUP=yes
+
+## Backups can be done local, pulled from another server or pushed to a backup server. Available options are [local,pull,push].
+## Pulled backups are the safest option, as the backup server contains the RSA key and cannot be compromised by another server.
+BACKUP_TYPE=local
+
+###### BACKUP STORAGE
+
+## Storage paths of the backups (absolute paths of the local or remote system)
+SQL_STORAGE="${HOME}/obackup-storage/sql-local"
+FILE_STORAGE="${HOME}/obackup-storage/files-local"
+
+## Encryption
+ENCRYPTION=yes
+
+## Backup encryption needs a temporary storage space in order to encrypt files before sending them (absolute paths of the local or remote system)
+CRYPT_STORAGE="${HOME}/obackup-storage/crypt-local"
+ 
+## GPG recipient (pubkey for this recipient must exist, see gpg2 --list-keys or gpg --list-keys
+GPG_RECIPIENT="John Doe"
+
+## Create backup directories if they do not exist
+CREATE_DIRS=yes
+
+## Keep absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
+## You should leave this enabled if you intend to use 'backup task division' functionality of oBackup, or everything will end up in the same directory.
+KEEP_ABSOLUTE_PATHS=yes
+
+## Generate an alert if backup size is lower than given value in Kb (this can also help identifying empty mount dirs).
+BACKUP_SIZE_MINIMUM=1024
+
+## Check backup size before proceeding
+GET_BACKUP_SIZE=yes
+
+## Generate an alert if storage free space is lower than given value in Kb.
+## Keep in mind that disabling backup file size test will only test min space against SQL backup size.
+SQL_WARN_MIN_SPACE=1048576
+FILE_WARN_MIN_SPACE=1048576
+
+###### REMOTE ONLY OPTIONS
+
+## In case of pulled or pushed backups, remote system URI needs to be supplied.
+REMOTE_SYSTEM_URI="ssh://backupuser@remote.system.tld:22/"
+
+## You can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
+SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa"
+
+## ssh compression should be used unless your remote connection is good enough (LAN)
+SSH_COMPRESSION=yes
+
+## Ignore ssh known hosts verification. DANGER WILL ROBINSON DANGER: This can lead to security risks. Only enable if you know what you're doing.
+SSH_IGNORE_KNOWN_HOSTS=no
+
+## Remote rsync executable path. Leave this empty in most cases
+RSYNC_REMOTE_PATH=""
+
+## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
+REMOTE_HOST_PING=yes
+
+## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
+REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
+
+## If enabled, commands will be executed as superuser on remote side. See documentation for /etc/sudoers configuration ("find", "du", "tee" and "rsync" need to be allowed). Requiretty needs to be disabled.
+SUDO_EXEC=no
+
+###### DATABASE SPECIFIC OPTIONS
+
+## Database backup user
+SQL_USER=root
+
+## Enabling the following option will save all databases on local or remote given SQL instance except the ones specified in the exclude list.
+## Every found database will be backed up as separate backup task.
+DATABASES_ALL=yes
+DATABASES_ALL_EXCLUDE_LIST="test;information_schema;zarafa_prod"
+## Alternatively, if DATABASES_ALL=no, you can specify a list of databases to backup separated by spaces.
+DATABASES_LIST="mysql"
+
+## Max backup execution time per Database task. Soft max exec time generates a warning only. Hard max exec time generates a warning and stops current backup task.
+## If a task gets stopped, next one in the task list gets executed. Time is specified in seconds.
+SOFT_MAX_EXEC_TIME_DB_TASK=3600
+HARD_MAX_EXEC_TIME_DB_TASK=7200
+
+## mysqldump options (ex: --extended-insert, --single-transaction, --quick...). See MySQL / MariaDB manual
+## default option: --opt
+MYSQLDUMP_OPTIONS="--opt --single-transaction"
+
+## Preferred SQL dump compression. Compression methods can be xz, lzma, pigz or gzip (will fallback from xz to gzip depending if available)
+## Generally, level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
+COMPRESSION_LEVEL=3
+
+###### FILES SPECIFIC OPTIONS
+
+## File backups are divided in tasks. Every directory in DIRECTORY_LIST will be processed as a unique task.
+## Every subdirectory of each directory in RECURSIVE_DIRECTORY_LIST will be processed as a unique task.
+## Example: RECURSIVE_DIRECTORY_LIST="/home;/var" will create backup tasks tasks "/home/dir1, "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/something".
+## You can exclude directories from the avove backup task creation, ex: avoid backing up "/home/dir2" by adding it to RECURSIVE_EXCLUDE_LIST.
+
+## Directories backup list. List of semicolon separated directories that will be backed up.
+DIRECTORY_LIST="${HOME}/obackup-testdata/testData"
+RECURSIVE_DIRECTORY_LIST="${HOME}/obackup-testdata/testDataRecursive"
+RECURSIVE_EXCLUDE_LIST="${HOME}/obackup-testdata/testDataRecursive/Excluded"
+
+## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns)
+RSYNC_PATTERN_FIRST=include
+
+## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work).
+## Paths are relative to sync dirs. List elements are separated by a semicolon.
+RSYNC_INCLUDE_PATTERN=""
+RSYNC_EXCLUDE_PATTERN="*.ded"
+#RSYNC_EXCLUDE_PATTERN="tmp;archives"
+
+## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
+## This file has to be in the same directory as the config file
+## Paths are relative to sync dirs. One element per line.
+RSYNC_INCLUDE_FROM=""
+RSYNC_EXCLUDE_FROM=""
+#RSYNC_EXCLUDE_FROM="exclude.list"
+
+## List separator char. You may set an alternative separator char for your directories lists above.
+PATH_SEPARATOR_CHAR=";"
+
+## Preserve basic linux permissions
+PRESERVE_PERMISSIONS=yes
+PRESERVE_OWNER=yes
+PRESERVE_GROUP=yes
+## On MACOS X, does not work and will be ignored 
+PRESERVE_EXECUTABILITY=yes
+
+## Preserve ACLS. Make sure source and target FS can hold same ACLs or you'll get loads of errors.
+PRESERVE_ACL=no
+## Preserve Xattr. MAke sure source and target FS can hold same Xattr or you'll get loads of errors.
+PRESERVE_XATTR=no
+
+## Transforms symlinks into referent files/dirs
+COPY_SYMLINKS=yes
+## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root.
+KEEP_DIRLINKS=yes
+## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
+PRESERVE_HARDLINKS=no
+
+
+## Let RSYNC compress file transfers. Do not use this on local-local backup schemes. Also, this is not useful if SSH compression is enabled.
+RSYNC_COMPRESS=no
+
+## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
+SOFT_MAX_EXEC_TIME_FILE_TASK=3600
+HARD_MAX_EXEC_TIME_FILE_TASK=7200
+
+## Keep partial uploads that can be resumed on next run, experimental feature
+PARTIAL=no
+
+## Delete files on destination that vanished from source. Do not turn this on unless you enabled backup rotation  or a snapshotting FS like zfs to keep those vanished files on the destination.
+DELETE_VANISHED_FILES=no
+
+## Use delta copy algortithm (usefull when local paths are network drives), defaults to yes
+DELTA_COPIES=yes
+
+## Bandwidth limit Kbytes / second for file backups. Leave 0 to disable limitation.
+BANDWIDTH=0
+
+## Paranoia option. Don't change this unless you read the documentation.
+RSYNC_EXECUTABLE=rsync
+
+###### ALERT OPTIONS
+
+## Alert email addresses separated by a space character
+DESTINATION_MAILS=""
+
+## Windows specific (msys / cygwin environment) only mail options (used with mailsend.exe from muquit, http://github.com/muquit/mailsend or from sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail/
+SENDER_MAIL="alert@your.system.tld"
+SMTP_SERVER=smtp.your.isp.tld
+SMTP_PORT=25
+# encryption can be tls, ssl or none
+SMTP_ENCRYPTION=none
+SMTP_USER=
+SMTP_PASSWORD=
+
+###### GENERAL BACKUP OPTIONS
+
+## Max execution time of whole backup process. Soft max exec time generates a warning only.
+## Hard max exec time generates a warning and stops the whole backup execution.
+SOFT_MAX_EXEC_TIME_TOTAL=30000
+HARD_MAX_EXEC_TIME_TOTAL=36000
+
+## Log a message every KEEP_LOGGING seconds just to know the task is still alive
+KEEP_LOGGING=1801
+
+## Backup Rotation. You may rotate backups if you don't use snapshots on your backup server.
+ROTATE_SQL_BACKUPS=yes
+ROTATE_SQL_COPIES=7
+ROTATE_FILE_BACKUPS=yes
+ROTATE_FILE_COPIES=7
+
+###### EXECUTION HOOKS
+
+## Commands can will be run before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set).
+## This is useful to make a snapshot before backing up data, or even handle snapshots of backed up data.
+LOCAL_RUN_BEFORE_CMD=""
+LOCAL_RUN_AFTER_CMD=""
+
+REMOTE_RUN_BEFORE_CMD=""
+REMOTE_RUN_AFTER_CMD=""
+
+## Max execution time of commands before they get force killed. Leave 0 if you don't want this to happen. Time is specified in seconds.
+MAX_EXEC_TIME_PER_CMD_BEFORE=0
+MAX_EXEC_TIME_PER_CMD_AFTER=0
+
+## Stops whole backup execution if one of the above commands fail
+STOP_ON_CMD_ERROR=no
+
+## Run local and remote after backup cmd's even on failure
+RUN_AFTER_CMD_ON_ERROR=yes

dev/tests/conf/max-exec-time.conf

@@ -0,0 +1,222 @@
+#!/usr/bin/env bash
+
+###### obackup - Local or Remote, push or pull backup script for files & mysql databases
+###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr)
+###### obackup v2.1x config file rev 2016081701
+
+###### GENERAL BACKUP OPTIONS
+
+## Backup identification string.
+INSTANCE_ID="local-test"
+
+## Log file location. Leaving this empty will create log file at /var/log/obackup.INSTANCE_ID.log (or current directory if /var/log doesn't exist).
+LOGFILE=""
+
+## Elements to backup
+SQL_BACKUP=yes
+FILE_BACKUP=yes
+
+## Backups can be done local, pulled from another server or pushed to a backup server. Available options are [local,pull,push].
+## Pulled backups are the safest option, as the backup server contains the RSA key and cannot be compromised by another server.
+BACKUP_TYPE=local
+
+###### BACKUP STORAGE
+
+## Storage paths of the backups (absolute paths of the local or remote system)
+SQL_STORAGE="${HOME}/obackup-storage/sql"
+FILE_STORAGE="${HOME}/obackup-storage/files"
+
+## Backup encryption using GPG and duplicity. Feature not ready yet.
+ENCRYPTION=no
+
+## Create backup directories if they do not exist
+CREATE_DIRS=yes
+
+## Keep absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
+## You should leave this enabled if you intend to use 'backup task division' functionality of oBackup, or everything will end up in the same directory.
+KEEP_ABSOLUTE_PATHS=yes
+
+## Generate an alert if backup size is lower than given value in Kb (this can also help identifying empty mount dirs).
+BACKUP_SIZE_MINIMUM=1024
+
+## Check backup size before proceeding
+GET_BACKUP_SIZE=yes
+
+## Generate an alert if storage free space is lower than given value in Kb.
+## Keep in mind that disabling backup file size test will only test min space against SQL backup size.
+SQL_WARN_MIN_SPACE=1048576
+FILE_WARN_MIN_SPACE=1048576
+
+###### REMOTE ONLY OPTIONS
+
+## In case of pulled or pushed backups, remote system URI needs to be supplied.
+REMOTE_SYSTEM_URI="ssh://backupuser@remote.system.tld:22/"
+
+## You can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
+SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa"
+
+## ssh compression should be used unless your remote connection is good enough (LAN)
+SSH_COMPRESSION=yes
+
+## Ignore ssh known hosts verification. DANGER WILL ROBINSON DANGER: This can lead to security risks. Only enable if you know what you're doing.
+SSH_IGNORE_KNOWN_HOSTS=no
+
+## Remote rsync executable path. Leave this empty in most cases
+RSYNC_REMOTE_PATH=""
+
+## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
+REMOTE_HOST_PING=yes
+
+## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
+REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
+
+## If enabled, commands will be executed as superuser on remote side. See documentation for /etc/sudoers configuration ("find", "du", "tee" and "rsync" need to be allowed). Requiretty needs to be disabled.
+SUDO_EXEC=no
+
+###### DATABASE SPECIFIC OPTIONS
+
+## Database backup user
+SQL_USER=root
+
+## Enabling the following option will save all databases on local or remote given SQL instance except the ones specified in the exclude list.
+## Every found database will be backed up as separate backup task.
+DATABASES_ALL=yes
+DATABASES_ALL_EXCLUDE_LIST="test;information_schema"
+## Alternatively, if DATABASES_ALL=no, you can specify a list of databases to backup separated by spaces.
+DATABASES_LIST="mysql"
+
+## Max backup execution time per Database task. Soft max exec time generates a warning only. Hard max exec time generates a warning and stops current backup task.
+## If a task gets stopped, next one in the task list gets executed. Time is specified in seconds.
+SOFT_MAX_EXEC_TIME_DB_TASK=1000
+HARD_MAX_EXEC_TIME_DB_TASK=1000
+
+## mysqldump options (ex: --extended-insert, --single-transaction, --quick...). See MySQL / MariaDB manual
+## default option: --opt
+MYSQLDUMP_OPTIONS="--opt --single-transaction"
+
+## Preferred SQL dump compression. Compression methods can be xz, lzma, pigz or gzip (will fallback from xz to gzip depending if available)
+## Generally, level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
+COMPRESSION_LEVEL=3
+
+###### FILES SPECIFIC OPTIONS
+
+## File backups are divided in tasks. Every directory in DIRECTORY_LIST will be processed as a unique task.
+## Every subdirectory of each directory in RECURSIVE_DIRECTORY_LIST will be processed as a unique task.
+## Example: RECURSIVE_DIRECTORY_LIST="/home;/var" will create backup tasks tasks "/home/dir1, "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/something".
+## You can exclude directories from the avove backup task creation, ex: avoid backing up "/home/dir2" by adding it to RECURSIVE_EXCLUDE_LIST.
+
+## Directories backup list. List of semicolon separated directories that will be backed up.
+DIRECTORY_LIST="${HOME}/obackup-testdata/testData"
+RECURSIVE_DIRECTORY_LIST="${HOME}/obackup-testdata/testDataRecursive"
+RECURSIVE_EXCLUDE_LIST="${HOME}/obackup-testdata/testDataRecursive/Excluded"
+
+## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns)
+RSYNC_PATTERN_FIRST=include
+
+## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work).
+## Paths are relative to sync dirs. List elements are separated by a semicolon.
+RSYNC_INCLUDE_PATTERN=""
+RSYNC_EXCLUDE_PATTERN="*.ded"
+#RSYNC_EXCLUDE_PATTERN="tmp;archives"
+
+## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
+## This file has to be in the same directory as the config file
+## Paths are relative to sync dirs. One element per line.
+RSYNC_INCLUDE_FROM=""
+RSYNC_EXCLUDE_FROM=""
+#RSYNC_EXCLUDE_FROM="exclude.list"
+
+## List separator char. You may set an alternative separator char for your directories lists above.
+PATH_SEPARATOR_CHAR=";"
+
+## Preserve basic linux permissions
+PRESERVE_PERMISSIONS=yes
+PRESERVE_OWNER=yes
+PRESERVE_GROUP=yes
+## On MACOS X, does not work and will be ignored 
+PRESERVE_EXECUTABILITY=yes
+
+## Preserve ACLS. Make sure source and target FS can hold same ACLs or you'll get loads of errors.
+PRESERVE_ACL=no
+## Preserve Xattr. MAke sure source and target FS can hold same Xattr or you'll get loads of errors.
+PRESERVE_XATTR=no
+
+## Transforms symlinks into referent files/dirs
+COPY_SYMLINKS=yes
+## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root.
+KEEP_DIRLINKS=yes
+## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
+PRESERVE_HARDLINKS=no
+
+
+## Let RSYNC compress file transfers. Do not use this on local-local backup schemes. Also, this is not useful if SSH compression is enabled.
+RSYNC_COMPRESS=no
+
+## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
+SOFT_MAX_EXEC_TIME_FILE_TASK=1000
+HARD_MAX_EXEC_TIME_FILE_TASK=1000
+
+## Keep partial uploads that can be resumed on next run, experimental feature
+PARTIAL=no
+
+## Delete files on destination that vanished from source. Do not turn this on unless you enabled backup rotation  or a snapshotting FS like zfs to keep those vanished files on the destination.
+DELETE_VANISHED_FILES=no
+
+## Use delta copy algortithm (usefull when local paths are network drives), defaults to yes
+DELTA_COPIES=yes
+
+## Bandwidth limit Kbytes / second for file backups. Leave 0 to disable limitation.
+BANDWIDTH=0
+
+## Paranoia option. Don't change this unless you read the documentation.
+RSYNC_EXECUTABLE=rsync
+
+###### ALERT OPTIONS
+
+## Alert email addresses separated by a space character
+DESTINATION_MAILS=""
+
+## Windows specific (msys / cygwin environment) only mail options (used with mailsend.exe from muquit, http://github.com/muquit/mailsend or from sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail/
+SENDER_MAIL="alert@your.system.tld"
+SMTP_SERVER=smtp.your.isp.tld
+SMTP_PORT=25
+# encryption can be tls, ssl or none
+SMTP_ENCRYPTION=none
+SMTP_USER=
+SMTP_PASSWORD=
+
+###### GENERAL BACKUP OPTIONS
+
+## Max execution time of whole backup process. Soft max exec time generates a warning only.
+## Hard max exec time generates a warning and stops the whole backup execution.
+SOFT_MAX_EXEC_TIME_TOTAL=1000
+HARD_MAX_EXEC_TIME_TOTAL=1
+
+## Log a message every KEEP_LOGGING seconds just to know the task is still alive
+KEEP_LOGGING=1801
+
+## Backup Rotation. You may rotate backups if you don't use snapshots on your backup server.
+ROTATE_SQL_BACKUPS=no
+ROTATE_SQL_COPIES=7
+ROTATE_FILE_BACKUPS=no
+ROTATE_FILE_COPIES=7
+
+###### EXECUTION HOOKS
+
+## Commands can will be run before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set).
+## This is useful to make a snapshot before backing up data, or even handle snapshots of backed up data.
+LOCAL_RUN_BEFORE_CMD=""
+LOCAL_RUN_AFTER_CMD=""
+
+REMOTE_RUN_BEFORE_CMD=""
+REMOTE_RUN_AFTER_CMD=""
+
+## Max execution time of commands before they get force killed. Leave 0 if you don't want this to happen. Time is specified in seconds.
+MAX_EXEC_TIME_PER_CMD_BEFORE=0
+MAX_EXEC_TIME_PER_CMD_AFTER=0
+
+## Stops whole backup execution if one of the above commands fail
+STOP_ON_CMD_ERROR=no
+
+## Run local and remote after backup cmd's even on failure
+RUN_AFTER_CMD_ON_ERROR=yes

dev/tests/conf/old.conf

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+###### Remote (or local) backup script for files & databases
+###### (C) 2013 by Ozy de Jong (www.badministrateur.com)
+###### Config file rev 0408201301
+
+## Backup identification, any string you want
+BACKUP_ID="really-old-config-file"
+
+## General backup options
+BACKUP_SQL=yes
+BACKUP_FILES=yes
+
+## Local storage paths
+LOCAL_SQL_STORAGE="${HOME}/obackup-storage/sql-old"
+LOCAL_FILE_STORAGE="${HOME}/obackup-storage/files-old"
+## Keep the absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
+## You should leave this enabled if you use recursive directories backup lists or they'll all end in the same path.
+LOCAL_STORAGE_KEEP_ABSOLUTE_PATHS=yes
+## Generate an alert if backup size is lower than given value in Kb, useful for local mounted backups.
+BACKUP_SIZE_MINIMUM=1024
+## Generate an alert if local storage free space is lower than given value in Kb.
+LOCAL_STORAGE_WARN_MIN_SPACE=1048576
+
+## If enabled, file backups will be processed with sudo command. See documentation for /etc/sudoers configuration ("find", "du" and "rsync" need to be allowed). Requiretty needs to be disabled.
+SUDO_EXEC=no
+## Paranoia option. Don't change this unless you read the documentation and still feel concerned about security issues.
+RSYNC_EXECUTABLE=rsync
+
+## Remote options (will make backups of remote system through ssh tunnel, public RSA key need to be put into /home/.ssh/authorized_keys in remote users home directory)
+REMOTE_BACKUP=yes
+SSH_RSA_PRIVATE_KEY=${HOME}/.ssh/id_rsa_local
+REMOTE_USER=root
+REMOTE_HOST=localhost
+REMOTE_PORT=22
+## ssh compression should be used unless your remote connection is good enough (LAN)
+SSH_COMPRESSION=yes
+## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
+REMOTE_HOST_PING=yes
+## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
+REMOTE_3RD_PARTY_HOST="www.kernel.org www.google.com"
+
+## Databases options
+SQL_USER=root
+## Save all databases except the ones specified in the exlude list. Every found database will be backed up as separate task (see documentation for explanation about tasks)
+DATABASES_ALL=yes
+DATABASES_ALL_EXCLUDE_LIST="test;information_schema;zarafa_prod"
+# Alternatively, you can specifiy a manual list of databases to backup separated by spaces
+DATABASES_LIST=""
+## Max backup execution time per DB task. Soft is warning only. Hard is warning, stopping backup task and processing next one. Time is specified in seconds
+SOFT_MAX_EXEC_TIME_DB_TASK=3600
+HARD_MAX_EXEC_TIME_DB_TASK=7200
+## Preferred sql dump compression. Can be set to xz, lzma or gzip.
+## Generally, xz level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
+COMPRESSION_PROGRAM=xz
+COMPRESSION_LEVEL=3
+## Dump compression should be done on remote side but can also be done locally to lower remote system usage (will take more bandwidth, check for ssh compression)
+COMPRESSION_REMOTE=yes
+
+## Path separator. You can set whatever seperator you want in your directories list below. You may change this in case you have some esoteric filenames (try to use unconventional separators like | ).
+PATH_SEPARATOR_CHAR=";"
+## File backup lists. Double quoted directory list separated by the $PATH_SEPARATOR_CHAR. Every directory will be processed as task (see documentation for explanation about tasks)
+DIRECTORIES_SIMPLE_LIST="${HOME}/obackup-testdata/testData"
+## Recurse directory list separated by the $PATH_SEPARATOR_CHAR. Will create a backup task per subdirectory (one level only), eg RECURSE_LIST="/home /var" will create tasks "/home/dir1", "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/whatever"
+DIRECTORIES_RECURSE_LIST="${HOME}/obackup-testdata/testDataRecursive"
+## You can optionally exclude directories from RECURSE_LIST tasks, eg on the above example you could exclude /home/dir2 by adding it to RECURSE_EXCLUDE_LIST
+DIRECTORIES_RECURSE_EXCLUDE_LIST="${HOME}/obackup-testdata/testDataRecursive/Excluded"
+## Be aware that every recurse list will have it's own root (exclude pattern is relative from /home/web for /home/web/{recursedir})
+RSYNC_EXCLUDE_PATTERN="*.ded"
+## Preserve ACLS. Make sure target FS can hold ACLs or you'll get loads of errors.
+PRESERVE_ACL=no
+## Preserve Xattr
+PRESERVE_XATTR=no
+## Let RSYNC compress file transfers. Do not use if you already enabled SSH compression.
+RSYNC_COMPRESS=yes
+## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
+SOFT_MAX_EXEC_TIME_FILE_TASK=3600
+HARD_MAX_EXEC_TIME_FILE_TASK=7200
+
+## Alert email adresses separated by a space character
+DESTINATION_MAILS="your@mail.tld"
+
+## Max execution time of whole backup process. Soft is warning only. Hard is warning and stopping whole backup process.
+SOFT_MAX_EXEC_TIME_TOTAL=30000
+HARD_MAX_EXEC_TIME_TOTAL=36000
+
+## Backup Rotation in case you don't use a snapshot aware file system like zfs or btrfs to perform a snapshot before every backup
+ROTATE_BACKUPS=yes
+ROTATE_COPIES=7
+
+## Commands that will be run  before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set to yes). Very usefull to initiate snapshots.
+## Set max execution time to 0 if you want these commands not to get stopped, else set a value in seconds after which execution will be stopped.
+LOCAL_RUN_BEFORE_CMD=""
+LOCAL_RUN_AFTER_CMD=""
+
+REMOTE_RUN_BEFORE_CMD=""
+REMOTE_RUN_AFTER_CMD=""
+
+MAX_EXEC_TIME_PER_CMD_BEFORE=0
+MAX_EXEC_TIME_PER_CMD_AFTER=0
+

dev/tests/conf/pull.conf

@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+
+###### obackup - Local or Remote, push or pull backup script for files & mysql databases
+###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr)
+###### obackup v2.1x config file rev 2016081701
+
+###### GENERAL BACKUP OPTIONS
+
+## Backup identification string.
+INSTANCE_ID="pull-test"
+
+## Log file location. Leaving this empty will create log file at /var/log/obackup.INSTANCE_ID.log (or current directory if /var/log doesn't exist).
+LOGFILE=""
+
+## Elements to backup
+SQL_BACKUP=yes
+FILE_BACKUP=yes
+
+## Backups can be done local, pulled from another server or pushed to a backup server. Available options are [local,pull,push].
+## Pulled backups are the safest option, as the backup server contains the RSA key and cannot be compromised by another server.
+BACKUP_TYPE=pull
+
+###### BACKUP STORAGE
+
+## Storage paths of the backups (absolute paths of the local or remote system)
+SQL_STORAGE="${HOME}/obackup-storage/sql-pull"
+FILE_STORAGE="${HOME}/obackup-storage/files-pull"
+
+## Encryption
+ENCRYPTION=no
+
+## Backup encryption needs a temporary storage space in order to encrypt files before sending them (absolute paths of the local or remote system)
+CRYPT_STORAGE="${HOME}/obackup-storage/crypt-pull"
+
+## GPG recipient (pubkey for this recipient must exist, see gpg2 --list-keys or gpg --list-keys
+GPG_RECIPIENT="John Doe"
+
+## Create backup directories if they do not exist
+CREATE_DIRS=yes
+
+## Keep absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
+## You should leave this enabled if you intend to use 'backup task division' functionality of oBackup, or everything will end up in the same directory.
+KEEP_ABSOLUTE_PATHS=yes
+
+## Generate an alert if backup size is lower than given value in Kb (this can also help identifying empty mount dirs).
+BACKUP_SIZE_MINIMUM=1024
+
+## Check backup size before proceeding
+GET_BACKUP_SIZE=yes
+
+## Generate an alert if storage free space is lower than given value in Kb.
+## Keep in mind that disabling backup file size test will only test min space against SQL backup size.
+SQL_WARN_MIN_SPACE=1048576
+FILE_WARN_MIN_SPACE=1048576
+
+###### REMOTE ONLY OPTIONS
+
+## In case of pulled or pushed backups, remote system URI needs to be supplied.
+REMOTE_SYSTEM_URI="ssh://root@localhost:49999/"
+
+## You can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
+SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa_local"
+
+## Alternatively, you may specify an SSH password file (less secure). Needs sshpass utility installed.
+SSH_PASSWORD_FILE=""
+
+_REMOTE_TOKEN=SomeAlphaNumericToken9
+
+## ssh compression should be used unless your remote connection is good enough (LAN)
+SSH_COMPRESSION=yes
+
+## Ignore ssh known hosts verification. DANGER WILL ROBINSON DANGER: This can lead to security risks. Only enable if you know what you're doing.
+SSH_IGNORE_KNOWN_HOSTS=no
+
+## Remote rsync executable path. Leave this empty in most cases
+RSYNC_REMOTE_PATH=""
+
+## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
+REMOTE_HOST_PING=yes
+
+## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
+REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
+
+## If enabled, commands will be executed as superuser on remote side. See documentation for /etc/sudoers configuration ("find", "du", "tee" and "rsync" need to be allowed). Requiretty needs to be disabled.
+SUDO_EXEC=no
+
+###### DATABASE SPECIFIC OPTIONS
+
+## Database backup user
+SQL_USER=root
+
+## Enabling the following option will save all databases on local or remote given SQL instance except the ones specified in the exclude list.
+## Every found database will be backed up as separate backup task.
+DATABASES_ALL=yes
+DATABASES_ALL_EXCLUDE_LIST="test;information_schema;zarafa_prod"
+## Alternatively, if DATABASES_ALL=no, you can specify a list of databases to backup separated by spaces.
+DATABASES_LIST=mysql
+
+## Max backup execution time per Database task. Soft max exec time generates a warning only. Hard max exec time generates a warning and stops current backup task.
+## If a task gets stopped, next one in the task list gets executed. Time is specified in seconds.
+SOFT_MAX_EXEC_TIME_DB_TASK=3600
+HARD_MAX_EXEC_TIME_DB_TASK=7200
+
+## mysqldump options (ex: --extended-insert, --single-transaction, --quick...). See MySQL / MariaDB manual
+## default option: --opt
+MYSQLDUMP_OPTIONS="--opt --single-transaction"
+
+## Preferred SQL dump compression. Compression methods can be xz, lzma, pigz or gzip (will fallback from xz to gzip depending if available)
+## Generally, level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
+COMPRESSION_LEVEL=3
+
+###### FILES SPECIFIC OPTIONS
+
+## File backups are divided in tasks. Every directory in DIRECTORY_LIST will be processed as a unique task.
+## Every subdirectory of each directory in RECURSIVE_DIRECTORY_LIST will be processed as a unique task.
+## Example: RECURSIVE_DIRECTORY_LIST="/home;/var" will create backup tasks tasks "/home/dir1, "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/something".
+## You can exclude directories from the avove backup task creation, ex: avoid backing up "/home/dir2" by adding it to RECURSIVE_EXCLUDE_LIST.
+
+## Directories backup list. List of semicolon separated directories that will be backed up.
+DIRECTORY_LIST=/root/obackup-testdata/testData
+RECURSIVE_DIRECTORY_LIST=/root/obackup-testdata/testDataRecursive
+RECURSIVE_EXCLUDE_LIST="${HOME}/obackup-testdata/testDataRecursive/Excluded"
+
+## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns)
+RSYNC_PATTERN_FIRST=include
+
+## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work).
+## Paths are relative to sync dirs. List elements are separated by a semicolon.
+RSYNC_INCLUDE_PATTERN=""
+RSYNC_EXCLUDE_PATTERN="*.ded"
+#RSYNC_EXCLUDE_PATTERN="tmp;archives"
+
+## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
+## This file has to be in the same directory as the config file
+## Paths are relative to sync dirs. One element per line.
+RSYNC_INCLUDE_FROM=""
+RSYNC_EXCLUDE_FROM=""
+#RSYNC_EXCLUDE_FROM="exclude.list"
+
+## List separator char. You may set an alternative separator char for your directories lists above.
+PATH_SEPARATOR_CHAR=";"
+
+## Preserve basic linux permissions
+PRESERVE_PERMISSIONS=yes
+PRESERVE_OWNER=yes
+PRESERVE_GROUP=yes
+## On MACOS X, does not work and will be ignored 
+PRESERVE_EXECUTABILITY=yes
+
+## Preserve ACLS. Make sure source and target FS can hold same ACLs or you'll get loads of errors.
+PRESERVE_ACL=no
+## Preserve Xattr. MAke sure source and target FS can hold same Xattr or you'll get loads of errors.
+PRESERVE_XATTR=no
+
+## Transforms symlinks into referent files/dirs
+COPY_SYMLINKS=yes
+## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root.
+KEEP_DIRLINKS=yes
+## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
+PRESERVE_HARDLINKS=no
+
+
+## Let RSYNC compress file transfers. Do not use this on local-local backup schemes. Also, this is not useful if SSH compression is enabled.
+RSYNC_COMPRESS=no
+
+## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
+SOFT_MAX_EXEC_TIME_FILE_TASK=3600
+HARD_MAX_EXEC_TIME_FILE_TASK=7200
+
+## Keep partial uploads that can be resumed on next run, experimental feature
+PARTIAL=no
+
+## Delete files on destination that vanished from source. Do not turn this on unless you enabled backup rotation  or a snapshotting FS like zfs to keep those vanished files on the destination.
+DELETE_VANISHED_FILES=no
+
+## Use delta copy algortithm (usefull when local paths are network drives), defaults to yes
+DELTA_COPIES=yes
+
+## Bandwidth limit Kbytes / second for file backups. Leave 0 to disable limitation.
+BANDWIDTH=0
+
+## Paranoia option. Don't change this unless you read the documentation.
+RSYNC_EXECUTABLE=rsync
+
+###### ALERT OPTIONS
+
+## Alert email addresses separated by a space character
+DESTINATION_MAILS=""
+
+## Windows specific (msys / cygwin environment) only mail options (used with mailsend.exe from muquit, http://github.com/muquit/mailsend or from sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail/
+SENDER_MAIL="alert@your.system.tld"
+SMTP_SERVER=smtp.your.isp.tld
+SMTP_PORT=25
+# encryption can be tls, ssl or none
+SMTP_ENCRYPTION=none
+SMTP_USER=
+SMTP_PASSWORD=
+
+###### GENERAL BACKUP OPTIONS
+
+## Max execution time of whole backup process. Soft max exec time generates a warning only.
+## Hard max exec time generates a warning and stops the whole backup execution.
+SOFT_MAX_EXEC_TIME_TOTAL=30000
+HARD_MAX_EXEC_TIME_TOTAL=36000
+
+## Log a message every KEEP_LOGGING seconds just to know the task is still alive
+KEEP_LOGGING=1801
+
+## Backup Rotation. You may rotate backups if you don't use snapshots on your backup server.
+ROTATE_SQL_BACKUPS=yes
+ROTATE_SQL_COPIES=7
+ROTATE_FILE_BACKUPS=yes
+ROTATE_FILE_COPIES=7
+
+###### EXECUTION HOOKS
+
+## Commands can will be run before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set).
+## This is useful to make a snapshot before backing up data, or even handle snapshots of backed up data.
+LOCAL_RUN_BEFORE_CMD=""
+LOCAL_RUN_AFTER_CMD=""
+
+REMOTE_RUN_BEFORE_CMD=""
+REMOTE_RUN_AFTER_CMD=""
+
+## Max execution time of commands before they get force killed. Leave 0 if you don't want this to happen. Time is specified in seconds.
+MAX_EXEC_TIME_PER_CMD_BEFORE=0
+MAX_EXEC_TIME_PER_CMD_AFTER=0
+
+## Stops whole backup execution if one of the above commands fail
+STOP_ON_CMD_ERROR=no
+
+## Run local and remote after backup cmd's even on failure
+RUN_AFTER_CMD_ON_ERROR=yes

dev/tests/conf/push.conf

@@ -0,0 +1,233 @@
+#!/usr/bin/env bash
+
+###### obackup - Local or Remote, push or pull backup script for files & mysql databases
+###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr)
+###### obackup v2.1x config file rev 2016081701
+
+###### GENERAL BACKUP OPTIONS
+
+## Backup identification string.
+INSTANCE_ID="push-test"
+
+## Log file location. Leaving this empty will create log file at /var/log/obackup.INSTANCE_ID.log (or current directory if /var/log doesn't exist).
+LOGFILE=""
+
+## Elements to backup
+SQL_BACKUP=yes
+FILE_BACKUP=yes
+
+## Backups can be done local, pulled from another server or pushed to a backup server. Available options are [local,pull,push].
+## Pulled backups are the safest option, as the backup server contains the RSA key and cannot be compromised by another server.
+BACKUP_TYPE=push
+
+###### BACKUP STORAGE
+
+## Storage paths of the backups (absolute paths of the local or remote system)
+SQL_STORAGE="${HOME}/obackup-storage/sql-push"
+FILE_STORAGE="${HOME}/obackup-storage/files-push"
+
+## Encryption
+ENCRYPTION=no
+
+## Backup encryption needs a temporary storage space in order to encrypt files before sending them (absolute paths of the local or remote system)
+CRYPT_STORAGE="${HOME}/obackup-storage/crypt-push"
+
+## GPG recipient (pubkey for this recipient must exist, see gpg2 --list-keys or gpg --list-keys
+GPG_RECIPIENT="John Doe"
+
+## Create backup directories if they do not exist
+CREATE_DIRS=yes
+
+## Keep absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
+## You should leave this enabled if you intend to use 'backup task division' functionality of oBackup, or everything will end up in the same directory.
+KEEP_ABSOLUTE_PATHS=yes
+
+## Generate an alert if backup size is lower than given value in Kb (this can also help identifying empty mount dirs).
+BACKUP_SIZE_MINIMUM=1024
+
+## Check backup size before proceeding
+GET_BACKUP_SIZE=yes
+
+## Generate an alert if storage free space is lower than given value in Kb.
+## Keep in mind that disabling backup file size test will only test min space against SQL backup size.
+SQL_WARN_MIN_SPACE=1048576
+FILE_WARN_MIN_SPACE=1048576
+
+###### REMOTE ONLY OPTIONS
+
+## In case of pulled or pushed backups, remote system URI needs to be supplied.
+REMOTE_SYSTEM_URI="ssh://root@localhost:49999/"
+
+## You can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
+SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa_local"
+
+## Alternatively, you may specify an SSH password file (less secure). Needs sshpass utility installed.
+SSH_PASSWORD_FILE=""
+
+_REMOTE_TOKEN=SomeAlphaNumericToken9
+
+## ssh compression should be used unless your remote connection is good enough (LAN)
+SSH_COMPRESSION=yes
+
+## Ignore ssh known hosts verification. DANGER WILL ROBINSON DANGER: This can lead to security risks. Only enable if you know what you're doing.
+SSH_IGNORE_KNOWN_HOSTS=no
+
+## Remote rsync executable path. Leave this empty in most cases
+RSYNC_REMOTE_PATH=""
+
+## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
+REMOTE_HOST_PING=yes
+
+## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
+REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
+
+## If enabled, commands will be executed as superuser on remote side. See documentation for /etc/sudoers configuration ("find", "du", "tee" and "rsync" need to be allowed). Requiretty needs to be disabled.
+SUDO_EXEC=no
+
+###### DATABASE SPECIFIC OPTIONS
+
+## Database backup user
+SQL_USER=root
+
+## Enabling the following option will save all databases on local or remote given SQL instance except the ones specified in the exclude list.
+## Every found database will be backed up as separate backup task.
+DATABASES_ALL=yes
+DATABASES_ALL_EXCLUDE_LIST="test;information_schema;zarafa_prod"
+## Alternatively, if DATABASES_ALL=no, you can specify a list of databases to backup separated by spaces.
+DATABASES_LIST=mysql
+
+## Max backup execution time per Database task. Soft max exec time generates a warning only. Hard max exec time generates a warning and stops current backup task.
+## If a task gets stopped, next one in the task list gets executed. Time is specified in seconds.
+SOFT_MAX_EXEC_TIME_DB_TASK=3600
+HARD_MAX_EXEC_TIME_DB_TASK=7200
+
+## mysqldump options (ex: --extended-insert, --single-transaction, --quick...). See MySQL / MariaDB manual
+## default option: --opt
+MYSQLDUMP_OPTIONS="--opt --single-transaction"
+
+## Preferred SQL dump compression. Compression methods can be xz, lzma, pigz or gzip (will fallback from xz to gzip depending if available)
+## Generally, level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
+COMPRESSION_LEVEL=3
+
+###### FILES SPECIFIC OPTIONS
+
+## File backups are divided in tasks. Every directory in DIRECTORY_LIST will be processed as a unique task.
+## Every subdirectory of each directory in RECURSIVE_DIRECTORY_LIST will be processed as a unique task.
+## Example: RECURSIVE_DIRECTORY_LIST="/home;/var" will create backup tasks tasks "/home/dir1, "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/something".
+## You can exclude directories from the avove backup task creation, ex: avoid backing up "/home/dir2" by adding it to RECURSIVE_EXCLUDE_LIST.
+
+## Directories backup list. List of semicolon separated directories that will be backed up.
+DIRECTORY_LIST=/root/obackup-testdata/testData
+RECURSIVE_DIRECTORY_LIST=/root/obackup-testdata/testDataRecursive
+RECURSIVE_EXCLUDE_LIST="${HOME}/obackup-testdata/testDataRecursive/Excluded"
+
+## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns)
+RSYNC_PATTERN_FIRST=include
+
+## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work).
+## Paths are relative to sync dirs. List elements are separated by a semicolon.
+RSYNC_INCLUDE_PATTERN=""
+RSYNC_EXCLUDE_PATTERN="*.ded"
+#RSYNC_EXCLUDE_PATTERN="tmp;archives"
+
+## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
+## This file has to be in the same directory as the config file
+## Paths are relative to sync dirs. One element per line.
+RSYNC_INCLUDE_FROM=""
+RSYNC_EXCLUDE_FROM=""
+#RSYNC_EXCLUDE_FROM="exclude.list"
+
+## List separator char. You may set an alternative separator char for your directories lists above.
+PATH_SEPARATOR_CHAR=";"
+
+## Preserve basic linux permissions
+PRESERVE_PERMISSIONS=yes
+PRESERVE_OWNER=yes
+PRESERVE_GROUP=yes
+## On MACOS X, does not work and will be ignored 
+PRESERVE_EXECUTABILITY=yes
+
+## Preserve ACLS. Make sure source and target FS can hold same ACLs or you'll get loads of errors.
+PRESERVE_ACL=no
+## Preserve Xattr. MAke sure source and target FS can hold same Xattr or you'll get loads of errors.
+PRESERVE_XATTR=no
+
+## Transforms symlinks into referent files/dirs
+COPY_SYMLINKS=yes
+## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root.
+KEEP_DIRLINKS=yes
+## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
+PRESERVE_HARDLINKS=no
+
+
+## Let RSYNC compress file transfers. Do not use this on local-local backup schemes. Also, this is not useful if SSH compression is enabled.
+RSYNC_COMPRESS=no
+
+## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
+SOFT_MAX_EXEC_TIME_FILE_TASK=3600
+HARD_MAX_EXEC_TIME_FILE_TASK=7200
+
+## Keep partial uploads that can be resumed on next run, experimental feature
+PARTIAL=no
+
+## Delete files on destination that vanished from source. Do not turn this on unless you enabled backup rotation  or a snapshotting FS like zfs to keep those vanished files on the destination.
+DELETE_VANISHED_FILES=no
+
+## Use delta copy algortithm (usefull when local paths are network drives), defaults to yes
+DELTA_COPIES=yes
+
+## Bandwidth limit Kbytes / second for file backups. Leave 0 to disable limitation.
+BANDWIDTH=0
+
+## Paranoia option. Don't change this unless you read the documentation.
+RSYNC_EXECUTABLE=rsync
+
+###### ALERT OPTIONS
+
+## Alert email addresses separated by a space character
+DESTINATION_MAILS=""
+
+## Windows specific (msys / cygwin environment) only mail options (used with mailsend.exe from muquit, http://github.com/muquit/mailsend or from sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail/
+SENDER_MAIL="alert@your.system.tld"
+SMTP_SERVER=smtp.your.isp.tld
+SMTP_PORT=25
+# encryption can be tls, ssl or none
+SMTP_ENCRYPTION=none
+SMTP_USER=
+SMTP_PASSWORD=
+
+###### GENERAL BACKUP OPTIONS
+
+## Max execution time of whole backup process. Soft max exec time generates a warning only.
+## Hard max exec time generates a warning and stops the whole backup execution.
+SOFT_MAX_EXEC_TIME_TOTAL=30000
+HARD_MAX_EXEC_TIME_TOTAL=36000
+
+## Log a message every KEEP_LOGGING seconds just to know the task is still alive
+KEEP_LOGGING=1801
+
+## Backup Rotation. You may rotate backups if you don't use snapshots on your backup server.
+ROTATE_SQL_BACKUPS=yes
+ROTATE_SQL_COPIES=7
+ROTATE_FILE_BACKUPS=yes
+ROTATE_FILE_COPIES=7
+
+###### EXECUTION HOOKS
+
+## Commands can will be run before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set).
+## This is useful to make a snapshot before backing up data, or even handle snapshots of backed up data.
+LOCAL_RUN_BEFORE_CMD=""
+LOCAL_RUN_AFTER_CMD=""
+
+REMOTE_RUN_BEFORE_CMD=""
+REMOTE_RUN_AFTER_CMD=""
+
+## Max execution time of commands before they get force killed. Leave 0 if you don't want this to happen. Time is specified in seconds.
+MAX_EXEC_TIME_PER_CMD_BEFORE=0
+MAX_EXEC_TIME_PER_CMD_AFTER=0
+
+## Stops whole backup execution if one of the above commands fail
+STOP_ON_CMD_ERROR=no
+
+## Run local and remote after backup cmd's even on failure
+RUN_AFTER_CMD_ON_ERROR=yes

dev/tests/run_tests.sh

@@ -0,0 +1,926 @@
+#!/usr/bin/env bash
+
+#TODO Encrypted Pull runs on F25 fail for decryption
+
+## obackup basic tests suite 2017020903
+
+OBACKUP_DIR="$(pwd)"
+OBACKUP_DIR=${OBACKUP_DIR%%/dev*}
+DEV_DIR="$OBACKUP_DIR/dev"
+TESTS_DIR="$DEV_DIR/tests"
+
+CONF_DIR="$TESTS_DIR/conf"
+LOCAL_CONF="local.conf"
+PULL_CONF="pull.conf"
+PUSH_CONF="push.conf"
+
+OLD_CONF="old.conf"
+TMP_OLD_CONF="tmp.old.conf"
+MAX_EXEC_CONF="max-exec-time.conf"
+
+OBACKUP_EXECUTABLE="obackup.sh"
+OBACKUP_DEV_EXECUTABLE="dev/n_obackup.sh"
+OBACKUP_UPGRADE="upgrade-v1.x-2.1x.sh"
+TMP_FILE="$DEV_DIR/tmp"
+
+SOURCE_DIR="${HOME}/obackup-testdata"
+TARGET_DIR="${HOME}/obackup-storage"
+
+TARGET_DIR_SQL_LOCAL="$TARGET_DIR/sql-local"
+TARGET_DIR_FILE_LOCAL="$TARGET_DIR/files-local"
+TARGET_DIR_CRYPT_LOCAL="$TARGET_DIR/crypt-local"
+
+TARGET_DIR_SQL_PULL="$TARGET_DIR/sql-pull"
+TARGET_DIR_FILE_PULL="$TARGET_DIR/files-pull"
+TARGET_DIR_CRYPT_PULL="$TARGET_DIR/crypt-pull"
+
+TARGET_DIR_SQL_PUSH="$TARGET_DIR/sql-push"
+TARGET_DIR_FILE_PUSH="$TARGET_DIR/files-push"
+TARGET_DIR_CRYPT_PUSH="$TARGET_DIR/crypt-push"
+
+
+SIMPLE_DIR="testData"
+RECURSIVE_DIR="testDataRecursive"
+
+S_DIR_1="dir rect ory"
+R_EXCLUDED_DIR="Excluded"
+R_DIR_1="a"
+R_DIR_2="b"
+R_DIR_3="c d"
+
+S_FILE_1="some file"
+R_FILE_1="file_1"
+R_FILE_2="file 2"
+R_FILE_3="file 3"
+N_FILE_1="non recurse file"
+
+EXCLUDED_FILE="exclu.ded"
+
+DATABASE_1="mysql.sql.xz"
+DATABASE_2="performance_schema.sql.xz"
+DATABASE_EXCLUDED="information_schema.sql.xz"
+
+CRYPT_EXTENSION=".obackup.gpg"
+ROTATE_1_EXTENSION=".obackup.1"
+
+PASSFILE="passfile"
+CRYPT_TESTFILE="testfile"
+
+# Later populated variables
+OBACKUP_VERSION=2.x
+OBACKUP_MIN_VERSION=x
+OBACKUP_IS_STABLE=maybe
+
+function SetupSSH {
+        echo -e  'y\n'| ssh-keygen -t rsa -b 2048 -N "" -f "${HOME}/.ssh/id_rsa_local"
+        if ! grep "$(cat ${HOME}/.ssh/id_rsa_local.pub)" "${HOME}/.ssh/authorized_keys"; then
+		echo "from=\"*\",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command=\"/usr/local/bin/ssh_filter.sh SomeAlphaNumericToken9\" $(cat ${HOME}/.ssh/id_rsa_local.pub)" >> "${HOME}/.ssh/authorized_keys"
+        fi
+	chmod 600 "${HOME}/.ssh/authorized_keys"
+
+        # Add localhost to known hosts so self connect works
+        if [ -z "$(ssh-keygen -F localhost)" ]; then
+                ssh-keyscan -H localhost >> "${HOME}/.ssh/known_hosts"
+        fi
+
+        # Update remote conf files with SSH port
+        sed -i.tmp 's#ssh://.*@localhost:[0-9]*/#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/#' "$CONF_DIR/$PULL_CONF"
+        sed -i.tmp 's#ssh://.*@localhost:[0-9]*/#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/#' "$CONF_DIR/$PUSH_CONF"
+}
+
+function RemoveSSH {
+        local pubkey
+
+        if [ -f "${HOME}/.ssh/id_rsa_local" ]; then
+
+                pubkey=$(cat "${HOME}/.ssh/id_rsa_local.pub")
+		sed -i.bak "s|.*$pubkey.*||g" "${HOME}/.ssh/authorized_keys"
+                rm -f "${HOME}/.ssh/{id_rsa_local.pub,id_rsa_local}"
+        fi
+}
+
+function SetupGPG {
+	if type gpg2 > /dev/null; then
+		CRYPT_TOOL=gpg2
+	elif type gpg > /dev/null; then
+		CRYPT_TOOL=gpg
+	else
+		echo "No gpg support"
+		assertEquals "Failed to detect gpg" "1" $?
+		return
+	fi
+
+	echo "Crypt tool=$CRYPT_TOOL"
+
+	if ! $CRYPT_TOOL --list-keys | grep "John Doe" > /dev/null; then
+
+		cat >gpgcommand <<EOF
+%echo Generating a GPG Key
+Key-Type: RSA
+Key-Length: 4096
+Name-Real: John Doe
+Name-Comment: obackup-test-key
+Name-Email: john@example.com
+Expire-Date: 0
+Passphrase: PassPhrase123
+# Do a commit here, so that we can later print "done" :-)
+%commit
+%echo done
+EOF
+
+		if type apt-get > /dev/null 2>&1; then
+			sudo apt-get install rng-tools
+		fi
+
+		# Setup fast entropy
+		if type rngd > /dev/null 2>&1; then
+			$SUDO_CMD rngd -r /dev/urandom
+		else
+			echo "No rngd support"
+		fi
+
+		$CRYPT_TOOL --batch --gen-key gpgcommand
+		echo "Currently owned $CRYPT_TOOL keys"
+		echo $($CRYPT_TOOL --list-keys)
+		rm -f gpgcommand
+
+	fi
+
+	echo "PassPhrase123" > "$TESTS_DIR/$PASSFILE"
+}
+
+function oneTimeSetUp () {
+	START_TIME=$SECONDS
+
+	source "$DEV_DIR/ofunctions.sh"
+	GetLocalOS
+
+	echo "Detected OS: $LOCAL_OS"
+
+        # Set some travis related changes
+        if [ "$TRAVIS_RUN" == true ]; then
+        echo "Running with travis settings"
+                REMOTE_USER="travis"
+		RHOST_PING="no"
+                SetConfFileValue "$CONF_DIR/$PULL_CONF" "REMOTE_3RD_PARTY_HOSTS" ""
+                SetConfFileValue "$CONF_DIR/$PUSH_CONF" "REMOTE_3RD_PARTY_HOSTS" ""
+		# Config value didn't have S at the end in old files
+                SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_3RD_PARTY_HOST" ""
+
+                SetConfFileValue "$CONF_DIR/$PULL_CONF" "REMOTE_HOST_PING" "no"
+                SetConfFileValue "$CONF_DIR/$PUSH_CONF" "REMOTE_HOST_PING" "no"
+                SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_HOST_PING" "no"
+        else
+            	echo "Running with local settings"
+                REMOTE_USER="root"
+		RHOST_PING="yes"
+                SetConfFileValue "$CONF_DIR/$PULL_CONF" "REMOTE_3RD_PARTY_HOSTS" "\"www.kernel.org www.google.com\""
+                SetConfFileValue "$CONF_DIR/$PUSH_CONF" "REMOTE_3RD_PARTY_HOSTS" "\"www.kernel.org www.google.com\""
+		# Config value didn't have S at the end in old files
+                SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_3RD_PARTY_HOST" "\"www.kernel.org www.google.com\""
+
+                SetConfFileValue "$CONF_DIR/$PULL_CONF" "REMOTE_HOST_PING" "yes"
+                SetConfFileValue "$CONF_DIR/$PUSH_CONF" "REMOTE_HOST_PING" "yes"
+                SetConfFileValue "$CONF_DIR/$OLD_CONF" "REMOTE_HOST_PING" "yes"
+        fi
+
+        # Get default ssh port from env
+        if [ "$SSH_PORT" == "" ]; then
+                SSH_PORT=22
+        fi
+
+
+	#TODO: Assuming that macos has the same syntax than bsd here
+        if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then
+                SUDO_CMD=""
+        elif [ "$LOCAL_OS" == "BSD" ] || [ "$LOCAL_OS" == "MacOSX" ]; then
+                SUDO_CMD=""
+        else
+                SUDO_CMD="sudo"
+        fi
+
+
+	SetupGPG
+	if [ "$SKIP_REMOTE" != "yes" ]; then
+		SetupSSH
+	fi
+
+	# Get OBACKUP version
+        OBACKUP_VERSION=$(GetConfFileValue "$OBACKUP_DIR/$OBACKUP_DEV_EXECUTABLE" "PROGRAM_VERSION")
+        OBACKUP_VERSION="${OBACKUP_VERSION##*=}"
+        OBACKUP_MIN_VERSION="${OBACKUP_VERSION:2:1}"
+
+        OBACKUP_IS_STABLE=$(GetConfFileValue "$OBACKUP_DIR/$OBACKUP_DEV_EXECUTABLE" "IS_STABLE")
+
+        echo "Running with $OBACKUP_VERSION ($OBACKUP_MIN_VERSION) STABLE=$OBACKUP_IS_STABLE"
+
+	# Set basic values that could get changed later
+	for i in "$LOCAL_CONF" "$PULL_CONF" "$PUSH_CONF"; do
+		SetConfFileValue "$CONF_DIR/$i" "ENCRYPTION" "no"
+		SetConfFileValue "$CONF_DIR/$i" "DATABASES_ALL" "yes"
+		SetConfFileValue "$CONF_DIR/$i" "DATABASES_LIST" "mysql"
+		SetConfFileValue "$CONF_DIR/$i" "FILE_BACKUP" "yes"
+		SetConfFileValue "$CONF_DIR/$i" "DIRECTORY_LIST" "${HOME}/obackup-testdata/testData"
+		SetConfFileValue "$CONF_DIR/$i" "RECURSIVE_DIRECTORY_LIST" "${HOME}/obackup-testdata/testDataRecursive"
+		SetConfFileValue "$CONF_DIR/$i" "SQL_BACKUP" "yes"
+	done
+}
+
+function oneTimeTearDown () {
+	SetConfFileValue "$OBACKUP_DIR/$OBACKUP_EXECUTABLE" "IS_STABLE" "$OBACKUP_IS_STABLE"
+
+	RemoveSSH
+
+	#TODO: uncomment this when dev is done
+	#rm -rf "$SOURCE_DIR"
+	#rm -rf "$TARGET_DIR"
+	rm -f "$TMP_FILE"
+
+	cd "$OBACKUP_DIR"
+	$SUDO_CMD ./install.sh --remove --silent --no-stats
+	assertEquals "Uninstall failed" "0" $?
+
+        ELAPSED_TIME=$(($SECONDS - $START_TIME))
+        echo "It took $ELAPSED_TIME seconds to run these tests."
+}
+
+
+function setUp () {
+	rm -rf "$SOURCE_DIR"
+	rm -rf "$TARGET_DIR"
+
+	mkdir -p "$SOURCE_DIR/$SIMPLE_DIR/$S_DIR_1"
+	mkdir -p "$SOURCE_DIR/$RECURSIVE_DIR/$R_EXCLUDED_DIR"
+	mkdir -p "$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_1"
+	mkdir -p "$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_2"
+	mkdir -p "$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_3"
+
+	touch "$SOURCE_DIR/$SIMPLE_DIR/$S_DIR_1/$S_FILE_1"
+	touch "$SOURCE_DIR/$SIMPLE_DIR/$EXCLUDED_FILE"
+	touch "$SOURCE_DIR/$RECURSIVE_DIR/$N_FILE_1"
+	touch "$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_1/$R_FILE_1"
+	touch "$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_2/$R_FILE_2"
+	dd if=/dev/urandom of="$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_3/$R_FILE_3" bs=1048576 count=2
+	assertEquals "dd file creation" "0" $?
+	touch "$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_3/$EXCLUDED_FILE"
+
+	FilePresence=(
+	"$SOURCE_DIR/$SIMPLE_DIR/$S_DIR_1/$S_FILE_1"
+	"$SOURCE_DIR/$RECURSIVE_DIR/$N_FILE_1"
+	"$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_1/$R_FILE_1"
+	"$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_2/$R_FILE_2"
+	"$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_3/$R_FILE_3"
+	)
+
+	DatabasePresence=(
+	"$DATABASE_1"
+	"$DATABASE_2"
+	)
+
+	FileExcluded=(
+	"$SOURCE_DIR/$SIMPLE_DIR/$EXCLUDED_FILE"
+	"$SOURCE_DIR/$RECURSIVE_DIR/$R_DIR_3/$R_EXCLUDED_FILE"
+	)
+
+	DatabaseExcluded=(
+	"$DATABASE_EXCLUDED"
+	)
+
+	DirectoriesExcluded=(
+	"$RECURSIVE_DIR/$R_EXCLUDED_DIR"
+	)
+}
+
+function test_Merge () {
+	cd "$DEV_DIR"
+	./merge.sh
+	assertEquals "Merging code" "0" $?
+
+	cd "$OBACKUP_DIR"
+	$SUDO_CMD ./install.sh --silent --no-stats
+	assertEquals "Install failed" "0" $?
+
+	# Set obackup version to stable while testing to avoid warning message
+        SetConfFileValue "$OBACKUP_DIR/$OBACKUP_EXECUTABLE" "IS_STABLE" "yes"
+}
+
+# Keep this function to check GPG behavior depending on OS. (GPG 2.1 / GPG 2.0x / GPG 1.4 don't behave the same way)
+function test_GPG () {
+	echo "Encrypting file"
+	$CRYPT_TOOL --out "$TESTS_DIR/$CRYPT_TESTFILE$CRYPT_EXTENSION" --recipient "John Doe" --batch --yes --encrypt "$TESTS_DIR/$PASSFILE"
+	assertEquals "Encrypt file" "0" $?
+
+
+        # Detect if GnuPG >= 2.1 that does not allow automatic pin entry anymore
+        cryptToolVersion=$($CRYPT_TOOL --version | head -1 | awk '{print $3}')
+        cryptToolMajorVersion=${cryptToolVersion%%.*}
+        cryptToolSubVersion=${cryptToolVersion#*.}
+        cryptToolSubVersion=${cryptToolSubVersion%.*}
+
+        if [ $cryptToolMajorVersion -eq 2 ] && [ $cryptToolSubVersion -ge 1 ]; then
+                additionalParameters="--pinentry-mode loopback"
+        fi
+
+	if [ "$CRYPT_TOOL" == "gpg2" ]; then
+                options="--batch --yes"
+        elif [ "$CRYPT_TOOL" == "gpg" ]; then
+                options="--no-use-agent --batch"
+        fi
+
+
+	echo "Decrypt using passphrase file"
+	$CRYPT_TOOL $options --out "$TESTS_DIR/$CRYPT_TESTFILE" --batch --yes $additionalParameters --passphrase-file="$TESTS_DIR/$PASSFILE" --decrypt "$TESTS_DIR/$CRYPT_TESTFILE$CRYPT_EXTENSION"
+	assertEquals "Decrypt file using passfile" "0" $?
+
+	echo "Decrypt using passphrase"
+	$CRYPT_TOOL $options --out "$TESTS_DIR/$CRYPT_TESTFILE" --batch --yes $additionalParameters --passphrase PassPhrase123 --decrypt "$TESTS_DIR/$CRYPT_TESTFILE$CRYPT_EXTENSION"
+	assertEquals "Decrypt file using passphrase" "0" $?
+
+	echo "Decrypt using passphrase file with cat"
+	$CRYPT_TOOL $options --out "$TESTS_DIR/$CRYPT_TESTFILE" --batch --yes $additionalParameters --passphrase $(cat "$TESTS_DIR/$PASSFILE") --decrypt "$TESTS_DIR/$CRYPT_TESTFILE$CRYPT_EXTENSION"
+	assertEquals "Decrypt file using passphrase" "0" $?
+}
+
+function test_LocalRun () {
+	SetConfFileValue "$CONF_DIR/$LOCAL_CONF" "ENCRYPTION" "no"
+
+	# Basic return code tests. Need to go deep into file presence testing
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$LOCAL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${FilePresence[@]}"; do
+		[ -f "$TARGET_DIR_FILE_LOCAL/$file" ]
+		assertEquals "File Presence [$TARGET_DIR_FILE_LOCAL/$file]" "0" $?
+	done
+
+	for file in "${FileExcluded[@]}"; do
+		[ -f "$TARGET_DIR_FILE_LOCAL/$file" ]
+		assertEquals "File Excluded [$TARGET_DIR_FILE_LOCAL/$file]" "1" $?
+	done
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_LOCAL/$file" ]
+		assertEquals "Database Presence [$TARGET_DIR_SQL_LOCAL/$file]" "0" $?
+	done
+
+	for file in "${DatabaseExcluded[@]}"; do
+		[ -f "$TARGET_DIR_SQL_LOCAL/$file" ]
+		assertEquals "Database Excluded [$TARGET_DIR_SQL_LOCAL/$file]" "1" $?
+	done
+
+	for directory in "${DirectoriesExcluded[@]}"; do
+		[ -d "$TARGET_DIR_FILE_LOCAL/$directory" ]
+		assertEquals "Directory Excluded [$TARGET_DIR_FILE_LOCAL/$directory]" "1" $?
+	done
+
+	diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-local/$SOURCE_DIR" | grep -i Exclu
+	[ $(diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-local/$SOURCE_DIR" | grep -i Exclu | wc -l) -eq 2 ]
+	assertEquals "Diff should only output excluded files" "0" $?
+
+	# Tests presence of rotated files
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$LOCAL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_LOCAL/$file$ROTATE_1_EXTENSION" ]
+		assertEquals "Database rotated Presence [$TARGET_DIR_SQL_LOCAL/$file]" "0" $?
+	done
+
+	[ -d "$TARGET_DIR_FILE_LOCAL/$(dirname $SOURCE_DIR)$ROTATE_1_EXTENSION" ]
+	assertEquals "File rotated Presence [$TARGET_DIR_FILE_LOCAL/$(dirname $SOURCE_DIR)$ROTATE_1_EXTENSION]" "0" $?
+
+}
+
+function test_PullRun () {
+	if [ "$SKIP_REMOTE" == "yes" ]; then
+		return 0
+	fi
+
+	SetConfFileValue "$CONF_DIR/$PULL_CONF" "ENCRYPTION" "no"
+
+	# Basic return code tests. Need to go deep into file presence testing
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PULL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${FilePresence[@]}"; do
+		[ -f "$TARGET_DIR_FILE_PULL/$file" ]
+		assertEquals "File Presence [$TARGET_DIR_FILE_PULL/$file]" "0" $?
+	done
+
+	for file in "${FileExcluded[@]}"; do
+		[ -f "$TARGET_DIR_FILE_PULL/$file" ]
+		assertEquals "File Excluded [$TARGET_DIR_FILE_PULL/$file]" "1" $?
+	done
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PULL/$file" ]
+		assertEquals "Database Presence [$TARGET_DIR_SQL_PULL/$file]" "0" $?
+	done
+
+	for file in "${DatabaseExcluded[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PULL/$file" ]
+		assertEquals "Database Excluded [$TARGET_DIR_SQL_PULL/$file]" "1" $?
+	done
+
+	for directory in "${DirectoriesExcluded[@]}"; do
+		[ -d "$TARGET_DIR_FILE_PULL/$directory" ]
+		assertEquals "Directory Excluded [$TARGET_DIR_FILE_PULL/$directory]" "1" $?
+	done
+
+	diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-pull/$SOURCE_DIR" | grep -i Exclu
+	[ $(diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-pull/$SOURCE_DIR" | grep -i Exclu | wc -l) -eq 2 ]
+	assertEquals "Diff should only output excluded files" "0" $?
+
+	# Tests presence of rotated files
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PULL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PULL/$file$ROTATE_1_EXTENSION" ]
+		assertEquals "Database rotated Presence [$TARGET_DIR_SQL_PULL/$file$ROTATE_1_EXTENSION]" "0" $?
+	done
+
+	[ -d "$TARGET_DIR_FILE_PULL/$(dirname $SOURCE_DIR)$ROTATE_1_EXTENSION" ]
+	assertEquals "File rotated Presence [$TARGET_DIR_FILE_PULL/$(dirname $SOURCE_DIR)$ROTATE_1_EXTENSION]" "0" $?
+
+}
+
+function test_PushRun () {
+	if [ "$SKIP_REMOTE" == "yes" ]; then
+		return 0
+	fi
+
+	SetConfFileValue "$CONF_DIR/$PUSH_CONF" "ENCRYPTION" "no"
+
+	# Basic return code tests. Need to go deep into file presence testing
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PUSH_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${FilePresence[@]}"; do
+		[ -f "$TARGET_DIR_FILE_PUSH/$file" ]
+		assertEquals "File Presence [$TARGET_DIR_FILE_PUSH/$file]" "0" $?
+	done
+
+	for file in "${FileExcluded[@]}"; do
+		[ -f "$TARGET_DIR_FILE_PUSH/$file" ]
+		assertEquals "File Excluded [$TARGET_DIR_FILE_PUSH/$file]" "1" $?
+	done
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PUSH/$file" ]
+		assertEquals "Database Presence [$TARGET_DIR_SQL_PUSH/$file]" "0" $?
+	done
+
+	for file in "${DatabaseExcluded[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PUSH/$file" ]
+		assertEquals "Database Excluded [$TARGET_DIR_SQL_PUSH/$file]" "1" $?
+	done
+
+	for directory in "${DirectoriesExcluded[@]}"; do
+		[ -d "$TARGET_DIR_FILE_PUSH/$directory" ]
+		assertEquals "Directory Excluded [$TARGET_DIR_FILE_PUSH/$directory]" "1" $?
+	done
+
+	diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-push/$SOURCE_DIR" | grep -i Exclu
+	[ $(diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-push/$SOURCE_DIR" | grep -i Exclu | wc -l) -eq 2 ]
+	assertEquals "Diff should only output excluded files" "0" $?
+
+	# Tests presence of rotated files
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PUSH_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PUSH/$file$ROTATE_1_EXTENSION" ]
+		assertEquals "Database rotated Presence [$TARGET_DIR_SQL_PUSH/$file$ROTATE_1_EXTENSION]" "0" $?
+	done
+
+	[ -d "$TARGET_DIR_FILE_PUSH/$(dirname $SOURCE_DIR)$ROTATE_1_EXTENSION" ]
+	assertEquals "File rotated Presence [$TARGET_DIR_FILE_PUSH/$(dirname $SOURCE_DIR)$ROTATE_1_EXTENSION]" "0" $?
+
+}
+
+function test_EncryptLocalRun () {
+	SetConfFileValue "$CONF_DIR/$LOCAL_CONF" "ENCRYPTION" "yes"
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$LOCAL_CONF"
+
+	for file in "${FilePresence[@]}"; do
+		[ -f "$TARGET_DIR_CRYPT_LOCAL/$file$CRYPT_EXTENSION" ]
+		assertEquals "File Presence [$TARGET_DIR_CRYPT_LOCAL/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+# TODO: Exclusion lists don't work with encrypted files yet
+#	for file in "${FileExcluded[@]}"; do
+#		[ -f "$TARGET_DIR_CRYPT_LOCAL/$file$CRYPT_EXTENSION" ]
+#		assertEquals "File Excluded [$TARGET_DIR_CRYPT_LOCAL/$file$CRYPT_EXTENSION]" "1" $?
+#	done
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_LOCAL/$file$CRYPT_EXTENSION" ]
+		assertEquals "Database Presence [$TARGET_DIR_SQL_LOCAL/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+#	for file in "${DatabaseExcluded[@]}"; do
+#		[ -f "$TARGET_DIR_SQL_LOCAL/$file$CRYPT_EXTENSION" ]
+#		assertEquals "Database Excluded [$TARGET_DIR_SQL_LOCAL/$file$CRYPT_EXTENSION]" "1" $?
+#	done
+
+#	for directory in "${DirectoriesExcluded[@]}"; do
+#		[ -d "$TARGET_DIR_CRYPT_LOCAL/$directory" ]
+#		assertEquals "Directory Excluded [$TARGET_DIR_CRYPT_LOCAL/$directory]" "1" $?
+#	done
+
+	diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-local/$SOURCE_DIR" | grep -i Exclu
+	[ $(diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-local/$SOURCE_DIR" | grep -i Exclu | wc -l) -eq 5 ]
+	assertEquals "Diff should only output excluded files" "0" $?
+
+	# Tests presence of rotated files
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$LOCAL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_LOCAL/$file$CRYPT_EXTENSION$ROTATE_1_EXTENSION" ]
+		assertEquals "Database rotated Presence [$TARGET_DIR_SQL_LOCAL/$file$CRYPT_EXTENSION$ROTATE_1_EXTENSION]" "0" $?
+	done
+
+	[ -d "$TARGET_DIR_CRYPT_LOCAL/$(dirname $SOURCE_DIR)$CRYPT_EXTENSION$ROTATE_1_EXTENSION" ]
+	assertEquals "File rotated Presence [$TARGET_DIR_CRYPT_LOCAL/$(dirname $SOURCE_DIR)$CRYPT_EXTENSION$ROTATE_1_EXTENSION]" "0" $?
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE --decrypt="$TARGET_DIR_SQL_LOCAL" --passphrase-file="$TESTS_DIR/$PASSFILE"
+	assertEquals "Decrypt sql storage in [$TARGET_DIR_SQL_LOCAL]" "0" $?
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE --decrypt="$TARGET_DIR_CRYPT_LOCAL" --passphrase-file="$TESTS_DIR/$PASSFILE"
+	assertEquals "Decrypt file storage in [$TARGET_DIR_CRYPT_LOCAL]" "0" $?
+
+	SetConfFileValue "$CONF_DIR/$LOCAL_CONF" "ENCRYPTION" "no"
+}
+
+function test_EncryptPullRun () {
+	if [ "$SKIP_REMOTE" == "yes" ]; then
+		return 0
+	fi
+
+	# Basic return code tests. Need to go deep into file presence testing
+	SetConfFileValue "$CONF_DIR/$PULL_CONF" "ENCRYPTION" "yes"
+
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PULL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${FilePresence[@]}"; do
+		[ -f "$TARGET_DIR_CRYPT_PULL/$file$CRYPT_EXTENSION" ]
+		assertEquals "File Presence [$TARGET_DIR_CRYPT_PULL/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+#	for file in "${FileExcluded[@]}"; do
+#		[ -f "$TARGET_DIR_CRYPT_PULL/$file$CRYPT_EXTENSION" ]
+#		assertEquals "File Excluded [$TARGET_DIR_CRYPT_PULL/$file$CRYPT_EXTENSION]" "1" $?
+#	done
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PULL/$file$CRYPT_EXTENSION" ]
+		assertEquals "Database Presence [$TARGET_DIR_SQL_PULL/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+#	for file in "${DatabaseExcluded[@]}"; do
+#		[ -f "$TARGET_DIR_SQL_PULL/$file$CRYPT_EXTENSION" ]
+#		assertEquals "Database Excluded [$TARGET_DIR_SQL_PULL/$file$CRYPT_EXTENSION]" "1" $?
+#	done
+
+#	for directory in "${DirectoriesExcluded[@]}"; do
+#		[ -d "$TARGET_DIR_FILE_PULL/$directory" ]
+#		assertEquals "Directory Excluded [$TARGET_DIR_FILE_PULL/$directory]" "1" $?
+#	done
+
+	# Only excluded files should be listed here
+	diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-pull/$SOURCE_DIR" | grep -i Exclu
+	[ $(diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-pull/$SOURCE_DIR" | grep -i Exclu | wc -l) -eq 2 ]
+	assertEquals "Diff should only output excluded files" "0" $?
+
+	# Tests presence of rotated files
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PULL_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PULL/$file$CRYPT_EXTENSION$ROTATE_1_EXTENSION" ]
+		assertEquals "Database rotated Presence [$TARGET_DIR_SQL_PULL/$file$CRYPT_EXTENSION$ROTATE_1_EXTENSION]" "0" $?
+	done
+
+	[ -d "$TARGET_DIR_FILE_CRYPT/$(dirname $SOURCE_DIR)$CRYPT_EXTENSION$ROTATE_1_EXTENSION" ]
+	assertEquals "File rotated Presence [$TARGET_DIR_FILE_CRYPT/$(dirname $SOURCE_DIR)$CRYPT_EXTENSION$ROTATE_1_EXTENSION]" "0" $?
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE --decrypt="$TARGET_DIR_SQL_PULL" --passphrase-file="$TESTS_DIR/$PASSFILE"
+	assertEquals "Decrypt sql storage in [$TARGET_DIR_SQL_PULL]" "0" $?
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE --decrypt="$TARGET_DIR_CRYPT_PULL" --passphrase-file="$TESTS_DIR/$PASSFILE"
+	assertEquals "Decrypt file storage in [$TARGET_DIR_CRYPT_PULL]" "0" $?
+
+	SetConfFileValue "$CONF_DIR/$PULL_CONF" "ENCRYPTION" "no"
+}
+
+function test_EncryptPushRun () {
+	if [ "$SKIP_REMOTE" == "yes" ]; then
+		return 0
+	fi
+
+	# Basic return code tests. Need to go deep into file presence testing
+	SetConfFileValue "$CONF_DIR/$PUSH_CONF" "ENCRYPTION" "yes"
+
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PUSH_CONF"
+	assertEquals "Return code" "0" $?
+
+	# Same here, why do we check for crypt extension in file_push instead of file_crypt
+	for file in "${FilePresence[@]}"; do
+		[ -f "$TARGET_DIR_FILE_PUSH/$file$CRYPT_EXTENSION" ]
+		assertEquals "File Presence [$TARGET_DIR_FILE_PUSH/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+#	for file in "${FileExcluded[@]}"; do
+#		[ -f "$TARGET_DIR_FILE_PUSH/$file$CRYPT_EXTENSION" ]
+#		assertEquals "File Excluded [$TARGET_DIR_FILE_PUSH/$file$CRYPT_EXTENSION]" "1" $?
+#	done
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PUSH/$file$CRYPT_EXTENSION" ]
+		assertEquals "Database Presence [$TARGET_DIR_SQL_PUSH/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+#	for file in "${DatabaseExcluded[@]}"; do
+#		[ -f "$TARGET_DIR_SQL_PUSH/$file$CRYPT_EXTENSION" ]
+#		assertEquals "Database Excluded [$TARGET_DIR_SQL_PUSH/$file$CRYPT_EXTENSION]" "1" $?
+#	done
+
+#	for directory in "${DirectoriesExcluded[@]}"; do
+#		[ -d "$TARGET_DIR_FILE_PUSH/$directory" ]
+#		assertEquals "Directory Excluded [$TARGET_DIR_FILE_PUSH/$directory]" "1" $?
+#	done
+
+	diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-push/$SOURCE_DIR" | grep -i Exclu
+	[ $(diff -qr "$SOURCE_DIR" "$TARGET_DIR/files-push/$SOURCE_DIR" | grep -i Exclu | wc -l) -eq 5 ]
+	assertEquals "Diff should only output excluded files" "0" $?
+	# Tests presence of rotated files
+
+	cd "$OBACKUP_DIR"
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$PUSH_CONF"
+	assertEquals "Return code" "0" $?
+
+	for file in "${DatabasePresence[@]}"; do
+		[ -f "$TARGET_DIR_SQL_PUSH/$file$CRYPT_EXTENSION$ROTATE_1_EXTENSION" ]
+		assertEquals "Database rotated Presence [$TARGET_DIR_SQL_PUSH/$file$CRYPT_EXTENSION]" "0" $?
+	done
+
+	[ -d "$TARGET_DIR_FILE_PUSH/$(dirname $SOURCE_DIR)$CRYPT_EXTENSION$ROTATE_1_EXTENSION" ]
+	assertEquals "File rotated Presence [$TARGET_DIR_FILE_PUSH/$(dirname $SOURCE_DIR)$CRYPT_EXTENSION$ROTATE_1_EXTENSION]" "0" $?
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE --decrypt="$TARGET_DIR_SQL_PUSH" --passphrase-file="$TESTS_DIR/$PASSFILE" --verbose
+	assertEquals "Decrypt sql storage in [$TARGET_DIR_SQL_PUSH]" "0" $?
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE --decrypt="$TARGET_DIR_FILE_PUSH" --passphrase-file="$TESTS_DIR/$PASSFILE"
+	assertEquals "Decrypt file storage in [$TARGET_DIR_FILE_PUSH]" "0" $?
+
+	SetConfFileValue "$CONF_DIR/$PUSH_CONF" "ENCRYPTION" "no"
+}
+
+function test_missing_databases () {
+	cd "$OBACKUP_DIR"
+
+	# Prepare files for missing databases
+	for i in "$LOCAL_CONF" "$PUSH_CONF" "$PULL_CONF"; do
+		SetConfFileValue "$CONF_DIR/$i" "DATABASES_ALL" "no"
+		SetConfFileValue "$CONF_DIR/$i" "DATABASES_LIST" "\"zorglub;mysql\""
+		SetConfFileValue "$CONF_DIR/$i" "SQL_BACKUP" "yes"
+		SetConfFileValue "$CONF_DIR/$i" "FILE_BACKUP" "no"
+
+		REMOTE_HOST=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$i"
+		assertEquals "Missing databases should trigger error with [$i]" "1" $?
+
+		SetConfFileValue "$CONF_DIR/$i" "DATABASES_ALL" "yes"
+		SetConfFileValue "$CONF_DIR/$i" "DATABASES_LIST" "mysql"
+		SetConfFileValue "$CONF_DIR/$i" "FILE_BACKUP" "yes"
+
+	done
+
+	for i in "$LOCAL_CONF" "$PUSH_CONF" "$PULL_CONF"; do
+		SetConfFileValue "$CONF_DIR/$i" "DIRECTORY_LIST" "${HOME}/obackup-testdata/nonPresentData"
+		SetConfFileValue "$CONF_DIR/$i" "RECURSIVE_DIRECTORY_LIST" "${HOME}/obackup-testdata/nonPresentDataRecursive"
+		SetConfFileValue "$CONF_DIR/$i" "SQL_BACKUP" "no"
+		SetConfFileValue "$CONF_DIR/$i" "FILE_BACKUP" "yes"
+
+		REMOTE_HOST=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$i"
+		assertEquals "Missing files should trigger error with [$i]" "1" $?
+		echo "glob"
+		return
+		echo "nope"
+		SetConfFileValue "$CONF_DIR/$i" "DIRECTORY_LIST" "${HOME}/obackup-testdata/testData"
+		SetConfFileValue "$CONF_DIR/$i" "RECURSIVE_DIRECTORY_LIST" "${HOME}/obackup-testdata/testDataRecursive"
+		SetConfFileValue "$CONF_DIR/$i" "SQL_BACKUP" "yes"
+	done
+}
+
+function test_timed_execution () {
+	cd "$OBACKUP_DIR"
+
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_DB_TASK" 1
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_TOTAL" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_TOTAL" 1000
+
+	SLEEP_TIME=2 REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$MAX_EXEC_CONF"
+	assertEquals "Soft max exec time db reached in obackup Return code" "2" $?
+
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_DB_TASK" 1
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_TOTAL" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_TOTAL" 1000
+
+	SLEEP_TIME=2 REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$MAX_EXEC_CONF"
+	assertEquals "Hard max exec time db reached in obackup Return code" "1" $?
+
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_FILE_TASK" 1
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_TOTAL" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_TOTAL" 1000
+
+	SLEEP_TIME=2 REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$MAX_EXEC_CONF"
+	assertEquals "Soft max exec time file reached in obackup Return code" "2" $?
+
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_FILE_TASK" 1
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_TOTAL" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_TOTAL" 1000
+
+	SLEEP_TIME=2 REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$MAX_EXEC_CONF"
+	assertEquals "Hard max exec time file reached in obackup Return code" "1" $?
+
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_TOTAL" 1
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_TOTAL" 1000
+
+	SLEEP_TIME=1.5 REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$MAX_EXEC_CONF"
+	assertEquals "Soft max exec time total reached in obackup Return code" "2" $?
+
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_DB_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_FILE_TASK" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "SOFT_MAX_EXEC_TIME_TOTAL" 1000
+	SetConfFileValue "$CONF_DIR/$MAX_EXEC_CONF" "HARD_MAX_EXEC_TIME_TOTAL" 1
+
+	SLEEP_TIME=2 REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$MAX_EXEC_CONF"
+	assertEquals "Hard max exec time total reached in obackup Return code" "1" $?
+}
+
+function test_WaitForTaskCompletion () {
+	local pids
+	# Standard wait
+	sleep 1 &
+	pids="$!"
+	sleep 2 &
+	pids="$pids;$!"
+	WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
+	assertEquals "WaitForTaskCompletion test 1" "0" $?
+
+	# Standard wait with warning
+	sleep 2 &
+	pids="$!"
+	sleep 5 &
+	pids="$pids;$!"
+
+	WaitForTaskCompletion $pids 3 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
+	assertEquals "WaitForTaskCompletion test 2" "0" $?
+
+	# Both pids are killed
+	sleep 5 &
+	pids="$!"
+	sleep 5 &
+	pids="$pids;$!"
+
+	WaitForTaskCompletion $pids 0 2 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
+	assertEquals "WaitForTaskCompletion test 3" "2" $?
+
+	# One of two pids are killed
+	sleep 2 &
+	pids="$!"
+	sleep 10 &
+	pids="$pids;$!"
+
+	WaitForTaskCompletion $pids 0 3 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
+	assertEquals "WaitForTaskCompletion test 4" "1" $?
+
+	# Count since script begin, the following should output two warnings and both pids should get killed
+	sleep 20 &
+	pids="$!"
+	sleep 20 &
+	pids="$pids;$!"
+
+	WaitForTaskCompletion $pids 3 5 $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
+	assertEquals "WaitForTaskCompletion test 5" "2" $?
+}
+
+function test_ParallelExec () {
+	local cmd
+
+	# Test if parallelExec works correctly in array mode
+
+	cmd="sleep 2;sleep 2;sleep 2;sleep 2"
+	ParallelExec 4 "$cmd"
+	assertEquals "ParallelExec test 1" "0" $?
+
+	cmd="sleep 2;du /none;sleep 2"
+	ParallelExec 2 "$cmd"
+	assertEquals "ParallelExec test 2" "1" $?
+
+	cmd="sleep 4;du /none;sleep 3;du /none;sleep 2"
+	ParallelExec 3 "$cmd"
+	assertEquals "ParallelExec test 3" "2" $?
+
+	# Test if parallelExec works correctly in file mode
+
+	echo "sleep 2" > "$TMP_FILE"
+	echo "sleep 2" >> "$TMP_FILE"
+	echo "sleep 2" >> "$TMP_FILE"
+	echo "sleep 2" >> "$TMP_FILE"
+	ParallelExec 4 "$TMP_FILE" true
+	assertEquals "ParallelExec test 4" "0" $?
+
+	echo "sleep 2" > "$TMP_FILE"
+	echo "du /nome" >> "$TMP_FILE"
+	echo "sleep 2" >> "$TMP_FILE"
+	ParallelExec 2 "$TMP_FILE" true
+	assertEquals "ParallelExec test 5" "1" $?
+
+	echo "sleep 4" > "$TMP_FILE"
+	echo "du /none" >> "$TMP_FILE"
+	echo "sleep 3" >> "$TMP_FILE"
+	echo "du /none" >> "$TMP_FILE"
+	echo "sleep 2" >> "$TMP_FILE"
+	ParallelExec 3 "$TMP_FILE" true
+	assertEquals "ParallelExec test 6" "2" $?
+
+	#function ParallelExec $numberOfProcesses $commandsArg $readFromFile $softTime $HardTime $sleepTime $keepLogging $counting $Spinner $noError $callerName
+	# Test if parallelExec works correctly in array mode with full  time control
+
+	cmd="sleep 5;sleep 5;sleep 5;sleep 5;sleep 5"
+	ParallelExec 4 "$cmd" false 1 0 .05 3600 true true false ${FUNCNAME[0]}
+	assertEquals "ParallelExec full test 1" "0" $?
+
+	cmd="sleep 2;du /none;sleep 2;sleep 2;sleep 4"
+	ParallelExec 2 "$cmd" false 0 0 .1 2 true false false ${FUNCNAME[0]}
+	assertEquals "ParallelExec full test 2" "1" $?
+
+	cmd="sleep 4;du /none;sleep 3;du /none;sleep 2"
+	ParallelExec 3 "$cmd" false 1 2 .05 7000 true true false ${FUNCNAME[0]}
+	assertNotEquals "ParallelExec full test 3" "0" $?
+
+}
+
+function test_UpgradeConfPullRun () {
+
+	# Basic return code tests. Need to go deep into file presence testing
+	cd "$OBACKUP_DIR"
+
+
+	# Make a security copy of the old config file
+	cp "$CONF_DIR/$OLD_CONF" "$CONF_DIR/$TMP_OLD_CONF"
+
+	./$OBACKUP_UPGRADE "$CONF_DIR/$TMP_OLD_CONF"
+	assertEquals "Conf file upgrade" "0" $?
+
+        # Update remote conf files with SSH port
+        sed -i.tmp 's#ssh://.*@localhost:[0-9]*/#ssh://'$REMOTE_USER'@localhost:'$SSH_PORT'/#' "$CONF_DIR/$TMP_OLD_CONF"
+
+	REMOTE_HOST_PING=$RHOST_PING ./$OBACKUP_EXECUTABLE "$CONF_DIR/$TMP_OLD_CONF"
+	assertEquals "Upgraded conf file execution test" "0" $?
+
+	rm -f "$CONF_DIR/$TMP_OLD_CONF"
+	rm -f "$CONF_DIR/$TMP_OLD_CONF.save"
+}
+
+. "$TESTS_DIR/shunit2/shunit2"
+

dev/tests/shunit2/shunit2_test.sh

@@ -0,0 +1,124 @@
+#! /bin/sh
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2008 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit test suite runner.
+#
+# This script runs all the unit tests that can be found, and generates a nice
+# report of the tests.
+
+MY_NAME=`basename $0`
+MY_PATH=`dirname $0`
+
+PREFIX='shunit2_test_'
+SHELLS='/bin/sh /bin/bash /bin/dash /bin/ksh /bin/pdksh /bin/zsh'
+TESTS=''
+for test in ${PREFIX}[a-z]*.sh; do
+  TESTS="${TESTS} ${test}"
+done
+
+# load common unit test functions
+. ../lib/versions
+. ./shunit2_test_helpers
+
+usage()
+{
+  echo "usage: ${MY_NAME} [-e key=val ...] [-s shell(s)] [-t test(s)]"
+}
+
+env=''
+
+# process command line flags
+while getopts 'e:hs:t:' opt; do
+  case ${opt} in
+    e)  # set an environment variable
+      key=`expr "${OPTARG}" : '\([^=]*\)='`
+      val=`expr "${OPTARG}" : '[^=]*=\(.*\)'`
+      if [ -z "${key}" -o -z "${val}" ]; then
+        usage
+        exit 1
+      fi
+      eval "${key}='${val}'"
+      export ${key}
+      env="${env:+${env} }${key}"
+      ;;
+    h) usage; exit 0 ;;  # output help
+    s) shells=${OPTARG} ;;  # list of shells to run
+    t) tests=${OPTARG} ;;  # list of tests to run
+    *) usage; exit 1 ;;
+  esac
+done
+shift `expr ${OPTIND} - 1`
+
+# fill shells and/or tests
+shells=${shells:-${SHELLS}}
+tests=${tests:-${TESTS}}
+
+# error checking
+if [ -z "${tests}" ]; then
+  th_error 'no tests found to run; exiting'
+  exit 1
+fi
+
+cat <<EOF
+#------------------------------------------------------------------------------
+# System data
+#
+
+# test run info
+shells: ${shells}
+tests: ${tests}
+EOF
+for key in ${env}; do
+  eval "echo \"${key}=\$${key}\""
+done
+echo
+
+# output system data
+echo "# system info"
+echo "$ date"
+date
+echo
+
+echo "$ uname -mprsv"
+uname -mprsv
+
+#
+# run tests
+#
+
+for shell in ${shells}; do
+  echo
+
+  # check for existance of shell
+  if [ ! -x ${shell} ]; then
+    th_warn "unable to run tests with the ${shell} shell"
+    continue
+  fi
+
+  cat <<EOF
+
+#------------------------------------------------------------------------------
+# Running the test suite with ${shell}
+#
+EOF
+
+  SHUNIT_SHELL=${shell}  # pass shell onto tests
+  shell_name=`basename ${shell}`
+  shell_version=`versions_shellVersion "${shell}"`
+
+  echo "shell name: ${shell_name}"
+  echo "shell version: ${shell_version}"
+
+  # execute the tests
+  for suite in ${tests}; do
+    suiteName=`expr "${suite}" : "${PREFIX}\(.*\).sh"`
+    echo
+    echo "--- Executing the '${suiteName}' test suite ---"
+    ( exec ${shell} ./${suite} 2>&1; )
+  done
+done

dev/tests/shunit2/shunit2_test_asserts.sh

@@ -0,0 +1,206 @@
+#! /bin/sh
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2008 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+#
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit test for assert functions
+
+# load test helpers
+. ./shunit2_test_helpers
+
+#------------------------------------------------------------------------------
+# suite tests
+#
+
+commonEqualsSame()
+{
+  fn=$1
+
+  ( ${fn} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'equal' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} "${MSG}" 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'equal; with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} 'abc def' 'abc def' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'equal with spaces' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'not equal' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} '' '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'null values' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} arg1 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+commonNotEqualsSame()
+{
+  fn=$1
+
+  ( ${fn} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not same' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} "${MSG}" 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not same, with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'same' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} '' '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'null values' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} arg1 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( ${fn} arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+testAssertEquals()
+{
+  commonEqualsSame 'assertEquals'
+}
+
+testAssertNotEquals()
+{
+  commonNotEqualsSame 'assertNotEquals'
+}
+
+testAssertSame()
+{
+  commonEqualsSame 'assertSame'
+}
+
+testAssertNotSame()
+{
+  commonNotEqualsSame 'assertNotSame'
+}
+
+testAssertNull()
+{
+  ( assertNull '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'null' $? "${stdoutF}" "${stderrF}"
+
+  ( assertNull "${MSG}" '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'null, with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( assertNull 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'not null' $? "${stdoutF}" "${stderrF}"
+
+  ( assertNull >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( assertNull arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+testAssertNotNull()
+{
+  ( assertNotNull 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not null' $? "${stdoutF}" "${stderrF}"
+
+  ( assertNotNull "${MSG}" 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not null, with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( assertNotNull 'x"b' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not null, with double-quote' $? \
+      "${stdoutF}" "${stderrF}"
+
+  ( assertNotNull "x'b" >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not null, with single-quote' $? \
+      "${stdoutF}" "${stderrF}"
+
+  ( assertNotNull 'x$b' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not null, with dollar' $? \
+      "${stdoutF}" "${stderrF}"
+
+  ( assertNotNull 'x`b' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'not null, with backtick' $? \
+      "${stdoutF}" "${stderrF}"
+
+  ( assertNotNull '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'null' $? "${stdoutF}" "${stderrF}"
+
+  # there is no test for too few arguments as $1 might actually be null
+
+  ( assertNotNull arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+testAssertTrue()
+{
+  ( assertTrue 0 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'true' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue "${MSG}" 0 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'true, with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue '[ 0 -eq 0 ]' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'true condition' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue 1 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'false' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue '[ 0 -eq 1 ]' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'false condition' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'null' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( assertTrue arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+testAssertFalse()
+{
+  ( assertFalse 1 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'false' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse "${MSG}" 1 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'false, with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse '[ 0 -eq 1 ]' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertTrueWithNoOutput 'false condition' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse 0 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'true' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse '[ 0 -eq 0 ]' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'true condition' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'true condition' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( assertFalse arg1 arg2 arg3 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+#------------------------------------------------------------------------------
+# suite functions
+#
+
+oneTimeSetUp()
+{
+  th_oneTimeSetUp
+
+  MSG='This is a test message'
+}
+
+# load and run shUnit2
+[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0
+. ${TH_SHUNIT}

dev/tests/shunit2/shunit2_test_failures.sh

@@ -0,0 +1,86 @@
+#! /bin/sh
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2008 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+#
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit test for failure functions
+
+# load common unit-test functions
+. ./shunit2_test_helpers
+
+#-----------------------------------------------------------------------------
+# suite tests
+#
+
+testFail()
+{
+  ( fail >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'fail' $? "${stdoutF}" "${stderrF}"
+
+  ( fail "${MSG}" >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'fail with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( fail arg1 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+testFailNotEquals()
+{
+  ( failNotEquals 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'same' $? "${stdoutF}" "${stderrF}"
+
+  ( failNotEquals "${MSG}" 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'same with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( failNotEquals 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'not same' $? "${stdoutF}" "${stderrF}"
+
+  ( failNotEquals '' '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'null values' $? "${stdoutF}" "${stderrF}"
+
+  ( failNotEquals >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( failNotEquals arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+testFailSame()
+{
+  ( failSame 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'same' $? "${stdoutF}" "${stderrF}"
+
+  ( failSame "${MSG}" 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'same with msg' $? "${stdoutF}" "${stderrF}"
+
+  ( failSame 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'not same' $? "${stdoutF}" "${stderrF}"
+
+  ( failSame '' '' >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithOutput 'null values' $? "${stdoutF}" "${stderrF}"
+
+  ( failSame >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too few arguments' $? "${stdoutF}" "${stderrF}"
+
+  ( failSame arg1 arg2 arg3 arg4 >"${stdoutF}" 2>"${stderrF}" )
+  th_assertFalseWithError 'too many arguments' $? "${stdoutF}" "${stderrF}"
+}
+
+#-----------------------------------------------------------------------------
+# suite functions
+#
+
+oneTimeSetUp()
+{
+  th_oneTimeSetUp
+
+  MSG='This is a test message'
+}
+
+# load and run shUnit2
+[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0
+. ${TH_SHUNIT}

dev/tests/shunit2/shunit2_test_helpers

@@ -0,0 +1,229 @@
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2008 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+#
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit test common functions
+
+# treat unset variables as an error when performing parameter expansion
+set -u
+
+# set shwordsplit for zsh
+[ -n "${ZSH_VERSION:-}" ] && setopt shwordsplit
+
+#
+# constants
+#
+
+# path to shUnit2 library. can be overridden by setting SHUNIT_INC
+TH_SHUNIT=${SHUNIT_INC:-./shunit2}
+
+# configure debugging. set the DEBUG environment variable to any
+# non-empty value to enable debug output, or TRACE to enable trace
+# output.
+TRACE=${TRACE:+'th_trace '}
+[ -n "${TRACE}" ] && DEBUG=1
+[ -z "${TRACE}" ] && TRACE=':'
+
+DEBUG=${DEBUG:+'th_debug '}
+[ -z "${DEBUG}" ] && DEBUG=':'
+
+#
+# variables
+#
+
+th_RANDOM=0
+
+#
+# functions
+#
+
+# message functions
+th_trace() { echo "${MY_NAME}:TRACE $@" >&2; }
+th_debug() { echo "${MY_NAME}:DEBUG $@" >&2; }
+th_info() { echo "${MY_NAME}:INFO $@" >&2; }
+th_warn() { echo "${MY_NAME}:WARN $@" >&2; }
+th_error() { echo "${MY_NAME}:ERROR $@" >&2; }
+th_fatal() { echo "${MY_NAME}:FATAL $@" >&2; }
+
+# output subtest name
+th_subtest() { echo " $@" >&2; }
+
+th_oneTimeSetUp()
+{
+  # these files will be cleaned up automatically by shUnit2
+  stdoutF="${SHUNIT_TMPDIR}/stdout"
+  stderrF="${SHUNIT_TMPDIR}/stderr"
+  returnF="${SHUNIT_TMPDIR}/return"
+  expectedF="${SHUNIT_TMPDIR}/expected"
+}
+
+# generate a random number
+th_generateRandom()
+{
+  tfgr_random=${th_RANDOM}
+
+  while [ "${tfgr_random}" = "${th_RANDOM}" ]; do
+    if [ -n "${RANDOM:-}" ]; then
+      # $RANDOM works
+      tfgr_random=${RANDOM}${RANDOM}${RANDOM}$$
+    elif [ -r '/dev/urandom' ]; then
+      tfgr_random=`od -vAn -N4 -tu4 </dev/urandom |sed 's/^[^0-9]*//'`
+    else
+      tfgr_date=`date '+%H%M%S'`
+      tfgr_random=`expr ${tfgr_date} \* $$`
+      unset tfgr_date
+    fi
+    [ "${tfgr_random}" = "${th_RANDOM}" ] && sleep 1
+  done
+
+  th_RANDOM=${tfgr_random}
+  unset tfgr_random
+}
+
+# this section returns the data section from the specified section of a file. a
+# datasection is defined by a [header], one or more lines of data, and then a
+# blank line.
+th_getDataSect()
+{
+  th_sgrep "\\[$1\\]" "$2" |sed '1d'
+}
+
+# this function greps a section from a file. a section is defined as a group of
+# lines preceeded and followed by blank lines.
+th_sgrep()
+{
+  th_pattern_=$1
+  shift
+
+  sed -e '/./{H;$!d;}' -e "x;/${th_pattern_}/"'!d;' $@ |sed '1d'
+
+  unset th_pattern_
+}
+
+# Custom assert that checks for true return value (0), and no output to STDOUT
+# or STDERR. If a non-zero return value is encountered, the output of STDERR
+# will be output.
+#
+# Args:
+#  th_test_: string: name of the subtest
+#  th_rtrn_: integer: the return value of the subtest performed
+#  th_stdout_: string: filename where stdout was redirected to
+#  th_stderr_: string: filename where stderr was redirected to
+th_assertTrueWithNoOutput()
+{
+  th_test_=$1
+  th_rtrn_=$2
+  th_stdout_=$3
+  th_stderr_=$4
+
+  assertTrue "${th_test_}; expected return value of zero" ${th_rtrn_}
+  [ ${th_rtrn_} -ne ${SHUNIT_TRUE} ] && cat "${th_stderr_}"
+  assertFalse "${th_test_}; expected no output to STDOUT" \
+      "[ -s '${th_stdout_}' ]"
+  assertFalse "${th_test_}; expected no output to STDERR" \
+      "[ -s '${th_stderr_}' ]"
+
+  unset th_test_ th_rtrn_ th_stdout_ th_stderr_
+}
+
+# Custom assert that checks for non-zero return value, output to STDOUT, but no
+# output to STDERR.
+#
+# Args:
+#  th_test_: string: name of the subtest
+#  th_rtrn_: integer: the return value of the subtest performed
+#  th_stdout_: string: filename where stdout was redirected to
+#  th_stderr_: string: filename where stderr was redirected to
+th_assertFalseWithOutput()
+{
+  th_test_=$1
+  th_rtrn_=$2
+  th_stdout_=$3
+  th_stderr_=$4
+
+  assertFalse "${th_test_}; expected non-zero return value" ${th_rtrn_}
+  assertTrue "${th_test_}; expected output to STDOUT" \
+      "[ -s '${th_stdout_}' ]"
+  assertFalse "${th_test_}; expected no output to STDERR" \
+      "[ -s '${th_stderr_}' ]"
+  [ -s "${th_stdout_}" -a ! -s "${th_stderr_}" ] || \
+      _th_showOutput ${SHUNIT_FALSE} "${th_stdout_}" "${th_stderr_}"
+
+  unset th_test_ th_rtrn_ th_stdout_ th_stderr_
+}
+
+# Custom assert that checks for non-zero return value, no output to STDOUT, but
+# output to STDERR.
+#
+# Args:
+#  th_test_: string: name of the subtest
+#  th_rtrn_: integer: the return value of the subtest performed
+#  th_stdout_: string: filename where stdout was redirected to
+#  th_stderr_: string: filename where stderr was redirected to
+th_assertFalseWithError()
+{
+  th_test_=$1
+  th_rtrn_=$2
+  th_stdout_=$3
+  th_stderr_=$4
+
+  assertFalse "${th_test_}; expected non-zero return value" ${th_rtrn_}
+  assertFalse "${th_test_}; expected no output to STDOUT" \
+      "[ -s '${th_stdout_}' ]"
+  assertTrue "${th_test_}; expected output to STDERR" \
+      "[ -s '${th_stderr_}' ]"
+  [ ! -s "${th_stdout_}" -a -s "${th_stderr_}" ] || \
+      _th_showOutput ${SHUNIT_FALSE} "${th_stdout_}" "${th_stderr_}"
+
+  unset th_test_ th_rtrn_ th_stdout_ th_stderr_
+}
+
+# Some shells, zsh on Solaris in particular, return immediately from a sub-shell
+# when a non-zero return value is encountered. To properly catch these values,
+# they are either written to disk, or recognized as an error the file is empty.
+th_clearReturn() { cp /dev/null "${returnF}"; }
+th_queryReturn()
+{
+  if [ -s "${returnF}" ]; then
+    th_return=`cat "${returnF}"`
+  else
+    th_return=${SHUNIT_ERROR}
+  fi
+}
+
+# Providing external and internal calls to the showOutput helper function.
+th_showOutput() { _th_showOutput $@; }
+_th_showOutput()
+{
+  _th_return_=$1
+  _th_stdout_=$2
+  _th_stderr_=$3
+
+  isSkipping
+  if [ $? -eq ${SHUNIT_FALSE} -a ${_th_return_} != ${SHUNIT_TRUE} ]; then
+    if [ -n "${_th_stdout_}" -a -s "${_th_stdout_}" ]; then
+      echo '>>> STDOUT' >&2
+      cat "${_th_stdout_}" >&2
+    fi
+    if [ -n "${_th_stderr_}" -a -s "${_th_stderr_}" ]; then
+      echo '>>> STDERR' >&2
+      cat "${_th_stderr_}" >&2
+    fi
+    if [ -n "${_th_stdout_}" -o -n "${_th_stderr_}" ]; then
+      echo '<<< end output' >&2
+    fi
+  fi
+
+  unset _th_return_ _th_stdout_ _th_stderr_
+}
+
+#
+# main
+#
+
+${TRACE} 'trace output enabled'
+${DEBUG} 'debug output enabled'

dev/tests/shunit2/shunit2_test_macros.sh

@@ -0,0 +1,246 @@
+#! /bin/sh
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2008 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit test for macros.
+
+# load test helpers
+. ./shunit2_test_helpers
+
+#------------------------------------------------------------------------------
+# suite tests
+#
+
+testAssertEquals()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_EQUALS_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_EQUALS_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_EQUALS_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_EQUALS_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testAssertNotEquals()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_NOT_EQUALS_} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NOT_EQUALS_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_NOT_EQUALS_} '"some msg"' 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NOT_EQUALS_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testSame()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_SAME_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_SAME_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_SAME_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_SAME_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testNotSame()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_NOT_SAME_} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NOT_SAME_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_NOT_SAME_} '"some msg"' 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NOT_SAME_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testNull()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_NULL_} 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NULL_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_NULL_} '"some msg"' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NULL_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testNotNull()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_NOT_NULL_} '' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NOT_NULL_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_NOT_NULL_} '"some msg"' '""' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_NOT_NULL_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stdoutF}" "${stderrF}" >&2
+}
+
+testAssertTrue()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_TRUE_} ${SHUNIT_FALSE} >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_TRUE_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+
+  ( ${_ASSERT_TRUE_} '"some msg"' ${SHUNIT_FALSE} >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_TRUE_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testAssertFalse()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_ASSERT_FALSE_} ${SHUNIT_TRUE} >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_FALSE_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_ASSERT_FALSE_} '"some msg"' ${SHUNIT_TRUE} >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_ASSERT_FALSE_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testFail()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_FAIL_} >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_FAIL_} '"some msg"' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testFailNotEquals()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_FAIL_NOT_EQUALS_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_NOT_EQUALS_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_FAIL_NOT_EQUALS_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_NOT_EQUALS_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testFailSame()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_FAIL_SAME_} 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_SAME_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_FAIL_SAME_} '"some msg"' 'x' 'x' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_SAME_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testFailNotSame()
+{
+  # start skipping if LINENO not available
+  [ -z "${LINENO:-}" ] && startSkipping
+
+  ( ${_FAIL_NOT_SAME_} 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_NOT_SAME_ failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+
+  ( ${_FAIL_NOT_SAME_} '"some msg"' 'x' 'y' >"${stdoutF}" 2>"${stderrF}" )
+  grep '^ASSERT:\[[0-9]*\] *' "${stdoutF}" >/dev/null
+  rtrn=$?
+  assertTrue '_FAIL_NOT_SAME_ w/ msg failure' ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+#------------------------------------------------------------------------------
+# suite functions
+#
+
+oneTimeSetUp()
+{
+  th_oneTimeSetUp
+}
+
+# load and run shUnit2
+[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0
+. ${TH_SHUNIT}

dev/tests/shunit2/shunit2_test_misc.sh

@@ -0,0 +1,160 @@
+#! /bin/sh
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2008 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+#
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit tests of miscellaneous things
+
+# load test helpers
+. ./shunit2_test_helpers
+
+#------------------------------------------------------------------------------
+# suite tests
+#
+
+# Note: the test script is prefixed with '#' chars so that shUnit2 does not
+# incorrectly interpret the embedded functions as real functions.
+testUnboundVariable()
+{
+  unittestF="${SHUNIT_TMPDIR}/unittest"
+  sed 's/^#//' >"${unittestF}" <<EOF
+## treat unset variables as an error when performing parameter expansion
+#set -u
+#
+#boom() { x=\$1; }  # this function goes boom if no parameters are passed!
+#test_boom()
+#{
+#   assertEquals 1 1
+#   boom  # No parameter given
+#   assertEquals 0 \$?
+#}
+#. ${TH_SHUNIT}
+EOF
+  ( exec ${SHUNIT_SHELL:-sh} "${unittestF}" >"${stdoutF}" 2>"${stderrF}" )
+  assertFalse 'expected a non-zero exit value' $?
+  grep '^ASSERT:Unknown failure' "${stdoutF}" >/dev/null
+  assertTrue 'assert message was not generated' $?
+  grep '^Ran [0-9]* test' "${stdoutF}" >/dev/null
+  assertTrue 'test count message was not generated' $?
+  grep '^FAILED' "${stdoutF}" >/dev/null
+  assertTrue 'failure message was not generated' $?
+}
+
+testIssue7()
+{
+  ( assertEquals 'Some message.' 1 2 >"${stdoutF}" 2>"${stderrF}" )
+  diff "${stdoutF}" - >/dev/null <<EOF
+ASSERT:Some message. expected:<1> but was:<2>
+EOF
+  rtrn=$?
+  assertEquals ${SHUNIT_TRUE} ${rtrn}
+  [ ${rtrn} -ne ${SHUNIT_TRUE} ] && cat "${stderrF}" >&2
+}
+
+testPrepForSourcing()
+{
+  assertEquals '/abc' `_shunit_prepForSourcing '/abc'`
+  assertEquals './abc' `_shunit_prepForSourcing './abc'`
+  assertEquals './abc' `_shunit_prepForSourcing 'abc'`
+}
+
+testEscapeCharInStr()
+{
+  actual=`_shunit_escapeCharInStr '\' ''`
+  assertEquals '' "${actual}"
+  assertEquals 'abc\\' `_shunit_escapeCharInStr '\' 'abc\'`
+  assertEquals 'abc\\def' `_shunit_escapeCharInStr '\' 'abc\def'`
+  assertEquals '\\def' `_shunit_escapeCharInStr '\' '\def'`
+
+  actual=`_shunit_escapeCharInStr '"' ''`
+  assertEquals '' "${actual}"
+  assertEquals 'abc\"' `_shunit_escapeCharInStr '"' 'abc"'`
+  assertEquals 'abc\"def' `_shunit_escapeCharInStr '"' 'abc"def'`
+  assertEquals '\"def' `_shunit_escapeCharInStr '"' '"def'`
+
+  actual=`_shunit_escapeCharInStr '$' ''`
+  assertEquals '' "${actual}"
+  assertEquals 'abc\$' `_shunit_escapeCharInStr '$' 'abc$'`
+  assertEquals 'abc\$def' `_shunit_escapeCharInStr '$' 'abc$def'`
+  assertEquals '\$def' `_shunit_escapeCharInStr '$' '$def'`
+
+#  actual=`_shunit_escapeCharInStr "'" ''`
+#  assertEquals '' "${actual}"
+#  assertEquals "abc\\'" `_shunit_escapeCharInStr "'" "abc'"`
+#  assertEquals "abc\\'def" `_shunit_escapeCharInStr "'" "abc'def"`
+#  assertEquals "\\'def" `_shunit_escapeCharInStr "'" "'def"`
+
+#  # must put the backtick in a variable so the shell doesn't misinterpret it
+#  # while inside a backticked sequence (e.g. `echo '`'` would fail).
+#  backtick='`'
+#  actual=`_shunit_escapeCharInStr ${backtick} ''`
+#  assertEquals '' "${actual}"
+#  assertEquals '\`abc' \
+#      `_shunit_escapeCharInStr "${backtick}" ${backtick}'abc'`
+#  assertEquals 'abc\`' \
+#      `_shunit_escapeCharInStr "${backtick}" 'abc'${backtick}`
+#  assertEquals 'abc\`def' \
+#      `_shunit_escapeCharInStr "${backtick}" 'abc'${backtick}'def'`
+}
+
+testEscapeCharInStr_specialChars()
+{
+  # make sure our forward slash doesn't upset sed
+  assertEquals '/' `_shunit_escapeCharInStr '\' '/'`
+
+  # some shells escape these differently
+  #assertEquals '\\a' `_shunit_escapeCharInStr '\' '\a'`
+  #assertEquals '\\b' `_shunit_escapeCharInStr '\' '\b'`
+}
+
+# Test the various ways of declaring functions.
+#
+# Prefixing (then stripping) with comment symbol so these functions aren't
+# treated as real functions by shUnit2.
+testExtractTestFunctions()
+{
+  f="${SHUNIT_TMPDIR}/extract_test_functions"
+  sed 's/^#//' <<EOF >"${f}"
+#testABC() { echo 'ABC'; }
+#test_def() {
+#  echo 'def'
+#}
+#testG3 ()
+#{
+#  echo 'G3'
+#}
+#function test4() { echo '4'; }
+#	test5() { echo '5'; }
+#some_test_function() { echo 'some func'; }
+#func_with_test_vars() {
+#  testVariable=1234
+#}
+EOF
+
+  actual=`_shunit_extractTestFunctions "${f}"`
+  assertEquals 'testABC test_def testG3 test4 test5' "${actual}"
+}
+
+#------------------------------------------------------------------------------
+# suite functions
+#
+
+setUp()
+{
+  for f in ${expectedF} ${stdoutF} ${stderrF}; do
+    cp /dev/null ${f}
+  done
+}
+
+oneTimeSetUp()
+{
+  th_oneTimeSetUp
+}
+
+# load and run shUnit2
+[ -n "${ZSH_VERSION:-}" ] && SHUNIT_PARENT=$0
+. ${TH_SHUNIT}

dev/tests/shunit2/shunit2_test_standalone.sh

@@ -0,0 +1,41 @@
+#! /bin/sh
+# $Id$
+# vim:et:ft=sh:sts=2:sw=2
+#
+# Copyright 2010 Kate Ward. All Rights Reserved.
+# Released under the LGPL (GNU Lesser General Public License)
+# Author: kate.ward@forestent.com (Kate Ward)
+#
+# shUnit2 unit test for standalone operation.
+#
+# This unit test is purely to test that calling shunit2 directly, while passing
+# the name of a unit test script, works. When run, this script determines if it
+# is running as a standalone program, and calls main() if it is.
+
+ARGV0=`basename "$0"`
+
+# load test helpers
+. ./shunit2_test_helpers
+
+#------------------------------------------------------------------------------
+# suite tests
+#
+
+testStandalone()
+{
+  assertTrue ${SHUNIT_TRUE}
+}
+
+#------------------------------------------------------------------------------
+# main
+#
+
+main()
+{
+  ${TH_SHUNIT} "${ARGV0}"
+}
+
+# are we running as a standalone?
+if [ "${ARGV0}" = 'shunit2_test_standalone.sh' ]; then
+  if [ $# -gt 0 ]; then main "$@"; else main; fi
+fi

exclude.list.example

@@ -0,0 +1,7 @@
+.AppleDouble/
+._*
+.DS_Store
+Thumbs.db
+System Volume Information
+$Recycle.Bin
+

host_backup.conf

@@ -1,163 +0,0 @@
-#!/bin/bash
-
-###### Remote (or local) backup script for files & databases
-###### (L) 2013 by Ozy de Jong (www.netpower.fr)
-###### Config file rev 0211201301
-
-## ---------- GENERAL BACKUP OPTIONS
-
-## Backup identification name. 
-BACKUP_ID="your backup name"
-
-## Log file location. Leaving this empty will create log file at /var/log/obackup_version_BACKUP_ID.log (or current directory if /var/log doesn't exist)
-LOGFILE=""
-
-## Backup databases
-BACKUP_SQL=no
-## Backup files
-BACKUP_FILES=yes
-
-## ---------- LOCAL BACKUP STORAGE OPTIONS
-
-## Local storage paths where to put backups
-LOCAL_SQL_STORAGE="/home/storage/backup/sql"
-LOCAL_FILE_STORAGE="/home/storage/backup/files"
-
-## Create backup directories if they do not exist
-CREATE_DIRS=yes
-
-## Keep absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
-## You should leave this enabled if you intend to use 'backup task division' functionality of OBackup, or everything will end up in the same directory.
-LOCAL_STORAGE_KEEP_ABSOLUTE_PATHS=yes
-
-## Generate an alert if backup size is lower than given value in Kb (this can also help identifying empty mount dirs)
-BACKUP_SIZE_MINIMUM=1024
-
-## Generate an alert if local storage free space is lower than given value in Kb.
-LOCAL_STORAGE_WARN_MIN_SPACE=1048576
-
-## ---------- MISC OPTIONS
-
-## Bandwidth limit Kbytes / second for file backups. Leave 0 to disable limitation.
-BANDWIDTH=0
-
-## If enabled, file backups will be processed as superuser. See documentation for /etc/sudoers configuration ("find", "du" and "rsync" need to be allowed). Requiretty needs to be disabled.
-SUDO_EXEC=no
-## Paranoia option. Don't change this unless you read the documentation.
-RSYNC_EXECUTABLE=rsync
-
-## ---------- REMOTE BACKUP OPTIONS
-
-## The following options allow this Obackup instance to connect to a remote system via an ssh tunnel.
-## Needs public RSA key need to be put into ~/.ssh/authorized_keys in remote users home directory.
-REMOTE_BACKUP=no
-SSH_RSA_PRIVATE_KEY=~/.ssh/id_rsa
-REMOTE_USER=backupuser
-REMOTE_HOST=yourhost.local
-REMOTE_PORT=22
-## ssh compression should be used unless your remote connection is good enough (LAN)
-SSH_COMPRESSION=yes
-## Remote rsync executable path. Leave this empty in most cases
-REMOTE_RSYNC_PATH=""
-
-## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
-REMOTE_HOST_PING=yes
-## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
-REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
-
-## ---------- DATABASE BACKUP OPTIONS
-
-## Database backup user
-SQL_USER=backupuser
-
-## Enabling the following option will save all databases on local or remote given SQL instance except the ones specified in the exlude list.
-## Every found database will be backed up as separate backup task.
-DATABASES_ALL=yes
-DATABASES_ALL_EXCLUDE_LIST="test;mysql"
-## Alternatively, if DATABASES_ALL=no, you can specifiy a list of databases to backup separated by spaces.
-DATABASES_LIST=""
-
-## Max backup execution time per Database task. Soft max exec time generates a warning only. Hard max ecec time generates a warning and stops current backup task.
-## If a task gets stopped, next one in the task list gets executed. Time is specified in seconds.
-SOFT_MAX_EXEC_TIME_DB_TASK=3600
-HARD_MAX_EXEC_TIME_DB_TASK=7200
-
-## Preferred SQL dump compression. Can be set to xz, lzma or gzip.
-## Generally, xz level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
-COMPRESSION_PROGRAM=xz
-COMPRESSION_LEVEL=3
-
-## SQL Dump compression should be done on remote side but can also be done locally to lower remote system usage (will take more bandwidth, check for ssh compression)
-COMPRESSION_REMOTE=yes
-
-## ---------- FILES BACKUP OPTIONS
-
-## Directories backup list. List of semicolon separated directories that will be backed up recursively. Every directory will be processed as one backup task.
-DIRECTORIES_SIMPLE_LIST="/var/named;/var/lib"
-
-## There's a special backup schema in Obackup called 'backup task division'  which creates one backup task per level 1 subdirectory of a directory.
-## This is VERY usefull to backup multiple virtualhosts as separate tasks without having to specifiy each one separately.
-## This may also be usefull dividing big data directories in subdirectories tasks.
-
-## Directories backup task division backup: Semicolon separated directories of which every level 1 subdirectory will be backed up recursively as a spearate backup task.
-## Example: "/home;/var" will create tasks "/home/dir1", "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/whatever"
-DIRECTORIES_RECURSE_LIST="/home"
-## You may optionally exclude subdirectories from task division. On the above example you could exclude /home/dir2 by adding it to DIRECTORIES_RECURSE_EXCLUDE_LIST
-DIRECTORIES_RECURSE_EXCLUDE_LIST="/home/backupuser;/home/lost+found"
-
-## Rsync exclude patterns, used by simple and division lists
-RSYNC_EXCLUDE_PATTERN="*/tmp;*/ftp/www/cache/cachefs;*/sessions"
-
-## List separator char. You may set an alternative seperator char for your directories lists above.
-PATH_SEPARATOR_CHAR=";"
-
-## Preserve ACLS. Make sure source and target FS can hold same ACLs or you'll get loads of errors.
-PRESERVE_ACL=no
-## Preserve Xattr. MAke sure source and target FS can hold same Xattr or you'll get loads of errors.
-PRESERVE_XATTR=no
-
-## Let RSYNC compress file transfers. Do not use this on local-local backup schemes. Also, this is not usefull if SSH compression is enabled.
-RSYNC_COMPRESS=no
-
-## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
-SOFT_MAX_EXEC_TIME_FILE_TASK=3600
-HARD_MAX_EXEC_TIME_FILE_TASK=7200
-
-## ---------- ALERT OPTIONS
-
-## Alert email adresses separated by a space character
-DESTINATION_MAILS="your@mail.address"
-
-## Windows (MSYS environment) only mail options (used by sendemail.exe)
-SENDER_MAIL="alert@your.system"
-SMTP_SERVER=smtp.your.isp.com
-SMTP_USER=
-SMTP_PASSWORD=
-
-## ---------- GENERAL BACKUP OPTIONS
-
-## Max execution time of whole backup process. Soft max exec time generates a warning only.
-## Hard max exec time generates a warning and stops the whole backup execution.
-SOFT_MAX_EXEC_TIME_TOTAL=30000
-HARD_MAX_EXEC_TIME_TOTAL=36000
-
-## Backup Rotation. You may rotate backups if you don't use snapshots on your backup server.
-ROTATE_BACKUPS=no
-ROTATE_COPIES=7
-
-## ---------- EXECUTION HOOKS
-
-## Commands can will be run before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set).
-## This is usefull to make a snapshot before backing up data, or even handle snapshots of backed up data.
-LOCAL_RUN_BEFORE_CMD=""
-LOCAL_RUN_AFTER_CMD=""
-
-REMOTE_RUN_BEFORE_CMD=""
-REMOTE_RUN_AFTER_CMD=""
-
-## Max execution time of commands before they get force killed. Leave 0 if you don't wan't this to happen. Time is specified in seconds.
-MAX_EXEC_TIME_PER_CMD_BEFORE=0
-MAX_EXEC_TIME_PER_CMD_AFTER=0
-
-## Stops whole backup execution if one of the above commands fail
-STOP_ON_CMD_ERROR=no

host_backup.conf.example

@@ -0,0 +1,252 @@
+#!/usr/bin/env bash
+
+###### obackup - Local or Remote, push or pull backup script for files & mysql databases
+###### (C) 2013-2016 by Orsiris de Jong (www.netpower.fr)
+###### obackup v2.1x config file rev 2017020901
+
+###### GENERAL BACKUP OPTIONS
+
+## Backup identification string.
+INSTANCE_ID="test-backup"
+
+## Log file location. Leaving this empty will create log file at /var/log/obackup.INSTANCE_ID.log (or current directory if /var/log doesn't exist).
+LOGFILE=""
+
+## Elements to backup
+SQL_BACKUP=yes
+FILE_BACKUP=yes
+
+## Backups can be done local, pulled from another server or pushed to a backup server. Available options are [local,pull,push].
+## Pulled backups are the safest option, as the backup server contains the RSA key and cannot be compromised by another server.
+BACKUP_TYPE=local
+
+###### BACKUP STORAGE
+
+## Storage paths of the backups (absolute paths of the local or remote system). Please use ${HOME} instead of ~ if needed.
+SQL_STORAGE="/home/storage/backup/sql"
+FILE_STORAGE="/home/storage/backup/files"
+
+## Backup encryption using GPG and rsync.
+## Push backups get encrypted locally in CRYPT_STORAGE before they are sent to the remote system
+## Local and pull backups get encrypted after backup, in CRYPT_STORAGE
+ENCRYPTION=no
+
+## Backup encryption needs a temporary storage space in order to encrypt files before sending them (absolute paths of the local or remote system)
+## In case of a pull backup, an encrypted copy of FILE_BACKUP goes here
+CRYPT_STORAGE=/home/storage/backup/crpyt
+
+## GPG recipient (pubkey for this recipient must exist, see gpg2 --list-keys or gpg --list-keys
+GPG_RECIPIENT="John Doe"
+
+## Use n CPUs for encryption / decryption where n is an integer. Defaults to 1
+PARALLEL_ENCRYPTION_PROCESSES=
+
+## Create backup directories if they do not exist
+CREATE_DIRS=yes
+
+## Keep absolute source path in your backup, eg: /your/backup/storage/the/remote/server/files
+## You should leave this enabled if you intend to use 'backup task division' functionality of oBackup, or everything will end up in the same directory.
+KEEP_ABSOLUTE_PATHS=yes
+
+## Generate an alert if backup size is lower than given value in Kb (this can also help identifying empty mount dirs).
+BACKUP_SIZE_MINIMUM=1024
+
+## Check backup size before proceeding
+GET_BACKUP_SIZE=yes
+
+## Generate an alert if storage free space is lower than given value in Kb.
+## Keep in mind that disabling backup file size test will only test min space against SQL backup size.
+SQL_WARN_MIN_SPACE=1048576
+FILE_WARN_MIN_SPACE=1048576
+
+###### REMOTE ONLY OPTIONS
+
+## In case of pulled or pushed backups, remote system URI needs to be supplied.
+REMOTE_SYSTEM_URI="ssh://backupuser@remote.system.tld:22/"
+
+## You can specify a RSA key (please use full path). If not defined, the default ~/.ssh/id_rsa will be used. See documentation for further information.
+SSH_RSA_PRIVATE_KEY="${HOME}/.ssh/id_rsa"
+
+## Alternatively, you may specify an SSH password file (less secure). Needs sshpass utility installed.
+SSH_PASSWORD_FILE=""
+
+## When using ssh filter, you must specify a remote token matching the one setup in authorized_keys
+_REMOTE_TOKEN=SomeAlphaNumericToken9
+
+## ssh compression should be used unless your remote connection is good enough (LAN)
+SSH_COMPRESSION=yes
+
+## Ignore ssh known hosts verification. DANGER WILL ROBINSON DANGER: This can lead to security risks. Only enable if you know what you're doing.
+## Works on Redhat / CentOS, doesn't work on Debian / Ubunutu
+SSH_IGNORE_KNOWN_HOSTS=no
+
+## Remote rsync executable path. Leave this empty in most cases
+RSYNC_REMOTE_PATH=""
+
+## Check for connectivity to remote host before launching remote backup tasks. Be sure the hosts responds to ping. Failing to ping will skip current task.
+REMOTE_HOST_PING=yes
+
+## Check for internet access by pinging one or more 3rd party hosts before remote backup tasks. Leave empty if you don't want this check to be be performed. Failing to ping will skip current task.
+REMOTE_3RD_PARTY_HOSTS="www.kernel.org www.google.com"
+
+## If enabled, commands will be executed as superuser on remote side. See documentation for /etc/sudoers configuration ("find", "du", "tee" and "rsync" need to be allowed). Requiretty needs to be disabled.
+SUDO_EXEC=no
+
+###### DATABASE SPECIFIC OPTIONS
+
+## Database backup user (should be the same you are running obackup with)
+SQL_USER=root
+
+## Enabling the following option will save all databases on local or remote given SQL instance except the ones specified in the exclude list.
+## Every found database will be backed up as separate backup task.
+DATABASES_ALL=yes
+DATABASES_ALL_EXCLUDE_LIST="test;mysql"
+## Alternatively, if DATABASES_ALL=no, you can specify a list of databases to backup separated by semi-colons.
+#DATABASES_LIST="somedatabase"
+
+## Max backup execution time per Database task. Soft max exec time generates a warning only. Hard max exec time generates a warning and stops current backup task.
+## If a task gets stopped, next one in the task list gets executed. Time is specified in seconds.
+SOFT_MAX_EXEC_TIME_DB_TASK=3600
+HARD_MAX_EXEC_TIME_DB_TASK=7200
+
+## mysqldump options (ex: --extended-insert, --single-transaction, --quick...). See MySQL / MariaDB manual
+## default option: --opt
+MYSQLDUMP_OPTIONS="--opt --single-transaction"
+
+## Preferred SQL dump compression. Compression methods can be xz, lzma, pigz or gzip (will fallback from xz to gzip depending if available)
+## Generally, level 5 is a good compromise between cpu, memory hunger and compress ratio. Gzipped files are set to be rsyncable.
+## If you use encryption, compression will only bring small benefits as GPG already has pretty good compression included
+COMPRESSION_LEVEL=3
+
+###### FILES SPECIFIC OPTIONS
+
+## File backups are divided in tasks. Every directory in DIRECTORY_LIST will be processed as a unique task.
+## Every subdirectory of each directory in RECURSIVE_DIRECTORY_LIST will be processed as a unique task.
+## Example: RECURSIVE_DIRECTORY_LIST="/home;/var" will create backup tasks tasks "/home/dir1, "/home/dir2", ... "/home/dirN", "/var/log", "/var/lib"... "/var/something".
+## You can exclude directories from the avove backup task creation, ex: avoid backing up "/home/dir2" by adding it to RECURSIVE_EXCLUDE_LIST.
+## Please use ${HOME} instead of ~ if needed.
+
+## Directories backup list. List of semicolon separated directories that will be backed up.
+DIRECTORY_LIST="/var/named"
+RECURSIVE_DIRECTORY_LIST="/home"
+RECURSIVE_EXCLUDE_LIST="/home/backupuser;/home/lost+found"
+
+## Rsync exclude / include order (the option set here will be set first, eg: include will make include then exclude patterns)
+RSYNC_PATTERN_FIRST=include
+
+## List of files / directories to incldue / exclude from sync on both sides (see rsync patterns, wildcards work).
+## Paths are relative to sync dirs. List elements are separated by a semicolon.
+RSYNC_INCLUDE_PATTERN=""
+RSYNC_EXCLUDE_PATTERN=""
+#RSYNC_EXCLUDE_PATTERN="tmp;archives"
+
+## Files that contains lists of files / directories to include / exclude from sync on both sides. Leave this empty if you don't want to use an exclusion file.
+## This file has to be in the same directory as the config file
+## Paths are relative to sync dirs. One element per line.
+RSYNC_INCLUDE_FROM=""
+RSYNC_EXCLUDE_FROM=""
+#RSYNC_EXCLUDE_FROM="exclude.list"
+
+## List separator char. You may set an alternative separator char for your directories lists above.
+PATH_SEPARATOR_CHAR=";"
+
+## Optional arguments passed to rsync executable. The following are already managed by the program and shoul never be passed here
+## -rltD -n -P -o -g --executability -A -X -zz -L -K -H -8 -u -i --stats --checksum --bwlimit --partial --partial-dir --exclude --exclude-from --include--from --no-whole-file --whole-file --list-only
+RSYNC_OPTIONAL_ARGS=""
+
+## Preserve basic linux permissions
+PRESERVE_PERMISSIONS=yes
+PRESERVE_OWNER=yes
+PRESERVE_GROUP=yes
+## On MACOS X, does not work and will be ignored 
+PRESERVE_EXECUTABILITY=yes
+
+## Preserve ACLS. Make sure source and target FS can hold same ACLs or you'll get loads of errors.
+PRESERVE_ACL=no
+## Preserve Xattr. MAke sure source and target FS can hold same Xattr or you'll get loads of errors.
+PRESERVE_XATTR=no
+
+## Transforms symlinks into referent files/dirs
+COPY_SYMLINKS=yes
+## Treat symlinked dirs as dirs. CAUTION: This also follows symlinks outside of the replica root.
+KEEP_DIRLINKS=yes
+## Preserve hard links. Make sure source and target FS can manage hard links or you will lose them.
+PRESERVE_HARDLINKS=no
+
+
+## Let RSYNC compress file transfers. Do not use this on local-local backup schemes. Also, this is not useful if SSH compression is enabled.
+RSYNC_COMPRESS=no
+
+## Max execution time per file backup task. Soft is warning only. Hard is warning, stopping backup and processing next one one file list. Tilme is specified in seconds
+SOFT_MAX_EXEC_TIME_FILE_TASK=3600
+HARD_MAX_EXEC_TIME_FILE_TASK=7200
+
+## Keep partial uploads that can be resumed on next run, experimental feature
+PARTIAL=no
+
+## Delete files on destination that vanished from source. Do not turn this on unless you enabled backup rotation  or a snapshotting FS like zfs to keep those vanished files on the destination.
+DELETE_VANISHED_FILES=no
+
+## Use delta copy algortithm (usefull when local paths are network drives), defaults to yes
+DELTA_COPIES=yes
+
+## Bandwidth limit Kbytes / second for file backups. Leave 0 to disable limitation.
+BANDWIDTH=0
+
+## Paranoia option. Don't change this unless you read the documentation.
+RSYNC_EXECUTABLE=rsync
+
+###### ALERT OPTIONS
+
+## Alert email addresses separated by a space character
+DESTINATION_MAILS="your@mail.address"
+
+## Optional change of mail body encoding (using iconv)
+## By default, all mails are sent in UTF-8 format without header (because of maximum compatibility of all platforms)
+## You may specify an optional encoding here (like "ISO-8859-1" or whatever iconv can handle)
+MAIL_BODY_CHARSET=""
+
+## Environment specific mail options (used with busybox sendemail, mailsend.exe from muquit, http://github.com/muquit/mailsend or sendemail.exe from Brandon Zehm, http://caspian.dotconf.net/menu/Software/SendEmail)
+SENDER_MAIL="alert@your.system.tld"
+SMTP_SERVER=smtp.your.isp.tld
+SMTP_PORT=25
+# encryption can be tls, ssl or none
+SMTP_ENCRYPTION=none
+SMTP_USER=
+SMTP_PASSWORD=
+
+###### GENERAL BACKUP OPTIONS
+
+## Max execution time of whole backup process. Soft max exec time generates a warning only.
+## Hard max exec time generates a warning and stops the whole backup execution.
+SOFT_MAX_EXEC_TIME_TOTAL=30000
+HARD_MAX_EXEC_TIME_TOTAL=36000
+
+## Log a message every KEEP_LOGGING seconds just to know the task is still alive
+KEEP_LOGGING=1801
+
+## Backup Rotation. You may rotate backups if you don't use snapshots on your backup server.
+ROTATE_SQL_BACKUPS=no
+ROTATE_SQL_COPIES=7
+ROTATE_FILE_BACKUPS=no
+ROTATE_FILE_COPIES=7
+
+###### EXECUTION HOOKS
+
+## Commands can will be run before and / or after backup execution (remote execution will only happen if REMOTE_BACKUP is set).
+## This is useful to make a snapshot before backing up data, or even handle snapshots of backed up data.
+LOCAL_RUN_BEFORE_CMD=""
+LOCAL_RUN_AFTER_CMD=""
+
+REMOTE_RUN_BEFORE_CMD=""
+REMOTE_RUN_AFTER_CMD=""
+
+## Max execution time of commands before they get force killed. Leave 0 if you don't want this to happen. Time is specified in seconds.
+MAX_EXEC_TIME_PER_CMD_BEFORE=0
+MAX_EXEC_TIME_PER_CMD_AFTER=0
+
+## Stops whole backup execution if one of the above commands fail
+STOP_ON_CMD_ERROR=no
+
+## Run local and remote after backup cmd's even on failure
+RUN_AFTER_CMD_ON_ERROR=no

install.sh

@@ -0,0 +1,472 @@
+#!/usr/bin/env bash
+
+## Installer script suitable for osync / obackup / pmocr
+
+_OFUNCTIONS_BOOTSTRAP=true
+
+PROGRAM=obackup
+
+PROGRAM_VERSION=$(grep "PROGRAM_VERSION=" $PROGRAM.sh)
+PROGRAM_VERSION=${PROGRAM_VERSION#*=}
+PROGRAM_BINARY=$PROGRAM".sh"
+PROGRAM_BATCH=$PROGRAM"-batch.sh"
+SSH_FILTER="ssh_filter.sh"
+
+SCRIPT_BUILD=2017031402
+
+## osync / obackup / pmocr / zsnap install script
+## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
+## Please adapt this to fit your distro needs
+
+# Get current install.sh path from http://stackoverflow.com/a/246128/2635443
+SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+CONF_DIR=$FAKEROOT/etc/$PROGRAM
+BIN_DIR="$FAKEROOT/usr/local/bin"
+SERVICE_DIR_INIT=$FAKEROOT/etc/init.d
+# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora
+SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system
+SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user
+
+if [ "$PROGRAM" == "osync" ]; then
+	SERVICE_NAME="osync-srv"
+elif [ "$PROGRAM" == "pmocr" ]; then
+	SERVICE_NAME="pmocr-srv"
+fi
+
+SERVICE_FILE_INIT="$SERVICE_NAME"
+SERVICE_FILE_SYSTEMD_SYSTEM="$SERVICE_NAME@.service"
+SERVICE_FILE_SYSTEMD_USER="$SERVICE_NAME@.service.user"
+
+## Generic code
+
+## Default log file
+if [ -w "$FAKEROOT/var/log" ]; then
+	LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log"
+elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
+	LOG_FILE="$HOME/$PROGRAM-install.log"
+else
+	LOG_FILE="./$PROGRAM-install.log"
+fi
+
+# QuickLogger subfunction, can be called directly
+function _QuickLogger {
+	local value="${1}"
+	local destination="${2}" # Destination: stdout, log, both
+
+	if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then
+		echo -e "$(date) - $value" >> "$LOG_FILE"
+	elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then
+		echo -e "$value"
+	fi
+}
+
+# Generic quick logging function
+function QuickLogger {
+	local value="${1}"
+
+	if [ "$_LOGGER_SILENT" == true ]; then
+		_QuickLogger "$value" "log"
+	else
+		_QuickLogger "$value" "stdout"
+	fi
+}
+## from https://gist.github.com/cdown/1163649
+function UrlEncode {
+	local length="${#1}"
+
+	local LANG=C
+	for (( i = 0; i < length; i++ )); do
+		local c="${1:i:1}"
+		case $c in
+			[a-zA-Z0-9.~_-])
+			printf "$c"
+			;;
+			*)
+			printf '%%%02X' "'$c"
+			;;
+		esac
+	done
+}
+function GetLocalOS {
+	local localOsVar
+	local localOsName
+	local localOsVer
+
+	# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
+	if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
+		localOsVar="BusyBox"
+	else
+		# Detecting the special ubuntu userland in Windows 10 bash
+		if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
+			localOsVar="Microsoft"
+		else
+			localOsVar="$(uname -spior 2>&1)"
+			if [ $? != 0 ]; then
+				localOsVar="$(uname -v 2>&1)"
+				if [ $? != 0 ]; then
+					localOsVar="$(uname)"
+				fi
+			fi
+		fi
+	fi
+
+	case $localOsVar in
+		# Android uname contains both linux and android, keep it before linux entry
+		*"Android"*)
+		LOCAL_OS="Android"
+		;;
+		*"Linux"*)
+		LOCAL_OS="Linux"
+		;;
+		*"BSD"*)
+		LOCAL_OS="BSD"
+		;;
+		*"MINGW32"*|*"MSYS"*)
+		LOCAL_OS="msys"
+		;;
+		*"CYGWIN"*)
+		LOCAL_OS="Cygwin"
+		;;
+		*"Microsoft"*)
+		LOCAL_OS="WinNT10"
+		;;
+		*"Darwin"*)
+		LOCAL_OS="MacOSX"
+		;;
+		*"BusyBox"*)
+		LOCAL_OS="BusyBox"
+		;;
+		*)
+		if [ "$IGNORE_OS_TYPE" == "yes" ]; then
+			Logger "Running on unknown local OS [$localOsVar]." "WARN"
+			return
+		fi
+		if [ "$_OFUNCTIONS_VERSION" != "" ]; then
+			Logger "Running on >> $localOsVar << not supported. Please report to the author." "ERROR"
+		fi
+		exit 1
+		;;
+	esac
+	if [ "$_OFUNCTIONS_VERSION" != "" ]; then
+		Logger "Local OS: [$localOsVar]." "DEBUG"
+	fi
+
+	# Get linux versions
+	if [ -f "/etc/os-release" ]; then
+		localOsName=$(GetConfFileValue "/etc/os-release" "NAME")
+		localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION")
+	fi
+
+	# Add a global variable for statistics in installer
+	LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
+}
+function GetConfFileValue () {
+        local file="${1}"
+        local name="${2}"
+        local value
+
+        value=$(grep "^$name=" "$file")
+        if [ $? == 0 ]; then
+                value="${value##*=}"
+                echo "$value"
+        else
+		Logger "Cannot get value for [$name] in config file [$file]." "ERROR"
+        fi
+}
+
+function SetLocalOSSettings {
+	USER=root
+
+	# LOCAL_OS and LOCAL_OS_FULL are global variables set at GetLocalOS
+
+	case $LOCAL_OS in
+		*"BSD"*)
+		GROUP=wheel
+		;;
+		*"MacOSX"*)
+		GROUP=admin
+		;;
+		*"msys"*|*"Cygwin"*)
+		USER=""
+		GROUP=""
+		;;
+		*)
+		GROUP=root
+		;;
+	esac
+
+	if [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "BusyBox" ]; then
+		QuickLogger "Cannot be installed on [$LOCAL_OS]. Please use $PROGRAM.sh directly."
+		exit 1
+	fi
+
+	if ([ "$USER" != "" ] && [ "$(whoami)" != "$USER" ] && [ "$FAKEROOT" == "" ]); then
+		QuickLogger "Must be run as $USER."
+		exit 1
+	fi
+
+	OS=$(UrlEncode "$LOCAL_OS_FULL")
+}
+
+function GetInit {
+	if [ -f /sbin/init ]; then
+		if file /sbin/init | grep systemd > /dev/null; then
+			init="systemd"
+		else
+			init="initV"
+		fi
+	else
+		QuickLogger "Can't detect initV or systemd. Service files won't be installed. You can still run $PROGRAM manually or via cron."
+		init="none"
+	fi
+}
+
+function CreateDir {
+	local dir="${1}"
+
+	if [ ! -d "$dir" ]; then
+		mkdir "$dir"
+		if [ $? == 0 ]; then
+			QuickLogger "Created directory [$dir]."
+		else
+			QuickLogger "Cannot create directory [$dir]."
+			exit 1
+		fi
+	fi
+}
+
+function CopyFile {
+	local sourcePath="${1}"
+	local destPath="${2}"
+	local fileName="${3}"
+	local fileMod="${4}"
+	local fileUser="${5}"
+	local fileGroup="${6}"
+	local overwrite="${7:-false}"
+
+	local userGroup=""
+	local oldFileName
+
+	if [ -f "$destPath/$fileName" ] && [ $overwrite == false ]; then
+		oldFileName="$fileName"
+		fileName="$oldFileName.new"
+		cp "$sourcePath/$oldFileName" "$destPath/$fileName"
+	else
+		cp "$sourcePath/$fileName" "$destPath"
+	fi
+
+	if [ $? != 0 ]; then
+		QuickLogger "Cannot copy [$fileName] to [$destPath]. Make sure to run install script in the directory containing all other files."
+		QuickLogger "Also make sure you have permissions to write to [$BIN_DIR]."
+		exit 1
+	else
+		QuickLogger "Copied [$fileName] to [$destPath]."
+		if [ "$fileMod" != "" ]; then
+			chmod "$fileMod" "$destPath/$fileName"
+			if [ $? != 0 ]; then
+				QuickLogger "Cannot set file permissions of [$destPath/$fileName] to [$fileMod]."
+				exit 1
+			else
+				QuickLogger "Set file permissions to [$fileMod] on [$destPath/$fileName]."
+			fi
+		fi
+
+		if [ "$fileUser" != "" ]; then
+			userGroup="$fileUser"
+
+			if [ "$fileGroup" != "" ]; then
+				userGroup="$userGroup"":$fileGroup"
+			fi
+
+			chown "$userGroup" "$destPath/$fileName"
+			if [ $? != 0 ]; then
+				QuickLogger "Could not set file ownership on [$destPath/$fileName] to [$userGroup]."
+				exit 1
+			else
+				QuickLogger "Set file ownership on [$destPath/$fileName] to [$userGroup]."
+			fi
+		fi
+	fi
+}
+
+function CopyExampleFiles {
+	exampleFiles=()
+	exampleFiles[0]="sync.conf.example"		# osync
+	exampleFiles[1]="host_backup.conf.example"	# obackup
+	exampleFiles[2]="exclude.list.example"		# osync & obackup
+	exampleFiles[3]="snapshot.conf.example"		# zsnap
+	exampleFiles[4]="default.conf"			# pmocr
+
+	for file in "${exampleFiles[@]}"; do
+		if [ -f "$SCRIPT_PATH/$file" ]; then
+			CopyFile "$SCRIPT_PATH" "$CONF_DIR" "$file" "" "" "" false
+		fi
+	done
+}
+
+function CopyProgram {
+	binFiles=()
+	binFiles[0]="$PROGRAM_BINARY"
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
+		binFiles[1]="$PROGRAM_BATCH"
+		binFiles[2]="$SSH_FILTER"
+	fi
+
+	local user=""
+	local group=""
+
+	if ([ "$USER" != "" ] && [ "$FAKEROOT" == "" ]); then
+		user="$USER"
+	fi
+	if ([ "$GROUP" != "" ] && [ "$FAKEROOT" == "" ]); then
+		group="$GROUP"
+	fi
+
+	for file in "${binFiles[@]}"; do
+		CopyFile "$SCRIPT_PATH" "$BIN_DIR" "$file" 755 "$user" "$group" true
+	done
+}
+
+function CopyServiceFiles {
+	if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
+		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true
+		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true
+
+		QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
+		QuickLogger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR."
+		QuickLogger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]."
+		QuickLogger "In userland, active with [systemctl --user start $SERVICE_NAME@instance.conf]."
+	elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then
+		CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true
+
+		QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
+		QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]."
+		QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]."
+	else
+		QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation."
+	fi
+}
+
+function Statistics {
+	if type wget > /dev/null; then
+		wget -qO- "$STATS_LINK" > /dev/null 2>&1
+		if [ $? == 0 ]; then
+			return 0
+		fi
+	fi
+
+	if type curl > /dev/null; then
+		curl "$STATS_LINK" -o /dev/null > /dev/null 2>&1
+		if [ $? == 0 ]; then
+			return 0
+		fi
+	fi
+
+	QuickLogger "Neiter wget nor curl could be used for. Cannot run statistics. Use the provided link please."
+	return 1
+}
+
+function RemoveFile {
+	local file="${1}"
+
+	if [ -f "$file" ]; then
+		rm -f "$file"
+		if [ $? != 0 ]; then
+			QuickLogger "Could not remove file [$file]."
+		else
+			QuickLogger "Removed file [$file]."
+		fi
+	else
+		QuickLogger "File [$file] not found. Skipping."
+	fi
+}
+
+function RemoveAll {
+	RemoveFile "$BIN_DIR/$PROGRAM_BINARY"
+	RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
+	if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then		# Check if any other program requiring ssh filter is present before removal
+		RemoveFile "$BIN_DIR/$SSH_FILTER"
+	else
+		QuickLogger "Skipping removal of [$BIN_DIR/$SSH_FILTER] because other programs present that need it."
+	fi
+	RemoveFile "$SERVICE_DIR_SYSTEMD_SYSTEM/$SERVICE_FILE_SYSTEMD_SYSTEM"
+	RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM"
+	RemoveFile "$SERVICE_DIR_INIT/$SERVICE_FILE_INIT"
+
+	QuickLogger "Skipping configuration files in [$CONF_DIR]. You may remove this directory manually."
+}
+
+function Usage {
+	echo "Installs $PROGRAM into $BIN_DIR"
+	echo "options:"
+	echo "--silent		Will log and bypass user interaction."
+	echo "--no-stats	Used with --silent in order to refuse sending anonymous install stats."
+	exit 127
+}
+
+_LOGGER_SILENT=false
+_STATS=1
+ACTION="install"
+
+for i in "$@"
+do
+	case $i in
+		--silent)
+		_LOGGER_SILENT=true
+		;;
+		--no-stats)
+		_STATS=0
+		;;
+		--remove)
+		ACTION="uninstall"
+		;;
+		--help|-h|-?)
+		Usage
+	esac
+done
+
+if [ "$FAKEROOT" != "" ]; then
+	mkdir -p "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_USER" "$BIN_DIR"
+fi
+
+GetLocalOS
+SetLocalOSSettings
+GetInit
+
+STATS_LINK="http://instcount.netpower.fr?program=$PROGRAM&version=$PROGRAM_VERSION&os=$OS&action=$ACTION"
+
+if [ "$ACTION" == "uninstall" ]; then
+	RemoveAll
+	QuickLogger "$PROGRAM uninstalled."
+else
+	CreateDir "$CONF_DIR"
+	CreateDir "$BIN_DIR"
+	CopyExampleFiles
+	CopyProgram
+	CopyServiceFiles
+	QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
+	if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
+		QuickLogger ""
+		QuickLogger "If connecting remotely, consider setup ssh filter to enhance security."
+		QuickLogger ""
+	fi
+fi
+
+if [ $_STATS -eq 1 ]; then
+	if [ $_LOGGER_SILENT == true ]; then
+		Statistics
+	else
+		QuickLogger "In order to make usage statistics, the script would like to connect to $STATS_LINK"
+		read -r -p "No data except those in the url will be send. Allow [Y/n] " response
+		case $response in
+			[nN])
+			exit
+			;;
+			*)
+			Statistics
+			exit $?
+			;;
+		esac
+	fi
+fi

obackup-batch.sh

@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+SUBPROGRAM=obackup
+PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
+AUTHOR="(L) 2013-2016 by Orsiris de Jong"
+CONTACT="http://www.netpower.fr - ozy@netpower.fr"
+PROGRAM_BUILD=2016120401
+
+## Runs an osync /obackup instance for every conf file found
+## If an instance fails, run it again if time permits
+
+if ! type "$BASH" > /dev/null; then
+	echo "Please run this script only with bash shell. Tested on bash >= 3.2"
+	exit 127
+fi
+
+## If maximum execution time is not reached, failed instances will be rerun. Max exec time is in seconds. Example is set to 10 hours.
+MAX_EXECUTION_TIME=36000
+
+## Specifies the number of total runs an instance may get
+MAX_RUNS=3
+
+## Log file path
+if [ -w /var/log ]; then
+	LOG_FILE=/var/log/$SUBPROGRAM-batch.log
+else
+	LOG_FILE=./$SUBPROGRAM-batch.log
+fi
+
+# No need to edit under this line ##############################################################
+
+function _logger {
+	local value="${1}" # What to log
+	echo -e "$value" >> "$LOG_FILE"
+}
+
+function Logger {
+	local value="${1}" # What to log
+	local level="${2}" # Log level: DEBUG, NOTICE, WARN, ERROR, CRITIAL
+
+	prefix="$(date) - "
+
+	if [ "$level" == "CRITICAL" ]; then
+		_logger "$prefix\e[41m$value\e[0m"
+	elif [ "$level" == "ERROR" ]; then
+		_logger "$prefix\e[91m$value\e[0m"
+	elif [ "$level" == "WARN" ]; then
+		_logger "$prefix\e[93m$value\e[0m"
+	elif [ "$level" == "NOTICE" ]; then
+		_logger "$prefix$value"
+	elif [ "$level" == "DEBUG" ]; then
+		if [ "$DEBUG" == "yes" ]; then
+			_logger "$prefix$value"
+		fi
+	else
+		_logger "\e[41mLogger function called without proper loglevel.\e[0m"
+		_logger "$prefix$value"
+	fi
+}
+
+function CheckEnvironment {
+	## osync / obackup executable full path can be set here if it cannot be found on the system
+	if ! type $SUBPROGRAM.sh > /dev/null 2>&1
+	then
+		if [ -f /usr/local/bin/$SUBPROGRAM.sh ]
+		then
+			SUBPROGRAM_EXECUTABLE=/usr/local/bin/$SUBPROGRAM.sh
+		else
+			Logger "Could not find [/usr/local/bin/$SUBPROGRAM.sh]" "CRITICAL"
+			( >&2 echo "Could not find [/usr/local/bin/$SUBPROGRAM.sh]" )
+			exit 1
+		fi
+	else
+		SUBPROGRAM_EXECUTABLE=$(type -p $SUBPROGRAM.sh)
+	fi
+
+	if [ "$CONF_FILE_PATH" == "" ]; then
+		Usage
+	fi
+}
+
+function Batch {
+	local runs=1 # Number of batch runs
+	local runList # Actual conf file list to run
+	local runAgainList # List of failed conf files sto run again
+
+	local confFile
+	local result
+
+	local i
+
+	# Using -e because find will accept directories or files
+	if [ ! -e "$CONF_FILE_PATH" ]; then
+		Logger "Cannot find conf file path [$CONF_FILE_PATH]." "CRITICAL"
+		Usage
+	else
+		# Ugly hack to read files into an array while preserving special characters
+		runList=()
+		while IFS= read -d $'\0' -r file; do runList+=("$file"); done < <(find "$CONF_FILE_PATH" -maxdepth 1 -iname "*.conf" -print0)
+
+		while ([ $MAX_EXECUTION_TIME -gt $SECONDS ] || [ $MAX_EXECUTION_TIME -eq 0 ]) && [ "${#runList[@]}" -gt 0 ] && [ $runs -le $MAX_RUNS ]; do
+			runAgainList=()
+			Logger "Sequential run n°$runs of $SUBPROGRAM instances for:" "NOTICE"
+			for confFile in "${runList[@]}"; do
+				Logger "$(basename $confFile)" "NOTICE"
+			done
+			for confFile in "${runList[@]}"; do
+				$SUBPROGRAM_EXECUTABLE "$confFile" --silent $opts &
+				wait $!
+				result=$?
+				if [ $result != 0 ]; then
+					if [ $result == 1 ] || [ $result == 128 ]; then # Do not handle exit code 128 because it is already handled here
+						Logger "Instance $(basename $confFile) failed with exit code [$result]." "ERROR"
+						runAgainList+=("$confFile")
+					elif [ $result == 2 ]; then
+						Logger "Instance $(basename $confFile) finished with warnings." "WARN"
+					fi
+				else
+					Logger "Instance $(basename $confFile) succeed." "NOTICE"
+				fi
+			done
+			runList=("${runAgainList[@]}")
+			runs=$(($runs + 1))
+		done
+	fi
+}
+
+function Usage {
+	echo "$PROGRAM $PROGRAM_BUILD"
+	echo $AUTHOR
+	echo $CONTACT
+	echo ""
+	echo "Batch script to sequentially run osync or obackup instances and rerun failed ones."
+	echo "Usage: $PROGRAM.sh [OPTIONS] [$SUBPROGRAM OPTIONS]"
+	echo ""
+	echo "[OPTIONS]"
+	echo "--path=/path/to/conf      Path to osync / obackup conf files, defaults to /etc/osync or /etc/obackup"
+	echo "--max-runs=X              Number of max runs per instance, (defaults to 3)"
+	echo "--max-exec-time=X         Retry failed instances only if max execution time not reached (defaults to 36000 seconds). Set to 0 to bypass execution time check"
+	echo "[$SUBPROGRAM OPTIONS]"
+	echo "Specify whatever options $PROGRAM accepts. Example"
+	echo "$PROGRAM.sh --path=/etc/$SUBPROGRAM --no-maxtime"
+	echo ""
+	echo "No output will be written to stdout/stderr."
+	echo "Verify log file in [$LOG_FILE]."
+	exit 128
+}
+
+opts=""
+for i in "$@"
+do
+	case $i in
+		--path=*)
+		CONF_FILE_PATH=${i##*=}
+		;;
+		--max-runs=*)
+		MAX_RUNS=${i##*=}
+		;;
+		--max-exec-time=*)
+		MAX_EXECUTION_TIME=${i##*=}
+		;;
+		--help|-h|-?)
+		Usage
+		;;
+		*)
+		opts="$opts$i "
+		;;
+	esac
+done
+
+CheckEnvironment
+Logger "$(date) $SUBPROGRAM batch run" "NOTICE"
+Batch

ssh_filter.sh

@@ -1,79 +1,53 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-##### Obackup / Osync ssh command filter build 2306201301
+##### osync / obackup ssh command filter
-##### This script should be located in /usr/local/bin in the remote system that will be backed up / synced
+##### This script should be located in /usr/local/bin in the remote system to sync / backup
 ##### It will filter the commands that can be run remotely via ssh.
 ##### Please chmod 755 and chown root:root this file
 
-## If enabled, execution of "sudo" command will be allowed.
+##### Any command that has env _REMOTE_TOKEN= with the corresponding token in it will be run
+##### Any other command will return a "syntax error"
+##### For details, see ssh_filter.log
+
+SCRIPT_BUILD=2017020802
+
+## Allow sudo
 SUDO_EXEC=yes
-## Paranoia option. Don't change this unless you read the documentation and still feel concerned about security issues.
-RSYNC_EXECUTABLE=rsync
-## Enable other commands, useful for remote execution hooks like remotely creating snapshots.
-CMD1=
-CMD2=
-CMD3=
 
-LOG_FILE=~/.ssh/ssh_filter.log
+## Log all valid commands too
+_DEBUG=no
 
-function Log
+## Set remote token in authorized_keys
-{
+if [ "$1" != "" ]; then
+	_REMOTE_TOKEN="${1}"
+fi
+
+LOG_FILE="${HOME}/.ssh/ssh_filter.log"
+
+function Log {
 	DATE=$(date)
-	echo "$DATE - $1" >> $LOG_FILE
+	echo "$DATE - $1" >> "$LOG_FILE"
 }
 
-function Go
+function Go {
-{
+	if [ "$_DEBUG" == "yes" ]; then
-	eval $SSH_ORIGINAL_COMMAND
+		Log "Executing [$SSH_ORIGINAL_COMMAND]."
-}
-
-case ${SSH_ORIGINAL_COMMAND%% *} in
-	"$RSYNC_EXECUTABLE")
-	Go ;;
-	"mysqldump")
-	Go ;;
-	"mysql")
-	Go ;;
-	"echo")
-	Go ;;
-	"find")
-	Go ;;
-	"du")
-	Go ;;
-	"$CMD1")
-	Go ;;
-	"$CMD2")
-	Go ;;
-	"$CMD3")
-	Go ;;
-	"sudo")
-	if [ "$SUDO_EXEC" == "yes" ]
-	then
-		if [[ "$SSH_ORIGINAL_COMMAND" == "sudo $RSYNC_EXECUTABLE"* ]]
-		then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo du"* ]]
-		then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo find"* ]]
-		then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo $CMD1"* ]]
-		then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo $CMD2"* ]]
-		then
-			Go
-		elif [[ "$SSH_ORIGINAL_COMMAND" == "sudo $CMD3"* ]]
-		then
-			Go
-		else
-			Log "Command [$SSH_ORIGINAL_COMMAND] not allowed."
-		fi
-	else
-		Log "Command [$SSH_ORIGINAL_COMMAND] not allowed. sudo not enabled."
 	fi
+	eval "$SSH_ORIGINAL_COMMAND"
+}
+
+case "${SSH_ORIGINAL_COMMAND}" in
+	*"env _REMOTE_TOKEN=$_REMOTE_TOKEN"*)
+		if [ "$SUDO_EXEC" != "yes" ] && [[ $SSH_ORIGINAL_COMMAND == *"sudo "* ]]; then
+			Log "Command [$SSH_ORIGINAL_COMMAND] contains sudo which is not allowed."
+			echo "Syntax error unexpected end of file"
+			exit 1
+		fi
+	Go
 	;;
 	*)
 	Log "Command [$SSH_ORIGINAL_COMMAND] not allowed."
+	echo "Syntax error near unexpected token"
+	exit 1
+	;;
 esac

upgrade-v1.x-2.1x.sh

@@ -0,0 +1,339 @@
+#!/usr/bin/env bash
+
+PROGRAM="obackup.upgrade"
+SUBPROGRAM="obackup"
+AUTHOR="(C) 2016 by Orsiris de Jong"
+CONTACT="http://www.netpower.fr/obacup - ozy@netpower.fr"
+OLD_PROGRAM_VERSION="v1.x"
+NEW_PROGRAM_VERSION="v2.1x"
+CONFIG_FILE_VERSION=2017020901
+PROGRAM_BUILD=2016113001
+
+if ! type "$BASH" > /dev/null; then
+        echo "Please run this script only with bash shell. Tested on bash >= 3.2"
+        exit 127
+fi
+
+# Defines all keywords / value sets in obackup configuration files
+# bash does not support two dimensional arrays, so we declare two arrays:
+# ${KEYWORDS[index]}=${VALUES[index]}
+
+KEYWORDS=(
+INSTANCE_ID
+LOGFILE
+SQL_BACKUP
+FILE_BACKUP
+BACKUP_TYPE
+SQL_STORAGE
+FILE_STORAGE
+ENCRYPTION
+CRYPT_STORAGE
+GPG_RECIPIENT
+PARALLEL_ENCRYPTION_PROCESSES
+CREATE_DIRS
+KEEP_ABSOLUTE_PATHS
+BACKUP_SIZE_MINIMUM
+GET_BACKUP_SIZE
+SQL_WARN_MIN_SPACE
+FILE_WARN_MIN_SPACE
+REMOTE_SYSTEM_URI
+SSH_RSA_PRIVATE_KEY
+SSH_PASSWORD_FILE
+_REMOTE_TOKEN
+SSH_COMPRESSION
+SSH_IGNORE_KNOWN_HOSTS
+RSYNC_REMOTE_PATH
+REMOTE_HOST_PING
+REMOTE_3RD_PARTY_HOSTS
+SUDO_EXEC
+SQL_USER
+DATABASES_ALL
+DATABASES_ALL_EXCLUDE_LIST
+DATABASES_LIST
+SOFT_MAX_EXEC_TIME_DB_TASK
+HARD_MAX_EXEC_TIME_DB_TASK
+MYSQLDUMP_OPTIONS
+COMPRESSION_LEVEL
+DIRECTORY_LIST
+RECURSIVE_DIRECTORY_LIST
+RECURSIVE_EXCLUDE_LIST
+RSYNC_PATTERN_FIRST
+RSYNC_INCLUDE_PATTERN
+RSYNC_EXCLUDE_PATTERN
+RSYNC_INCLUDE_FROM
+RSYNC_EXCLUDE_FROM
+PATH_SEPARATOR_CHAR
+RSYNC_OPTIONAL_ARGS
+PRESERVE_PERMISSIONS
+PRESERVE_OWNER
+PRESERVE_GROUP
+PRESERVE_EXECUTABILITY
+PRESERVE_ACL
+PRESERVE_XATTR
+COPY_SYMLINKS
+KEEP_DIRLINKS
+PRESERVE_HARDLINKS
+RSYNC_COMPRESS
+SOFT_MAX_EXEC_TIME_FILE_TASK
+HARD_MAX_EXEC_TIME_FILE_TASK
+PARTIAL
+DELETE_VANISHED_FILES
+DELTA_COPIES
+BANDWIDTH
+RSYNC_EXECUTABLE
+DESTINATION_MAILS
+MAIL_BODY_CHARSET
+SENDER_MAIL
+SMTP_SERVER
+SMTP_PORT
+SMTP_ENCRYPTION
+SMTP_USER
+SMTP_PASSWORD
+SOFT_MAX_EXEC_TIME_TOTAL
+HARD_MAX_EXEC_TIME_TOTAL
+KEEP_LOGGING
+ROTATE_SQL_BACKUPS
+ROTATE_SQL_COPIES
+ROTATE_FILE_BACKUPS
+ROTATE_FILE_COPIES
+LOCAL_RUN_BEFORE_CMD
+LOCAL_RUN_AFTER_CMD
+REMOTE_RUN_BEFORE_CMD
+REMOTE_RUN_AFTER_CMD
+MAX_EXEC_TIME_PER_CMD_BEFORE
+MAX_EXEC_TIME_PER_CMD_AFTER
+STOP_ON_CMD_ERROR
+RUN_AFTER_CMD_ON_ERROR
+)
+
+VALUES=(
+test-backup
+''
+yes
+yes
+local
+/home/storage/sql
+/home/storage/files
+no
+/home/storage/crypt
+'Your Name used with GPG signature'
+''
+yes
+yes
+1024
+yes
+1048576
+1048576
+ssh://backupuser@remote.system.tld:22/
+${HOME}/.ssh/id_rsa
+''
+SomeAlphaNumericToken9
+yes
+no
+''
+yes
+'www.kernel.org www.google.com'
+no
+root
+yes
+test
+''
+3600
+7200
+'--opt --single-transaction'
+3
+/some/path
+/home
+/home/backupuser\;/host/lost+found
+include
+''
+''
+''
+''
+\;
+''
+yes
+yes
+yes
+yes
+no
+no
+yes
+yes
+no
+no
+3600
+7200
+no
+no
+yes
+0
+rsync
+infrastructure@example.com
+''
+sender@example.com
+smtp.isp.tld
+25
+none
+''
+''
+30000
+36000
+1801
+no
+7
+no
+7
+''
+''
+''
+''
+0
+0
+no
+no
+)
+
+function Usage {
+	echo "$PROGRAM $PROGRAM_BUILD"
+	echo $AUTHOR
+	echo $CONTACT
+	echo ""
+	echo "This script migrates $SUBPROGRAM $OLD_PROGRAM_VERSION config files to $NEW_PROGRAM_VERSION."
+	echo ""
+	echo "Usage: $0 /path/to/config_file.conf"
+	echo "Please make sure the config file is writable."
+	exit 128
+}
+
+function LoadConfigFile {
+	local config_file="${1}"
+
+	if [ ! -f "$config_file" ]; then
+		echo "Cannot load configuration file [$config_file]. Sync cannot start."
+		exit 1
+	elif [[ "$1" != *".conf" ]]; then
+		echo "Wrong configuration file supplied [$config_file]. Sync cannot start."
+		exit 1
+	else
+		egrep '^#|^[^ ]*=[^;&]*'  "$config_file" > "./$SUBPROGRAM.$FUNCNAME.$$"
+		source "./$SUBPROGRAM.$FUNCNAME.$$"
+		rm -f "./$SUBPROGRAM.$FUNCNAME.$$"
+	fi
+}
+
+function RewriteOldConfigFiles {
+	local config_file="${1}"
+
+	if ! grep "BACKUP_ID=" $config_file > /dev/null && ! grep "INSTANCE_ID=" $config_file > /dev/null; then
+		echo "File [$config_file] does not seem to be a obackup config file."
+		exit 1
+	fi
+
+	echo "Backing up [$config_file] as [$config_file.save]"
+	cp -p "$config_file" "$config_file.save"
+	if [ $? != 0 ]; then
+		echo "Cannot backup config file."
+		exit 1
+	fi
+
+	echo "Rewriting config file $config_file"
+
+	sed -i'.tmp' 's/^BACKUP_ID=/INSTANCE_ID=/g' "$config_file"
+	sed -i'.tmp' 's/^BACKUP_SQL=/SQL_BACKUP=/g' "$config_file"
+	sed -i'.tmp' 's/^BACKUP_FILES=/FILE_BACKUP=/g' "$config_file"
+	sed -i'.tmp' 's/^LOCAL_SQL_STORAGE=/SQL_STORAGE=/g' "$config_file"
+	sed -i'.tmp' 's/^LOCAL_FILE_STORAGE=/FILE_STORAGE=/g' "$config_file"
+
+	sed -i'.tmp' 's/^DISABLE_GET_BACKUP_FILE_SIZE=no/GET_BACKUP_SIZE=yes/g' "$config_file"
+	sed -i'.tmp' 's/^DISABLE_GET_BACKUP_FILE_SIZE=yes/GET_BACKUP_SIZE=no/g' "$config_file"
+	sed -i'.tmp' 's/^LOCAL_STORAGE_KEEP_ABSOLUTE_PATHS=/KEEP_ABSOLUTE_PATHS=/g' "$config_file"
+	sed -i'.tmp' 's/^LOCAL_STORAGE_WARN_MIN_SPACE=/SQL_WARN_MIN_SPACE=/g' "$config_file"
+	if ! grep "^FILE_WARN_MIN_SPACE=" "$config_file" > /dev/null; then
+		VALUE=$(cat $config_file | grep "SQL_WARN_MIN_SPACE=")
+		VALUE=${VALUE#*=}
+		sed -i'.tmp' '/^SQL_WARN_MIN_SPACE=*/a\'$'\n''FILE_WARN_MIN_SPACE='$VALUE'\'$'\n''' "$config_file"
+	fi
+	sed -i'.tmp' 's/^DIRECTORIES_SIMPLE_LIST=/DIRECTORY_LIST=/g' "$config_file"
+	sed -i'.tmp' 's/^DIRECTORIES_RECURSE_LIST=/RECURSIVE_DIRECTORY_LIST=/g' "$config_file"
+	sed -i'.tmp' 's/^DIRECTORIES_RECURSE_EXCLUDE_LIST=/RECURSIVE_EXCLUDE_LIST=/g' "$config_file"
+	sed -i'.tmp' 's/^ROTATE_BACKUPS=/ROTATE_SQL_BACKUPS=/g' "$config_file"
+	if ! grep "^ROTATE_FILE_BACKUPS=" "$config_file" > /dev/null; then
+		VALUE=$(cat $config_file | grep "ROTATE_SQL_BACKUPS=")
+		VALUE=${VALUE#*=}
+		sed -i'.tmp' '/^ROTATE_SQL_BACKUPS=*/a\'$'\n''ROTATE_FILE_BACKUPS='$VALUE'\'$'\n''' "$config_file"
+	fi
+	sed -i'.tmp' 's/^ROTATE_COPIES=/ROTATE_SQL_COPIES=/g' "$config_file"
+	if ! grep "^ROTATE_FILE_COPIES=" "$config_file" > /dev/null; then
+		VALUE=$(cat $config_file | grep "ROTATE_SQL_COPIES=")
+		VALUE=${VALUE#*=}
+		sed -i'.tmp' '/^ROTATE_SQL_COPIES=*/a\'$'\n''ROTATE_FILE_COPIES='$VALUE'\'$'\n''' "$config_file"
+	fi
+	REMOTE_BACKUP=$(cat $config_file | grep "REMOTE_BACKUP=")
+	REMOTE_BACKUP=${REMOTE_BACKUP#*=}
+	if [ "$REMOTE_BACKUP" == "yes" ]; then
+		REMOTE_USER=$(cat $config_file | grep "REMOTE_USER=")
+		REMOTE_USER=${REMOTE_USER#*=}
+		REMOTE_HOST=$(cat $config_file | grep "REMOTE_HOST=")
+		REMOTE_HOST=${REMOTE_HOST#*=}
+		REMOTE_PORT=$(cat $config_file | grep "REMOTE_PORT=")
+		REMOTE_PORT=${REMOTE_PORT#*=}
+
+		REMOTE_SYSTEM_URI="ssh://$REMOTE_USER@$REMOTE_HOST:$REMOTE_PORT/"
+
+		sed -i'.tmp' 's#^REMOTE_BACKUP=yes#REMOTE_SYSTEM_URI='$REMOTE_SYSTEM_URI'#g' "$config_file"
+		sed -i'.tmp' '/^REMOTE_USER==*/d' "$config_file"
+		sed -i'.tmp' '/^REMOTE_HOST==*/d' "$config_file"
+		sed -i'.tmp' '/^REMOTE_PORT==*/d' "$config_file"
+
+		sed -i'.tmp' '/^INSTANCE_ID=*/a\'$'\n''BACKUP_TYPE=pull\'$'\n''' "$config_file"
+	else
+		if ! grep "^BACKUP_TYPE=" "$config_file" > /dev/null; then
+			sed -i'.tmp' '/^INSTANCE_ID=*/a\'$'\n''BACKUP_TYPE=local\'$'\n''' "$config_file"
+		fi
+	fi
+	sed -i'.tmp' 's/^REMOTE_3RD_PARTY_HOST=/REMOTE_3RD_PARTY_HOSTS=/g' "$config_file"
+}
+
+function AddMissingConfigOptions {
+	local config_file="${1}"
+	local counter=0
+
+	while [ $counter -lt ${#KEYWORDS[@]} ]; do
+		if ! grep "^${KEYWORDS[$counter]}=" > /dev/null "$config_file"; then
+			echo "${KEYWORDS[$counter]} not found"
+			if [ $counter -gt 0 ]; then
+				sed -i'.tmp' '/^'${KEYWORDS[$((counter-1))]}'=*/a\'$'\n'${KEYWORDS[$counter]}'="'"${VALUES[$counter]}"'"\'$'\n''' "$config_file"
+				if [ $? -ne 0 ]; then
+					echo "Cannot add missing ${[KEYWORDS[$counter]}."
+					exit 1
+				fi
+			else
+				sed -i'.tmp' '/onfig file rev*/a\'$'\n'${KEYWORDS[$counter]}'="'"${VALUES[$counter]}"'"\'$'\n''' "$config_file"
+			fi
+			echo "Added missing ${KEYWORDS[$counter]} config option with default option [${VALUES[$counter]}]"
+		fi
+		counter=$((counter+1))
+	done
+}
+
+function UpdateConfigHeader {
+	local config_file="${1}"
+
+	# "onfig file rev" to deal with earlier variants of the file
+        sed -i'.tmp' 's/.*onfig file rev.*/##### '$SUBPROGRAM' config file rev '$CONFIG_FILE_VERSION' '$NEW_PROGRAM_VERSION'/' "$config_file"
+
+	rm -f "$config_file.tmp"
+}
+
+if [ "$1" != "" ] && [ -f "$1" ] && [ -w "$1" ]; then
+	CONF_FILE="$1"
+	# Make sure there is no ending slash
+	CONF_FILE="${CONF_FILE%/}"
+	LoadConfigFile "$CONF_FILE"
+	RewriteOldConfigFiles "$CONF_FILE"
+	AddMissingConfigOptions "$CONF_FILE"
+	UpdateConfigHeader "$CONF_FILE"
+else
+	Usage
+fi