Linux/obackup
1
0
Fork 0
mirror of https://github.com/deajan/obackup.git synced 2024-11-15 04:03:41 +01:00
Code Issues Releases Wiki Activity

Rebuilt targets

Browse Source
This commit is contained in:
deajan 2018-11-06 15:33:16 +01:00
parent 9cf5574488
commit 768d17c2cb
4 changed files with 941 additions and 402 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

249
dev/debug_obackup.sh
View File

@ -7,7 +7,7 @@ PROGRAM="obackup"
AUTHOR="(C) 2013-2018 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
PROGRAM_VERSION=2.1-RC1
PROGRAM_BUILD=2018093008
PROGRAM_BUILD=2018110601
IS_STABLE=no
#### Execution order #__WITH_PARANOIA_DEBUG
@ -34,22 +34,10 @@ IS_STABLE=no
# RsyncPatterns #__WITH_PARANOIA_DEBUG
# FilesBackup #__WITH_PARANOIA_DEBUG
_OFUNCTIONS_VERSION=2.3.0-RC1
_OFUNCTIONS_BUILD=2018100105
_OFUNCTIONS_VERSION=2.3.0-RC2
_OFUNCTIONS_BUILD=2018110502
_OFUNCTIONS_BOOTSTRAP=true
## To use in a program, define the following variables:
## PROGRAM=program-name
## INSTANCE_ID=program-instance-name
## _DEBUG=yes/no
## _LOGGER_SILENT=true/false
## _LOGGER_VERBOSE=true/false
## _LOGGER_ERR_ONLY=true/false
## _LOGGER_PREFIX="date"/"time"/""
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
exit 127
@ -116,7 +104,9 @@ else
LOG_FILE="/tmp/$PROGRAM.log"
fi
#### RUN_DIR SUBSET ####
## Default directory where to store temporary run files
if [ -w /tmp ]; then
RUN_DIR=/tmp
elif [ -w /var/tmp ]; then
@ -125,8 +115,38 @@ else
RUN_DIR=.
fi
#### PoorMansRandomGenerator SUBSET ####
# Get a random number on Windows BusyBox alike, also works on most Unixes
## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator.
if [ "$_REMOTE_EXECUTION" == true ]; then
mkdir -p "$RUN_DIR/$PROGRAM.remote"
RUN_DIR="$RUN_DIR/$PROGRAM.remote"
fi
#### RUN_DIR SUBSET END ####
# Get a random number on Windows BusyBox alike, also works on most Unixes that have dd, if dd is not found, then return $RANDOM
function PoorMansRandomGenerator {
local digits="${1}" # The number of digits to generate
local number
local isFirst=true
if type dd >/dev/null 2>&1; then
# Some read bytes can't be used, se we read twice the number of required bytes
dd if=/dev/urandom bs=$digits count=2 2> /dev/null | while read -r -n1 char; do
if [ $isFirst == false ] || [ $(printf "%d" "'$char") != "0" ]; then
number=$number$(printf "%d" "'$char")
isFirst=false
fi
if [ ${#number} -ge $digits ]; then
echo ${number:0:$digits}
break;
fi
done
elif [ "$RANDOM" -ne 0 ]; then
echo $RANDOM
else
Logger "Cannot generate random number." "ERROR"
fi
}
function PoorMansRandomGenerator {
local digits="${1}" # The number of digits to generate
local number
@ -140,10 +160,9 @@ function PoorMansRandomGenerator {
fi
done
}
#### PoorMansRandomGenerator SUBSET END ####
# Initial TSTMAP value before function declaration
TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4)
TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 5)
# Default alert attachment filename
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log"
@ -153,13 +172,6 @@ set -o pipefail
set -o errtrace
function Dummy {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
sleep $SLEEP_TIME
}
# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array
# usage: joinString separaratorChar Array
function joinString {
@ -198,6 +210,8 @@ function RemoteLogger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -274,6 +288,8 @@ function Logger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -338,6 +354,26 @@ function Logger {
fi
}
# Function is busybox compatible since busybox ash does not understand direct regex, we use expr
function IsInteger {
local value="${1}"
if type expr > /dev/null 2>&1; then
expr "$value" : '^[0-9]\{1,\}$' > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo 1
else
echo 0
fi
else
if [[ $value =~ ^[0-9]+$ ]]; then
echo 1
else
echo 0
fi
fi
}
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds {
local pid="${1}" # Parent pid to kill childs
@ -405,6 +441,33 @@ function KillAllChilds {
return $errorcount
}
function CleanUp {
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp"
fi
}
function GenericTrapQuit {
local exitcode=0
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
WARN_ALERT=true
exitcode=2
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
ERROR_ALERT=true
exitcode=1
fi
CleanUp
exit $exitcode
}
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
function SendAlert {
local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
@ -803,10 +866,11 @@ function ExecTasks {
local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries
local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors
__CheckArguments 1-18 $# "$@" #__WITH_PARANOIA_DEBUG
local i
Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 1-18 $# "$@" #__WITH_PARANOIA_DEBUG
Logger "${FUNCNAME[0]} id [$id] called by [${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} < ${FUNCNAME[5]} < ${FUNCNAME[6]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
# Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode
if [ "$_DEBUG" == "yes" ]; then
@ -822,9 +886,6 @@ function ExecTasks {
done
fi
# Change '-' to '_' in task id
id="${id/-/_}"
# Expand validExitCodes into array
IFS=';' read -r -a validExitCodes <<< "$validExitCodes"
@ -861,16 +922,10 @@ function ExecTasks {
local newPidsArray # New array of currently running pids for next iteration
local pidsTimeArray # Array containing execution begin time of pids
local executeCommand # Boolean to check if currentCommand can be executed given a condition
local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG
local functionMode
if [ $counting == true ]; then
local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once
else
local softAlert=false
fi
local failedPidsList # List containing failed pids with exit code separated by semicolons (eg : 2355:1;4534:2;2354:3)
# Initialise global variable
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
@ -1031,17 +1086,17 @@ function ExecTasks {
# Check for valid exit codes
if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then
if [ $noErrorLogsAtAll != true ]; then
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR"
if [ "$functionMode" == "ParallelExec" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
fi
fi
errorcount=$((errorcount+1))
# Welcome to variable variable bash hell
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$pid:$retval\""
if [ "$failedPidsList" == "" ]; then
failedPidsList="$pid:$retval"
else
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
failedPidsList="$failedPidsList;$pid:$retval"
fi
else
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
@ -1203,6 +1258,8 @@ function ExecTasks {
# Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$failedPidsList\""
if [ $mainItemCount -eq 1 ]; then
return $retval
else
@ -1210,16 +1267,6 @@ function ExecTasks {
fi
}
function CleanUp {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp"
fi
}
# Usage: var=$(StripSingleQuotes "$var")
function StripSingleQuotes {
local string="${1}"
@ -1258,40 +1305,19 @@ function EscapeDoubleQuotes {
echo "${value//\"/\\\"}"
}
function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
if [[ $value =~ ^-?[0-9]+([.][0-9]+)?$ ]]; then
echo 1
else
echo 0
fi
}
# Usage [ $(IsNumeric $var) -eq 1 ]
function IsNumeric {
local value="${1}"
if [[ $value =~ ^[0-9]+([.][0-9]+)?$ ]]; then
echo 1
else
echo 0
fi
}
# Function is busybox compatible since busybox ash does not understand direct regex, we use expr
function IsInteger {
local value="${1}"
if type expr > /dev/null 2>&1; then
expr "$value" : "^[0-9]\+$" > /dev/null 2>&1
expr "$value" : '^[-+]\{0,1\}[0-9]*\.\{0,1\}[0-9]\{1,\}$' > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo 1
else
echo 0
fi
else
if [[ $value =~ ^[0-9]+$ ]]; then
if [[ $value =~ ^[-+]?[0-9]+([.][0-9]+)?$ ]]; then
echo 1
else
echo 0
@ -1299,6 +1325,12 @@ function IsInteger {
fi
}
function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
echo $(IsNumeric "$value")
}
# Converts human readable sizes into integer kilobyte sizes
# Usage numericSize="$(HumanToNumeric $humanSize)"
function HumanToNumeric {
@ -1395,6 +1427,8 @@ function GetLocalOS {
# There is no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
elif set -o | grep "winxp" > /dev/null; then
localOsVar="BusyBox-w32"
else
# Detecting the special ubuntu userland in Windows 10 bash
if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
@ -1427,7 +1461,7 @@ function GetLocalOS {
*"CYGWIN"*)
LOCAL_OS="Cygwin"
;;
*"Microsoft"*)
*"Microsoft"*|*"MS/Windows"*)
LOCAL_OS="WinNT10"
;;
*"Darwin"*)
@ -1458,7 +1492,8 @@ function GetLocalOS {
fi
# Get Host info for Windows
if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then localOsVar="$(uname -a)"
if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then
localOsVar="$localOsVar $(uname -a)"
if [ "$PROGRAMW6432" != "" ]; then
LOCAL_OS_BITNESS=64
LOCAL_OS_FAMILY="Windows"
@ -1472,6 +1507,9 @@ function GetLocalOS {
# Get Host info for Unix
else
LOCAL_OS_FAMILY="Unix"
fi
if [ "$LOCAL_OS_FAMILY" == "Unix" ]; then
if uname -m | grep '64' > /dev/null 2>&1; then
LOCAL_OS_BITNESS=64
else
@ -2088,14 +2126,14 @@ function InitLocalOSDependingSettings {
function InitRemoteOSDependingSettings {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
if [ "$REMOTE_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then
if [ "$REMOTE_OS" == "msys" ] || [ "$REMOTE_OS" == "Cygwin" ]; then
REMOTE_FIND_CMD=$(dirname $BASH)/find
else
REMOTE_FIND_CMD=find
fi
## Stat command has different syntax on Linux and FreeBSD/MacOSX
if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then
if [ "$REMOTE_OS" == "MacOSX" ] || [ "$REMOTE_OS" == "BSD" ]; then
REMOTE_STAT_CMD="stat -f \"%Sm\""
REMOTE_STAT_CTIME_MTIME_CMD="stat -f \\\"%N;%c;%m\\\""
else
@ -2274,13 +2312,26 @@ function SetConfFileValue () {
local value="${3}"
local separator="${4:-#}"
if grep "^$name=" "$file" > /dev/null; then
if [ -f "$file" ]; then
if grep "^$name=" "$file" > /dev/null 2>&1; then
# Using -i.tmp for BSD compat
sed -i.tmp "s$separator^$name=.*$separator$name=$value$separator" "$file"
if [ $? -ne 0 ]; then
Logger "Cannot update value [$name] to [$value] in config file [$file]." "ERROR"
fi
rm -f "$file.tmp"
Logger "Set [$name] to [$value] in config file [$file]." "DEBUG"
else
Logger "Cannot set value [$name] to [$value] in config file [$file]." "ERROR"
echo "$name=$value" >> "$file"
if [ $? -ne 0 ]; then
Logger "Cannot create value [$name] to [$value] in config file [$file]." "ERROR"
fi
fi
else
echo "$name=$value" > "$file"
if [ $? -ne 0 ]; then
Logger "Config file [$file] does not exist. Failed to create it witn value [$name]." "ERROR"
fi
fi
}
@ -2709,7 +2760,7 @@ function _ListRecursiveBackupDirectoriesRemote {
$SSH_CMD env _REMOTE_TOKEN=$_REMOTE_TOKEN \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env RECURSIVE_DIRECTORY_LIST="'$RECURSIVE_DIRECTORY_LIST'" env PATH_SEPARATOR_CHAR="'$PATH_SEPARATOR_CHAR'" \
env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
## allow function call checks #__WITH_PARANOIA_DEBUG
@ -2777,6 +2828,8 @@ function RemoteLogger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -3002,7 +3055,8 @@ function _GetDirectoriesSizeRemote {
# Error output is different from stdout because not all files in list may fail at once
$SSH_CMD env _REMOTE_TOKEN=$_REMOTE_TOKEN \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" dirList="'$dirList'" \
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
dirList="'$dirList'" \
$COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" &
## allow function call checks #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG
@ -3069,6 +3123,8 @@ function RemoteLogger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -3205,7 +3261,7 @@ function _CreateDirectoryRemote {
$SSH_CMD env _REMOTE_TOKEN=$_REMOTE_TOKEN \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env _REMOTE_EXECUTION=true env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env dirToCreate="'$dirToCreate'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
## allow function call checks #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG
@ -3272,6 +3328,8 @@ function RemoteLogger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -3433,7 +3491,7 @@ function GetDiskSpaceRemote {
$SSH_CMD env _REMOTE_TOKEN=$_REMOTE_TOKEN \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env _REMOTE_EXECUTION=true env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env DF_CMD="'$DF_CMD'" \
env pathToCheck="'$pathToCheck'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" &
## allow function call checks #__WITH_PARANOIA_DEBUG
@ -3501,6 +3559,8 @@ function RemoteLogger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -4144,7 +4204,7 @@ function Rsync {
## Manage to backup recursive directories lists files only (not recursing into subdirectories)
if [ $recursive == false ]; then
# Fixes symlinks to directories in target cannot be deleted when backing up root directory without recursion
rsyncArgs="$RSYNC_DEFAULT_NONRECURSIVE_ARGS -k"
rsyncArgs="$RSYNC_DEFAULT_ARGS -f '- /*/*/'"
else
rsyncArgs="$RSYNC_DEFAULT_ARGS"
fi
@ -4188,7 +4248,8 @@ function FilesBackup {
local backupTask
local backupTasks
local destinationDir
local withoutCryptPath
local encryptDir
IFS=$PATH_SEPARATOR_CHAR read -r -a backupTasks <<< "$FILE_BACKUP_TASKS"
@ -4202,6 +4263,7 @@ function FilesBackup {
else
destinationDir=$(dirname "$FILE_STORAGE/${backupTask#/}/")
fi
encryptDir="$FILE_STORAGE/${backupTask#/}"
else
destinationDir="$FILE_STORAGE"
encryptDir="$FILE_STORAGE"
@ -4228,7 +4290,7 @@ function FilesBackup {
IFS=$PATH_SEPARATOR_CHAR read -r -a backupTasks <<< "$RECURSIVE_DIRECTORY_LIST"
for backupTask in "${backupTasks[@]}"; do
# Backup recursive directories withouht recursion
# Backup recursive directories without recursion
if [ "$KEEP_ABSOLUTE_PATHS" != "no" ]; then
# Fix for backup of '/'
@ -4273,6 +4335,7 @@ function FilesBackup {
else
destinationDir=$(dirname "$FILE_STORAGE/${backupTask#/}/")
fi
encryptDir="$FILE_STORAGE/${backupTask#/}"
else
destinationDir="$FILE_STORAGE"
encryptDir="$FILE_STORAGE"
@ -4399,7 +4462,7 @@ function _RotateBackupsRemote {
$SSH_CMD env _REMOTE_TOKEN=$_REMOTE_TOKEN \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env _REMOTE_EXECUTION=true env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" env rotateCopies="'$rotateCopies'" env backupPath="'$backupPath'" \
$COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
## allow function call checks #__WITH_PARANOIA_DEBUG
@ -4467,6 +4530,8 @@ function RemoteLogger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -4628,9 +4693,6 @@ function Init {
local hosturiandpath
local hosturi
trap TrapStop INT QUIT TERM HUP
trap TrapQuit EXIT
## Test if target dir is a ssh uri, and if yes, break it down it its values
if [ "${REMOTE_SYSTEM_URI:0:6}" == "ssh://" ] && [ "$BACKUP_TYPE" != "local" ]; then
REMOTE_OPERATION="yes"
@ -4764,6 +4826,9 @@ function Usage {
exit 128
}
#### SCRIPT ENTRY POINT ####
trap TrapQuit EXIT
# Command line argument flags
_DRYRUN=false
no_maxtime=false

640
install.sh
View File

@ -10,15 +10,431 @@ PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh"
SSH_FILTER="ssh_filter.sh"
SCRIPT_BUILD=2018090301
SCRIPT_BUILD=2018100201
INSTANCE_ID="installer-$SCRIPT_BUILD"
## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
## Please adapt this to fit your distro needs
_OFUNCTIONS_VERSION=2.3.0-RC2
_OFUNCTIONS_BUILD=2018110502
_OFUNCTIONS_BOOTSTRAP=true
if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
exit 127
fi
## Correct output of sort command (language agnostic sorting)
export LC_ALL=C
## Default umask for file creation
umask 0077
# Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
# Environment variables that can be overriden by programs
_DRYRUN=false
_LOGGER_SILENT=false
_LOGGER_VERBOSE=false
_LOGGER_ERR_ONLY=false
_LOGGER_PREFIX="date"
if [ "$KEEP_LOGGING" == "" ]; then
KEEP_LOGGING=1801
fi
# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags
ERROR_ALERT=false
WARN_ALERT=false
## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no
_LOGGER_VERBOSE=false
else
trap 'TrapError ${LINENO} $?' ERR
_LOGGER_VERBOSE=true
fi
if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=.05
fi
SCRIPT_PID=$$
LOCAL_USER=$(whoami)
LOCAL_HOST=$(hostname)
if [ "$PROGRAM" == "" ]; then
PROGRAM="ofunctions"
fi
## Default log file until config file is loaded
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM.log"
elif [ -w . ]; then
LOG_FILE="./$PROGRAM.log"
else
LOG_FILE="/tmp/$PROGRAM.log"
fi
#### RUN_DIR SUBSET ####
## Default directory where to store temporary run files
if [ -w /tmp ]; then
RUN_DIR=/tmp
elif [ -w /var/tmp ]; then
RUN_DIR=/var/tmp
else
RUN_DIR=.
fi
## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator.
if [ "$_REMOTE_EXECUTION" == true ]; then
mkdir -p "$RUN_DIR/$PROGRAM.remote"
RUN_DIR="$RUN_DIR/$PROGRAM.remote"
fi
#### RUN_DIR SUBSET END ####
# Get a random number on Windows BusyBox alike, also works on most Unixes that have dd, if dd is not found, then return $RANDOM
function PoorMansRandomGenerator {
local digits="${1}" # The number of digits to generate
local number
local isFirst=true
if type dd >/dev/null 2>&1; then
# Some read bytes can't be used, se we read twice the number of required bytes
dd if=/dev/urandom bs=$digits count=2 2> /dev/null | while read -r -n1 char; do
if [ $isFirst == false ] || [ $(printf "%d" "'$char") != "0" ]; then
number=$number$(printf "%d" "'$char")
isFirst=false
fi
if [ ${#number} -ge $digits ]; then
echo ${number:0:$digits}
break;
fi
done
elif [ "$RANDOM" -ne 0 ]; then
echo $RANDOM
else
Logger "Cannot generate random number." "ERROR"
fi
}
function PoorMansRandomGenerator {
local digits="${1}" # The number of digits to generate
local number
# Some read bytes can't be used, se we read twice the number of required bytes
dd if=/dev/urandom bs=$digits count=2 2> /dev/null | while read -r -n1 char; do
number=$number$(printf "%d" "'$char")
if [ ${#number} -ge $digits ]; then
echo ${number:0:$digits}
break;
fi
done
}
# Initial TSTMAP value before function declaration
TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 5)
# Default alert attachment filename
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log"
# Set error exit code if a piped command fails
set -o pipefail
set -o errtrace
# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array
# usage: joinString separaratorChar Array
function joinString {
local IFS="$1"; shift; echo "$*";
}
# Sub function of Logger
function _Logger {
local logValue="${1}" # Log to file
local stdValue="${2}" # Log to screeen
local toStdErr="${3:-false}" # Log to stderr instead of stdout
if [ "$logValue" != "" ]; then
echo -e "$logValue" >> "$LOG_FILE"
# Build current log file for alerts if we have a sufficient environment
if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then
echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP"
fi
fi
if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then
if [ $toStdErr == true ]; then
# Force stderr color in subshell
(>&2 echo -e "$stdValue")
else
echo -e "$stdValue"
fi
fi
}
# Remote logger similar to below Logger, without log to file and alert flags
function RemoteLogger {
local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
prefix=""
fi
if [ "$level" == "CRITICAL" ]; then
_Logger "" "$prefix\e[1;33;41m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "WARN" ]; then
_Logger "" "$prefix\e[33m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "NOTICE" ]; then
if [ $_LOGGER_ERR_ONLY != true ]; then
_Logger "" "$prefix$value"
fi
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value"
return
elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then
_Logger "" "$prefix$value"
return
fi
else
_Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true
_Logger "" "Value was: $prefix$value" true
fi
}
# General log function with log levels:
# Environment variables
# _LOGGER_SILENT: Disables any output to stdout & stderr
# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel
# _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout
# Loglevels
# Except for VERBOSE, all loglevels are ALWAYS sent to log file
# CRITICAL, ERROR, WARN sent to stderr, color depending on level, level also logged
# NOTICE sent to stdout
# VERBOSE sent to stdout if _LOGGER_VERBOSE = true
# ALWAYS is sent to stdout unless _LOGGER_SILENT = true
# DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes
# SIMPLE is a wrapper for QuickLogger that does not use advanced functionality
function Logger {
local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="$(date '+%Y-%m-%d %H:%M:%S') - "
else
prefix=""
fi
## Obfuscate _REMOTE_TOKEN in logs (for ssh_filter usage only in osync and obackup)
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/__(o_O)__}"
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
ERROR_ALERT=true
# ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag.
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "ERROR" ]; then
_Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true
ERROR_ALERT=true
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "WARN" ]; then
_Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true
WARN_ALERT=true
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "NOTICE" ]; then
if [ "$_LOGGER_ERR_ONLY" != true ]; then
_Logger "$prefix$value" "$prefix$value"
fi
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix($level):$value" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
_Logger "$prefix$value" "$prefix$value"
return
elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then
_Logger "$prefix$value" "$prefix$value"
return
fi
elif [ "$level" == "SIMPLE" ]; then
if [ "$_LOGGER_SILENT" == true ]; then
_Logger "$preix$value"
else
_Logger "$preix$value" "$prefix$value"
fi
return
else
_Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true
_Logger "Value was: $prefix$value" "Value was: $prefix$value" true
fi
}
# Function is busybox compatible since busybox ash does not understand direct regex, we use expr
function IsInteger {
local value="${1}"
if type expr > /dev/null 2>&1; then
expr "$value" : '^[0-9]\{1,\}$' > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo 1
else
echo 0
fi
else
if [[ $value =~ ^[0-9]+$ ]]; then
echo 1
else
echo 0
fi
fi
}
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds {
local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" # Should parent be killed too ?
# Paranoid checks, we can safely assume that $pid should not be 0 nor 1
if [ $(IsInteger "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Bogus pid given [$pid]." "CRITICAL"
return 1
fi
if kill -0 "$pid" > /dev/null 2>&1; then
if children="$(pgrep -P "$pid")"; then
if [[ "$pid" == *"$children"* ]]; then
Logger "Bogus pgrep implementation." "CRITICAL"
children="${children/$pid/}"
fi
for child in $children; do
KillChilds "$child" true
done
fi
fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if [ "$self" == true ]; then
# We need to check for pid again because it may have disappeared after recursive function call
if kill -0 "$pid" > /dev/null 2>&1; then
kill -s TERM "$pid"
Logger "Sent SIGTERM to process [$pid]." "DEBUG"
if [ $? != 0 ]; then
sleep 15
Logger "Sending SIGTERM to process [$pid] failed." "DEBUG"
kill -9 "$pid"
if [ $? != 0 ]; then
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
return 1
fi # Simplify the return 0 logic here
else
return 0
fi
else
return 0
fi
else
return 0
fi
}
function KillAllChilds {
local pids="${1}" # List of parent pids to kill separated by semi-colon
local self="${2:-false}" # Should parent be killed too ?
local errorcount=0
IFS=';' read -a pidsArray <<< "$pids"
for pid in "${pidsArray[@]}"; do
KillChilds $pid $self
if [ $? != 0 ]; then
errorcount=$((errorcount+1))
fi
done
return $errorcount
}
function CleanUp {
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp"
fi
}
function GenericTrapQuit {
local exitcode=0
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
WARN_ALERT=true
exitcode=2
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
ERROR_ALERT=true
exitcode=1
fi
CleanUp
exit $exitcode
}
# Get current install.sh path from http://stackoverflow.com/a/246128/2635443
SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@ -86,186 +502,6 @@ else
LOG_FILE="./$PROGRAM-install.log"
fi
#### RemoteLogger SUBSET ####
# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array
# usage: joinString separaratorChar Array
function joinString {
local IFS="$1"; shift; echo "$*";
}
# Sub function of Logger
function _Logger {
local logValue="${1}" # Log to file
local stdValue="${2}" # Log to screeen
local toStdErr="${3:-false}" # Log to stderr instead of stdout
if [ "$logValue" != "" ]; then
echo -e "$logValue" >> "$LOG_FILE"
# Build current log file for alerts if we have a sufficient environment
if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then
echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP"
fi
fi
if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then
if [ $toStdErr == true ]; then
# Force stderr color in subshell
(>&2 echo -e "$stdValue")
else
echo -e "$stdValue"
fi
fi
}
# Remote logger similar to below Logger, without log to file and alert flags
function RemoteLogger {
local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
prefix=""
fi
if [ "$level" == "CRITICAL" ]; then
_Logger "" "$prefix\e[1;33;41m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "ERROR" ]; then
_Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "WARN" ]; then
_Logger "" "$prefix\e[33m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "NOTICE" ]; then
if [ $_LOGGER_ERR_ONLY != true ]; then
_Logger "" "$prefix$value"
fi
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value"
return
elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then
_Logger "" "$prefix$value"
return
fi
elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG
_Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG
return #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG
else
_Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true
_Logger "" "Value was: $prefix$value" true
fi
}
#### RemoteLogger SUBSET END ####
# General log function with log levels:
# Environment variables
# _LOGGER_SILENT: Disables any output to stdout & stderr
# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel
# _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout
# Loglevels
# Except for VERBOSE, all loglevels are ALWAYS sent to log file
# CRITICAL, ERROR, WARN sent to stderr, color depending on level, level also logged
# NOTICE sent to stdout
# VERBOSE sent to stdout if _LOGGER_VERBOSE = true
# ALWAYS is sent to stdout unless _LOGGER_SILENT = true
# DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes
# SIMPLE is a wrapper for QuickLogger that does not use advanced functionality
function Logger {
local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="$(date '+%Y-%m-%d %H:%M:%S') - "
else
prefix=""
fi
## Obfuscate _REMOTE_TOKEN in logs (for ssh_filter usage only in osync and obackup)
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/__(o_O)__}"
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
ERROR_ALERT=true
# ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag.
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "ERROR" ]; then
_Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true
ERROR_ALERT=true
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "WARN" ]; then
_Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true
WARN_ALERT=true
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "NOTICE" ]; then
if [ "$_LOGGER_ERR_ONLY" != true ]; then
_Logger "$prefix$value" "$prefix$value"
fi
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix($level):$value" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
_Logger "$prefix$value" "$prefix$value"
return
elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then
_Logger "$prefix$value" "$prefix$value"
return
fi
elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG
_Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG
return #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG
elif [ "$level" == "SIMPLE" ]; then
if [ "$_LOGGER_SILENT" == true ]; then
_Logger "$preix$value"
else
_Logger "$preix$value" "$prefix$value"
fi
return
else
_Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true
_Logger "Value was: $prefix$value" "Value was: $prefix$value" true
fi
}
## Modified version of https://gist.github.com/cdown/1163649
function UrlEncode {
local length="${#1}"
@ -293,6 +529,8 @@ function GetLocalOS {
# There is no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
elif set -o | grep "winxp" > /dev/null; then
localOsVar="BusyBox-w32"
else
# Detecting the special ubuntu userland in Windows 10 bash
if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
@ -325,7 +563,7 @@ function GetLocalOS {
*"CYGWIN"*)
LOCAL_OS="Cygwin"
;;
*"Microsoft"*)
*"Microsoft"*|*"MS/Windows"*)
LOCAL_OS="WinNT10"
;;
*"Darwin"*)
@ -356,7 +594,8 @@ function GetLocalOS {
fi
# Get Host info for Windows
if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then localOsVar="$(uname -a)"
if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then
localOsVar="$localOsVar $(uname -a)"
if [ "$PROGRAMW6432" != "" ]; then
LOCAL_OS_BITNESS=64
LOCAL_OS_FAMILY="Windows"
@ -370,6 +609,9 @@ function GetLocalOS {
# Get Host info for Unix
else
LOCAL_OS_FAMILY="Unix"
fi
if [ "$LOCAL_OS_FAMILY" == "Unix" ]; then
if uname -m | grep '64' > /dev/null 2>&1; then
LOCAL_OS_BITNESS=64
else
@ -658,19 +900,27 @@ function Usage {
exit 127
}
function TrapQuit {
local exitcode=0
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
WARN_ALERT=true
exitcode=2
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
ERROR_ALERT=true
exitcode=1
fi
CleanUp
exit $exitcode
}
############################## Script entry point
if [ "$LOGFILE" == "" ]; then
if [ -w /var/log ]; then
LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log"
else
LOG_FILE="./$PROGRAM.$INSTANCE_ID.log"
fi
else
LOG_FILE="$LOGFILE"
fi
trap TrapQuit TERM EXIT HUP QUIT
if [ ! -w "$(dirname $LOG_FILE)" ]; then
echo "Cannot write to log [$(dirname $LOG_FILE)]."
else
@ -695,7 +945,7 @@ else
if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "pmocr" ]; then
CopyServiceFiles
fi
Logger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM" "SIMPLE"
Logger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM_BINARY" "SIMPLE"
if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
echo ""
Logger "If connecting remotely, consider setup ssh filter to enhance security." "SIMPLE"

191
obackup-batch.sh
View File

@ -1,9 +1,9 @@
#!/usr/bin/env bash
SUBPROGRAM=obackup
PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
AUTHOR="(L) 2013-2017 by Orsiris de Jong"
AUTHOR="(L) 2013-2018 by Orsiris de Jong"
CONTACT="http://www.netpower.fr - ozy@netpower.fr"
PROGRAM_BUILD=2016120401
PROGRAM_BUILD=2018100201
## Runs an osync /obackup instance for every conf file found
## If an instance fails, run it again if time permits
@ -26,34 +26,191 @@ else
LOG_FILE=./$SUBPROGRAM-batch.log
fi
## Default directory where to store temporary run files
if [ -w /tmp ]; then
RUN_DIR=/tmp
elif [ -w /var/tmp ]; then
RUN_DIR=/var/tmp
else
RUN_DIR=.
fi
trap TrapQuit TERM EXIT HUP QUIT
# No need to edit under this line ##############################################################
function _logger {
local value="${1}" # What to log
echo -e "$value" >> "$LOG_FILE"
#### RemoteLogger SUBSET ####
# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array
# usage: joinString separaratorChar Array
function joinString {
local IFS="$1"; shift; echo "$*";
}
function Logger {
local value="${1}" # What to log
local level="${2}" # Log level: DEBUG, NOTICE, WARN, ERROR, CRITIAL
# Sub function of Logger
function _Logger {
local logValue="${1}" # Log to file
local stdValue="${2}" # Log to screeen
local toStdErr="${3:-false}" # Log to stderr instead of stdout
prefix="$(date) - "
if [ "$logValue" != "" ]; then
echo -e "$logValue" >> "$LOG_FILE"
# Build current log file for alerts if we have a sufficient environment
if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then
echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP"
fi
fi
if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then
if [ $toStdErr == true ]; then
# Force stderr color in subshell
(>&2 echo -e "$stdValue")
else
echo -e "$stdValue"
fi
fi
}
# Remote logger similar to below Logger, without log to file and alert flags
function RemoteLogger {
local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="R $(date) - "
else
prefix=""
fi
if [ "$level" == "CRITICAL" ]; then
_logger "$prefix\e[41m$value\e[0m"
_Logger "" "$prefix\e[1;33;41m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "ERROR" ]; then
_logger "$prefix\e[91m$value\e[0m"
_Logger "" "$prefix\e[31m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "WARN" ]; then
_logger "$prefix\e[93m$value\e[0m"
_Logger "" "$prefix\e[33m$value\e[0m" true
if [ $_DEBUG == "yes" ]; then
_Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true
fi
return
elif [ "$level" == "NOTICE" ]; then
_logger "$prefix$value"
if [ $_LOGGER_ERR_ONLY != true ]; then
_Logger "" "$prefix$value"
fi
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
_Logger "" "$prefix$value"
return
elif [ "$level" == "DEBUG" ]; then
if [ "$DEBUG" == "yes" ]; then
_logger "$prefix$value"
if [ "$_DEBUG" == "yes" ]; then
_Logger "" "$prefix$value"
return
fi
else
_logger "\e[41mLogger function called without proper loglevel.\e[0m"
_logger "$prefix$value"
_Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true
_Logger "" "Value was: $prefix$value" true
fi
}
#### RemoteLogger SUBSET END ####
# General log function with log levels:
# Environment variables
# _LOGGER_SILENT: Disables any output to stdout & stderr
# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel
# _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout
# Loglevels
# Except for VERBOSE, all loglevels are ALWAYS sent to log file
# CRITICAL, ERROR, WARN sent to stderr, color depending on level, level also logged
# NOTICE sent to stdout
# VERBOSE sent to stdout if _LOGGER_VERBOSE = true
# ALWAYS is sent to stdout unless _LOGGER_SILENT = true
# DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes
# SIMPLE is a wrapper for QuickLogger that does not use advanced functionality
function Logger {
local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="$(date '+%Y-%m-%d %H:%M:%S') - "
else
prefix=""
fi
## Obfuscate _REMOTE_TOKEN in logs (for ssh_filter usage only in osync and obackup)
value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/__(o_O)__}"
value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/__(o_O)__}"
if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
ERROR_ALERT=true
# ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag.
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "ERROR" ]; then
_Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true
ERROR_ALERT=true
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "WARN" ]; then
_Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true
WARN_ALERT=true
echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP"
return
elif [ "$level" == "NOTICE" ]; then
if [ "$_LOGGER_ERR_ONLY" != true ]; then
_Logger "$prefix$value" "$prefix$value"
fi
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix($level):$value" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
_Logger "$prefix$value" "$prefix$value"
return
elif [ "$level" == "DEBUG" ]; then
if [ "$_DEBUG" == "yes" ]; then
_Logger "$prefix$value" "$prefix$value"
return
fi
elif [ "$level" == "SIMPLE" ]; then
if [ "$_LOGGER_SILENT" == true ]; then
_Logger "$preix$value"
else
_Logger "$preix$value" "$prefix$value"
fi
return
else
_Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true
_Logger "Value was: $prefix$value" "Value was: $prefix$value" true
fi
}

243
obackup.sh
View File

@ -7,26 +7,14 @@ PROGRAM="obackup"
AUTHOR="(C) 2013-2018 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr"
PROGRAM_VERSION=2.1-RC1
PROGRAM_BUILD=2018093008
PROGRAM_BUILD=2018110601
IS_STABLE=no
_OFUNCTIONS_VERSION=2.3.0-RC1
_OFUNCTIONS_BUILD=2018100105
_OFUNCTIONS_VERSION=2.3.0-RC2
_OFUNCTIONS_BUILD=2018110502
_OFUNCTIONS_BOOTSTRAP=true
## To use in a program, define the following variables:
## PROGRAM=program-name
## INSTANCE_ID=program-instance-name
## _DEBUG=yes/no
## _LOGGER_SILENT=true/false
## _LOGGER_VERBOSE=true/false
## _LOGGER_ERR_ONLY=true/false
## _LOGGER_PREFIX="date"/"time"/""
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP
if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
exit 127
@ -89,7 +77,9 @@ else
LOG_FILE="/tmp/$PROGRAM.log"
fi
#### RUN_DIR SUBSET ####
## Default directory where to store temporary run files
if [ -w /tmp ]; then
RUN_DIR=/tmp
elif [ -w /var/tmp ]; then
@ -98,8 +88,38 @@ else
RUN_DIR=.
fi
#### PoorMansRandomGenerator SUBSET ####
# Get a random number on Windows BusyBox alike, also works on most Unixes
## Special note when remote target is on the same host as initiator (happens for unit tests): we'll have to differentiate RUN_DIR so remote CleanUp won't affect initiator.
if [ "$_REMOTE_EXECUTION" == true ]; then
mkdir -p "$RUN_DIR/$PROGRAM.remote"
RUN_DIR="$RUN_DIR/$PROGRAM.remote"
fi
#### RUN_DIR SUBSET END ####
# Get a random number on Windows BusyBox alike, also works on most Unixes that have dd, if dd is not found, then return $RANDOM
function PoorMansRandomGenerator {
local digits="${1}" # The number of digits to generate
local number
local isFirst=true
if type dd >/dev/null 2>&1; then
# Some read bytes can't be used, se we read twice the number of required bytes
dd if=/dev/urandom bs=$digits count=2 2> /dev/null | while read -r -n1 char; do
if [ $isFirst == false ] || [ $(printf "%d" "'$char") != "0" ]; then
number=$number$(printf "%d" "'$char")
isFirst=false
fi
if [ ${#number} -ge $digits ]; then
echo ${number:0:$digits}
break;
fi
done
elif [ "$RANDOM" -ne 0 ]; then
echo $RANDOM
else
Logger "Cannot generate random number." "ERROR"
fi
}
function PoorMansRandomGenerator {
local digits="${1}" # The number of digits to generate
local number
@ -113,10 +133,9 @@ function PoorMansRandomGenerator {
fi
done
}
#### PoorMansRandomGenerator SUBSET END ####
# Initial TSTMAP value before function declaration
TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4)
TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 5)
# Default alert attachment filename
ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log"
@ -126,12 +145,6 @@ set -o pipefail
set -o errtrace
function Dummy {
sleep $SLEEP_TIME
}
# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array
# usage: joinString separaratorChar Array
function joinString {
@ -170,6 +183,8 @@ function RemoteLogger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -241,6 +256,8 @@ function Logger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -300,6 +317,26 @@ function Logger {
fi
}
# Function is busybox compatible since busybox ash does not understand direct regex, we use expr
function IsInteger {
local value="${1}"
if type expr > /dev/null 2>&1; then
expr "$value" : '^[0-9]\{1,\}$' > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo 1
else
echo 0
fi
else
if [[ $value =~ ^[0-9]+$ ]]; then
echo 1
else
echo 0
fi
fi
}
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds {
local pid="${1}" # Parent pid to kill childs
@ -365,6 +402,33 @@ function KillAllChilds {
return $errorcount
}
function CleanUp {
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp"
fi
}
function GenericTrapQuit {
local exitcode=0
# Get ERROR / WARN alert flags from subprocesses that call Logger
if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
WARN_ALERT=true
exitcode=2
fi
if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
ERROR_ALERT=true
exitcode=1
fi
CleanUp
exit $exitcode
}
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
function SendAlert {
local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
@ -744,6 +808,7 @@ function ExecTasks {
local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries
local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors
local i
@ -761,9 +826,6 @@ function ExecTasks {
done
fi
# Change '-' to '_' in task id
id="${id/-/_}"
# Expand validExitCodes into array
IFS=';' read -r -a validExitCodes <<< "$validExitCodes"
@ -800,15 +862,9 @@ function ExecTasks {
local newPidsArray # New array of currently running pids for next iteration
local pidsTimeArray # Array containing execution begin time of pids
local executeCommand # Boolean to check if currentCommand can be executed given a condition
local functionMode
if [ $counting == true ]; then
local softAlert=false # Does a soft alert need to be triggered, if yes, send an alert once
else
local softAlert=false
fi
local failedPidsList # List containing failed pids with exit code separated by semicolons (eg : 2355:1;4534:2;2354:3)
# Initialise global variable
eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
@ -968,17 +1024,17 @@ function ExecTasks {
# Check for valid exit codes
if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then
if [ $noErrorLogsAtAll != true ]; then
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR"
if [ "$functionMode" == "ParallelExec" ]; then
Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
fi
fi
errorcount=$((errorcount+1))
# Welcome to variable variable bash hell
if [ "$(eval echo \"\$WAIT_FOR_TASK_COMPLETION_$id\")" == "" ]; then
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$pid:$retval\""
if [ "$failedPidsList" == "" ]; then
failedPidsList="$pid:$retval"
else
eval "WAIT_FOR_TASK_COMPLETION_$id=\";$pid:$retval\""
failedPidsList="$failedPidsList;$pid:$retval"
fi
else
Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
@ -1132,6 +1188,8 @@ function ExecTasks {
# Return exit code if only one process was monitored, else return number of errors
# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value
eval "WAIT_FOR_TASK_COMPLETION_$id=\"$failedPidsList\""
if [ $mainItemCount -eq 1 ]; then
return $retval
else
@ -1139,15 +1197,6 @@ function ExecTasks {
fi
}
function CleanUp {
if [ "$_DEBUG" != "yes" ]; then
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP"
# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp"
fi
}
# Usage: var=$(StripSingleQuotes "$var")
function StripSingleQuotes {
local string="${1}"
@ -1186,40 +1235,19 @@ function EscapeDoubleQuotes {
echo "${value//\"/\\\"}"
}
function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
if [[ $value =~ ^-?[0-9]+([.][0-9]+)?$ ]]; then
echo 1
else
echo 0
fi
}
# Usage [ $(IsNumeric $var) -eq 1 ]
function IsNumeric {
local value="${1}"
if [[ $value =~ ^[0-9]+([.][0-9]+)?$ ]]; then
echo 1
else
echo 0
fi
}
# Function is busybox compatible since busybox ash does not understand direct regex, we use expr
function IsInteger {
local value="${1}"
if type expr > /dev/null 2>&1; then
expr "$value" : "^[0-9]\+$" > /dev/null 2>&1
expr "$value" : '^[-+]\{0,1\}[0-9]*\.\{0,1\}[0-9]\{1,\}$' > /dev/null 2>&1
if [ $? -eq 0 ]; then
echo 1
else
echo 0
fi
else
if [[ $value =~ ^[0-9]+$ ]]; then
if [[ $value =~ ^[-+]?[0-9]+([.][0-9]+)?$ ]]; then
echo 1
else
echo 0
@ -1227,6 +1255,12 @@ function IsInteger {
fi
}
function IsNumericExpand {
eval "local value=\"${1}\"" # Needed eval so variable variables can be processed
echo $(IsNumeric "$value")
}
# Converts human readable sizes into integer kilobyte sizes
# Usage numericSize="$(HumanToNumeric $humanSize)"
function HumanToNumeric {
@ -1323,6 +1357,8 @@ function GetLocalOS {
# There is no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
elif set -o | grep "winxp" > /dev/null; then
localOsVar="BusyBox-w32"
else
# Detecting the special ubuntu userland in Windows 10 bash
if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
@ -1355,7 +1391,7 @@ function GetLocalOS {
*"CYGWIN"*)
LOCAL_OS="Cygwin"
;;
*"Microsoft"*)
*"Microsoft"*|*"MS/Windows"*)
LOCAL_OS="WinNT10"
;;
*"Darwin"*)
@ -1386,7 +1422,8 @@ function GetLocalOS {
fi
# Get Host info for Windows
if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then localOsVar="$(uname -a)"
if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then
localOsVar="$localOsVar $(uname -a)"
if [ "$PROGRAMW6432" != "" ]; then
LOCAL_OS_BITNESS=64
LOCAL_OS_FAMILY="Windows"
@ -1400,6 +1437,9 @@ function GetLocalOS {
# Get Host info for Unix
else
LOCAL_OS_FAMILY="Unix"
fi
if [ "$LOCAL_OS_FAMILY" == "Unix" ]; then
if uname -m | grep '64' > /dev/null 2>&1; then
LOCAL_OS_BITNESS=64
else
@ -1943,14 +1983,14 @@ function InitLocalOSDependingSettings {
# Gets executed regardless of the need of remote connections. It is just that this code needs to get executed after we know if there is a remote os, and if yes, which one
function InitRemoteOSDependingSettings {
if [ "$REMOTE_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then
if [ "$REMOTE_OS" == "msys" ] || [ "$REMOTE_OS" == "Cygwin" ]; then
REMOTE_FIND_CMD=$(dirname $BASH)/find
else
REMOTE_FIND_CMD=find
fi
## Stat command has different syntax on Linux and FreeBSD/MacOSX
if [ "$LOCAL_OS" == "MacOSX" ] || [ "$LOCAL_OS" == "BSD" ]; then
if [ "$REMOTE_OS" == "MacOSX" ] || [ "$REMOTE_OS" == "BSD" ]; then
REMOTE_STAT_CMD="stat -f \"%Sm\""
REMOTE_STAT_CTIME_MTIME_CMD="stat -f \\\"%N;%c;%m\\\""
else
@ -2129,13 +2169,26 @@ function SetConfFileValue () {
local value="${3}"
local separator="${4:-#}"
if grep "^$name=" "$file" > /dev/null; then
if [ -f "$file" ]; then
if grep "^$name=" "$file" > /dev/null 2>&1; then
# Using -i.tmp for BSD compat
sed -i.tmp "s$separator^$name=.*$separator$name=$value$separator" "$file"
if [ $? -ne 0 ]; then
Logger "Cannot update value [$name] to [$value] in config file [$file]." "ERROR"
fi
rm -f "$file.tmp"
Logger "Set [$name] to [$value] in config file [$file]." "DEBUG"
else
Logger "Cannot set value [$name] to [$value] in config file [$file]." "ERROR"
echo "$name=$value" >> "$file"
if [ $? -ne 0 ]; then
Logger "Cannot create value [$name] to [$value] in config file [$file]." "ERROR"
fi
fi
else
echo "$name=$value" > "$file"
if [ $? -ne 0 ]; then
Logger "Config file [$file] does not exist. Failed to create it witn value [$name]." "ERROR"
fi
fi
}
@ -2556,7 +2609,7 @@ function _ListRecursiveBackupDirectoriesRemote {
$SSH_CMD env _REMOTE_TOKEN=$_REMOTE_TOKEN \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env RECURSIVE_DIRECTORY_LIST="'$RECURSIVE_DIRECTORY_LIST'" env PATH_SEPARATOR_CHAR="'$PATH_SEPARATOR_CHAR'" \
env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
@ -2620,6 +2673,8 @@ function RemoteLogger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -2837,7 +2892,8 @@ function _GetDirectoriesSizeRemote {
# Error output is different from stdout because not all files in list may fail at once
$SSH_CMD env _REMOTE_TOKEN=$_REMOTE_TOKEN \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" dirList="'$dirList'" \
env _REMOTE_EXECUTION="true" env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
dirList="'$dirList'" \
$COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" &
## allow debugging from command line with _DEBUG=yes
@ -2900,6 +2956,8 @@ function RemoteLogger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -3028,7 +3086,7 @@ function _CreateDirectoryRemote {
$SSH_CMD env _REMOTE_TOKEN=$_REMOTE_TOKEN \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env _REMOTE_EXECUTION=true env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env dirToCreate="'$dirToCreate'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 &
## allow debugging from command line with _DEBUG=yes
@ -3091,6 +3149,8 @@ function RemoteLogger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -3244,7 +3304,7 @@ function GetDiskSpaceRemote {
$SSH_CMD env _REMOTE_TOKEN=$_REMOTE_TOKEN \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env _REMOTE_EXECUTION=true env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env DF_CMD="'$DF_CMD'" \
env pathToCheck="'$pathToCheck'" $COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" &
@ -3308,6 +3368,8 @@ function RemoteLogger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -3937,7 +3999,7 @@ function Rsync {
## Manage to backup recursive directories lists files only (not recursing into subdirectories)
if [ $recursive == false ]; then
# Fixes symlinks to directories in target cannot be deleted when backing up root directory without recursion
rsyncArgs="$RSYNC_DEFAULT_NONRECURSIVE_ARGS -k"
rsyncArgs="$RSYNC_DEFAULT_ARGS -f '- /*/*/'"
else
rsyncArgs="$RSYNC_DEFAULT_ARGS"
fi
@ -3980,7 +4042,8 @@ function FilesBackup {
local backupTask
local backupTasks
local destinationDir
local withoutCryptPath
local encryptDir
IFS=$PATH_SEPARATOR_CHAR read -r -a backupTasks <<< "$FILE_BACKUP_TASKS"
@ -3994,6 +4057,7 @@ function FilesBackup {
else
destinationDir=$(dirname "$FILE_STORAGE/${backupTask#/}/")
fi
encryptDir="$FILE_STORAGE/${backupTask#/}"
else
destinationDir="$FILE_STORAGE"
encryptDir="$FILE_STORAGE"
@ -4020,7 +4084,7 @@ function FilesBackup {
IFS=$PATH_SEPARATOR_CHAR read -r -a backupTasks <<< "$RECURSIVE_DIRECTORY_LIST"
for backupTask in "${backupTasks[@]}"; do
# Backup recursive directories withouht recursion
# Backup recursive directories without recursion
if [ "$KEEP_ABSOLUTE_PATHS" != "no" ]; then
# Fix for backup of '/'
@ -4065,6 +4129,7 @@ function FilesBackup {
else
destinationDir=$(dirname "$FILE_STORAGE/${backupTask#/}/")
fi
encryptDir="$FILE_STORAGE/${backupTask#/}"
else
destinationDir="$FILE_STORAGE"
encryptDir="$FILE_STORAGE"
@ -4188,7 +4253,7 @@ function _RotateBackupsRemote {
$SSH_CMD env _REMOTE_TOKEN=$_REMOTE_TOKEN \
env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILENT="'$_LOGGER_SILENT'" env _LOGGER_VERBOSE="'$_LOGGER_VERBOSE'" env _LOGGER_PREFIX="'$_LOGGER_PREFIX'" env _LOGGER_ERR_ONLY="'$_LOGGER_ERR_ONLY'" \
env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env _REMOTE_EXECUTION=true env PROGRAM="'$PROGRAM'" env SCRIPT_PID="'$SCRIPT_PID'" env TSTAMP="'$TSTAMP'" \
env REMOTE_FIND_CMD="'$REMOTE_FIND_CMD'" env rotateCopies="'$rotateCopies'" env backupPath="'$backupPath'" \
$COMMAND_SUDO' bash -s' << 'ENDSSH' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
@ -4252,6 +4317,8 @@ function RemoteLogger {
local level="${2}" # Log level
local retval="${3:-undef}" # optional return value of command
local prefix
if [ "$_LOGGER_PREFIX" == "time" ]; then
prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
@ -4406,9 +4473,6 @@ function Init {
local hosturiandpath
local hosturi
trap TrapStop INT QUIT TERM HUP
trap TrapQuit EXIT
## Test if target dir is a ssh uri, and if yes, break it down it its values
if [ "${REMOTE_SYSTEM_URI:0:6}" == "ssh://" ] && [ "$BACKUP_TYPE" != "local" ]; then
REMOTE_OPERATION="yes"
@ -4540,6 +4604,9 @@ function Usage {
exit 128
}
#### SCRIPT ENTRY POINT ####
trap TrapQuit EXIT
# Command line argument flags
_DRYRUN=false
no_maxtime=false