1
0
mirror of https://github.com/deajan/obackup.git synced 2024-11-14 19:53:42 +01:00

Rebuilt targets

This commit is contained in:
deajan 2017-06-09 09:58:51 +02:00
parent 0ee0bf52fd
commit f15f216989
4 changed files with 348 additions and 195 deletions

View File

@ -35,8 +35,8 @@ IS_STABLE=no
# FilesBackup #__WITH_PARANOIA_DEBUG
_OFUNCTIONS_VERSION=2.1-RC3+dev
_OFUNCTIONS_BUILD=2017031401
_OFUNCTIONS_VERSION=2.1.4-rc1
_OFUNCTIONS_BUILD=2017060901
_OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@ -61,6 +61,9 @@ fi
## Correct output of sort command (language agnostic sorting)
export LC_ALL=C
## Default umask for file creation
umask 0077
# Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
@ -97,6 +100,8 @@ if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as envi
fi
SCRIPT_PID=$$
# TODO: Check if %N works on MacOS
TSTAMP=$(date '+%Y%m%dT%H%M%S.%N')
LOCAL_USER=$(whoami)
@ -288,7 +293,7 @@ function Logger {
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix:$value" "$prefix$value"
_Logger "$prefix($level):$value" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
@ -339,15 +344,29 @@ function KillChilds {
local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" # Should parent be killed too ?
# Warning: pgrep does not exist in cygwin, have this checked in CheckEnvironment
if children="$(pgrep -P "$pid")"; then
for child in $children; do
Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG
KillChilds "$child" true
done
# Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
if [ $(IsNumeric "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Bogus pid given [$pid]." "CRITICAL"
return 1
fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if kill -0 "$pid" > /dev/null 2>&1; then
# Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
if children="$(pgrep -P "$pid")"; then
if [[ "$pid" == *"$children"* ]]; then
Logger "Bogus pgrep implementation." "CRITICAL"
children="${children/$pid/}"
fi
for child in $children; do
Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG
KillChilds "$child" true
done
fi
fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if [ "$self" == true ]; then
# We need to check for pid again because it may have disappeared after recursive function call
if kill -0 "$pid" > /dev/null 2>&1; then
kill -s TERM "$pid"
Logger "Sent SIGTERM to process [$pid]." "DEBUG"
@ -409,6 +428,7 @@ function SendAlert {
fi
eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
if [ $? != 0 ]; then
Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
attachment=false
@ -657,6 +677,22 @@ function Spinner {
fi
}
function _PerfProfiler { #__WITH_PARANOIA_DEBUG
local perfString #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan) #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
for i in $(pgrep -P $$); do #__WITH_PARANOIA_DEBUG
perfString="$perfString\n"$(ps -p $i -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan | tail -1) #__WITH_PARANOIA_DEBUG
done #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
if type iostat > /dev/null 2>&1; then #__WITH_PARANOIA_DEBUG
perfString="$perfString\n"$(iostat) #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
} #__WITH_PARANOIA_DEBUG
# Time control function for background processes, suitable for multiple synchronous processes
# Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
@ -789,6 +825,11 @@ function WaitForTaskCompletion {
pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
done
Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
@ -954,6 +995,10 @@ function ParallelExec {
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
done
return $errorCount
@ -1139,7 +1184,7 @@ function GetLocalOS {
*"BSD"*)
LOCAL_OS="BSD"
;;
*"MINGW32"*|*"MSYS"*)
*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
LOCAL_OS="msys"
;;
*"CYGWIN"*)
@ -1165,9 +1210,6 @@ function GetLocalOS {
exit 1
;;
esac
if [ "$_OFUNCTIONS_VERSION" != "" ]; then
Logger "Local OS: [$localOsVar]." "DEBUG"
fi
# Get linux versions
if [ -f "/etc/os-release" ]; then
@ -1177,8 +1219,68 @@ function GetLocalOS {
# Add a global variable for statistics in installer
LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
if [ "$_OFUNCTIONS_VERSION" != "" ]; then
Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG"
fi
}
#__BEGIN_WITH_PARANOIA_DEBUG
function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing
if [ "$_DEBUG" == "yes" ]; then
local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
local numberOfGivenArguments="${2}" # Number of arguments that have been passed
local minArgs
local maxArgs
# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${3} and count
callerName="${FUNCNAME[1]}"
local iterate=3
local fetchArguments=true
local argList=""
local countedArguments
while [ $fetchArguments == true ]; do
cmd='argument=${'$iterate'}'
eval $cmd
if [ "$argument" == "" ]; then
fetchArguments=false
else
argList="$argList[Argument $((iterate-2)): $argument] "
iterate=$((iterate+1))
fi
done
countedArguments=$((iterate-3))
if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
minArgs=$numberOfArguments
maxArgs=$numberOfArguments
else
IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
fi
Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
Logger "$callerName arguments: $argList" "ERROR"
else
if [ ! -z "$argList" ]; then
Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
fi
fi
fi
}
#__END_WITH_PARANOIA_DEBUG
function GetRemoteOS {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
@ -1189,13 +1291,16 @@ function GetRemoteOS {
local remoteOsVar
$SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
$SSH_CMD env LC_ALL=C env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
function GetOs {
local localOsVar
local localOsName
local localOsVer
local osInfo="/etc/os-release"
# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
@ -1214,9 +1319,11 @@ function GetOs {
fi
fi
# Get linux versions
if [ -f "/etc/os-release" ]; then
localOsName=$(GetConfFileValue "/etc/os-release" "NAME")
localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION")
if [ -f "$osInfo" ]; then
localOsName=$(grep "^NAME=" "$osInfo")
localOsName="${localOsName##*=}"
localOsVer=$(grep "^VERSION=" "$osInfo")
localOsVer="${localOsVer##*=}"
fi
echo "$localOsVar ($localOsName $localOsVer)"
@ -1238,7 +1345,7 @@ ENDSSH
*"BSD"*)
REMOTE_OS="BSD"
;;
*"MINGW32"*|*"MSYS"*)
*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
REMOTE_OS="msys"
;;
*"CYGWIN"*)
@ -1323,7 +1430,7 @@ function RunRemoteCommand {
fi
Logger "Running command [$command] on remote host." "NOTICE"
cmd=$SSH_CMD' "env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
@ -1434,62 +1541,6 @@ function CheckConnectivity3rdPartyHosts {
fi #__WITH_PARANOIA_DEBUG
}
#__BEGIN_WITH_PARANOIA_DEBUG
function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing
if [ "$_DEBUG" == "yes" ]; then
local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
local numberOfGivenArguments="${2}" # Number of arguments that have been passed
local minArgs
local maxArgs
# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${3} and count
callerName="${FUNCNAME[1]}"
local iterate=3
local fetchArguments=true
local argList=""
local countedArguments
while [ $fetchArguments == true ]; do
cmd='argument=${'$iterate'}'
eval $cmd
if [ "$argument" == "" ]; then
fetchArguments=false
else
argList="$argList[Argument $((iterate-2)): $argument] "
iterate=$((iterate+1))
fi
done
countedArguments=$((iterate-3))
if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
minArgs=$numberOfArguments
maxArgs=$numberOfArguments
else
IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
fi
Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
Logger "$callerName arguments: $argList" "ERROR"
else
if [ ! -z "$argList" ]; then
Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
fi
fi
fi
}
#__END_WITH_PARANOIA_DEBUG
function RsyncPatternsAdd {
local patternType="${1}" # exclude or include
local pattern="${2}"
@ -1620,13 +1671,13 @@ function PostInit {
# Define remote commands
if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then
SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
elif [ -f "$SSH_PASSWORD_FILE" ]; then
SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS -p $REMOTE_PORT"
SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT"
else
SSH_PASSWORD=""
SSH_CMD=""
@ -1664,23 +1715,18 @@ function SetCompression {
COMPRESSION_PROGRAM="| pigz -c$compressionString"
COMPRESSION_EXTENSION=.gz
# obackup specific
if [ "$LOCAL_OS" != "MacOSX" ]; then
COMPRESSION_OPTIONS=--rsyncable
fi
COMPRESSION_OPTIONS=--rsyncable
elif type gzip > /dev/null 2>&1
then
COMPRESSION_PROGRAM="| gzip -c$compressionString"
COMPRESSION_EXTENSION=.gz
# obackup specific
if [ "$LOCAL_OS" != "MacOSX" ]; then
COMPRESSION_OPTIONS=--rsyncable
fi
COMPRESSION_OPTIONS=--rsyncable
else
COMPRESSION_PROGRAM=
COMPRESSION_EXTENSION=
fi
fi
ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
}
function InitLocalOSDependingSettings {
@ -1844,35 +1890,46 @@ function ParentPid {
# Neat version compare function found at http://stackoverflow.com/a/4025065/2635443
# Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2
vercomp () {
if [[ $1 == $2 ]]
then
return 0
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++))
do
if [[ -z ${ver2[i]} ]]
then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]}))
then
return 1
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
return 2
fi
done
return 0
function VerComp () {
if [ "$1" == "" ] || [ "$2" == "" ]; then
Logger "Bogus Vercomp values [$1] and [$2]." "WARN"
return 1
fi
if [[ $1 == $2 ]]
then
echo 0
return
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++))
do
if [[ -z ${ver2[i]} ]]
then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]}))
then
echo 1
return
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
echo 2
return
fi
done
echo 0
return
}
function GetConfFileValue () {
@ -1905,6 +1962,29 @@ function SetConfFileValue () {
fi
}
# Function can replace [ -f /some/file* ] tests
# Modified version of http://stackoverflow.com/a/6364244/2635443
function WildcardFileExists () {
local file="${1}"
local exists=0
for f in $file; do
## Check if the glob gets expanded to existing files.
## If not, f here will be exactly the pattern above
## and the exists test will evaluate to false.
if [ -e "$f" ]; then
exists=1
break
fi
done
if [ $exists -eq 1 ]; then
echo 1
else
echo 0
fi
}
# If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap
_OFUNCTIONS_BOOTSTRAP=true
[ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1
@ -4436,6 +4516,8 @@ else
Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
fi
# Switching rundir to
if [ "$IS_STABLE" != "yes" ]; then
Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN"
fi

View File

@ -12,7 +12,7 @@ PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh"
SSH_FILTER="ssh_filter.sh"
SCRIPT_BUILD=2017031402
SCRIPT_BUILD=2017041701
## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
@ -122,7 +122,7 @@ function GetLocalOS {
*"BSD"*)
LOCAL_OS="BSD"
;;
*"MINGW32"*|*"MSYS"*)
*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
LOCAL_OS="msys"
;;
*"CYGWIN"*)
@ -148,9 +148,6 @@ function GetLocalOS {
exit 1
;;
esac
if [ "$_OFUNCTIONS_VERSION" != "" ]; then
Logger "Local OS: [$localOsVar]." "DEBUG"
fi
# Get linux versions
if [ -f "/etc/os-release" ]; then
@ -160,6 +157,10 @@ function GetLocalOS {
# Add a global variable for statistics in installer
LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
if [ "$_OFUNCTIONS_VERSION" != "" ]; then
Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG"
fi
}
function GetConfFileValue () {
local file="${1}"
@ -331,7 +332,9 @@ function CopyProgram {
function CopyServiceFiles {
if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then
CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true
CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true
if [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM_USER" ]; then
CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true
fi
QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]."
QuickLogger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR."
@ -340,9 +343,9 @@ function CopyServiceFiles {
elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then
CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true
QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]."
QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]."
QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]."
QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]."
QuickLogger "Can be activated with [service $SERVICE_FILE_INIT start]."
QuickLogger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]."
else
QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation."
fi
@ -384,7 +387,11 @@ function RemoveFile {
function RemoveAll {
RemoveFile "$BIN_DIR/$PROGRAM_BINARY"
RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
fi
if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then # Check if any other program requiring ssh filter is present before removal
RemoveFile "$BIN_DIR/$SSH_FILTER"
else
@ -402,6 +409,7 @@ function Usage {
echo "options:"
echo "--silent Will log and bypass user interaction."
echo "--no-stats Used with --silent in order to refuse sending anonymous install stats."
echo "--remove Remove the program."
exit 127
}
@ -444,7 +452,9 @@ else
CreateDir "$BIN_DIR"
CopyExampleFiles
CopyProgram
CopyServiceFiles
if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "pmocr" ]; then
CopyServiceFiles
fi
QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
QuickLogger ""

View File

@ -1,7 +1,7 @@
#!/usr/bin/env bash
SUBPROGRAM=obackup
PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
AUTHOR="(L) 2013-2016 by Orsiris de Jong"
AUTHOR="(L) 2013-2017 by Orsiris de Jong"
CONTACT="http://www.netpower.fr - ozy@netpower.fr"
PROGRAM_BUILD=2016120401
@ -119,15 +119,15 @@ function Batch {
fi
done
runList=("${runAgainList[@]}")
runs=$(($runs + 1))
runs=$((runs + 1))
done
fi
}
function Usage {
echo "$PROGRAM $PROGRAM_BUILD"
echo $AUTHOR
echo $CONTACT
echo "$AUTHOR"
echo "$CONTACT"
echo ""
echo "Batch script to sequentially run osync or obackup instances and rerun failed ones."
echo "Usage: $PROGRAM.sh [OPTIONS] [$SUBPROGRAM OPTIONS]"

View File

@ -12,8 +12,8 @@ IS_STABLE=no
_OFUNCTIONS_VERSION=2.1-RC3+dev
_OFUNCTIONS_BUILD=2017031401
_OFUNCTIONS_VERSION=2.1.4-rc1
_OFUNCTIONS_BUILD=2017060901
_OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@ -38,6 +38,9 @@ fi
## Correct output of sort command (language agnostic sorting)
export LC_ALL=C
## Default umask for file creation
umask 0077
# Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
@ -70,6 +73,8 @@ if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as envi
fi
SCRIPT_PID=$$
# TODO: Check if %N works on MacOS
TSTAMP=$(date '+%Y%m%dT%H%M%S.%N')
LOCAL_USER=$(whoami)
@ -255,7 +260,7 @@ function Logger {
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix:$value" "$prefix$value"
_Logger "$prefix($level):$value" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
@ -301,14 +306,28 @@ function KillChilds {
local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" # Should parent be killed too ?
# Warning: pgrep does not exist in cygwin, have this checked in CheckEnvironment
if children="$(pgrep -P "$pid")"; then
for child in $children; do
KillChilds "$child" true
done
# Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
if [ $(IsNumeric "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Bogus pid given [$pid]." "CRITICAL"
return 1
fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if kill -0 "$pid" > /dev/null 2>&1; then
# Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
if children="$(pgrep -P "$pid")"; then
if [[ "$pid" == *"$children"* ]]; then
Logger "Bogus pgrep implementation." "CRITICAL"
children="${children/$pid/}"
fi
for child in $children; do
KillChilds "$child" true
done
fi
fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if [ "$self" == true ]; then
# We need to check for pid again because it may have disappeared after recursive function call
if kill -0 "$pid" > /dev/null 2>&1; then
kill -s TERM "$pid"
Logger "Sent SIGTERM to process [$pid]." "DEBUG"
@ -368,6 +387,7 @@ function SendAlert {
fi
eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
if [ $? != 0 ]; then
Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
attachment=false
@ -615,6 +635,7 @@ function Spinner {
}
# Time control function for background processes, suitable for multiple synchronous processes
# Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
# Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached
@ -739,6 +760,8 @@ function WaitForTaskCompletion {
pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
done
@ -897,6 +920,7 @@ function ParallelExec {
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
done
return $errorCount
@ -1081,7 +1105,7 @@ function GetLocalOS {
*"BSD"*)
LOCAL_OS="BSD"
;;
*"MINGW32"*|*"MSYS"*)
*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
LOCAL_OS="msys"
;;
*"CYGWIN"*)
@ -1107,9 +1131,6 @@ function GetLocalOS {
exit 1
;;
esac
if [ "$_OFUNCTIONS_VERSION" != "" ]; then
Logger "Local OS: [$localOsVar]." "DEBUG"
fi
# Get linux versions
if [ -f "/etc/os-release" ]; then
@ -1119,9 +1140,14 @@ function GetLocalOS {
# Add a global variable for statistics in installer
LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
if [ "$_OFUNCTIONS_VERSION" != "" ]; then
Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG"
fi
}
function GetRemoteOS {
if [ "$REMOTE_OPERATION" != "yes" ]; then
@ -1130,13 +1156,16 @@ function GetRemoteOS {
local remoteOsVar
$SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
$SSH_CMD env LC_ALL=C env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
function GetOs {
local localOsVar
local localOsName
local localOsVer
local osInfo="/etc/os-release"
# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
@ -1155,9 +1184,11 @@ function GetOs {
fi
fi
# Get linux versions
if [ -f "/etc/os-release" ]; then
localOsName=$(GetConfFileValue "/etc/os-release" "NAME")
localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION")
if [ -f "$osInfo" ]; then
localOsName=$(grep "^NAME=" "$osInfo")
localOsName="${localOsName##*=}"
localOsVer=$(grep "^VERSION=" "$osInfo")
localOsVer="${localOsVer##*=}"
fi
echo "$localOsVar ($localOsName $localOsVer)"
@ -1179,7 +1210,7 @@ ENDSSH
*"BSD"*)
REMOTE_OS="BSD"
;;
*"MINGW32"*|*"MSYS"*)
*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
REMOTE_OS="msys"
;;
*"CYGWIN"*)
@ -1262,7 +1293,7 @@ function RunRemoteCommand {
fi
Logger "Running command [$command] on remote host." "NOTICE"
cmd=$SSH_CMD' "env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
@ -1365,7 +1396,6 @@ function CheckConnectivity3rdPartyHosts {
fi
}
function RsyncPatternsAdd {
local patternType="${1}" # exclude or include
local pattern="${2}"
@ -1491,13 +1521,13 @@ function PostInit {
# Define remote commands
if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then
SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
elif [ -f "$SSH_PASSWORD_FILE" ]; then
SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS -p $REMOTE_PORT"
SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT"
else
SSH_PASSWORD=""
SSH_CMD=""
@ -1535,23 +1565,18 @@ function SetCompression {
COMPRESSION_PROGRAM="| pigz -c$compressionString"
COMPRESSION_EXTENSION=.gz
# obackup specific
if [ "$LOCAL_OS" != "MacOSX" ]; then
COMPRESSION_OPTIONS=--rsyncable
fi
COMPRESSION_OPTIONS=--rsyncable
elif type gzip > /dev/null 2>&1
then
COMPRESSION_PROGRAM="| gzip -c$compressionString"
COMPRESSION_EXTENSION=.gz
# obackup specific
if [ "$LOCAL_OS" != "MacOSX" ]; then
COMPRESSION_OPTIONS=--rsyncable
fi
COMPRESSION_OPTIONS=--rsyncable
else
COMPRESSION_PROGRAM=
COMPRESSION_EXTENSION=
fi
fi
ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
}
function InitLocalOSDependingSettings {
@ -1713,35 +1738,46 @@ function ParentPid {
# Neat version compare function found at http://stackoverflow.com/a/4025065/2635443
# Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2
vercomp () {
if [[ $1 == $2 ]]
then
return 0
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++))
do
if [[ -z ${ver2[i]} ]]
then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]}))
then
return 1
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
return 2
fi
done
return 0
function VerComp () {
if [ "$1" == "" ] || [ "$2" == "" ]; then
Logger "Bogus Vercomp values [$1] and [$2]." "WARN"
return 1
fi
if [[ $1 == $2 ]]
then
echo 0
return
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++))
do
if [[ -z ${ver2[i]} ]]
then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]}))
then
echo 1
return
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
echo 2
return
fi
done
echo 0
return
}
function GetConfFileValue () {
@ -1774,6 +1810,29 @@ function SetConfFileValue () {
fi
}
# Function can replace [ -f /some/file* ] tests
# Modified version of http://stackoverflow.com/a/6364244/2635443
function WildcardFileExists () {
local file="${1}"
local exists=0
for f in $file; do
## Check if the glob gets expanded to existing files.
## If not, f here will be exactly the pattern above
## and the exists test will evaluate to false.
if [ -e "$f" ]; then
exists=1
break
fi
done
if [ $exists -eq 1 ]; then
echo 1
else
echo 0
fi
}
# If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap
_OFUNCTIONS_BOOTSTRAP=true
[ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1
@ -4226,6 +4285,8 @@ else
Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
fi
# Switching rundir to
if [ "$IS_STABLE" != "yes" ]; then
Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN"
fi