Linux/obackup
1
0
Fork 0
mirror of https://github.com/deajan/obackup.git synced 2026-02-05 22:55:14 +01:00
Code Issues Releases Wiki Activity

Rebuilt targets

Browse Source
This commit is contained in:
deajan
2017-06-09 09:58:51 +02:00
parent 0ee0bf52fd
commit f15f216989
4 changed files with 348 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

318
dev/debug_obackup.sh
View File

@@ -35,8 +35,8 @@ IS_STABLE=no
# FilesBackup #__WITH_PARANOIA_DEBUG
_OFUNCTIONS_VERSION=2.1-RC3+dev
_OFUNCTIONS_BUILD=2017031401
_OFUNCTIONS_VERSION=2.1.4-rc1
_OFUNCTIONS_BUILD=2017060901
_OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@@ -61,6 +61,9 @@ fi
## Correct output of sort command (language agnostic sorting)
export LC_ALL=C
## Default umask for file creation
umask 0077
# Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
@@ -97,6 +100,8 @@ if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as envi
fi
SCRIPT_PID=$$
# TODO: Check if %N works on MacOS
TSTAMP=$(date '+%Y%m%dT%H%M%S.%N')
LOCAL_USER=$(whoami)
@@ -288,7 +293,7 @@ function Logger {
return
elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix:$value" "$prefix$value"
_Logger "$prefix($level):$value" "$prefix$value"
fi
return
elif [ "$level" == "ALWAYS" ]; then
@@ -339,15 +344,29 @@ function KillChilds {
local pid="${1}" # Parent pid to kill childs
local self="${2:-false}" # Should parent be killed too ?
# Warning: pgrep does not exist in cygwin, have this checked in CheckEnvironment
if children="$(pgrep -P "$pid")"; then
for child in $children; do
Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG
KillChilds "$child" true
done
# Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1
if [ $(IsNumeric "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
Logger "Bogus pid given [$pid]." "CRITICAL"
return 1
fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if kill -0 "$pid" > /dev/null 2>&1; then
# Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment
if children="$(pgrep -P "$pid")"; then
if [[ "$pid" == *"$children"* ]]; then
Logger "Bogus pgrep implementation." "CRITICAL"
children="${children/$pid/}"
fi
for child in $children; do
Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG
KillChilds "$child" true
done
fi
fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if [ "$self" == true ]; then
# We need to check for pid again because it may have disappeared after recursive function call
if kill -0 "$pid" > /dev/null 2>&1; then
kill -s TERM "$pid"
Logger "Sent SIGTERM to process [$pid]." "DEBUG"
@@ -409,6 +428,7 @@ function SendAlert {
fi
eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE"
ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
if [ $? != 0 ]; then
Logger "Cannot create [$ALERT_LOG_FILE]" "WARN"
attachment=false
@@ -657,6 +677,22 @@ function Spinner {
fi
}
function _PerfProfiler { #__WITH_PARANOIA_DEBUG
local perfString #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan) #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
for i in $(pgrep -P $$); do #__WITH_PARANOIA_DEBUG
perfString="$perfString\n"$(ps -p $i -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan | tail -1) #__WITH_PARANOIA_DEBUG
done #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
if type iostat > /dev/null 2>&1; then #__WITH_PARANOIA_DEBUG
perfString="$perfString\n"$(iostat) #__WITH_PARANOIA_DEBUG
fi #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
} #__WITH_PARANOIA_DEBUG
# Time control function for background processes, suitable for multiple synchronous processes
# Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
@@ -789,6 +825,11 @@ function WaitForTaskCompletion {
pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
done
Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
@@ -954,6 +995,10 @@ function ParallelExec {
# Trivial wait time for bash to not eat up all CPU
sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
done
return $errorCount
@@ -1139,7 +1184,7 @@ function GetLocalOS {
*"BSD"*)
LOCAL_OS="BSD"
;;
*"MINGW32"*|*"MSYS"*)
*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
LOCAL_OS="msys"
;;
*"CYGWIN"*)
@@ -1165,9 +1210,6 @@ function GetLocalOS {
exit 1
;;
esac
if [ "$_OFUNCTIONS_VERSION" != "" ]; then
Logger "Local OS: [$localOsVar]." "DEBUG"
fi
# Get linux versions
if [ -f "/etc/os-release" ]; then
@@ -1177,8 +1219,68 @@ function GetLocalOS {
# Add a global variable for statistics in installer
LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
if [ "$_OFUNCTIONS_VERSION" != "" ]; then
Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG"
fi
}
#__BEGIN_WITH_PARANOIA_DEBUG
function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing
if [ "$_DEBUG" == "yes" ]; then
local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
local numberOfGivenArguments="${2}" # Number of arguments that have been passed
local minArgs
local maxArgs
# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${3} and count
callerName="${FUNCNAME[1]}"
local iterate=3
local fetchArguments=true
local argList=""
local countedArguments
while [ $fetchArguments == true ]; do
cmd='argument=${'$iterate'}'
eval $cmd
if [ "$argument" == "" ]; then
fetchArguments=false
else
argList="$argList[Argument $((iterate-2)): $argument] "
iterate=$((iterate+1))
fi
done
countedArguments=$((iterate-3))
if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
minArgs=$numberOfArguments
maxArgs=$numberOfArguments
else
IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
fi
Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
Logger "$callerName arguments: $argList" "ERROR"
else
if [ ! -z "$argList" ]; then
Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
fi
fi
fi
}
#__END_WITH_PARANOIA_DEBUG
function GetRemoteOS {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
@@ -1189,13 +1291,16 @@ function GetRemoteOS {
local remoteOsVar
$SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
$SSH_CMD env LC_ALL=C env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1
function GetOs {
local localOsVar
local localOsName
local localOsVer
local osInfo="/etc/os-release"
# There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox"
@@ -1214,9 +1319,11 @@ function GetOs {
fi
fi
# Get linux versions
if [ -f "/etc/os-release" ]; then
localOsName=$(GetConfFileValue "/etc/os-release" "NAME")
localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION")
if [ -f "$osInfo" ]; then
localOsName=$(grep "^NAME=" "$osInfo")
localOsName="${localOsName##*=}"
localOsVer=$(grep "^VERSION=" "$osInfo")
localOsVer="${localOsVer##*=}"
fi
echo "$localOsVar ($localOsName $localOsVer)"
@@ -1238,7 +1345,7 @@ ENDSSH
*"BSD"*)
REMOTE_OS="BSD"
;;
*"MINGW32"*|*"MSYS"*)
*"MINGW32"*|*"MINGW64"*|*"MSYS"*)
REMOTE_OS="msys"
;;
*"CYGWIN"*)
@@ -1323,7 +1430,7 @@ function RunRemoteCommand {
fi
Logger "Running command [$command] on remote host." "NOTICE"
cmd=$SSH_CMD' "env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1'
Logger "cmd: $cmd" "DEBUG"
eval "$cmd" &
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false
@@ -1434,62 +1541,6 @@ function CheckConnectivity3rdPartyHosts {
fi #__WITH_PARANOIA_DEBUG
}
#__BEGIN_WITH_PARANOIA_DEBUG
function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing
if [ "$_DEBUG" == "yes" ]; then
local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
local numberOfGivenArguments="${2}" # Number of arguments that have been passed
local minArgs
local maxArgs
# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${3} and count
callerName="${FUNCNAME[1]}"
local iterate=3
local fetchArguments=true
local argList=""
local countedArguments
while [ $fetchArguments == true ]; do
cmd='argument=${'$iterate'}'
eval $cmd
if [ "$argument" == "" ]; then
fetchArguments=false
else
argList="$argList[Argument $((iterate-2)): $argument] "
iterate=$((iterate+1))
fi
done
countedArguments=$((iterate-3))
if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
minArgs=$numberOfArguments
maxArgs=$numberOfArguments
else
IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
fi
Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
Logger "$callerName arguments: $argList" "ERROR"
else
if [ ! -z "$argList" ]; then
Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
fi
fi
fi
}
#__END_WITH_PARANOIA_DEBUG
function RsyncPatternsAdd {
local patternType="${1}" # exclude or include
local pattern="${2}"
@@ -1620,13 +1671,13 @@ function PostInit {
# Define remote commands
if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then
SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT"
elif [ -f "$SSH_PASSWORD_FILE" ]; then
SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS -p $REMOTE_PORT"
SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT"
SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT"
RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT"
else
SSH_PASSWORD=""
SSH_CMD=""
@@ -1664,23 +1715,18 @@ function SetCompression {
COMPRESSION_PROGRAM="| pigz -c$compressionString"
COMPRESSION_EXTENSION=.gz
# obackup specific
if [ "$LOCAL_OS" != "MacOSX" ]; then
COMPRESSION_OPTIONS=--rsyncable
fi
COMPRESSION_OPTIONS=--rsyncable
elif type gzip > /dev/null 2>&1
then
COMPRESSION_PROGRAM="| gzip -c$compressionString"
COMPRESSION_EXTENSION=.gz
# obackup specific
if [ "$LOCAL_OS" != "MacOSX" ]; then
COMPRESSION_OPTIONS=--rsyncable
fi
COMPRESSION_OPTIONS=--rsyncable
else
COMPRESSION_PROGRAM=
COMPRESSION_EXTENSION=
fi
fi
ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION"
}
function InitLocalOSDependingSettings {
@@ -1844,35 +1890,46 @@ function ParentPid {
# Neat version compare function found at http://stackoverflow.com/a/4025065/2635443
# Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2
vercomp () {
if [[ $1 == $2 ]]
then
return 0
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++))
do
if [[ -z ${ver2[i]} ]]
then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]}))
then
return 1
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
return 2
fi
done
return 0
function VerComp () {
if [ "$1" == "" ] || [ "$2" == "" ]; then
Logger "Bogus Vercomp values [$1] and [$2]." "WARN"
return 1
fi
if [[ $1 == $2 ]]
then
echo 0
return
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++))
do
if [[ -z ${ver2[i]} ]]
then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]}))
then
echo 1
return
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
echo 2
return
fi
done
echo 0
return
}
function GetConfFileValue () {
@@ -1905,6 +1962,29 @@ function SetConfFileValue () {
fi
}
# Function can replace [ -f /some/file* ] tests
# Modified version of http://stackoverflow.com/a/6364244/2635443
function WildcardFileExists () {
local file="${1}"
local exists=0
for f in $file; do
## Check if the glob gets expanded to existing files.
## If not, f here will be exactly the pattern above
## and the exists test will evaluate to false.
if [ -e "$f" ]; then
exists=1
break
fi
done
if [ $exists -eq 1 ]; then
echo 1
else
echo 0
fi
}
# If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap
_OFUNCTIONS_BOOTSTRAP=true
[ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1
@@ -4436,6 +4516,8 @@ else
Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
fi
# Switching rundir to
if [ "$IS_STABLE" != "yes" ]; then
Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN"
fi