From f15f2169890ef8742183187f83f41dcd5ca700b5 Mon Sep 17 00:00:00 2001 From: deajan Date: Fri, 9 Jun 2017 09:58:51 +0200 Subject: [PATCH] Rebuilt targets --- dev/debug_obackup.sh | 318 +++++++++++++++++++++++++++---------------- install.sh | 32 +++-- obackup-batch.sh | 8 +- obackup.sh | 185 ++++++++++++++++--------- 4 files changed, 348 insertions(+), 195 deletions(-) diff --git a/dev/debug_obackup.sh b/dev/debug_obackup.sh index 535b248..04f55d7 100755 --- a/dev/debug_obackup.sh +++ b/dev/debug_obackup.sh @@ -35,8 +35,8 @@ IS_STABLE=no # FilesBackup #__WITH_PARANOIA_DEBUG -_OFUNCTIONS_VERSION=2.1-RC3+dev -_OFUNCTIONS_BUILD=2017031401 +_OFUNCTIONS_VERSION=2.1.4-rc1 +_OFUNCTIONS_BUILD=2017060901 _OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr @@ -61,6 +61,9 @@ fi ## Correct output of sort command (language agnostic sorting) export LC_ALL=C +## Default umask for file creation +umask 0077 + # Standard alert mail body MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." @@ -97,6 +100,8 @@ if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as envi fi SCRIPT_PID=$$ + +# TODO: Check if %N works on MacOS TSTAMP=$(date '+%Y%m%dT%H%M%S.%N') LOCAL_USER=$(whoami) @@ -288,7 +293,7 @@ function Logger { return elif [ "$level" == "VERBOSE" ]; then if [ $_LOGGER_VERBOSE == true ]; then - _Logger "$prefix:$value" "$prefix$value" + _Logger "$prefix($level):$value" "$prefix$value" fi return elif [ "$level" == "ALWAYS" ]; then @@ -339,15 +344,29 @@ function KillChilds { local pid="${1}" # Parent pid to kill childs local self="${2:-false}" # Should parent be killed too ? - # Warning: pgrep does not exist in cygwin, have this checked in CheckEnvironment - if children="$(pgrep -P "$pid")"; then - for child in $children; do - Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG - KillChilds "$child" true - done + # Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1 + if [ $(IsNumeric "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then + Logger "Bogus pid given [$pid]." "CRITICAL" + return 1 fi - # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing + + if kill -0 "$pid" > /dev/null 2>&1; then + # Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment + if children="$(pgrep -P "$pid")"; then + if [[ "$pid" == *"$children"* ]]; then + Logger "Bogus pgrep implementation." "CRITICAL" + children="${children/$pid/}" + fi + for child in $children; do + Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG + KillChilds "$child" true + done + fi + fi + + # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing if [ "$self" == true ]; then + # We need to check for pid again because it may have disappeared after recursive function call if kill -0 "$pid" > /dev/null 2>&1; then kill -s TERM "$pid" Logger "Sent SIGTERM to process [$pid]." "DEBUG" @@ -409,6 +428,7 @@ function SendAlert { fi eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" if [ $? != 0 ]; then Logger "Cannot create [$ALERT_LOG_FILE]" "WARN" attachment=false @@ -657,6 +677,22 @@ function Spinner { fi } +function _PerfProfiler { #__WITH_PARANOIA_DEBUG + local perfString #__WITH_PARANOIA_DEBUG + #__WITH_PARANOIA_DEBUG + perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan) #__WITH_PARANOIA_DEBUG + #__WITH_PARANOIA_DEBUG + for i in $(pgrep -P $$); do #__WITH_PARANOIA_DEBUG + perfString="$perfString\n"$(ps -p $i -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan | tail -1) #__WITH_PARANOIA_DEBUG + done #__WITH_PARANOIA_DEBUG + #__WITH_PARANOIA_DEBUG + if type iostat > /dev/null 2>&1; then #__WITH_PARANOIA_DEBUG + perfString="$perfString\n"$(iostat) #__WITH_PARANOIA_DEBUG + fi #__WITH_PARANOIA_DEBUG + #__WITH_PARANOIA_DEBUG + Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG +} #__WITH_PARANOIA_DEBUG + # Time control function for background processes, suitable for multiple synchronous processes # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 @@ -789,6 +825,11 @@ function WaitForTaskCompletion { pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU sleep $sleepTime + + if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG + _PerfProfiler ##__WITH_PARANOIA_DEBUG + fi ##__WITH_PARANOIA_DEBUG + done Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG @@ -954,6 +995,10 @@ function ParallelExec { # Trivial wait time for bash to not eat up all CPU sleep $sleepTime + + if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG + _PerfProfiler ##__WITH_PARANOIA_DEBUG + fi ##__WITH_PARANOIA_DEBUG done return $errorCount @@ -1139,7 +1184,7 @@ function GetLocalOS { *"BSD"*) LOCAL_OS="BSD" ;; - *"MINGW32"*|*"MSYS"*) + *"MINGW32"*|*"MINGW64"*|*"MSYS"*) LOCAL_OS="msys" ;; *"CYGWIN"*) @@ -1165,9 +1210,6 @@ function GetLocalOS { exit 1 ;; esac - if [ "$_OFUNCTIONS_VERSION" != "" ]; then - Logger "Local OS: [$localOsVar]." "DEBUG" - fi # Get linux versions if [ -f "/etc/os-release" ]; then @@ -1177,8 +1219,68 @@ function GetLocalOS { # Add a global variable for statistics in installer LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" + + if [ "$_OFUNCTIONS_VERSION" != "" ]; then + Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" + fi } +#__BEGIN_WITH_PARANOIA_DEBUG +function __CheckArguments { + # Checks the number of arguments of a function and raises an error if some are missing + + if [ "$_DEBUG" == "yes" ]; then + local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments + local numberOfGivenArguments="${2}" # Number of arguments that have been passed + + local minArgs + local maxArgs + + # All arguments of the function to check are passed as array in ${3} (the function call waits for $@) + # If any of the arguments contains spaces, bash things there are two aguments + # In order to avoid this, we need to iterate over ${3} and count + + callerName="${FUNCNAME[1]}" + + local iterate=3 + local fetchArguments=true + local argList="" + local countedArguments + while [ $fetchArguments == true ]; do + cmd='argument=${'$iterate'}' + eval $cmd + if [ "$argument" == "" ]; then + fetchArguments=false + else + argList="$argList[Argument $((iterate-2)): $argument] " + iterate=$((iterate+1)) + fi + done + + countedArguments=$((iterate-3)) + + if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then + minArgs=$numberOfArguments + maxArgs=$numberOfArguments + else + IFS='-' read minArgs maxArgs <<< "$numberOfArguments" + fi + + Logger "Entering function [$callerName]." "PARANOIA_DEBUG" + + if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then + Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR" + Logger "$callerName arguments: $argList" "ERROR" + else + if [ ! -z "$argList" ]; then + Logger "$callerName arguments: $argList" "PARANOIA_DEBUG" + fi + fi + fi +} + +#__END_WITH_PARANOIA_DEBUG + function GetRemoteOS { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1189,13 +1291,16 @@ function GetRemoteOS { local remoteOsVar -$SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +$SSH_CMD env LC_ALL=C env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 + function GetOs { local localOsVar local localOsName local localOsVer + local osInfo="/etc/os-release" + # There's no good way to tell if currently running in BusyBox shell. Using sluggish way. if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then localOsVar="BusyBox" @@ -1214,9 +1319,11 @@ function GetOs { fi fi # Get linux versions - if [ -f "/etc/os-release" ]; then - localOsName=$(GetConfFileValue "/etc/os-release" "NAME") - localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION") + if [ -f "$osInfo" ]; then + localOsName=$(grep "^NAME=" "$osInfo") + localOsName="${localOsName##*=}" + localOsVer=$(grep "^VERSION=" "$osInfo") + localOsVer="${localOsVer##*=}" fi echo "$localOsVar ($localOsName $localOsVer)" @@ -1238,7 +1345,7 @@ ENDSSH *"BSD"*) REMOTE_OS="BSD" ;; - *"MINGW32"*|*"MSYS"*) + *"MINGW32"*|*"MINGW64"*|*"MSYS"*) REMOTE_OS="msys" ;; *"CYGWIN"*) @@ -1323,7 +1430,7 @@ function RunRemoteCommand { fi Logger "Running command [$command] on remote host." "NOTICE" - cmd=$SSH_CMD' "env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1' + cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false @@ -1434,62 +1541,6 @@ function CheckConnectivity3rdPartyHosts { fi #__WITH_PARANOIA_DEBUG } -#__BEGIN_WITH_PARANOIA_DEBUG -function __CheckArguments { - # Checks the number of arguments of a function and raises an error if some are missing - - if [ "$_DEBUG" == "yes" ]; then - local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments - local numberOfGivenArguments="${2}" # Number of arguments that have been passed - - local minArgs - local maxArgs - - # All arguments of the function to check are passed as array in ${3} (the function call waits for $@) - # If any of the arguments contains spaces, bash things there are two aguments - # In order to avoid this, we need to iterate over ${3} and count - - callerName="${FUNCNAME[1]}" - - local iterate=3 - local fetchArguments=true - local argList="" - local countedArguments - while [ $fetchArguments == true ]; do - cmd='argument=${'$iterate'}' - eval $cmd - if [ "$argument" == "" ]; then - fetchArguments=false - else - argList="$argList[Argument $((iterate-2)): $argument] " - iterate=$((iterate+1)) - fi - done - - countedArguments=$((iterate-3)) - - if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then - minArgs=$numberOfArguments - maxArgs=$numberOfArguments - else - IFS='-' read minArgs maxArgs <<< "$numberOfArguments" - fi - - Logger "Entering function [$callerName]." "PARANOIA_DEBUG" - - if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then - Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR" - Logger "$callerName arguments: $argList" "ERROR" - else - if [ ! -z "$argList" ]; then - Logger "$callerName arguments: $argList" "PARANOIA_DEBUG" - fi - fi - fi -} - -#__END_WITH_PARANOIA_DEBUG - function RsyncPatternsAdd { local patternType="${1}" # exclude or include local pattern="${2}" @@ -1620,13 +1671,13 @@ function PostInit { # Define remote commands if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then - SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" - SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" - RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" + SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" elif [ -f "$SSH_PASSWORD_FILE" ]; then - SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" - SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -P $REMOTE_PORT" - RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS -p $REMOTE_PORT" + SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT" else SSH_PASSWORD="" SSH_CMD="" @@ -1664,23 +1715,18 @@ function SetCompression { COMPRESSION_PROGRAM="| pigz -c$compressionString" COMPRESSION_EXTENSION=.gz # obackup specific - if [ "$LOCAL_OS" != "MacOSX" ]; then - COMPRESSION_OPTIONS=--rsyncable - fi + COMPRESSION_OPTIONS=--rsyncable elif type gzip > /dev/null 2>&1 then COMPRESSION_PROGRAM="| gzip -c$compressionString" COMPRESSION_EXTENSION=.gz # obackup specific - if [ "$LOCAL_OS" != "MacOSX" ]; then - COMPRESSION_OPTIONS=--rsyncable - fi + COMPRESSION_OPTIONS=--rsyncable else COMPRESSION_PROGRAM= COMPRESSION_EXTENSION= fi fi - ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" } function InitLocalOSDependingSettings { @@ -1844,35 +1890,46 @@ function ParentPid { # Neat version compare function found at http://stackoverflow.com/a/4025065/2635443 # Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2 -vercomp () { - if [[ $1 == $2 ]] - then - return 0 - fi - local IFS=. - local i ver1=($1) ver2=($2) - # fill empty fields in ver1 with zeros - for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) - do - ver1[i]=0 - done - for ((i=0; i<${#ver1[@]}; i++)) - do - if [[ -z ${ver2[i]} ]] - then - # fill empty fields in ver2 with zeros - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})) - then - return 1 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})) - then - return 2 - fi - done - return 0 +function VerComp () { + if [ "$1" == "" ] || [ "$2" == "" ]; then + Logger "Bogus Vercomp values [$1] and [$2]." "WARN" + return 1 + fi + + if [[ $1 == $2 ]] + then + echo 0 + return + fi + + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + echo 1 + return + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + echo 2 + return + fi + done + + echo 0 + return } function GetConfFileValue () { @@ -1905,6 +1962,29 @@ function SetConfFileValue () { fi } +# Function can replace [ -f /some/file* ] tests +# Modified version of http://stackoverflow.com/a/6364244/2635443 +function WildcardFileExists () { + local file="${1}" + local exists=0 + + for f in $file; do + ## Check if the glob gets expanded to existing files. + ## If not, f here will be exactly the pattern above + ## and the exists test will evaluate to false. + if [ -e "$f" ]; then + exists=1 + break + fi + done + + if [ $exists -eq 1 ]; then + echo 1 + else + echo 0 + fi +} + # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap _OFUNCTIONS_BOOTSTRAP=true [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1 @@ -4436,6 +4516,8 @@ else Logger "Script begin, logging to [$LOG_FILE]." "DEBUG" fi +# Switching rundir to + if [ "$IS_STABLE" != "yes" ]; then Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN" fi diff --git a/install.sh b/install.sh index 862153c..24018f9 100755 --- a/install.sh +++ b/install.sh @@ -12,7 +12,7 @@ PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SSH_FILTER="ssh_filter.sh" -SCRIPT_BUILD=2017031402 +SCRIPT_BUILD=2017041701 ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 @@ -122,7 +122,7 @@ function GetLocalOS { *"BSD"*) LOCAL_OS="BSD" ;; - *"MINGW32"*|*"MSYS"*) + *"MINGW32"*|*"MINGW64"*|*"MSYS"*) LOCAL_OS="msys" ;; *"CYGWIN"*) @@ -148,9 +148,6 @@ function GetLocalOS { exit 1 ;; esac - if [ "$_OFUNCTIONS_VERSION" != "" ]; then - Logger "Local OS: [$localOsVar]." "DEBUG" - fi # Get linux versions if [ -f "/etc/os-release" ]; then @@ -160,6 +157,10 @@ function GetLocalOS { # Add a global variable for statistics in installer LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" + + if [ "$_OFUNCTIONS_VERSION" != "" ]; then + Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" + fi } function GetConfFileValue () { local file="${1}" @@ -331,7 +332,9 @@ function CopyProgram { function CopyServiceFiles { if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true - CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true + if [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM_USER" ]; then + CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true + fi QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]." QuickLogger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR." @@ -340,9 +343,9 @@ function CopyServiceFiles { elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true - QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]." - QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]." - QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]." + QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]." + QuickLogger "Can be activated with [service $SERVICE_FILE_INIT start]." + QuickLogger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]." else QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation." fi @@ -384,7 +387,11 @@ function RemoveFile { function RemoveAll { RemoveFile "$BIN_DIR/$PROGRAM_BINARY" - RemoveFile "$BIN_DIR/$PROGRAM_BATCH" + + if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then + RemoveFile "$BIN_DIR/$PROGRAM_BATCH" + fi + if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then # Check if any other program requiring ssh filter is present before removal RemoveFile "$BIN_DIR/$SSH_FILTER" else @@ -402,6 +409,7 @@ function Usage { echo "options:" echo "--silent Will log and bypass user interaction." echo "--no-stats Used with --silent in order to refuse sending anonymous install stats." + echo "--remove Remove the program." exit 127 } @@ -444,7 +452,9 @@ else CreateDir "$BIN_DIR" CopyExampleFiles CopyProgram - CopyServiceFiles + if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "pmocr" ]; then + CopyServiceFiles + fi QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM" if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then QuickLogger "" diff --git a/obackup-batch.sh b/obackup-batch.sh index 5e7b366..dc6f771 100755 --- a/obackup-batch.sh +++ b/obackup-batch.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash SUBPROGRAM=obackup PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones -AUTHOR="(L) 2013-2016 by Orsiris de Jong" +AUTHOR="(L) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr - ozy@netpower.fr" PROGRAM_BUILD=2016120401 @@ -119,15 +119,15 @@ function Batch { fi done runList=("${runAgainList[@]}") - runs=$(($runs + 1)) + runs=$((runs + 1)) done fi } function Usage { echo "$PROGRAM $PROGRAM_BUILD" - echo $AUTHOR - echo $CONTACT + echo "$AUTHOR" + echo "$CONTACT" echo "" echo "Batch script to sequentially run osync or obackup instances and rerun failed ones." echo "Usage: $PROGRAM.sh [OPTIONS] [$SUBPROGRAM OPTIONS]" diff --git a/obackup.sh b/obackup.sh index 6fefb74..4605fe0 100755 --- a/obackup.sh +++ b/obackup.sh @@ -12,8 +12,8 @@ IS_STABLE=no -_OFUNCTIONS_VERSION=2.1-RC3+dev -_OFUNCTIONS_BUILD=2017031401 +_OFUNCTIONS_VERSION=2.1.4-rc1 +_OFUNCTIONS_BUILD=2017060901 _OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr @@ -38,6 +38,9 @@ fi ## Correct output of sort command (language agnostic sorting) export LC_ALL=C +## Default umask for file creation +umask 0077 + # Standard alert mail body MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." @@ -70,6 +73,8 @@ if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as envi fi SCRIPT_PID=$$ + +# TODO: Check if %N works on MacOS TSTAMP=$(date '+%Y%m%dT%H%M%S.%N') LOCAL_USER=$(whoami) @@ -255,7 +260,7 @@ function Logger { return elif [ "$level" == "VERBOSE" ]; then if [ $_LOGGER_VERBOSE == true ]; then - _Logger "$prefix:$value" "$prefix$value" + _Logger "$prefix($level):$value" "$prefix$value" fi return elif [ "$level" == "ALWAYS" ]; then @@ -301,14 +306,28 @@ function KillChilds { local pid="${1}" # Parent pid to kill childs local self="${2:-false}" # Should parent be killed too ? - # Warning: pgrep does not exist in cygwin, have this checked in CheckEnvironment - if children="$(pgrep -P "$pid")"; then - for child in $children; do - KillChilds "$child" true - done + # Paranoid checks, we can safely assume that $pid shouldn't be 0 nor 1 + if [ $(IsNumeric "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then + Logger "Bogus pid given [$pid]." "CRITICAL" + return 1 fi - # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing + + if kill -0 "$pid" > /dev/null 2>&1; then + # Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment + if children="$(pgrep -P "$pid")"; then + if [[ "$pid" == *"$children"* ]]; then + Logger "Bogus pgrep implementation." "CRITICAL" + children="${children/$pid/}" + fi + for child in $children; do + KillChilds "$child" true + done + fi + fi + + # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing if [ "$self" == true ]; then + # We need to check for pid again because it may have disappeared after recursive function call if kill -0 "$pid" > /dev/null 2>&1; then kill -s TERM "$pid" Logger "Sent SIGTERM to process [$pid]." "DEBUG" @@ -368,6 +387,7 @@ function SendAlert { fi eval "cat \"$LOG_FILE\" $COMPRESSION_PROGRAM > $ALERT_LOG_FILE" + ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" if [ $? != 0 ]; then Logger "Cannot create [$ALERT_LOG_FILE]" "WARN" attachment=false @@ -615,6 +635,7 @@ function Spinner { } + # Time control function for background processes, suitable for multiple synchronous processes # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 # Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached @@ -739,6 +760,8 @@ function WaitForTaskCompletion { pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU sleep $sleepTime + + done @@ -897,6 +920,7 @@ function ParallelExec { # Trivial wait time for bash to not eat up all CPU sleep $sleepTime + done return $errorCount @@ -1081,7 +1105,7 @@ function GetLocalOS { *"BSD"*) LOCAL_OS="BSD" ;; - *"MINGW32"*|*"MSYS"*) + *"MINGW32"*|*"MINGW64"*|*"MSYS"*) LOCAL_OS="msys" ;; *"CYGWIN"*) @@ -1107,9 +1131,6 @@ function GetLocalOS { exit 1 ;; esac - if [ "$_OFUNCTIONS_VERSION" != "" ]; then - Logger "Local OS: [$localOsVar]." "DEBUG" - fi # Get linux versions if [ -f "/etc/os-release" ]; then @@ -1119,9 +1140,14 @@ function GetLocalOS { # Add a global variable for statistics in installer LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" + + if [ "$_OFUNCTIONS_VERSION" != "" ]; then + Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" + fi } + function GetRemoteOS { if [ "$REMOTE_OPERATION" != "yes" ]; then @@ -1130,13 +1156,16 @@ function GetRemoteOS { local remoteOsVar -$SSH_CMD env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 +$SSH_CMD env LC_ALL=C env _REMOTE_TOKEN="$_REMOTE_TOKEN" bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" 2>&1 + function GetOs { local localOsVar local localOsName local localOsVer + local osInfo="/etc/os-release" + # There's no good way to tell if currently running in BusyBox shell. Using sluggish way. if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then localOsVar="BusyBox" @@ -1155,9 +1184,11 @@ function GetOs { fi fi # Get linux versions - if [ -f "/etc/os-release" ]; then - localOsName=$(GetConfFileValue "/etc/os-release" "NAME") - localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION") + if [ -f "$osInfo" ]; then + localOsName=$(grep "^NAME=" "$osInfo") + localOsName="${localOsName##*=}" + localOsVer=$(grep "^VERSION=" "$osInfo") + localOsVer="${localOsVer##*=}" fi echo "$localOsVar ($localOsName $localOsVer)" @@ -1179,7 +1210,7 @@ ENDSSH *"BSD"*) REMOTE_OS="BSD" ;; - *"MINGW32"*|*"MSYS"*) + *"MINGW32"*|*"MINGW64"*|*"MSYS"*) REMOTE_OS="msys" ;; *"CYGWIN"*) @@ -1262,7 +1293,7 @@ function RunRemoteCommand { fi Logger "Running command [$command] on remote host." "NOTICE" - cmd=$SSH_CMD' "env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1' + cmd=$SSH_CMD' "env LC_ALL=C env _REMOTE_TOKEN="'$_REMOTE_TOKEN'" $command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP'" 2>&1' Logger "cmd: $cmd" "DEBUG" eval "$cmd" & WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false @@ -1365,7 +1396,6 @@ function CheckConnectivity3rdPartyHosts { fi } - function RsyncPatternsAdd { local patternType="${1}" # exclude or include local pattern="${2}" @@ -1491,13 +1521,13 @@ function PostInit { # Define remote commands if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then - SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" - SCP_CMD="$(type -p scp) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" - RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" + SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" elif [ -f "$SSH_PASSWORD_FILE" ]; then - SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" - SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -P $REMOTE_PORT" - RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP $SSH_OPTS -p $REMOTE_PORT" + SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" + SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT" + RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT" else SSH_PASSWORD="" SSH_CMD="" @@ -1535,23 +1565,18 @@ function SetCompression { COMPRESSION_PROGRAM="| pigz -c$compressionString" COMPRESSION_EXTENSION=.gz # obackup specific - if [ "$LOCAL_OS" != "MacOSX" ]; then - COMPRESSION_OPTIONS=--rsyncable - fi + COMPRESSION_OPTIONS=--rsyncable elif type gzip > /dev/null 2>&1 then COMPRESSION_PROGRAM="| gzip -c$compressionString" COMPRESSION_EXTENSION=.gz # obackup specific - if [ "$LOCAL_OS" != "MacOSX" ]; then - COMPRESSION_OPTIONS=--rsyncable - fi + COMPRESSION_OPTIONS=--rsyncable else COMPRESSION_PROGRAM= COMPRESSION_EXTENSION= fi fi - ALERT_LOG_FILE="$ALERT_LOG_FILE$COMPRESSION_EXTENSION" } function InitLocalOSDependingSettings { @@ -1713,35 +1738,46 @@ function ParentPid { # Neat version compare function found at http://stackoverflow.com/a/4025065/2635443 # Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2 -vercomp () { - if [[ $1 == $2 ]] - then - return 0 - fi - local IFS=. - local i ver1=($1) ver2=($2) - # fill empty fields in ver1 with zeros - for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) - do - ver1[i]=0 - done - for ((i=0; i<${#ver1[@]}; i++)) - do - if [[ -z ${ver2[i]} ]] - then - # fill empty fields in ver2 with zeros - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})) - then - return 1 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})) - then - return 2 - fi - done - return 0 +function VerComp () { + if [ "$1" == "" ] || [ "$2" == "" ]; then + Logger "Bogus Vercomp values [$1] and [$2]." "WARN" + return 1 + fi + + if [[ $1 == $2 ]] + then + echo 0 + return + fi + + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + echo 1 + return + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + echo 2 + return + fi + done + + echo 0 + return } function GetConfFileValue () { @@ -1774,6 +1810,29 @@ function SetConfFileValue () { fi } +# Function can replace [ -f /some/file* ] tests +# Modified version of http://stackoverflow.com/a/6364244/2635443 +function WildcardFileExists () { + local file="${1}" + local exists=0 + + for f in $file; do + ## Check if the glob gets expanded to existing files. + ## If not, f here will be exactly the pattern above + ## and the exists test will evaluate to false. + if [ -e "$f" ]; then + exists=1 + break + fi + done + + if [ $exists -eq 1 ]; then + echo 1 + else + echo 0 + fi +} + # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap _OFUNCTIONS_BOOTSTRAP=true [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1 @@ -4226,6 +4285,8 @@ else Logger "Script begin, logging to [$LOG_FILE]." "DEBUG" fi +# Switching rundir to + if [ "$IS_STABLE" != "yes" ]; then Logger "This is an unstable dev build [$PROGRAM_BUILD]. Please use with caution." "WARN" fi