From 5d460c39165354db89b01cb53b795095c706c25e Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 30 Jul 2018 15:36:53 +0200 Subject: [PATCH] Rebuilt targets --- dev/debug_obackup.sh | 416 +++++++++++++++++++++++++++---------------- install.sh | 127 +++++++------ obackup.sh | 394 +++++++++++++++++++++++++--------------- 3 files changed, 589 insertions(+), 348 deletions(-) diff --git a/dev/debug_obackup.sh b/dev/debug_obackup.sh index aabf7ac..60d1404 100755 --- a/dev/debug_obackup.sh +++ b/dev/debug_obackup.sh @@ -7,7 +7,7 @@ PROGRAM="obackup" AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" PROGRAM_VERSION=2.1-beta5 -PROGRAM_BUILD=2018022401 +PROGRAM_BUILD=2018031501 IS_STABLE=no #### Execution order #__WITH_PARANOIA_DEBUG @@ -34,20 +34,10 @@ IS_STABLE=no # RsyncPatterns #__WITH_PARANOIA_DEBUG # FilesBackup #__WITH_PARANOIA_DEBUG - -#TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling" -#done: add checkRFC function (and use it for --destination-mails) -#done: ExecTasks still needs some better call argument list -#done: ExecTasks sub function relocate -#done: SendMail and SendEmail convert functions inverted, check on osync and obackup -#command line arguments don't take -AaqV for example - _OFUNCTIONS_VERSION=2.3.0-dev -_OFUNCTIONS_BUILD=2018031501 +_OFUNCTIONS_BUILD=2018070902 _OFUNCTIONS_BOOTSTRAP=true -## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr - ## To use in a program, define the following variables: ## PROGRAM=program-name ## INSTANCE_ID=program-instance-name @@ -110,9 +100,6 @@ fi SCRIPT_PID=$$ -# TODO: Check if %N works on MacOS -TSTAMP=$(date '+%Y%m%dT%H%M%S.%N') - LOCAL_USER=$(whoami) LOCAL_HOST=$(hostname) @@ -140,6 +127,46 @@ else RUN_DIR=. fi +#### PoorMansRandomGenerator SUBSET #### +# Get a random number on Windows BusyBox alike, also works on most Unixes +function PoorMansRandomGenerator { + local digits="${1}" # The number of digits to generate + + local minimum=1 + local maximum + local n=0 + + if [ "$digits" == "" ]; then + digits=5 + fi + + # Minimum already has a digit + for n in $(seq 1 $((digits-1))); do + minimum=$minimum"0" + maximum=$maximum"9" + done + maximum=$maximum"9" + + #n=0; while [ $n -lt $minimum ]; do n=$n$(dd if=/dev/urandom bs=100 count=1 2>/dev/null | tr -cd '0-9'); done; n=$(echo $n | sed -e 's/^0//') + # bs=19 since if real random strikes, having a 19 digits number is not supported + while [ $n -lt $minimum ] || [ $n -gt $maximum ]; do + if [ $n -lt $minimum ]; then + # Add numbers + n=$n$(dd if=/dev/urandom bs=19 count=1 2>/dev/null | tr -cd '0-9') + n=$(echo $n | sed -e 's/^0//') + if [ "$n" == "" ]; then + n=0 + fi + elif [ $n -gt $maximum ]; then + n=$(echo $n | sed 's/.$//') + fi + done + echo $n +} +#### PoorMansRandomGenerator SUBSET END #### + +# Initial TSTMAP value before function declaration +TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4) # Default alert attachment filename ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log" @@ -155,7 +182,6 @@ function Dummy { sleep $SLEEP_TIME } -#### Logger SUBSET #### # Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array # usage: joinString separaratorChar Array @@ -171,8 +197,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -207,7 +236,7 @@ function RemoteLogger { fi return elif [ "$level" == "ERROR" ]; then - _Logger "" "$prefix\e[91m$value\e[0m" true + _Logger "" "$prefix\e[31m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -229,7 +258,7 @@ function RemoteLogger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "" "$prefix$value" + _Logger "" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -262,6 +291,7 @@ function RemoteLogger { # VERBOSE sent to stdout if _LOGGER_VERBOSE = true # ALWAYS is sent to stdout unless _LOGGER_SILENT = true # DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes +# SIMPLE is a wrapper for QuickLogger that does not use advanced functionality function Logger { local value="${1}" # Sentence to log (in double quotes) local level="${2}" # Log level @@ -318,35 +348,18 @@ function Logger { _Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG return #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG + elif [ "$level" == "SIMPLE" ]; then + if [ "$_LOGGER_SILENT" == true ]; then + _Logger "$preix$value" + else + _Logger "$preix$value" "$prefix$value" + fi + return else _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } -#### Logger SUBSET END #### - -# QuickLogger subfunction, can be called directly -function _QuickLogger { - local value="${1}" - local destination="${2}" # Destination: stdout, log, both - - if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then - echo -e "$(date) - $value" >> "$LOG_FILE" - elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then - echo -e "$value" - fi -} - -# Generic quick logging function -function QuickLogger { - local value="${1}" - - if [ "$_LOGGER_SILENT" == true ]; then - _QuickLogger "$value" "log" - else - _QuickLogger "$value" "stdout" - fi -} # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X function KillChilds { @@ -360,7 +373,6 @@ function KillChilds { fi if kill -0 "$pid" > /dev/null 2>&1; then - # Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment if children="$(pgrep -P "$pid")"; then if [[ "$pid" == *"$children"* ]]; then Logger "Bogus pgrep implementation." "CRITICAL" @@ -482,6 +494,12 @@ function SendAlert { # encryption can be set to tls, ssl or none # smtpUser and smtpPassword are optional # SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword" + +# If text is received as attachment ATT00001.bin or noname, consider adding the following to /etc/mail.rc +#set ttycharset=iso-8859-1 +#set sendcharsets=iso-8859-1 +#set encoding=8bit + function SendEmail { local subject="${1}" local message="${2}" @@ -504,11 +522,16 @@ function SendEmail { local i - for i in "${destinationMails}"; do - if [ $(CheckRFC822 "$i") -ne 1 ]; then - Logger "Given email [$i] does not seem to be valid." "WARN" - fi - done + if [ "${destinationMails[@]}" != "" ]; then + for i in "${destinationMails[@]}"; do + if [ $(CheckRFC822 "$i") -ne 1 ]; then + Logger "Given email [$i] does not seem to be valid." "WARN" + fi + done + else + Logger "No valid email adresses given." "WARN" + return 1 + fi # Prior to sending an email, convert its body if needed if [ "$MAIL_BODY_CHARSET" != "" ]; then @@ -780,33 +803,33 @@ function ParallelExec { function ExecTasks { # Mandatory arguments - local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands + local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands # Optional arguments - local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id - local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false) - local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0 - local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0 - local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0 - local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0 - local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false) - local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1) - local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging - local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running - local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true) - local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks) + local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id + local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false) + local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0 + local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0 + local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0 + local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0 + local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false) + local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1) + local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging + local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running + local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true) + local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks) # Parallelism specific arguments - local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode. - local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not - local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing - local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries - local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors + local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode. + local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not + local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing + local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries + local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors local i - Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG - __CheckArguments 1-18 $# "$@" #__WITH_PARANOIA_DEBUG + Logger "${FUNCNAME[0]} called by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + __CheckArguments 1-18 $# "$@" #__WITH_PARANOIA_DEBUG # Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode if [ "$_DEBUG" == "yes" ]; then @@ -829,40 +852,40 @@ function ExecTasks { IFS=';' read -r -a validExitCodes <<< "$validExitCodes" # ParallelExec specific variables - local auxItemCount=0 # Number of conditional commands - local commandsArray=() # Array containing commands + local auxItemCount=0 # Number of conditional commands + local commandsArray=() # Array containing commands local commandsConditionArray=() # Array containing conditional commands - local currentCommand # Variable containing currently processed command - local currentCommandCondition # Variable containing currently processed conditional command - local commandsArrayPid=() # Array containing pids of commands currently run - local postponedRetryCount=0 # Number of current postponed commands retries - local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once) + local currentCommand # Variable containing currently processed command + local currentCommandCondition # Variable containing currently processed conditional command + local commandsArrayPid=() # Array containing pids of commands currently run + local postponedRetryCount=0 # Number of current postponed commands retries + local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once) local postponedCounter=0 - local isPostponedCommand=false # Is the current command from a postponed file ? - local postponedExecTime=0 # How much time has passed since last postponed condition was checked - local needsPostponing # Does currentCommand need to be postponed + local isPostponedCommand=false # Is the current command from a postponed file ? + local postponedExecTime=0 # How much time has passed since last postponed condition was checked + local needsPostponing # Does currentCommand need to be postponed local temp # Common variables - local pid # Current pid working on - local pidState # State of the process - local mainItemCount=0 # number of given items (pids or commands) - local readFromFile # Should we read pids / commands from a file (true) + local pid # Current pid working on + local pidState # State of the process + local mainItemCount=0 # number of given items (pids or commands) + local readFromFile # Should we read pids / commands from a file (true) local counter=0 - local log_ttime=0 # local time instance for comparaison + local log_ttime=0 # local time instance for comparaison - local seconds_begin=$SECONDS # Seconds since the beginning of the script - local exec_time=0 # Seconds since the beginning of this function + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function - local retval=0 # return value of monitored pid process - local subRetval=0 # return value of condition commands - local errorcount=0 # Number of pids that finished with errors - local pidsArray # Array of currently running pids - local newPidsArray # New array of currently running pids for next iteration - local pidsTimeArray # Array containing execution begin time of pids - local executeCommand # Boolean to check if currentCommand can be executed given a condition + local retval=0 # return value of monitored pid process + local subRetval=0 # return value of condition commands + local errorcount=0 # Number of pids that finished with errors + local pidsArray # Array of currently running pids + local newPidsArray # New array of currently running pids for next iteration + local pidsTimeArray # Array containing execution begin time of pids + local executeCommand # Boolean to check if currentCommand can be executed given a condition - local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG + local hasPids=false # Are any valable pids given to function ? #__WITH_PARANOIA_DEBUG local functionMode @@ -915,7 +938,7 @@ function ExecTasks { counter=$mainItemCount fi - Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG + Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG # soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop function _ExecTasksTimeCheck { @@ -1031,7 +1054,7 @@ function ExecTasks { # Check for valid exit codes if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then if [ $noErrorLogsAtAll != true ]; then - Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR" + Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" if [ "$functionMode" == "ParallelExec" ]; then Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" fi @@ -1047,22 +1070,22 @@ function ExecTasks { Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" fi fi - hasPids=true ##__WITH_PARANOIA_DEBUG + hasPids=true ##__WITH_PARANOIA_DEBUG fi done # hasPids can be false on last iteration in ParallelExec mode - if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then ##__WITH_PARANOIA_DEBUG - Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG - fi ##__WITH_PARANOIA_DEBUG + if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then ##__WITH_PARANOIA_DEBUG + Logger "No valable pids given." "ERROR" ##__WITH_PARANOIA_DEBUG + fi ##__WITH_PARANOIA_DEBUG pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU sleep $sleepTime - if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG - _PerfProfiler ##__WITH_PARANOIA_DEBUG - fi ##__WITH_PARANOIA_DEBUG + if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG + _PerfProfiler ##__WITH_PARANOIA_DEBUG + fi ##__WITH_PARANOIA_DEBUG } @@ -1251,6 +1274,13 @@ function EscapeSpaces { echo "${string// /\\ }" } +# Usage var=$(EscapeDoubleQuotes "$var") or var="$(EscapeDoubleQuotes "$var")" +function EscapeDoubleQuotes { + local value="${1}" + + echo "${value//\"/\\\"}" +} + function IsNumericExpand { eval "local value=\"${1}\"" # Needed eval so variable variables can be processed @@ -1272,27 +1302,23 @@ function IsNumeric { fi } -#### CheckRFC822 SUBSET #### -# Checks email address validity -function CheckRFC822 { - local mail="${1}" - local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" - - if [[ $mail =~ $rfc822 ]]; then - echo 1 - else - echo 0 - fi -} -#### CheckRFC822 SUBSET END #### - +# Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" - if [[ $value =~ ^[0-9]+$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[0-9]+$ ]]; then + echo 1 + else + echo 0 + fi fi } @@ -1327,12 +1353,26 @@ function HumanToNumeric { echo $value } -## from https://gist.github.com/cdown/1163649 +# Checks email address validity +function CheckRFC822 { + local mail="${1}" + local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" + + if [[ $mail =~ $rfc822 ]]; then + echo 1 + else + echo 0 + fi +} + +## Modified version of https://gist.github.com/cdown/1163649 function UrlEncode { local length="${#1}" + local i + local LANG=C - for (( i = 0; i < length; i++ )); do + for i in $(seq 0 $((length-1))); do local c="${1:i:1}" case $c in [a-zA-Z0-9.~_-]) @@ -1435,10 +1475,34 @@ function GetLocalOS { if [ -f "/etc/os-release" ]; then localOsName=$(GetConfFileValue "/etc/os-release" "NAME" true) localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION" true) + elif [ "$LOCAL_OS" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - # Add a global variable for statistics in installer - LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then localOsVar="$(uname -a)" + if [ "$PROGRAMW6432" != "" ]; then + LOCAL_OS_BITNESS=64 + LOCAL_OS_FAMILY="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + LOCAL_OS_BITNESS=32 + LOCAL_OS_FAMILY="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$LOCAL_OS" == "BusyBox" ]; then + LOCAL_OS_FAMILY="Unix" + fi + # Get Host info for Unix + else + LOCAL_OS_FAMILY="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + LOCAL_OS_BITNESS=64 + else + LOCAL_OS_BITNESS=32 + fi + fi + + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer) $LOCAL_OS_BITNESS-bit $LOCAL_OS_FAMILY" if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" @@ -1518,6 +1582,8 @@ function GetOs { local localOsVar local localOsName local localOsVer + local localOsBitness + local localOsFamily local osInfo="/etc/os-release" @@ -1544,9 +1610,36 @@ function GetOs { localOsName="${localOsName##*=}" localOsVer=$(grep "^VERSION=" "$osInfo") localOsVer="${localOsVer##*=}" + elif [ "$localOsVar" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - echo "$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + case $localOsVar in + *"MINGW32"*|*"MINGW64"*|*"MSYS"*|*"CYGWIN*"|*"Microsoft"*|*"WinNT10*") + if [ "$PROGRAMW6432" != "" ]; then + localOsBitness=64 + localOsFamily="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + localOsBitness=32 + localOsFamily="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$localOsVar" == "BusyBox" ]; then + localOsFamily="Unix" + fi + ;; + *) + localOsFamily="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + localOsBitness=64 + else + localOsBitness=32 + fi + ;; + esac + + echo "$localOsVar ($localOsName $localOsVer) $localOsBitness-bit $localOsFamily" } GetOs @@ -1554,6 +1647,9 @@ GetOs ENDSSH if [ $? != 0 ]; then Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT]." "CRITICAL" + if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + Logger "$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "ERROR" + fi exit 1 fi @@ -1972,7 +2068,7 @@ function InitLocalOSDependingSettings { ## Using mingw version of find instead of windows one ## Getting running processes is quite different ## Ping command is not the same - if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "Microsoft" ] || [ "$LOCAL_OS" == "WinNT10" ]; then FIND_CMD=$(dirname $BASH)/find PING_CMD='$SYSTEMROOT\system32\ping -n 2' @@ -2454,7 +2550,7 @@ function _ListDatabasesLocal { sqlCmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" Logger "Launching command [$sqlCmd]." "DEBUG" eval "$sqlCmd" & - ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK true $SLEEP_TIME $KEEP_LOGGING + ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING retval=$? if [ $retval -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" @@ -2480,7 +2576,7 @@ function _ListDatabasesRemote { sqlCmd="$SSH_CMD \"env _REMOTE_TOKEN=$_REMOTE_TOKEN mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP\" 2>&1" Logger "Command output: $sqlCmd" "DEBUG" eval "$sqlCmd" & - ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK true $SLEEP_TIME $KEEP_LOGGING + ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING retval=$? if [ $retval -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" @@ -2674,8 +2770,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -2710,7 +2809,7 @@ function RemoteLogger { fi return elif [ "$level" == "ERROR" ]; then - _Logger "" "$prefix\e[91m$value\e[0m" true + _Logger "" "$prefix\e[31m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -2732,7 +2831,7 @@ function RemoteLogger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "" "$prefix$value" + _Logger "" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -2960,8 +3059,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -2996,7 +3098,7 @@ function RemoteLogger { fi return elif [ "$level" == "ERROR" ]; then - _Logger "" "$prefix\e[91m$value\e[0m" true + _Logger "" "$prefix\e[31m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -3018,7 +3120,7 @@ function RemoteLogger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "" "$prefix$value" + _Logger "" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -3160,8 +3262,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -3196,7 +3301,7 @@ function RemoteLogger { fi return elif [ "$level" == "ERROR" ]; then - _Logger "" "$prefix\e[91m$value\e[0m" true + _Logger "" "$prefix\e[31m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -3218,7 +3323,7 @@ function RemoteLogger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "" "$prefix$value" + _Logger "" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -3386,8 +3491,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -3422,7 +3530,7 @@ function RemoteLogger { fi return elif [ "$level" == "ERROR" ]; then - _Logger "" "$prefix\e[91m$value\e[0m" true + _Logger "" "$prefix\e[31m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -3444,7 +3552,7 @@ function RemoteLogger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "" "$prefix$value" + _Logger "" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -3664,7 +3772,7 @@ function _BackupDatabaseLocalToLocal { Logger "Launching command [$drySqlCmd]." "DEBUG" eval "$drySqlCmd" & fi - ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING + ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK true $SLEEP_TIME $KEEP_LOGGING retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then if [ $_DRYRUN == false ]; then @@ -3713,7 +3821,7 @@ function _BackupDatabaseLocalToRemote { Logger "Launching command [$drySqlCmd]." "DEBUG" eval "$drySqlCmd" & fi - ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING + ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK true $SLEEP_TIME $KEEP_LOGGING retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then if [ $_DRYRUN == false ]; then @@ -3762,7 +3870,7 @@ function _BackupDatabaseRemoteToLocal { Logger "Launching command [$drySqlCmd]." "DEBUG" eval "$drySqlCmd" & fi - ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING + ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK true $SLEEP_TIME $KEEP_LOGGING retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then if [ $_DRYRUN == false ]; then @@ -3890,6 +3998,7 @@ function EncryptFiles { successCounter=$((successCounter+1)) fi fi + #TODO: This redirection does not work with busybox since there is no subshell support done < <($FIND_CMD "$filePath" $recursiveArgs -type f ! -name "*$cryptFileExtension" -print0) if [ $(IsNumeric $PARALLEL_ENCRYPTION_PROCESSES) -eq 1 ] && [ "$PARALLEL_ENCRYPTION_PROCESSES" != "1" ]; then @@ -4333,8 +4442,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -4369,7 +4481,7 @@ function RemoteLogger { fi return elif [ "$level" == "ERROR" ]; then - _Logger "" "$prefix\e[91m$value\e[0m" true + _Logger "" "$prefix\e[31m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -4391,7 +4503,7 @@ function RemoteLogger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "" "$prefix$value" + _Logger "" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then diff --git a/install.sh b/install.sh index d4fad03..5d2c0f4 100755 --- a/install.sh +++ b/install.sh @@ -12,7 +12,7 @@ PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SSH_FILTER="ssh_filter.sh" -SCRIPT_BUILD=2017041701 +SCRIPT_BUILD=2017072701 ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 @@ -21,6 +21,39 @@ SCRIPT_BUILD=2017041701 # Get current install.sh path from http://stackoverflow.com/a/246128/2635443 SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +_LOGGER_SILENT=false +_STATS=1 +ACTION="install" +FAKEROOT="" + +function GetCommandlineArguments { + for i in "$@"; do + case $i in + --prefix=*) + FAKEROOT="${i##*=}" + ;; + --silent) + _LOGGER_SILENT=true + ;; + --no-stats) + _STATS=0 + ;; + --remove) + ACTION="uninstall" + ;; + --help|-h|-?) + Usage + ;; + *) + Logger "Unknown option '$i'" "CRITICAL" + Usage + ;; + esac + done +} + +GetCommandlineArguments "$@" + CONF_DIR=$FAKEROOT/etc/$PROGRAM BIN_DIR="$FAKEROOT/usr/local/bin" SERVICE_DIR_INIT=$FAKEROOT/etc/init.d @@ -49,34 +82,15 @@ else LOG_FILE="./$PROGRAM-install.log" fi -# QuickLogger subfunction, can be called directly -function _QuickLogger { - local value="${1}" - local destination="${2}" # Destination: stdout, log, both - - if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then - echo -e "$(date) - $value" >> "$LOG_FILE" - elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then - echo -e "$value" - fi -} - -# Generic quick logging function -function QuickLogger { - local value="${1}" - - if [ "$_LOGGER_SILENT" == true ]; then - _QuickLogger "$value" "log" - else - _QuickLogger "$value" "stdout" - fi -} -## from https://gist.github.com/cdown/1163649 +include #### QuickLogger SUBSET #### +## Modified version of https://gist.github.com/cdown/1163649 function UrlEncode { local length="${#1}" + local i + local LANG=C - for (( i = 0; i < length; i++ )); do + for i in $(seq 0 $((length-1))); do local c="${1:i:1}" case $c in [a-zA-Z0-9.~_-]) @@ -153,10 +167,34 @@ function GetLocalOS { if [ -f "/etc/os-release" ]; then localOsName=$(GetConfFileValue "/etc/os-release" "NAME" true) localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION" true) + elif [ "$LOCAL_OS" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - # Add a global variable for statistics in installer - LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then localOsVar="$(uname -a)" + if [ "$PROGRAMW6432" != "" ]; then + LOCAL_OS_BITNESS=64 + LOCAL_OS_FAMILY="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + LOCAL_OS_BITNESS=32 + LOCAL_OS_FAMILY="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$LOCAL_OS" == "BusyBox" ]; then + LOCAL_OS_FAMILY="Unix" + fi + # Get Host info for Unix + else + LOCAL_OS_FAMILY="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + LOCAL_OS_BITNESS=64 + else + LOCAL_OS_BITNESS=32 + fi + fi + + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer) $LOCAL_OS_BITNESS-bit $LOCAL_OS_FAMILY" if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" @@ -234,7 +272,7 @@ function CreateDir { local dir="${1}" if [ ! -d "$dir" ]; then - mkdir "$dir" + mkdir -p "$dir" if [ $? == 0 ]; then QuickLogger "Created directory [$dir]." else @@ -338,8 +376,10 @@ function CopyProgram { function CopyServiceFiles { if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then + CreateDir "$SERVICE_DIR_SYSTEMD_SYSTEM" CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true - if [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM_USER" ]; then + if [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_USER" ]; then + CreateDir "$SERVICE_DIR_SYSTEMD_USER" CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true fi @@ -348,6 +388,7 @@ function CopyServiceFiles { QuickLogger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]." QuickLogger "In userland, active with [systemctl --user start $SERVICE_NAME@instance.conf]." elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then + CreateDir "$SERVICE_DIR_INIT" CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]." @@ -405,7 +446,7 @@ function RemoveAll { QuickLogger "Skipping removal of [$BIN_DIR/$SSH_FILTER] because other programs present that need it." fi RemoveFile "$SERVICE_DIR_SYSTEMD_SYSTEM/$SERVICE_FILE_SYSTEMD_SYSTEM" - RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_SYSTEM" + RemoveFile "$SERVICE_DIR_SYSTEMD_USER/$SERVICE_FILE_SYSTEMD_USER" RemoveFile "$SERVICE_DIR_INIT/$SERVICE_FILE_INIT" QuickLogger "Skipping configuration files in [$CONF_DIR]. You may remove this directory manually." @@ -417,34 +458,10 @@ function Usage { echo "--silent Will log and bypass user interaction." echo "--no-stats Used with --silent in order to refuse sending anonymous install stats." echo "--remove Remove the program." + echo "--prefix=/path Use prefix to install path." exit 127 } -_LOGGER_SILENT=false -_STATS=1 -ACTION="install" - -for i in "$@" -do - case $i in - --silent) - _LOGGER_SILENT=true - ;; - --no-stats) - _STATS=0 - ;; - --remove) - ACTION="uninstall" - ;; - --help|-h|-?) - Usage - esac -done - -if [ "$FAKEROOT" != "" ]; then - mkdir -p "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_DIR_SYSTEMD_USER" "$BIN_DIR" -fi - GetLocalOS SetLocalOSSettings GetInit diff --git a/obackup.sh b/obackup.sh index db3bf03..85797d4 100755 --- a/obackup.sh +++ b/obackup.sh @@ -7,24 +7,14 @@ PROGRAM="obackup" AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/obackup - ozy@netpower.fr" PROGRAM_VERSION=2.1-beta5 -PROGRAM_BUILD=2018022401 +PROGRAM_BUILD=2018031501 IS_STABLE=no - -#TODO: ExecTasks postponed arrays / files grow a lot. Consider having them "rolling" -#done: add checkRFC function (and use it for --destination-mails) -#done: ExecTasks still needs some better call argument list -#done: ExecTasks sub function relocate -#done: SendMail and SendEmail convert functions inverted, check on osync and obackup -#command line arguments don't take -AaqV for example - _OFUNCTIONS_VERSION=2.3.0-dev -_OFUNCTIONS_BUILD=2018031501 +_OFUNCTIONS_BUILD=2018070902 _OFUNCTIONS_BOOTSTRAP=true -## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr - ## To use in a program, define the following variables: ## PROGRAM=program-name ## INSTANCE_ID=program-instance-name @@ -83,9 +73,6 @@ fi SCRIPT_PID=$$ -# TODO: Check if %N works on MacOS -TSTAMP=$(date '+%Y%m%dT%H%M%S.%N') - LOCAL_USER=$(whoami) LOCAL_HOST=$(hostname) @@ -113,6 +100,46 @@ else RUN_DIR=. fi +#### PoorMansRandomGenerator SUBSET #### +# Get a random number on Windows BusyBox alike, also works on most Unixes +function PoorMansRandomGenerator { + local digits="${1}" # The number of digits to generate + + local minimum=1 + local maximum + local n=0 + + if [ "$digits" == "" ]; then + digits=5 + fi + + # Minimum already has a digit + for n in $(seq 1 $((digits-1))); do + minimum=$minimum"0" + maximum=$maximum"9" + done + maximum=$maximum"9" + + #n=0; while [ $n -lt $minimum ]; do n=$n$(dd if=/dev/urandom bs=100 count=1 2>/dev/null | tr -cd '0-9'); done; n=$(echo $n | sed -e 's/^0//') + # bs=19 since if real random strikes, having a 19 digits number is not supported + while [ $n -lt $minimum ] || [ $n -gt $maximum ]; do + if [ $n -lt $minimum ]; then + # Add numbers + n=$n$(dd if=/dev/urandom bs=19 count=1 2>/dev/null | tr -cd '0-9') + n=$(echo $n | sed -e 's/^0//') + if [ "$n" == "" ]; then + n=0 + fi + elif [ $n -gt $maximum ]; then + n=$(echo $n | sed 's/.$//') + fi + done + echo $n +} +#### PoorMansRandomGenerator SUBSET END #### + +# Initial TSTMAP value before function declaration +TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4) # Default alert attachment filename ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log" @@ -127,7 +154,6 @@ function Dummy { sleep $SLEEP_TIME } -#### Logger SUBSET #### # Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array # usage: joinString separaratorChar Array @@ -143,8 +169,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -179,7 +208,7 @@ function RemoteLogger { fi return elif [ "$level" == "ERROR" ]; then - _Logger "" "$prefix\e[91m$value\e[0m" true + _Logger "" "$prefix\e[31m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -201,7 +230,7 @@ function RemoteLogger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "" "$prefix$value" + _Logger "" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -229,6 +258,7 @@ function RemoteLogger { # VERBOSE sent to stdout if _LOGGER_VERBOSE = true # ALWAYS is sent to stdout unless _LOGGER_SILENT = true # DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes +# SIMPLE is a wrapper for QuickLogger that does not use advanced functionality function Logger { local value="${1}" # Sentence to log (in double quotes) local level="${2}" # Log level @@ -280,35 +310,18 @@ function Logger { _Logger "$prefix$value" "$prefix$value" return fi + elif [ "$level" == "SIMPLE" ]; then + if [ "$_LOGGER_SILENT" == true ]; then + _Logger "$preix$value" + else + _Logger "$preix$value" "$prefix$value" + fi + return else _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } -#### Logger SUBSET END #### - -# QuickLogger subfunction, can be called directly -function _QuickLogger { - local value="${1}" - local destination="${2}" # Destination: stdout, log, both - - if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then - echo -e "$(date) - $value" >> "$LOG_FILE" - elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then - echo -e "$value" - fi -} - -# Generic quick logging function -function QuickLogger { - local value="${1}" - - if [ "$_LOGGER_SILENT" == true ]; then - _QuickLogger "$value" "log" - else - _QuickLogger "$value" "stdout" - fi -} # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X function KillChilds { @@ -322,7 +335,6 @@ function KillChilds { fi if kill -0 "$pid" > /dev/null 2>&1; then - # Warning: pgrep is not native on cygwin, have this checked in CheckEnvironment if children="$(pgrep -P "$pid")"; then if [[ "$pid" == *"$children"* ]]; then Logger "Bogus pgrep implementation." "CRITICAL" @@ -441,6 +453,12 @@ function SendAlert { # encryption can be set to tls, ssl or none # smtpUser and smtpPassword are optional # SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword" + +# If text is received as attachment ATT00001.bin or noname, consider adding the following to /etc/mail.rc +#set ttycharset=iso-8859-1 +#set sendcharsets=iso-8859-1 +#set encoding=8bit + function SendEmail { local subject="${1}" local message="${2}" @@ -462,11 +480,16 @@ function SendEmail { local i - for i in "${destinationMails}"; do - if [ $(CheckRFC822 "$i") -ne 1 ]; then - Logger "Given email [$i] does not seem to be valid." "WARN" - fi - done + if [ "${destinationMails[@]}" != "" ]; then + for i in "${destinationMails[@]}"; do + if [ $(CheckRFC822 "$i") -ne 1 ]; then + Logger "Given email [$i] does not seem to be valid." "WARN" + fi + done + else + Logger "No valid email adresses given." "WARN" + return 1 + fi # Prior to sending an email, convert its body if needed if [ "$MAIL_BODY_CHARSET" != "" ]; then @@ -721,28 +744,28 @@ function ParallelExec { function ExecTasks { # Mandatory arguments - local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands + local mainInput="${1}" # Contains list of pids / commands separated by semicolons or filepath to list of pids / commands # Optional arguments - local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id - local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false) - local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0 - local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0 - local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0 - local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0 - local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false) - local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1) - local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging - local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running - local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true) - local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks) + local id="${2:-base}" # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id + local readFromFile="${3:-false}" # Is mainInput / auxInput a semicolon separated list (true) or a filepath (false) + local softPerProcessTime="${4:-0}" # Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0 + local hardPerProcessTime="${5:-0}" # Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0 + local softMaxTime="${6:-0}" # Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0 + local hardMaxTime="${7:-0}" # Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0 + local counting="${8:-true}" # Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false) + local sleepTime="${9:-.5}" # Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1) + local keepLogging="${10:-1800}" # Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging + local spinner="${11:-true}" # Show spinner (true) or do not show anything (false) while running + local noTimeErrorLog="${12:-false}" # Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true) + local noErrorLogsAtAll="${13:-false}" # Do not log any errros at all (useful for recursive ExecTasks checks) # Parallelism specific arguments - local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode. - local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not - local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing - local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries - local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors + local numberOfProcesses="${14:-0}" # Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode. + local auxInput="${15}" # Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not + local maxPostponeRetries="${16:-3}" # If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing + local minTimeBetweenRetries="${17:-300}" # Time (in seconds) between postponed command retries + local validExitCodes="${18:-0}" # Semi colon separated list of valid main command exit codes which will not trigger errors local i @@ -768,38 +791,38 @@ function ExecTasks { IFS=';' read -r -a validExitCodes <<< "$validExitCodes" # ParallelExec specific variables - local auxItemCount=0 # Number of conditional commands - local commandsArray=() # Array containing commands + local auxItemCount=0 # Number of conditional commands + local commandsArray=() # Array containing commands local commandsConditionArray=() # Array containing conditional commands - local currentCommand # Variable containing currently processed command - local currentCommandCondition # Variable containing currently processed conditional command - local commandsArrayPid=() # Array containing pids of commands currently run - local postponedRetryCount=0 # Number of current postponed commands retries - local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once) + local currentCommand # Variable containing currently processed command + local currentCommandCondition # Variable containing currently processed conditional command + local commandsArrayPid=() # Array containing pids of commands currently run + local postponedRetryCount=0 # Number of current postponed commands retries + local postponedItemCount=0 # Number of commands that have been postponed (keep at least one in order to check once) local postponedCounter=0 - local isPostponedCommand=false # Is the current command from a postponed file ? - local postponedExecTime=0 # How much time has passed since last postponed condition was checked - local needsPostponing # Does currentCommand need to be postponed + local isPostponedCommand=false # Is the current command from a postponed file ? + local postponedExecTime=0 # How much time has passed since last postponed condition was checked + local needsPostponing # Does currentCommand need to be postponed local temp # Common variables - local pid # Current pid working on - local pidState # State of the process - local mainItemCount=0 # number of given items (pids or commands) - local readFromFile # Should we read pids / commands from a file (true) + local pid # Current pid working on + local pidState # State of the process + local mainItemCount=0 # number of given items (pids or commands) + local readFromFile # Should we read pids / commands from a file (true) local counter=0 - local log_ttime=0 # local time instance for comparaison + local log_ttime=0 # local time instance for comparaison - local seconds_begin=$SECONDS # Seconds since the beginning of the script - local exec_time=0 # Seconds since the beginning of this function + local seconds_begin=$SECONDS # Seconds since the beginning of the script + local exec_time=0 # Seconds since the beginning of this function - local retval=0 # return value of monitored pid process - local subRetval=0 # return value of condition commands - local errorcount=0 # Number of pids that finished with errors - local pidsArray # Array of currently running pids - local newPidsArray # New array of currently running pids for next iteration - local pidsTimeArray # Array containing execution begin time of pids - local executeCommand # Boolean to check if currentCommand can be executed given a condition + local retval=0 # return value of monitored pid process + local subRetval=0 # return value of condition commands + local errorcount=0 # Number of pids that finished with errors + local pidsArray # Array of currently running pids + local newPidsArray # New array of currently running pids for next iteration + local pidsTimeArray # Array containing execution begin time of pids + local executeCommand # Boolean to check if currentCommand can be executed given a condition local functionMode @@ -968,7 +991,7 @@ function ExecTasks { # Check for valid exit codes if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then if [ $noErrorLogsAtAll != true ]; then - Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR" + Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG" if [ "$functionMode" == "ParallelExec" ]; then Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR" fi @@ -1179,6 +1202,13 @@ function EscapeSpaces { echo "${string// /\\ }" } +# Usage var=$(EscapeDoubleQuotes "$var") or var="$(EscapeDoubleQuotes "$var")" +function EscapeDoubleQuotes { + local value="${1}" + + echo "${value//\"/\\\"}" +} + function IsNumericExpand { eval "local value=\"${1}\"" # Needed eval so variable variables can be processed @@ -1200,27 +1230,23 @@ function IsNumeric { fi } -#### CheckRFC822 SUBSET #### -# Checks email address validity -function CheckRFC822 { - local mail="${1}" - local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" - - if [[ $mail =~ $rfc822 ]]; then - echo 1 - else - echo 0 - fi -} -#### CheckRFC822 SUBSET END #### - +# Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" - if [[ $value =~ ^[0-9]+$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[0-9]+$ ]]; then + echo 1 + else + echo 0 + fi fi } @@ -1255,12 +1281,26 @@ function HumanToNumeric { echo $value } -## from https://gist.github.com/cdown/1163649 +# Checks email address validity +function CheckRFC822 { + local mail="${1}" + local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$" + + if [[ $mail =~ $rfc822 ]]; then + echo 1 + else + echo 0 + fi +} + +## Modified version of https://gist.github.com/cdown/1163649 function UrlEncode { local length="${#1}" + local i + local LANG=C - for (( i = 0; i < length; i++ )); do + for i in $(seq 0 $((length-1))); do local c="${1:i:1}" case $c in [a-zA-Z0-9.~_-]) @@ -1363,10 +1403,34 @@ function GetLocalOS { if [ -f "/etc/os-release" ]; then localOsName=$(GetConfFileValue "/etc/os-release" "NAME" true) localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION" true) + elif [ "$LOCAL_OS" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - # Add a global variable for statistics in installer - LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "WinNT10" ]; then localOsVar="$(uname -a)" + if [ "$PROGRAMW6432" != "" ]; then + LOCAL_OS_BITNESS=64 + LOCAL_OS_FAMILY="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + LOCAL_OS_BITNESS=32 + LOCAL_OS_FAMILY="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$LOCAL_OS" == "BusyBox" ]; then + LOCAL_OS_FAMILY="Unix" + fi + # Get Host info for Unix + else + LOCAL_OS_FAMILY="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + LOCAL_OS_BITNESS=64 + else + LOCAL_OS_BITNESS=32 + fi + fi + + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer) $LOCAL_OS_BITNESS-bit $LOCAL_OS_FAMILY" if [ "$_OFUNCTIONS_VERSION" != "" ]; then Logger "Local OS: [$LOCAL_OS_FULL]." "DEBUG" @@ -1390,6 +1454,8 @@ function GetOs { local localOsVar local localOsName local localOsVer + local localOsBitness + local localOsFamily local osInfo="/etc/os-release" @@ -1416,9 +1482,36 @@ function GetOs { localOsName="${localOsName##*=}" localOsVer=$(grep "^VERSION=" "$osInfo") localOsVer="${localOsVer##*=}" + elif [ "$localOsVar" == "BusyBox" ]; then + localOsVer=`ls --help 2>&1 | head -1 | cut -f2 -d' '` + localOsName="BusyBox" fi - echo "$localOsVar ($localOsName $localOsVer)" + # Get Host info for Windows + case $localOsVar in + *"MINGW32"*|*"MINGW64"*|*"MSYS"*|*"CYGWIN*"|*"Microsoft"*|*"WinNT10*") + if [ "$PROGRAMW6432" != "" ]; then + localOsBitness=64 + localOsFamily="Windows" + elif [ "$PROGRAMFILES" != "" ]; then + localOsBitness=32 + localOsFamily="Windows" + # Case where running on BusyBox but no program files defined + elif [ "$localOsVar" == "BusyBox" ]; then + localOsFamily="Unix" + fi + ;; + *) + localOsFamily="Unix" + if uname -m | grep '64' > /dev/null 2>&1; then + localOsBitness=64 + else + localOsBitness=32 + fi + ;; + esac + + echo "$localOsVar ($localOsName $localOsVer) $localOsBitness-bit $localOsFamily" } GetOs @@ -1426,6 +1519,9 @@ GetOs ENDSSH if [ $? != 0 ]; then Logger "Cannot connect to remote system [$REMOTE_HOST] port [$REMOTE_PORT]." "CRITICAL" + if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then + Logger "$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")" "ERROR" + fi exit 1 fi @@ -1828,7 +1924,7 @@ function InitLocalOSDependingSettings { ## Using mingw version of find instead of windows one ## Getting running processes is quite different ## Ping command is not the same - if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then + if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ] || [ "$LOCAL_OS" == "Microsoft" ] || [ "$LOCAL_OS" == "WinNT10" ]; then FIND_CMD=$(dirname $BASH)/find PING_CMD='$SYSTEMROOT\system32\ping -n 2' @@ -2305,7 +2401,7 @@ function _ListDatabasesLocal { sqlCmd="mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;' > $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1" Logger "Launching command [$sqlCmd]." "DEBUG" eval "$sqlCmd" & - ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK true $SLEEP_TIME $KEEP_LOGGING + ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING retval=$? if [ $retval -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" @@ -2330,7 +2426,7 @@ function _ListDatabasesRemote { sqlCmd="$SSH_CMD \"env _REMOTE_TOKEN=$_REMOTE_TOKEN mysql -u $SQL_USER -Bse 'SELECT table_schema, round(sum( data_length + index_length ) / 1024) FROM information_schema.TABLES GROUP by table_schema;'\" > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP\" 2>&1" Logger "Command output: $sqlCmd" "DEBUG" eval "$sqlCmd" & - ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK true $SLEEP_TIME $KEEP_LOGGING + ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING retval=$? if [ $retval -eq 0 ]; then Logger "Listing databases succeeded." "NOTICE" @@ -2517,8 +2613,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -2553,7 +2652,7 @@ function RemoteLogger { fi return elif [ "$level" == "ERROR" ]; then - _Logger "" "$prefix\e[91m$value\e[0m" true + _Logger "" "$prefix\e[31m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -2575,7 +2674,7 @@ function RemoteLogger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "" "$prefix$value" + _Logger "" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -2791,8 +2890,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -2827,7 +2929,7 @@ function RemoteLogger { fi return elif [ "$level" == "ERROR" ]; then - _Logger "" "$prefix\e[91m$value\e[0m" true + _Logger "" "$prefix\e[31m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -2849,7 +2951,7 @@ function RemoteLogger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "" "$prefix$value" + _Logger "" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -2979,8 +3081,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -3015,7 +3120,7 @@ function RemoteLogger { fi return elif [ "$level" == "ERROR" ]; then - _Logger "" "$prefix\e[91m$value\e[0m" true + _Logger "" "$prefix\e[31m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -3037,7 +3142,7 @@ function RemoteLogger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "" "$prefix$value" + _Logger "" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -3193,8 +3298,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -3229,7 +3337,7 @@ function RemoteLogger { fi return elif [ "$level" == "ERROR" ]; then - _Logger "" "$prefix\e[91m$value\e[0m" true + _Logger "" "$prefix\e[31m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -3251,7 +3359,7 @@ function RemoteLogger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "" "$prefix$value" + _Logger "" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then @@ -3464,7 +3572,7 @@ function _BackupDatabaseLocalToLocal { Logger "Launching command [$drySqlCmd]." "DEBUG" eval "$drySqlCmd" & fi - ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING + ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK true $SLEEP_TIME $KEEP_LOGGING retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then if [ $_DRYRUN == false ]; then @@ -3512,7 +3620,7 @@ function _BackupDatabaseLocalToRemote { Logger "Launching command [$drySqlCmd]." "DEBUG" eval "$drySqlCmd" & fi - ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING + ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK true $SLEEP_TIME $KEEP_LOGGING retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then if [ $_DRYRUN == false ]; then @@ -3560,7 +3668,7 @@ function _BackupDatabaseRemoteToLocal { Logger "Launching command [$drySqlCmd]." "DEBUG" eval "$drySqlCmd" & fi - ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_TOTAL $HARD_MAX_EXEC_TIME_TOTAL true $SLEEP_TIME $KEEP_LOGGING + ExecTasks $! "${FUNCNAME[0]}" false 0 0 $SOFT_MAX_EXEC_TIME_DB_TASK $HARD_MAX_EXEC_TIME_DB_TASK true $SLEEP_TIME $KEEP_LOGGING retval=$? if [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" ]; then if [ $_DRYRUN == false ]; then @@ -3685,6 +3793,7 @@ function EncryptFiles { successCounter=$((successCounter+1)) fi fi + #TODO: This redirection does not work with busybox since there is no subshell support done < <($FIND_CMD "$filePath" $recursiveArgs -type f ! -name "*$cryptFileExtension" -print0) if [ $(IsNumeric $PARALLEL_ENCRYPTION_PROCESSES) -eq 1 ] && [ "$PARALLEL_ENCRYPTION_PROCESSES" != "1" ]; then @@ -4118,8 +4227,11 @@ function _Logger { if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" - # Current log file - echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then @@ -4154,7 +4266,7 @@ function RemoteLogger { fi return elif [ "$level" == "ERROR" ]; then - _Logger "" "$prefix\e[91m$value\e[0m" true + _Logger "" "$prefix\e[31m$value\e[0m" true if [ $_DEBUG == "yes" ]; then _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true fi @@ -4176,7 +4288,7 @@ function RemoteLogger { fi return elif [ "$level" == "ALWAYS" ]; then - _Logger "" "$prefix$value" + _Logger "" "$prefix$value" return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then