Linux/obackup
1
0
Fork 0
mirror of https://github.com/deajan/obackup.git synced 2025-01-12 15:13:53 +01:00
Code Issues Releases Wiki Activity

Updated ofunctions

Browse Source
This commit is contained in:
deajan 2016-08-06 13:52:17 +02:00
parent 759621c045
commit deb10dd3f0
1 changed files with 200 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

332
dev/ofunctions.sh
View File

@ -1,14 +1,17 @@
## FUNC_BUILD=2016071902-b #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016080601
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
#TODO: set _LOGGER_PREFIX in other apps, specially for osync daemon mode
#TODO: set _LOGGER_STDERR in other apps
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode ## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
if ! type "$BASH" > /dev/null; then if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2" echo "Please run this script only with bash shell. Tested on bash >= 3.2"
exit 127 exit 127
fi fi
#### obackup & osync specific code BEGIN ####
## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log will not be unnecessary big. ## Log a state message every $KEEP_LOGGING seconds. Should not be equal to soft or hard execution time so your log will not be unnecessary big.
KEEP_LOGGING=1801 KEEP_LOGGING=1801
@ -18,13 +21,12 @@ export LC_ALL=C
# Standard alert mail body # Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
#### obackup & osync specific code END #### # Environment variables that can be overriden by programs
#### MINIMAL-FUNCTION-SET BEGIN ####
# Environment variables
_DRYRUN=0 _DRYRUN=0
_SILENT=0 _SILENT=0
_LOGGER_PREFIX="date"
_LOGGER_STDERR=0
# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags # Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags
ERROR_ALERT=0 ERROR_ALERT=0
@ -38,7 +40,7 @@ fi #__WITH_PARANOIA_DEBUG
## allow debugging from command line with _DEBUG=yes ## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no _DEBUG=no
SLEEP_TIME=.1 SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys
_VERBOSE=0 _VERBOSE=0
else else
SLEEP_TIME=1 SLEEP_TIME=1
@ -80,7 +82,8 @@ ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.last.log"
function Dummy { function Dummy {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
sleep .1
sleep $SLEEP_TIME
} }
# Sub function of Logger # Sub function of Logger
@ -88,11 +91,11 @@ function _Logger {
local svalue="${1}" # What to log to stdout local svalue="${1}" # What to log to stdout
local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value local lvalue="${2:-$svalue}" # What to log to logfile, defaults to screen value
local evalue="${3}" # What to log to stderr local evalue="${3}" # What to log to stderr
echo -e "$lvalue" >> "$LOG_FILE" echo -e "$lvalue" >> "$LOG_FILE"
# <OSYNC SPECIFIC> Special case in daemon mode where systemctl doesn't need double timestamps if [ "$_LOGGER_STDERR" -eq 1 ]; then
if [ "$sync_on_changes" == "1" ]; then cat <<< "$evalue" 1>&2
cat <<< "$evalue" 1>&2 # Log to stderr in daemon mode
elif [ "$_SILENT" -eq 0 ]; then elif [ "$_SILENT" -eq 0 ]; then
echo -e "$svalue" echo -e "$svalue"
fi fi
@ -103,13 +106,13 @@ function Logger {
local value="${1}" # Sentence to log (in double quotes) local value="${1}" # Sentence to log (in double quotes)
local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL
# <OSYNC SPECIFIC> Special case in daemon mode we should timestamp instead of counting seconds if [ "$_LOGGER_PREFIX" == "time" ]; then
if [ "$sync_on_changes" == "1" ]; then prefix="TIME: $SECONDS - "
elif [ "$_LOGGER_PREFIX" == "date" ]; then
prefix="$(date) - " prefix="$(date) - "
else else
prefix="TIME: $SECONDS - " prefix=""
fi fi
# </OSYNC SPECIFIC>
if [ "$level" == "CRITICAL" ]; then if [ "$level" == "CRITICAL" ]; then
_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value" _Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value"
@ -147,6 +150,8 @@ function _QuickLogger {
local value="${1}" local value="${1}"
local destination="${2}" # Destination: stdout, log, both local destination="${2}" # Destination: stdout, log, both
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then if ([ "$destination" == "log" ] || [ "$destination" == "both" ]); then
echo -e "$(date) - $value" >> "$LOG_FILE" echo -e "$(date) - $value" >> "$LOG_FILE"
elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then elif ([ "$destination" == "stdout" ] || [ "$destination" == "both" ]); then
@ -158,6 +163,8 @@ function _QuickLogger {
function QuickLogger { function QuickLogger {
local value="${1}" local value="${1}"
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
if [ "$_SILENT" -eq 1 ]; then if [ "$_SILENT" -eq 1 ]; then
_QuickLogger "$value" "log" _QuickLogger "$value" "log"
else else
@ -167,18 +174,18 @@ function QuickLogger {
# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X # Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds { function KillChilds {
local pid="${1}" local pid="${1}" # Parent pid to kill
local self="${2:-false}" local self="${2:-false}"
if children="$(pgrep -P "$pid")"; then if children="$(pgrep -P "$pid")"; then
for child in $children; do for child in $children; do
Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG Logger "Launching KillChilds \"$child\" true" "DEBUG" #__WITH_PARANOIA_DEBUG
KillChilds "$child" true KillChilds "$child" true
done done
fi fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then
if ( [ "$self" == true ] && eval $PROCESS_TEST_CMD > /dev/null 2>&1); then
Logger "Sending SIGTERM to process [$pid]." "DEBUG" Logger "Sending SIGTERM to process [$pid]." "DEBUG"
kill -s SIGTERM "$pid" kill -s SIGTERM "$pid"
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -189,13 +196,31 @@ function KillChilds {
Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
return 1 return 1
fi fi
else
return 0
fi fi
return 0
else else
return 0 return 0
fi fi
} }
function KillAllChilds {
local pids="${1}" # List of parent pids to kill separated by semi-colon
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local errorcount=0
IFS=';' read -a pidsArray <<< "$pids"
for pid in "${pidsArray[@]}"; do
KillChilds $pid
if [ $? != 0 ]; then
errorcount=$((errorcount+1))
fi
done
return $errorcount
}
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
function SendAlert { function SendAlert {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
@ -337,7 +362,7 @@ function SendAlert {
fi fi
fi fi
# If function has not returned 0 yet, assume it's critical that no alert can be sent # If function has not returned 0 yet, assume it is critical that no alert can be sent
Logger "Cannot send alert (neither mutt, mail, sendmail, mailsend, sendemail or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue Logger "Cannot send alert (neither mutt, mail, sendmail, mailsend, sendemail or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue
# Delete tmp log file # Delete tmp log file
@ -472,7 +497,7 @@ function SendEmail {
fi fi
fi fi
# If function has not returned 0 yet, assume it's critical that no alert can be sent # If function has not returned 0 yet, assume it is critical that no alert can be sent
Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue
} }
@ -505,8 +530,6 @@ function LoadConfigFile {
CONFIG_FILE="$config_file" CONFIG_FILE="$config_file"
} }
#### MINIMAL-FUNCTION-SET END ####
function Spinner { function Spinner {
if [ $_SILENT -eq 1 ]; then if [ $_SILENT -eq 1 ]; then
return 0 return 0
@ -540,6 +563,139 @@ function Spinner {
esac esac
} }
function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
local caller_name="${4}" # Who called this function
local exit_on_error="${5:-false}" # Should the function exit on subprocess errors
Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 5 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
local errorcount=0 # Number of pids that finished with errors
local pidCount # number of given pids
IFS=';' read -a pidsArray <<< "$pids"
pidCount=${#pidsArray[@]}
while [ ${#pidsArray[@]} -gt 0 ]; do
newPidsArray=()
for pid in "${pidsArray[@]}"; do
if kill -0 $pid > /dev/null 2>&1; then
newPidsArray+=($pid)
else
wait $pid
result=$?
if [ $result -ne 0 ]; then
errorcount=$((errorcount+1))
Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG"
fi
fi
done
Spinner
exec_time=$(($SECONDS - $seconds_begin))
if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then
log_ttime=$exec_time
Logger "Current tasks still running with pids [${pidsArray[@]}]." "NOTICE"
fi
fi
if [ $exec_time -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
Logger "Max soft execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]." "WARN"
soft_alert=1
SendAlert
fi
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
Logger "Max hard execution time exceeded for task [$caller_name] with pids [${pidsArray[@]}]. Stopping task execution." "ERROR"
KillChilds $pid
if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE"
#return 0
else
errrorcount=$((errorcount+1))
#return 1
fi
fi
fi
pidsArray=("${newPidsArray[@]}")
sleep $SLEEP_TIME
done
Logger "${FUNCNAME[0]} ended for [$caller_name] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
if [ $exit_on_error == true ] && [ $errorcount -gt 0 ]; then
Logger "Stopping execution." "CRITICAL"
exit 1337
else
return $errorcount
fi
}
function WaitForCompletion {
local pid="${1}" # pid to wait for
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
local caller_name="${4}" # Who called this function
Logger "${FUNCNAME[0]} called by [$caller_name]" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 4 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
local log_time=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
while eval "$PROCESS_TEST_CMD" > /dev/null
do
Spinner
if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then
if [ $log_time -ne $SECONDS ]; then
log_time=$SECONDS
Logger "Current task still running." "NOTICE"
fi
fi
if [ $SECONDS -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then
Logger "Max soft execution time exceeded for script." "WARN"
soft_alert=1
SendAlert
fi
if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then
Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR"
KillChilds $pid
if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE"
return 0
else
return 1
fi
fi
fi
sleep $SLEEP_TIME
done
wait $pid
retval=$?
Logger "${FUNCNAME[0]} ended for [$caller_name] with status $retval." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
return $retval
}
#### MINIMAL-FUNCTION-SET END ####
# obsolete, use StripQuotes # obsolete, use StripQuotes
function SedStripQuotes { function SedStripQuotes {
echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g") echo $(echo $1 | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")
@ -720,108 +876,6 @@ function GetRemoteOS {
fi fi
} }
function WaitForTaskCompletion {
local pid="${1}" # pid to wait for
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
local caller_name="${4}" # Who called this function
Logger "${FUNCNAME[0]} called by [$caller_name]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 4 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
while eval "$PROCESS_TEST_CMD" > /dev/null
do
Spinner
exec_time=$(($SECONDS - $seconds_begin))
if [ $((($exec_time + 1) % $KEEP_LOGGING)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then
log_ttime=$exec_time
Logger "Current task still running with pid [$pid]." "NOTICE"
fi
fi
if [ $exec_time -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
Logger "Max soft execution time exceeded for task [$caller_name] with pid [$pid]." "WARN"
soft_alert=1
SendAlert
fi
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
Logger "Max hard execution time exceeded for task [$caller_name] with pid [$pid]. Stopping task execution." "ERROR"
KillChilds $pid
if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE"
return 0
else
return 1
fi
fi
fi
sleep $SLEEP_TIME
done
wait $pid
retval=$?
Logger "${FUNCNAME[0]} ended for [$caller_name] with status $retval." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
return $retval
}
function WaitForCompletion {
local pid="${1}" # pid to wait for
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
local hard_max_time="${3}" # If program with pid $pid takes longer than $hard_max_time seconds, will stop execution, unless $hard_max_time equals 0.
local caller_name="${4}" # Who called this function
Logger "${FUNCNAME[0]} called by [$caller_name]" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 4 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local soft_alert=0 # Does a soft alert need to be triggered, if yes, send an alert once
local log_time=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local retval=0 # return value of monitored pid process
while eval "$PROCESS_TEST_CMD" > /dev/null
do
Spinner
if [ $((($SECONDS + 1) % $KEEP_LOGGING)) -eq 0 ]; then
if [ $log_time -ne $SECONDS ]; then
log_time=$SECONDS
Logger "Current task still running." "NOTICE"
fi
fi
if [ $SECONDS -gt $soft_max_time ]; then
if [ $soft_alert -eq 0 ] && [ $soft_max_time != 0 ]; then
Logger "Max soft execution time exceeded for script." "WARN"
soft_alert=1
SendAlert
fi
if [ $SECONDS -gt $hard_max_time ] && [ $hard_max_time != 0 ]; then
Logger "Max hard execution time exceeded for script in [$caller_name]. Stopping current task execution." "ERROR"
KillChilds $pid
if [ $? == 0 ]; then
Logger "Task stopped successfully" "NOTICE"
return 0
else
return 1
fi
fi
fi
sleep $SLEEP_TIME
done
wait $pid
retval=$?
Logger "${FUNCNAME[0]} ended for [$caller_name] with status $retval." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
return $retval
}
function RunLocalCommand { function RunLocalCommand {
local command="${1}" # Command to run local command="${1}" # Command to run
local hard_max_time="${2}" # Max time to wait for command to compleet local hard_max_time="${2}" # Max time to wait for command to compleet
@ -891,24 +945,38 @@ function RunRemoteCommand {
function RunBeforeHook { function RunBeforeHook {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local pids=
if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then if [ "$LOCAL_RUN_BEFORE_CMD" != "" ]; then
RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE RunLocalCommand "$LOCAL_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE &
pids="$!"
fi fi
if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then if [ "$REMOTE_RUN_BEFORE_CMD" != "" ]; then
RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE RunRemoteCommand "$REMOTE_RUN_BEFORE_CMD" $MAX_EXEC_TIME_PER_CMD_BEFORE &
pids="$pids;$!"
fi
if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false
fi fi
} }
function RunAfterHook { function RunAfterHook {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local pids
if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then if [ "$LOCAL_RUN_AFTER_CMD" != "" ]; then
RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER RunLocalCommand "$LOCAL_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER &
pids="$!"
fi fi
if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then if [ "$REMOTE_RUN_AFTER_CMD" != "" ]; then
RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER RunRemoteCommand "$REMOTE_RUN_AFTER_CMD" $MAX_EXEC_TIME_PER_CMD_AFTER &
pids="$pids;$!"
fi
if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} false
fi fi
} }