From 4a9ebd0abc8dc0e272a0358eca99f86db0422616 Mon Sep 17 00:00:00 2001 From: deajan Date: Mon, 29 Aug 2016 18:26:47 +0200 Subject: [PATCH] Updated ofunctions --- dev/ofunctions.sh | 152 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 117 insertions(+), 35 deletions(-) diff --git a/dev/ofunctions.sh b/dev/ofunctions.sh index 82e5972..acf3fc2 100644 --- a/dev/ofunctions.sh +++ b/dev/ofunctions.sh @@ -1,9 +1,13 @@ #### MINIMAL-FUNCTION-SET BEGIN #### -## FUNC_BUILD=2016082606 -## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr +## FUNC_BUILD=2016082901 +## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr + +## To use in a program, define the following variables: +## PROGRAM=program-name +## INSTANCE_ID=program-instance-name +## _DEBUG=yes/no -## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode if ! type "$BASH" > /dev/null; then echo "Please run this script only with bash shell. Tested on bash >= 3.2" exit 127 @@ -16,20 +20,21 @@ export LC_ALL=C MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." # Environment variables that can be overriden by programs -_DRYRUN=0 -_SILENT=0 +_DRYRUN=false +_SILENT=false +_VERBOSE=false _LOGGER_PREFIX="date" -_LOGGER_STDERR=0 +_LOGGER_STDERR=false if [ "$KEEP_LOGGING" == "" ]; then KEEP_LOGGING=1801 fi # Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags -ERROR_ALERT=0 -WARN_ALERT=0 +ERROR_ALERT=false +WARN_ALERT=false -# Current log -CURRENT_LOG +# Log from current run +CURRENT_LOG= ## allow function call checks #__WITH_PARANOIA_DEBUG if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG @@ -40,11 +45,11 @@ fi #__WITH_PARANOIA_DEBUG if [ ! "$_DEBUG" == "yes" ]; then _DEBUG=no SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys - _VERBOSE=0 + _VERBOSE=false else SLEEP_TIME=1 trap 'TrapError ${LINENO} $?' ERR - _VERBOSE=1 + _VERBOSE=true fi SCRIPT_PID=$$ @@ -52,6 +57,10 @@ SCRIPT_PID=$$ LOCAL_USER=$(whoami) LOCAL_HOST=$(hostname) +if [ "$PROGRAM" == "" ]; then + PROGRAM="ofunctions" +fi + ## Default log file until config file is loaded if [ -w /var/log ]; then LOG_FILE="/var/log/$PROGRAM.log" @@ -94,17 +103,21 @@ function _Logger { echo -e "$lvalue" >> "$LOG_FILE" CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue" - if [ "$_LOGGER_STDERR" -eq 1 ]; then + if [ $_LOGGER_STDERR == true ]; then cat <<< "$evalue" 1>&2 - elif [ "$_SILENT" -eq 0 ]; then + elif [ "$_SILENT" == false ]; then echo -e "$svalue" fi } -# General log function with log levels +# General log function with log levels: +# CRITICAL, ERROR, WARN are colored in stdout, prefixed in stderr +# NOTICE is standard level +# VERBOSE is only sent to stdout / stderr if _VERBOSE=true +# DEBUG & PARANOIA_DEBUG are only sent if _DEBUG=yes function Logger { local value="${1}" # Sentence to log (in double quotes) - local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL + local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, VERBOSE, NOTICE, WARN, ERROR, CRITIAL if [ "$_LOGGER_PREFIX" == "time" ]; then prefix="TIME: $SECONDS - " @@ -116,19 +129,22 @@ function Logger { if [ "$level" == "CRITICAL" ]; then _Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value" - ERROR_ALERT=1 + ERROR_ALERT=true return elif [ "$level" == "ERROR" ]; then _Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value" - ERROR_ALERT=1 + ERROR_ALERT=true return elif [ "$level" == "WARN" ]; then _Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value" "$level:$value" - WARN_ALERT=1 + WARN_ALERT=true return elif [ "$level" == "NOTICE" ]; then _Logger "$prefix$value" return + elif [ "$level" == "VERBOSE" ] && [ $_VERBOSE == true ]; then + _Logger "$prefix$value" + return elif [ "$level" == "DEBUG" ]; then if [ "$_DEBUG" == "yes" ]; then _Logger "$prefix$value" @@ -165,7 +181,7 @@ function QuickLogger { __CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - if [ "$_SILENT" -eq 1 ]; then + if [ $_SILENT == true ]; then _QuickLogger "$value" "log" else _QuickLogger "$value" "stdout" @@ -262,9 +278,9 @@ function SendAlert { fi body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG" - if [ $ERROR_ALERT -eq 1 ]; then + if [ $ERROR_ALERT == true ]; then subject="Error alert for $INSTANCE_ID" - elif [ $WARN_ALERT -eq 1 ]; then + elif [ $WARN_ALERT == true ]; then subject="Warning alert for $INSTANCE_ID" else subject="Alert for $INSTANCE_ID" @@ -516,7 +532,7 @@ function TrapError { local job="$0" local line="$1" local code="${2:-1}" - if [ $_SILENT -eq 0 ]; then + if [ $_SILENT == false ]; then echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}" fi } @@ -542,7 +558,7 @@ function LoadConfigFile { } function Spinner { - if [ $_SILENT -eq 1 ]; then + if [ $_SILENT == true ]; then return 0 fi @@ -583,6 +599,7 @@ function joinString { # Time control function for background processes, suitable for multiple synchronous processes # Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 # Warning: Don't imbricate this function into another run if you plan to use the global variable output + function WaitForTaskCompletion { local pids="${1}" # pids to wait for, separated by semi-colon local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0. @@ -603,9 +620,13 @@ function WaitForTaskCompletion { local retval=0 # return value of monitored pid process local errorcount=0 # Number of pids that finished with errors + local pid # Current pid working on local pidCount # number of given pids local pidState # State of the process + local pidsArray # Array of currently running pids + local newPidsArray # New array of currently running pids + IFS=';' read -a pidsArray <<< "$pids" pidCount=${#pidsArray[@]} @@ -677,6 +698,7 @@ function WaitForTaskCompletion { done pidsArray=("${newPidsArray[@]}") + # Trivial wait time for bash to not eat up all CPU sleep $SLEEP_TIME done @@ -690,6 +712,66 @@ function WaitForTaskCompletion { fi } +# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs +# Returns the number of non zero exit codes from commands +function ParallelExec { + local numberOfProcesses="${1}" # Number of simultaneous commands to run + local commandsArg="${2}" # Semi-colon separated list of commands + + local pid + local runningPids=0 + local counter=0 + local commandsArray + local pidsArray + local newPidsArray + local retval + local retvalAll=0 + local pidState + local commandsArrayPid + + IFS=';' read -r -a commandsArray <<< "$commandsArg" + + Logger "Runnning ${#commandsArray[@]} commands in $numberOfProcesses simultaneous processes." "DEBUG" + + while [ $counter -lt "${#commandsArray[@]}" ] || [ ${#pidsArray[@]} -gt 0 ]; do + + while [ $counter -lt "${#commandsArray[@]}" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do + Logger "Running command [${commandsArray[$counter]}]." "DEBUG" + eval "${commandsArray[$counter]}" & + pid=$! + pidsArray+=($pid) + commandsArrayPid[$pid]="${commandsArray[$counter]}" + counter=$((counter+1)) + done + + + newPidsArray=() + for pid in "${pidsArray[@]}"; do + # Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :) + if kill -0 $pid > /dev/null 2>&1; then + pidState=$(ps -p$pid -o state= 2 > /dev/null) + if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then + newPidsArray+=($pid) + fi + else + # pid is dead, get it's exit code from wait command + wait $pid + retval=$? + if [ $retval -ne 0 ]; then + Logger "Command [${commandsArrayPid[$pid]}] failed with exit code [$retval]." "ERROR" + retvalAll=$((retvalAll+1)) + fi + fi + done + pidsArray=("${newPidsArray[@]}") + + # Trivial wait time for bash to not eat up all CPU + sleep $SLEEP_TIME + done + + return $retvalAll +} + function CleanUp { __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG @@ -877,7 +959,7 @@ function RunLocalCommand { local hard_max_time="${2}" # Max time to wait for command to compleet __CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG - if [ $_DRYRUN -ne 0 ]; then + if [ $_DRYRUN == true ]; then Logger "Dryrun: Local command [$command] not run." "NOTICE" return 0 fi @@ -892,7 +974,7 @@ function RunLocalCommand { Logger "Command failed." "ERROR" fi - if [ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]; then + if [ $_VERBOSE == true ] || [ $retval -ne 0 ]; then Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" fi @@ -910,7 +992,7 @@ function RunRemoteCommand { CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - if [ $_DRYRUN -ne 0 ]; then + if [ $_DRYRUN == true ]; then Logger "Dryrun: Local command [$command] not run." "NOTICE" return 0 fi @@ -927,7 +1009,7 @@ function RunRemoteCommand { Logger "Command failed." "ERROR" fi - if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ] && ([ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]) + if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ] && ([ $_VERBOSE == true ] || [ $retval -ne 0 ]) then Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE" fi @@ -1001,7 +1083,7 @@ function CheckConnectivity3rdPartyHosts { if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then - remote_3rd_party_success=0 + remote_3rd_party_success=false for i in $REMOTE_3RD_PARTY_HOSTS do eval "$PING_CMD $i > /dev/null 2>&1" & @@ -1009,11 +1091,11 @@ function CheckConnectivity3rdPartyHosts { if [ $? != 0 ]; then Logger "Cannot ping 3rd party host $i" "NOTICE" else - remote_3rd_party_success=1 + remote_3rd_party_success=true fi done - if [ $remote_3rd_party_success -ne 1 ]; then + if [ $remote_3rd_party_success == false ]; then Logger "No remote 3rd party host responded to ping. No internet ?" "ERROR" return 1 else @@ -1044,14 +1126,14 @@ function __CheckArguments { # In order to avoid this, we need to iterate over ${4} and count local iterate=4 - local fetchArguments=1 + local fetchArguments=true local argList="" local countedArguments - while [ $fetchArguments -eq 1 ]; do + while [ $fetchArguments == true ]; do cmd='argument=${'$iterate'}' eval $cmd if [ "$argument" = "" ]; then - fetchArguments=0 + fetchArguments=false else argList="$arg_list [Argument $(($iterate-3)): $argument]" iterate=$(($iterate+1)) @@ -1193,7 +1275,7 @@ function PreInit { ## Set rsync default arguments RSYNC_ARGS="-rltD" - if [ "$_DRYRUN" -eq 1 ]; then + if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" DRY_WARNING="/!\ DRY RUN" else